Weak scaling performance, Sphere benchmark, KNL, 1M particles/node
Performance in millions of particle-timesteps / second / node

Nodes	CPU/KNL (mpi,hyper)	Kokkos/KNL (mpi,thread,hyper)	Kokkos/serial/KNL (mpi,hyper)
1	287 (128,2)	293 (64,4,4)	259.4 (128,2)
2	230.8 (64,1)	264.3 (32,8,4)	214.8 (64,1)
4	199.9 (64,1)	238.8 (32,8,4)	186.4 (64,1)
8	176.6 (128,2)	202.8 (32,8,4)	166 (128,2)
16	175.5 (128,2)	184.8 (32,8,4)	161.8 (128,2)
32	156.1 (128,2)	169.4 (64,4,4)	153.6 (128,2)
64	137.9 (128,2)	143.6 (32,8,4)	134.9 (128,2)

Run commands and logfile links for column CPU/KNL

1	srun -n 128 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_knl -v x 64 -v y 40 -v z 40 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=cpu_knl.kind=weak.size=1M.node=1.mpi=128.hyper=2
2	srun -n 128 -C knl --ntasks-per-node 64 --cpu_bind=rank -c 4 ./spa_mutrino_knl -v x 64 -v y 40 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=cpu_knl.kind=weak.size=1M.node=2.mpi=64.hyper=1
4	srun -n 256 -C knl --ntasks-per-node 64 --cpu_bind=rank -c 4 ./spa_mutrino_knl -v x 64 -v y 80 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=cpu_knl.kind=weak.size=1M.node=4.mpi=64.hyper=1
8	srun -n 1024 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_knl -v x 128 -v y 80 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=cpu_knl.kind=weak.size=1M.node=8.mpi=128.hyper=2
16	srun -n 2048 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_knl -v x 128 -v y 80 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=cpu_knl.kind=weak.size=1M.node=16.mpi=128.hyper=2
32	srun -n 4096 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_knl -v x 128 -v y 160 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=cpu_knl.kind=weak.size=1M.node=32.mpi=128.hyper=2
64	srun -n 8192 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_knl -v x 256 -v y 160 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=cpu_knl.kind=weak.size=1M.node=64.mpi=128.hyper=2

Run commands and logfile links for column Kokkos/KNL

1	setenv OMP_NUM_THREADS 4; srun -n 64 -C knl --ntasks-per-node 64 --cpu_bind=cores -c 4 ./spa_mutrino_kokkos_knl -sf kk -k on t 4 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 40 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_knl.kind=weak.size=1M.node=1.mpi=64.thread=4.hyper=4
2	setenv OMP_NUM_THREADS 8; srun -n 64 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./spa_mutrino_kokkos_knl -sf kk -k on t 8 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_knl.kind=weak.size=1M.node=2.mpi=32.thread=8.hyper=4
4	setenv OMP_NUM_THREADS 8; srun -n 128 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./spa_mutrino_kokkos_knl -sf kk -k on t 8 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 80 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_knl.kind=weak.size=1M.node=4.mpi=32.thread=8.hyper=4
8	setenv OMP_NUM_THREADS 8; srun -n 256 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./spa_mutrino_kokkos_knl -sf kk -k on t 8 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_knl.kind=weak.size=1M.node=8.mpi=32.thread=8.hyper=4
16	setenv OMP_NUM_THREADS 8; srun -n 512 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./spa_mutrino_kokkos_knl -sf kk -k on t 8 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_knl.kind=weak.size=1M.node=16.mpi=32.thread=8.hyper=4
32	setenv OMP_NUM_THREADS 4; srun -n 2048 -C knl --ntasks-per-node 64 --cpu_bind=cores -c 4 ./spa_mutrino_kokkos_knl -sf kk -k on t 4 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_knl.kind=weak.size=1M.node=32.mpi=64.thread=4.hyper=4
64	setenv OMP_NUM_THREADS 8; srun -n 2048 -C knl --ntasks-per-node 32 --cpu_bind=cores -c 8 ./spa_mutrino_kokkos_knl -sf kk -k on t 8 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_knl.kind=weak.size=1M.node=64.mpi=32.thread=8.hyper=4

Run commands and logfile links for column Kokkos/serial/KNL

1	srun -n 128 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 40 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=1M.node=1.mpi=128.hyper=2
2	srun -n 128 -C knl --ntasks-per-node 64 --cpu_bind=rank -c 4 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=1M.node=2.mpi=64.hyper=1
4	srun -n 256 -C knl --ntasks-per-node 64 --cpu_bind=rank -c 4 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 80 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=1M.node=4.mpi=64.hyper=1
8	srun -n 1024 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 80 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=1M.node=8.mpi=128.hyper=2
16	srun -n 2048 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=1M.node=16.mpi=128.hyper=2
32	srun -n 4096 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=1M.node=32.mpi=128.hyper=2
64	srun -n 8192 -C knl --ntasks-per-node 128 --cpu_bind=threads -c 2 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.sphere.steps -log log.sparta.date=23Dec17.model=sphere.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=1M.node=64.mpi=128.hyper=2