Strong scaling performance, Free benchmark, Haswell, 64M particles
Performance in millions of particle-timesteps / second / node

Nodes	CPU (mpi,hyper)	Kokkos/OMP (mpi,hyper,thread)	Kokkos/serial (mpi,hyper)
1	237.8 (64,2)	218.1 (32,2,2)	220.9 (64,2)
2	245.9 (64,2)	230.8 (32,2,2)	229.1 (64,2)
4	265.1 (64,2)	249.5 (32,2,2)	247.8 (64,2)
8	301.8 (64,2)	286.3 (32,2,2)	280.4 (64,2)
16	362.1 (64,2)	350 (16,2,4)	335.2 (64,2)
32	384.3 (64,2)	396.3 (32,2,2)	376.9 (64,2)
64	386.8 (32,1)	380.1 (32,2,2)	372.1 (32,1)

Run commands and logfile links for column CPU

1	srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=cpu.kind=strong.size=64M.node=1.mpi=64.hyper=2
2	srun -n 128 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=cpu.kind=strong.size=64M.node=2.mpi=64.hyper=2
4	srun -n 256 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=cpu.kind=strong.size=64M.node=4.mpi=64.hyper=2
8	srun -n 512 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=cpu.kind=strong.size=64M.node=8.mpi=64.hyper=2
16	srun -n 1024 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=cpu.kind=strong.size=64M.node=16.mpi=64.hyper=2
32	srun -n 2048 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=cpu.kind=strong.size=64M.node=32.mpi=64.hyper=2
64	srun -n 2048 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./spa_mutrino_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=cpu.kind=strong.size=64M.node=64.mpi=32.hyper=1

Run commands and logfile links for column Kokkos/OMP

1	setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_omp.kind=strong.size=64M.node=1.mpi=32.thread=2.hyper=2
2	setenv OMP_NUM_THREADS 2; srun -n 64 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_omp.kind=strong.size=64M.node=2.mpi=32.thread=2.hyper=2
4	setenv OMP_NUM_THREADS 2; srun -n 128 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_omp.kind=strong.size=64M.node=4.mpi=32.thread=2.hyper=2
8	setenv OMP_NUM_THREADS 2; srun -n 256 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_omp.kind=strong.size=64M.node=8.mpi=32.thread=2.hyper=2
16	setenv OMP_NUM_THREADS 4; srun -n 256 -C haswell --ntasks-per-node 16 --cpu_bind=cores -c 4 ./spa_mutrino_kokkos_omp -sf kk -k on t 4 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_omp.kind=strong.size=64M.node=16.mpi=16.thread=4.hyper=2
32	setenv OMP_NUM_THREADS 2; srun -n 1024 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_omp.kind=strong.size=64M.node=32.mpi=32.thread=2.hyper=2
64	setenv OMP_NUM_THREADS 2; srun -n 2048 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_omp.kind=strong.size=64M.node=64.mpi=32.thread=2.hyper=2

Run commands and logfile links for column Kokkos/serial

1	srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_serial.kind=strong.size=64M.node=1.mpi=64.hyper=2
2	srun -n 128 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_serial.kind=strong.size=64M.node=2.mpi=64.hyper=2
4	srun -n 256 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_serial.kind=strong.size=64M.node=4.mpi=64.hyper=2
8	srun -n 512 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_serial.kind=strong.size=64M.node=8.mpi=64.hyper=2
16	srun -n 1024 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_serial.kind=strong.size=64M.node=16.mpi=64.hyper=2
32	srun -n 2048 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_serial.kind=strong.size=64M.node=32.mpi=64.hyper=2
64	srun -n 2048 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=mutrino.pkg=kokkos_serial.kind=strong.size=64M.node=64.mpi=32.hyper=1