Weak scaling performance, Collide benchmark, Haswell, 1M particles/node
Performance in millions of particle-timesteps / second / node

Nodes	CPU (mpi,hyper)	Kokkos/OMP (mpi,hyper,thread)	Kokkos/serial (mpi,hyper)
1	289.2 (64,2)	253.5 (32,2,2)	257.3 (64,2)
2	276.4 (64,2)	252 (32,2,2)	250.6 (64,2)
4	271 (64,2)	246.3 (32,2,2)	243.2 (64,2)
8	258 (64,2)	240.4 (32,2,2)	224.7 (32,1)
16	250.5 (64,2)	234.7 (32,2,2)	212.2 (64,2)
32	242.2 (64,2)	226.7 (32,2,2)	209.4 (64,2)
64	226.8 (32,1)	222.8 (32,2,2)	202.3 (32,1)

Run commands and logfile links for column CPU

1	srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 64 -v y 40 -v z 40 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=cpu.kind=weak.size=1M.node=1.mpi=64.hyper=2
2	srun -n 128 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 64 -v y 40 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=cpu.kind=weak.size=1M.node=2.mpi=64.hyper=2
4	srun -n 256 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 64 -v y 80 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=cpu.kind=weak.size=1M.node=4.mpi=64.hyper=2
8	srun -n 512 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 128 -v y 80 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=cpu.kind=weak.size=1M.node=8.mpi=64.hyper=2
16	srun -n 1024 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=cpu.kind=weak.size=1M.node=16.mpi=64.hyper=2
32	srun -n 2048 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_cpu -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=cpu.kind=weak.size=1M.node=32.mpi=64.hyper=2
64	srun -n 2048 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./spa_mutrino_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=cpu.kind=weak.size=1M.node=64.mpi=32.hyper=1

Run commands and logfile links for column Kokkos/OMP

1	setenv OMP_NUM_THREADS 2; srun -n 32 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 40 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_omp.kind=weak.size=1M.node=1.mpi=32.thread=2.hyper=2
2	setenv OMP_NUM_THREADS 2; srun -n 64 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_omp.kind=weak.size=1M.node=2.mpi=32.thread=2.hyper=2
4	setenv OMP_NUM_THREADS 2; srun -n 128 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 80 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_omp.kind=weak.size=1M.node=4.mpi=32.thread=2.hyper=2
8	setenv OMP_NUM_THREADS 2; srun -n 256 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_omp.kind=weak.size=1M.node=8.mpi=32.thread=2.hyper=2
16	setenv OMP_NUM_THREADS 2; srun -n 512 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_omp.kind=weak.size=1M.node=16.mpi=32.thread=2.hyper=2
32	setenv OMP_NUM_THREADS 2; srun -n 1024 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_omp.kind=weak.size=1M.node=32.mpi=32.thread=2.hyper=2
64	setenv OMP_NUM_THREADS 2; srun -n 2048 -C haswell --ntasks-per-node 32 --cpu_bind=cores -c 2 ./spa_mutrino_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_omp.kind=weak.size=1M.node=64.mpi=32.thread=2.hyper=2

Run commands and logfile links for column Kokkos/serial

1	srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 40 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=1M.node=1.mpi=64.hyper=2
2	srun -n 128 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=1M.node=2.mpi=64.hyper=2
4	srun -n 256 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 80 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=1M.node=4.mpi=64.hyper=2
8	srun -n 256 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 80 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=1M.node=8.mpi=32.hyper=1
16	srun -n 1024 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=1M.node=16.mpi=64.hyper=2
32	srun -n 2048 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=1M.node=32.mpi=64.hyper=2
64	srun -n 2048 -C haswell --ntasks-per-node 32 --cpu_bind=rank -c 2 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=1M.node=64.mpi=32.hyper=1