Weak scaling performance, Collide benchmark, 16M particles/node
Performance in millions of particle-timesteps / second / node

Nodes	SandyBridge	Haswell	Broadwell	KNL	K80-1	P100-1
1	68.95 (Kokkos/serial,mpi=16)	163.4 (Kokkos/serial,mpi=64,hyper=2)	176.8 (Kokkos/serial,mpi=72,hyper=2)	282.7 (Kokkos/serial/KNL,mpi=256,hyper=4)	151.6 (Kokkos/Cuda,mpi=2)	302.3 (Kokkos/Cuda,mpi=1)
2	68.32 (Kokkos/serial,mpi=16)	160.9 (Kokkos/serial,mpi=64,hyper=2)	176 (Kokkos/serial,mpi=72,hyper=2)	270.3 (Kokkos/serial/KNL,mpi=256,hyper=4)	140.1 (Kokkos/Cuda,mpi=2)	268.7 (Kokkos/Cuda,mpi=1)
4	70.3 (Kokkos/serial,mpi=16)	160.4 (Kokkos/serial,mpi=64,hyper=2)	175.5 (Kokkos/serial,mpi=72,hyper=2)	268.8 (Kokkos/serial/KNL,mpi=256,hyper=4)	141.1 (Kokkos/Cuda,mpi=2)	256.3 (Kokkos/Cuda,mpi=1)
8	69.07 (CPU,mpi=16)	157.1 (Kokkos/serial,mpi=64,hyper=2)	171.7 (Kokkos/serial,mpi=72,hyper=2)	268.5 (Kokkos/serial/KNL,mpi=256,hyper=4)	128.7 (Kokkos/Cuda,mpi=2)	243.6 (Kokkos/Cuda,mpi=1)
16	68.26 (CPU,mpi=16)	155.2 (Kokkos/serial,mpi=64,hyper=2)	172.4 (Kokkos/serial,mpi=72,hyper=2)	266.6 (Kokkos/serial/KNL,mpi=256,hyper=4)	None	None
32	67.6 (CPU,mpi=16)	154.3 (Kokkos/serial,mpi=64,hyper=2)	169.5 (Kokkos/serial,mpi=72,hyper=2)	262.6 (Kokkos/serial/KNL,mpi=256,hyper=4)	None	None
64	61.26 (Kokkos/serial,mpi=16)	152.1 (Kokkos/serial,mpi=64,hyper=2)	166.9 (Kokkos/serial,mpi=72,hyper=2)	260.8 (Kokkos/serial/KNL,mpi=256,hyper=4)	None	None

Run commands and logfile links for column SandyBridge

1	mpirun -n 16 -N 16 --bind-to core spa_chama_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=chama.pkg=kokkos_serial.kind=weak.size=16M.node=1.mpi=16
2	mpirun -n 32 -N 16 --bind-to core spa_chama_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=chama.pkg=kokkos_serial.kind=weak.size=16M.node=2.mpi=16
4	mpirun -n 64 -N 16 --bind-to core spa_chama_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=chama.pkg=kokkos_serial.kind=weak.size=16M.node=4.mpi=16
8	mpirun -n 128 -N 16 --bind-to core spa_chama_cpu -v x 256 -v y 160 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=chama.pkg=cpu.kind=weak.size=16M.node=8.mpi=16
16	mpirun -n 256 -N 16 --bind-to core spa_chama_cpu -v x 256 -v y 320 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=chama.pkg=cpu.kind=weak.size=16M.node=16.mpi=16
32	mpirun -n 512 -N 16 --bind-to core spa_chama_cpu -v x 512 -v y 320 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=chama.pkg=cpu.kind=weak.size=16M.node=32.mpi=16
64	mpirun -n 1024 -N 16 --bind-to core spa_chama_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 512 -v y 320 -v z 640 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=chama.pkg=kokkos_serial.kind=weak.size=16M.node=64.mpi=16

Run commands and logfile links for column Haswell

1	srun -n 64 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=16M.node=1.mpi=64.hyper=2
2	srun -n 128 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=16M.node=2.mpi=64.hyper=2
4	srun -n 256 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=16M.node=4.mpi=64.hyper=2
8	srun -n 512 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=16M.node=8.mpi=64.hyper=2
16	srun -n 1024 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 320 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=16M.node=16.mpi=64.hyper=2
32	srun -n 2048 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 512 -v y 320 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=16M.node=32.mpi=64.hyper=2
64	srun -n 4096 -C haswell --ntasks-per-node 64 --cpu_bind=rank -c 1 ./spa_mutrino_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 512 -v y 320 -v z 640 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial.kind=weak.size=16M.node=64.mpi=64.hyper=2

Run commands and logfile links for column Broadwell

1	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=serrano.pkg=kokkos_serial.kind=weak.size=16M.node=1.mpi=72.hyper=2
2	mpiexec -np 144 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=serrano.pkg=kokkos_serial.kind=weak.size=16M.node=2.mpi=72.hyper=2
4	mpiexec -np 288 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=serrano.pkg=kokkos_serial.kind=weak.size=16M.node=4.mpi=72.hyper=2
8	mpiexec -np 576 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=serrano.pkg=kokkos_serial.kind=weak.size=16M.node=8.mpi=72.hyper=2
16	mpiexec -np 1152 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 320 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=serrano.pkg=kokkos_serial.kind=weak.size=16M.node=16.mpi=72.hyper=2
32	mpiexec -np 2304 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 512 -v y 320 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=serrano.pkg=kokkos_serial.kind=weak.size=16M.node=32.mpi=72.hyper=2
64	mpiexec -np 4608 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 512 -v y 320 -v z 640 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=serrano.pkg=kokkos_serial.kind=weak.size=16M.node=64.mpi=72.hyper=2

Run commands and logfile links for column KNL

1	srun -n 256 -C knl --ntasks-per-node 256 --cpu_bind=threads -c 1 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=16M.node=1.mpi=256.hyper=4
2	srun -n 512 -C knl --ntasks-per-node 256 --cpu_bind=threads -c 1 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=16M.node=2.mpi=256.hyper=4
4	srun -n 1024 -C knl --ntasks-per-node 256 --cpu_bind=threads -c 1 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=16M.node=4.mpi=256.hyper=4
8	srun -n 2048 -C knl --ntasks-per-node 256 --cpu_bind=threads -c 1 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=16M.node=8.mpi=256.hyper=4
16	srun -n 4096 -C knl --ntasks-per-node 256 --cpu_bind=threads -c 1 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 320 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=16M.node=16.mpi=256.hyper=4
32	srun -n 8192 -C knl --ntasks-per-node 256 --cpu_bind=threads -c 1 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 512 -v y 320 -v z 320 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=16M.node=32.mpi=256.hyper=4
64	srun -n 16384 -C knl --ntasks-per-node 256 --cpu_bind=threads -c 1 ./spa_mutrino_kokkos_serial_knl -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 512 -v y 320 -v z 640 -v t 100 -in in.collide.steps -log log.sparta.date=23Dec17.model=collide.machine=mutrino.pkg=kokkos_serial_knl.kind=weak.size=16M.node=64.mpi=256.hyper=4

Run commands and logfile links for column K80-1

1	mpirun -np 2 --npersocket 1 --bind-to core spa_ride80_kokkos_cuda -sf kk -k on g 2 -pk kokkos reduction atomic comm threaded -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.gpu.steps -log log.sparta.date=23Dec17.model=collide.machine=ride80.pkg=kokkos_cuda.kind=weak.size=16M.node=1.mpi=2.gpu=2
2	mpirun -np 4 --npersocket 1 --bind-to core spa_ride80_kokkos_cuda -sf kk -k on g 2 -pk kokkos reduction atomic comm threaded -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.gpu.steps -log log.sparta.date=23Dec17.model=collide.machine=ride80.pkg=kokkos_cuda.kind=weak.size=16M.node=2.mpi=2.gpu=2
4	mpirun -np 8 --npersocket 1 --bind-to core spa_ride80_kokkos_cuda -sf kk -k on g 2 -pk kokkos reduction atomic comm threaded -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.gpu.steps -log log.sparta.date=23Dec17.model=collide.machine=ride80.pkg=kokkos_cuda.kind=weak.size=16M.node=4.mpi=2.gpu=2
8	mpirun -np 16 --npersocket 1 --bind-to core spa_ride80_kokkos_cuda -sf kk -k on g 2 -pk kokkos reduction atomic comm threaded -v x 256 -v y 160 -v z 320 -v t 100 -in in.collide.gpu.steps -log log.sparta.date=23Dec17.model=collide.machine=ride80.pkg=kokkos_cuda.kind=weak.size=16M.node=8.mpi=2.gpu=2
16	None
32	None
64	None

Run commands and logfile links for column P100-1

1	mpirun -np 1 --npernode 1 --bind-to core spa_ride100_kokkos_cuda -sf kk -k on g 1 -pk kokkos reduction atomic comm threaded -v x 128 -v y 80 -v z 160 -v t 100 -in in.collide.gpu.steps -log log.sparta.date=23Dec17.model=collide.machine=ride100.pkg=kokkos_cuda.kind=weak.size=16M.node=1.mpi=1.gpu=1
2	mpirun -np 2 --npernode 1 --bind-to core spa_ride100_kokkos_cuda -sf kk -k on g 1 -pk kokkos reduction atomic comm threaded -v x 128 -v y 160 -v z 160 -v t 100 -in in.collide.gpu.steps -log log.sparta.date=23Dec17.model=collide.machine=ride100.pkg=kokkos_cuda.kind=weak.size=16M.node=2.mpi=1.gpu=1
4	mpirun -np 4 --npernode 1 --bind-to core spa_ride100_kokkos_cuda -sf kk -k on g 1 -pk kokkos reduction atomic comm threaded -v x 256 -v y 160 -v z 160 -v t 100 -in in.collide.gpu.steps -log log.sparta.date=23Dec17.model=collide.machine=ride100.pkg=kokkos_cuda.kind=weak.size=16M.node=4.mpi=1.gpu=1
8	mpirun -np 8 --npernode 1 --bind-to core spa_ride100_kokkos_cuda -sf kk -k on g 1 -pk kokkos reduction atomic comm threaded -v x 256 -v y 160 -v z 320 -v t 100 -in in.collide.gpu.steps -log log.sparta.date=23Dec17.model=collide.machine=ride100.pkg=kokkos_cuda.kind=weak.size=16M.node=8.mpi=1.gpu=1
16	None
32	None
64	None