Single node performance, Free benchmark, Broadwell
Performance in millions of particle-timesteps / second

Nparticles	CPU (mpi,hyper)	Kokkos/OMP (mpi,hyper,thread)	Kokkos/serial (mpi,hyper)
32000	376.1 (32,1)	251 (16,2,2)	330.9 (32,1)
64000	569 (32,1)	390.4 (32,2,2)	489.5 (32,1)
128000	751 (32,2)	566.7 (32,2,2)	639.3 (32,1)
256000	956.4 (64,2)	693.1 (32,2,2)	824 (64,2)
512000	1020 (72,2)	787.7 (36,2,2)	875 (72,2)
1024000	522.6 (36,2)	481.7 (18,2,2)	490.6 (36,2)
2048000	452.5 (36,1)	425.8 (18,2,4)	434.3 (36,1)
4096000	419.4 (72,2)	398.2 (36,2,2)	397.4 (72,2)
8192000	331.4 (72,2)	313.4 (18,2,4)	315.5 (72,2)
16384000	287.2 (72,2)	265 (36,2,2)	269.2 (72,2)
32768000	265.4 (72,2)	243.9 (36,2,2)	246.4 (72,2)
65536000	253.7 (72,2)	232.2 (36,2,2)	235.5 (72,2)
131072000	246.7 (72,2)	226.2 (36,2,2)	229.3 (72,2)

Run commands and logfile links for column CPU

32000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 16 -v y 10 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=32K.node=1.mpi=32.hyper=1
64000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 16 -v y 20 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=64K.node=1.mpi=32.hyper=1
128000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 32 -v y 20 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=128K.node=1.mpi=32.hyper=2
256000	mpiexec -np 64 -npernode 64 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 32 -v y 20 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=256K.node=1.mpi=64.hyper=2
512000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 32 -v y 40 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=512K.node=1.mpi=72.hyper=2
1024000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 64 -v y 40 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=1M.node=1.mpi=36.hyper=2
2048000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 64 -v y 40 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=2M.node=1.mpi=36.hyper=1
4096000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 64 -v y 80 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=4M.node=1.mpi=72.hyper=2
8192000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 128 -v y 80 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=8M.node=1.mpi=72.hyper=2
16384000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 128 -v y 80 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=16M.node=1.mpi=72.hyper=2
32768000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 128 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=32M.node=1.mpi=72.hyper=2
65536000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=64M.node=1.mpi=72.hyper=2
131072000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_cpu -v x 256 -v y 160 -v z 320 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=cpu.kind=node.size=128M.node=1.mpi=72.hyper=2

Run commands and logfile links for column Kokkos/OMP

32000	mpiexec -np 16 -npernode 16 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 16 -v y 10 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=32K.node=1.mpi=16.thread=2.hyper=2
64000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 16 -v y 20 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=64K.node=1.mpi=32.thread=2.hyper=2
128000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 32 -v y 20 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=128K.node=1.mpi=32.thread=2.hyper=2
256000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 32 -v y 20 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=256K.node=1.mpi=32.thread=2.hyper=2
512000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 32 -v y 40 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=512K.node=1.mpi=36.thread=2.hyper=2
1024000	mpiexec -np 18 -npernode 18 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=1M.node=1.mpi=18.thread=2.hyper=2
2048000	mpiexec -np 18 -npernode 18 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 4 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=2M.node=1.mpi=18.thread=4.hyper=2
4096000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 80 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=4M.node=1.mpi=36.thread=2.hyper=2
8192000	mpiexec -np 18 -npernode 18 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 4 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=8M.node=1.mpi=18.thread=4.hyper=2
16384000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=16M.node=1.mpi=36.thread=2.hyper=2
32768000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=32M.node=1.mpi=36.thread=2.hyper=2
65536000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=64M.node=1.mpi=36.thread=2.hyper=2
131072000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to socket ./spa_serrano_kokkos_omp -sf kk -k on t 2 -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 320 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_omp.kind=node.size=128M.node=1.mpi=36.thread=2.hyper=2

Run commands and logfile links for column Kokkos/serial

32000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 16 -v y 10 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=32K.node=1.mpi=32.hyper=1
64000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 16 -v y 20 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=64K.node=1.mpi=32.hyper=1
128000	mpiexec -np 32 -npernode 32 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 32 -v y 20 -v z 20 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=128K.node=1.mpi=32.hyper=1
256000	mpiexec -np 64 -npernode 64 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 32 -v y 20 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=256K.node=1.mpi=64.hyper=2
512000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 32 -v y 40 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=512K.node=1.mpi=72.hyper=2
1024000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 40 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=1M.node=1.mpi=36.hyper=2
2048000	mpiexec -np 36 -npernode 36 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 40 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=2M.node=1.mpi=36.hyper=1
4096000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 64 -v y 80 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=4M.node=1.mpi=72.hyper=2
8192000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 80 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=8M.node=1.mpi=72.hyper=2
16384000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 80 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=16M.node=1.mpi=72.hyper=2
32768000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 128 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=32M.node=1.mpi=72.hyper=2
65536000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 160 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=64M.node=1.mpi=72.hyper=2
131072000	mpiexec -np 72 -npernode 72 --oversubscribe --bind-to core ./spa_serrano_kokkos_serial -sf kk -k on -pk kokkos reduction parallel/reduce comm classic -v x 256 -v y 160 -v z 320 -v t 100 -in in.free.steps -log log.sparta.date=23Dec17.model=free.machine=serrano.pkg=kokkos_serial.kind=node.size=128M.node=1.mpi=72.hyper=2