device: NVIDIA GB200
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
dtype: torch.bfloat16
torch: 2.12.0a0+5aff3928d8.nv26.05
transformer_engine: 2.15.0+42b84005
nvidia-cudnn-frontend: 1.23.0
nvidia-cutlass-dsl: 4.4.1
timing mode: CUDA graph replay

activation: swiglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=8192, fused C input N=None, activation output N=4096
fused cuDNN avg: 1.498 ms
unfused TE avg:  1.907 ms
speedup vs TE:   1.27x
fused samples:   [1.342, 1.4, 1.568, 1.575, 1.603]
TE samples:      [2.121, 1.932, 1.801, 1.82, 1.861]

activation: dswiglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=8192, activation output N=8192
fused cuDNN avg: 0.850 ms
unfused TE avg:  1.734 ms
speedup vs TE:   2.04x
fused samples:   [0.777, 0.782, 0.799, 0.913, 0.98]
TE samples:      [2.188, 1.634, 1.619, 1.623, 1.608]

activation: srelu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=None, activation output N=4096
fused cuDNN avg: 0.749 ms
unfused TE avg:  1.018 ms
speedup vs TE:   1.36x
fused samples:   [0.724, 0.708, 0.705, 0.798, 0.812]
TE samples:      [1.109, 1.111, 0.969, 0.954, 0.947]

activation: dsrelu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=4096, activation output N=4096
fused cuDNN avg: 0.720 ms
unfused TE avg:  1.329 ms
speedup vs TE:   1.85x
fused samples:   [0.716, 0.717, 0.705, 0.706, 0.755]
TE samples:      [1.47, 1.41, 1.296, 1.238, 1.229]

activation: geglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=8192, fused C input N=None, activation output N=4096
fused cuDNN avg: 1.500 ms
unfused TE avg:  1.864 ms
speedup vs TE:   1.24x
fused samples:   [1.356, 1.434, 1.558, 1.606, 1.546]
TE samples:      [2.025, 1.824, 1.808, 1.829, 1.833]

activation: dgeglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=8192, activation output N=8192
fused cuDNN avg: 0.870 ms
unfused TE avg:  1.837 ms
speedup vs TE:   2.11x
fused samples:   [0.783, 0.801, 0.888, 0.927, 0.953]
TE samples:      [2.473, 1.757, 1.658, 1.653, 1.642]

summary:
activation   cudnn_ms      te_ms   te/cudnn
swiglu          1.498      1.907      1.27x
dswiglu         0.850      1.734      2.04x
srelu           0.749      1.018      1.36x
dsrelu          0.720      1.329      1.85x
geglu           1.500      1.864      1.24x
dgeglu          0.870      1.837      2.11x
