--- /home/dev/tpshard-why-scratch/.prof/off/rank0.1789307140198018070.pt.trace.json.gz steps=199
AR stream=279    n=  6400 ( 32.16/step)      3788us/step  p50=    117 p90=    123 max=    179
AR stream=278    n=  6400 ( 32.16/step)      3739us/step  p50=    116 p90=    123 max=    181
AG stream=7      n=   199 (  1.00/step)      2358us/step  p50=   2463 p90=   2607 max=   2842
AR stream=7      n=   200 (  1.01/step)       644us/step  p50=    669 p90=    920 max=  19857
non-nccl kernel time: 17611us/step    nccl: 10529us/step

--- /home/dev/tpshard-why-scratch/.prof/off/rank1.1789307140711245338.pt.trace.json.gz steps=199
AG stream=7      n=   200 (  1.01/step)      3732us/step  p50=   2491 p90=   2646 max= 245535
AR stream=279    n=  6400 ( 32.16/step)      3576us/step  p50=    110 p90=    114 max=    174
AR stream=278    n=  6400 ( 32.16/step)      3563us/step  p50=    110 p90=    114 max=    175
AR stream=7      n=   200 (  1.01/step)       113us/step  p50=    110 p90=    116 max=    348
non-nccl kernel time: 12497us/step    nccl: 10984us/step

--- /home/dev/tpshard-why-scratch/.prof/on/rank0.1789307632901214918.pt.trace.json.gz steps=166
AR stream=7      n=   166 (  1.00/step)      6741us/step  p50=  11665 p90=  12681 max=  19829
AR stream=279    n=  5312 ( 32.00/step)      3798us/step  p50=    118 p90=    125 max=    211
AR stream=278    n=  5312 ( 32.00/step)      3692us/step  p50=    115 p90=    120 max=   1267
AG stream=7      n=   166 (  1.00/step)      2290us/step  p50=   2401 p90=   2468 max=   2600
AR stream=2370   n=   415 (  2.50/step)       825us/step  p50=    124 p90=   1083 max=   3977
AR stream=2371   n=   415 (  2.50/step)       296us/step  p50=    118 p90=    124 max=    165
non-nccl kernel time: 16611us/step    nccl: 17641us/step

--- /home/dev/tpshard-why-scratch/.prof/on/rank1.1789307664629770964.pt.trace.json.gz steps=166
AR stream=2370   n=   415 (  2.50/step)     22795us/step  p50=    111 p90=    120 max=3737091
AG stream=7      n=   166 (  1.00/step)      7377us/step  p50=   2914 p90=   3071 max= 769799
AR stream=279    n=  5312 ( 32.00/step)      3577us/step  p50=    111 p90=    115 max=    190
AR stream=278    n=  5312 ( 32.00/step)      3558us/step  p50=    110 p90=    114 max=    181
AR stream=2371   n=   415 (  2.50/step)       279us/step  p50=    110 p90=    117 max=    156
AR stream=7      n=   166 (  1.00/step)       112us/step  p50=    111 p90=    117 max=    128
non-nccl kernel time: 16373us/step    nccl: 37697us/step
