# mcg on real checkpoints, and the SASS harness's null control (#1861)
#
# Run in registry.arbi.work/arbi-serve:test-latest on 206 gpu0 (RTX 4090).
#   ./.mcgdrive2.sh -- see the commands inline below.
#
# The checkpoint arms are the point: gate 2a's synthetic arm reaches every
# width, but only a real quantised tensor shows the decode over the trellis a
# converter actually emitted.  mcg's fp16 -> int8 map is a FUNCTION, so
# 'outside-allowed-set = 0' here is exact equality, not containment.

wrote /tmp/mutated.cu
--- SASS null control (must FAIL) ---
baseline: /work/tools/int8_gemm/.baseline_9940127be.cu   width: K=4   new build: mul1 only
kernels compared: 159   IDENTICAL SASS: 156
  of which exl3_i8_gemm_kernel instantiations: 135
  REGRESSION gemm<4,128,128,64,1,8,2,8,false,false,false,false,false,false>: 1392 -> 1392 instructions, 39 positional mismatches, same multiset=False
  REGRESSION gemm<4,128,128,64,1,8,2,8,false,false,false,false,true,false>: 1768 -> 1768 instructions, 211 positional mismatches, same multiset=False
  REGRESSION gemm<4,128,128,64,1,8,2,8,false,false,false,false,true,true>: 2280 -> 2280 instructions, 274 positional mismatches, same multiset=False
SASS MUL1 IDENTITY K=4: FAIL
--- checkpoint arms ---
== /tmp/ckpt_minicpm4.log
checkpoint width census: {4: 168, 6: 1}

  K=4 cb=mcg
    ckpt down_proj                     k=4608   n=1536   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=6.284e-05
      control (n reversed inside each 16-tile): outside-allowed-set=7022994 max-excursion=244  (must be > 0)
    ckpt gate_proj                     k=1536   n=4608   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=6.305e-05
      control (n reversed inside each 16-tile): outside-allowed-set=7022626 max-excursion=244  (must be > 0)
    ckpt up_proj                       k=1536   n=4608   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=6.501e-05
      control (n reversed inside each 16-tile): outside-allowed-set=7022048 max-excursion=244  (must be > 0)
    ckpt k_proj                        k=1536   n=256    outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=6.406e-05
      control (n reversed inside each 16-tile): outside-allowed-set=390038 max-excursion=221  (must be > 0)
    synthetic random trellis           k=512    n=512    outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=5.284e-05
      control (n reversed inside each 16-tile): outside-allowed-set=260304 max-excursion=225  (must be > 0)
      mainloop s30  smem= 24576 B (K=4)  rel-err vs the exact int8 product = 2.41e-04  ok
      mainloop s27  smem= 49152 B (K=4)  rel-err vs the exact int8 product = 2.41e-04  ok
      mainloop s13  smem= 73728 B (K=4)  rel-err vs the exact int8 product = 2.41e-04  ok
      mainloop s31  smem= 20480 B (K=4)  rel-err vs the exact int8 product = 2.41e-04  ok
      control (reference rolled by one n-tile): rel-err = 1.42e+00  (must be >> 1e-3)

[g2a] compared 5 tensor(s) over widths [4] x codebooks ['mcg']
GATE 2a: PASS
== /tmp/ckpt_minicpm5.log
checkpoint width census: {5: 168, 6: 1}

  K=5 cb=mcg
    ckpt down_proj                     k=4608   n=1536   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=7.118e-05
      control (n reversed inside each 16-tile): outside-allowed-set=7019260 max-excursion=232  (must be > 0)
    ckpt gate_proj                     k=1536   n=4608   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=7.070e-05
      control (n reversed inside each 16-tile): outside-allowed-set=7019390 max-excursion=246  (must be > 0)
    ckpt up_proj                       k=1536   n=4608   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=6.892e-05
      control (n reversed inside each 16-tile): outside-allowed-set=7020640 max-excursion=247  (must be > 0)
    ckpt k_proj                        k=1536   n=256    outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=6.738e-05
      control (n reversed inside each 16-tile): outside-allowed-set=390152 max-excursion=223  (must be > 0)
    synthetic random trellis           k=512    n=512    outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=5.260e-05
      control (n reversed inside each 16-tile): outside-allowed-set=260190 max-excursion=230  (must be > 0)
      mainloop s30  smem= 26624 B (K=5)  rel-err vs the exact int8 product = 2.37e-04  ok
      mainloop s27  smem= 53248 B (K=5)  rel-err vs the exact int8 product = 2.37e-04  ok
      mainloop s13  smem= 79872 B (K=5)  rel-err vs the exact int8 product = 2.37e-04  ok
      mainloop s31  smem= 21504 B (K=5)  rel-err vs the exact int8 product = 2.37e-04  ok
      control (reference rolled by one n-tile): rel-err = 1.41e+00  (must be >> 1e-3)

[g2a] compared 5 tensor(s) over widths [5] x codebooks ['mcg']
GATE 2a: PASS
== /tmp/ckpt_dflash6.log
checkpoint width census: {6: 47}

  K=6 cb=mcg
    ckpt hidden_projection             k=5120   n=256    outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=8.002e-05
      control (n reversed inside each 16-tile): outside-allowed-set=1298964 max-excursion=237  (must be > 0)
    ckpt fc                            k=25600  n=5120   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=8.425e-05
      control (n reversed inside each 16-tile): outside-allowed-set=129890514 max-excursion=237  (must be > 0)
    ckpt kernel_projection             k=5120   n=1280   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=1.044e-04
      control (n reversed inside each 16-tile): outside-allowed-set=6486760 max-excursion=206  (must be > 0)
    ckpt down_proj                     k=17408  n=5120   outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=8.548e-05
      control (n reversed inside each 16-tile): outside-allowed-set=88321650 max-excursion=237  (must be > 0)
    synthetic random trellis           k=512    n=512    outside-allowed-set=0        max-excursion=0 unreachable-fp16=0 rel-MSE=5.267e-05
      control (n reversed inside each 16-tile): outside-allowed-set=260240 max-excursion=223  (must be > 0)
      mainloop s30  smem= 28672 B (K=6)  rel-err vs the exact int8 product = 2.40e-04  ok
      mainloop s27  smem= 57344 B (K=6)  rel-err vs the exact int8 product = 2.40e-04  ok
      mainloop s13  smem= 86016 B (K=6)  rel-err vs the exact int8 product = 2.40e-04  ok
      mainloop s31  smem= 22528 B (K=6)  rel-err vs the exact int8 product = 2.40e-04  ok
      control (reference rolled by one n-tile): rel-err = 1.41e+00  (must be >> 1e-3)

[g2a] compared 5 tensor(s) over widths [6] x codebooks ['mcg']
GATE 2a: PASS
--- gate 2b on mcg ---
  down_proj      M=256    k=4608   n=1536   rel(legB vs fp32)=2.514e-03  rel(int8 vs fp32)=1.181e-02  rel(int8 gemm vs rotated fp32)=1.180e-02
  down_proj      M=1024   k=4608   n=1536   rel(legB vs fp32)=2.513e-03  rel(int8 vs fp32)=1.183e-02  rel(int8 gemm vs rotated fp32)=1.182e-02
  down_proj      M=4096   k=4608   n=1536   rel(legB vs fp32)=2.511e-03  rel(int8 vs fp32)=1.180e-02  rel(int8 gemm vs rotated fp32)=1.179e-02
  gate_proj      M=256    k=1536   n=4608   rel(legB vs fp32)=2.046e-03  rel(int8 vs fp32)=1.142e-02  rel(int8 gemm vs rotated fp32)=1.142e-02
  gate_proj      M=1024   k=1536   n=4608   rel(legB vs fp32)=1.475e-03  rel(int8 vs fp32)=1.137e-02  rel(int8 gemm vs rotated fp32)=1.136e-02
  gate_proj      M=4096   k=1536   n=4608   rel(legB vs fp32)=2.048e-03  rel(int8 vs fp32)=1.136e-02  rel(int8 gemm vs rotated fp32)=1.134e-02
  up_proj        M=256    k=1536   n=4608   rel(legB vs fp32)=2.046e-03  rel(int8 vs fp32)=1.143e-02  rel(int8 gemm vs rotated fp32)=1.142e-02
  up_proj        M=1024   k=1536   n=4608   rel(legB vs fp32)=1.477e-03  rel(int8 vs fp32)=1.143e-02  rel(int8 gemm vs rotated fp32)=1.142e-02
  up_proj        M=4096   k=1536   n=4608   rel(legB vs fp32)=2.047e-03  rel(int8 vs fp32)=1.144e-02  rel(int8 gemm vs rotated fp32)=1.143e-02
  k_proj         M=256    k=1536   n=256    rel(legB vs fp32)=2.051e-03  rel(int8 vs fp32)=1.139e-02  rel(int8 gemm vs rotated fp32)=1.145e-02
  k_proj         M=1024   k=1536   n=256    rel(legB vs fp32)=2.059e-03  rel(int8 vs fp32)=1.136e-02  rel(int8 gemm vs rotated fp32)=1.141e-02
  k_proj         M=4096   k=1536   n=256    rel(legB vs fp32)=1.105e-03  rel(int8 vs fp32)=1.138e-02  rel(int8 gemm vs rotated fp32)=1.144e-02
  o_proj         M=256    k=2048   n=1536   rel(legB vs fp32)=1.675e-03  rel(int8 vs fp32)=1.184e-02  rel(int8 gemm vs rotated fp32)=1.185e-02
  o_proj         M=1024   k=2048   n=1536   rel(legB vs fp32)=1.674e-03  rel(int8 vs fp32)=1.180e-02  rel(int8 gemm vs rotated fp32)=1.177e-02
  o_proj         M=4096   k=2048   n=1536   rel(legB vs fp32)=2.387e-03  rel(int8 vs fp32)=1.177e-02  rel(int8 gemm vs rotated fp32)=1.175e-02
  q_proj         M=256    k=1536   n=2048   rel(legB vs fp32)=1.476e-03  rel(int8 vs fp32)=1.139e-02  rel(int8 gemm vs rotated fp32)=1.137e-02
  q_proj         M=1024   k=1536   n=2048   rel(legB vs fp32)=2.049e-03  rel(int8 vs fp32)=1.140e-02  rel(int8 gemm vs rotated fp32)=1.138e-02
  q_proj         M=4096   k=1536   n=2048   rel(legB vs fp32)=2.050e-03  rel(int8 vs fp32)=1.146e-02  rel(int8 gemm vs rotated fp32)=1.144e-02
  v_proj         M=256    k=1536   n=256    rel(legB vs fp32)=2.048e-03  rel(int8 vs fp32)=1.138e-02  rel(int8 gemm vs rotated fp32)=1.137e-02
  v_proj         M=1024   k=1536   n=256    rel(legB vs fp32)=2.059e-03  rel(int8 vs fp32)=1.140e-02  rel(int8 gemm vs rotated fp32)=1.139e-02
  v_proj         M=4096   k=1536   n=256    rel(legB vs fp32)=1.098e-03  rel(int8 vs fp32)=1.136e-02  rel(int8 gemm vs rotated fp32)=1.134e-02
[g2b] compared 21 linear(s)
GATE 2b: PASS
--- gate 1 mul1 ---
codebook indices = 65536   k_inv = 0.006767272949218750   int8 step = 4*k_inv = 0.027069091796875000
[1] fp16 mul1 arm vs independent reference : bitwise mismatches = 0
[2] integer decode  sat((bytesum-510+A)>>2)  : mismatches = 0
[3] control 'no +2'     differs on 16374 indices  (must be > 0)
    control 'signed dp4a' differs on 62452 indices  (must be > 0)
    integer  rel-MSE=5.974e-05  max|e|=0.012695  bias=-0.001572 = -0.157% of sigma
    no +2    rel-MSE=1.267e-04  max|e|=0.019440  bias=-0.008335 = -0.833% of sigma
    signed   rel-MSE=1.118e+01  max|e|=6.812500  bias=-3.056919 = -305.597% of sigma
    vs round-half-even: 8188 disagreements of 16385 ties, max |delta| = 1; all inside the d=2 (mod 4) tie set: True
    sigma(codebook) = 1.0003   distinct fp16 values = 913   distinct int8 levels = 238   saturated = 0
GATE 1: PASS
DONE2

# ---------------------------------------------------------------------------
# Shape sweep on a real mcg checkpoint: every tile shape x a spread of row
# counts, and BIT-IDENTITY across all of them.
#
#   I8_MODEL=/mnt/k8scache/models/MiniCPM5-1B-exl3-4bit I8_ONLY_BITS=4 \
#     I8_ONLY_CB=1 python3 sweep_shapes.py --linears o_proj gate_proj
#
# This is mcg's CLAMP-ELISION proof on real weights.  Shapes 70-73 keep the
# saturating path -- for mcg that is a TEN-bit read of the magic result
# followed by a clamp, where the served path takes eight bits and cannot
# saturate -- and the sweep requires every shape's output to be bit-identical.
# int32 accumulation does not reassociate, so this is also the batch-invariance
# property the shape pin exists to protect, as a structural guarantee.
# ---------------------------------------------------------------------------
  o_proj     k=2048   n=1536   M=1     s0=1.10e-02 s1=1.10e-02 s2=1.10e-02 s3=1.10e-02 s4=1.10e-02 s5=1.10e-02 s6=1.10e-02 s7=1.10e-02 s8=1.10e-02 s9=1.10e-02 s10=1.10e-02 s11=1.10e-02 s12=1.10e-02 s13=1.10e-02 s20=1.10e-02 s21=1.10e-02 s22=1.10e-02 s23=1.10e-02 s24=1.10e-02 s25=1.10e-02 s26=1.10e-02 s27=1.10e-02 s30=1.10e-02 s31=1.10e-02 s32=1.10e-02 s33=1.10e-02 s34=1.10e-02 s35=1.10e-02 s36=1.10e-02 s37=1.10e-02 s50=1.10e-02 s51=1.10e-02 s52=1.10e-02 s53=1.10e-02 s54=1.10e-02 s60=1.10e-02 s61=1.10e-02 s62=1.10e-02 s63=1.10e-02 s64=1.10e-02 s65=1.10e-02 s66=1.10e-02 s67=1.10e-02 s68=1.10e-02 s70=1.10e-02 s71=1.10e-02 s72=1.10e-02 s73=1.10e-02  bit-identical across all shapes: True
  o_proj     k=2048   n=1536   M=17    s0=1.18e-02 s1=1.18e-02 s2=1.18e-02 s3=1.18e-02 s4=1.18e-02 s5=1.18e-02 s6=1.18e-02 s7=1.18e-02 s8=1.18e-02 s9=1.18e-02 s10=1.18e-02 s11=1.18e-02 s12=1.18e-02 s13=1.18e-02 s20=1.18e-02 s21=1.18e-02 s22=1.18e-02 s23=1.18e-02 s24=1.18e-02 s25=1.18e-02 s26=1.18e-02 s27=1.18e-02 s30=1.18e-02 s31=1.18e-02 s32=1.18e-02 s33=1.18e-02 s34=1.18e-02 s35=1.18e-02 s36=1.18e-02 s37=1.18e-02 s50=1.18e-02 s51=1.18e-02 s52=1.18e-02 s53=1.18e-02 s54=1.18e-02 s60=1.18e-02 s61=1.18e-02 s62=1.18e-02 s63=1.18e-02 s64=1.18e-02 s65=1.18e-02 s66=1.18e-02 s67=1.18e-02 s68=1.18e-02 s70=1.18e-02 s71=1.18e-02 s72=1.18e-02 s73=1.18e-02  bit-identical across all shapes: True
  o_proj     k=2048   n=1536   M=63    s0=1.17e-02 s1=1.17e-02 s2=1.17e-02 s3=1.17e-02 s4=1.17e-02 s5=1.17e-02 s6=1.17e-02 s7=1.17e-02 s8=1.17e-02 s9=1.17e-02 s10=1.17e-02 s11=1.17e-02 s12=1.17e-02 s13=1.17e-02 s20=1.17e-02 s21=1.17e-02 s22=1.17e-02 s23=1.17e-02 s24=1.17e-02 s25=1.17e-02 s26=1.17e-02 s27=1.17e-02 s30=1.17e-02 s31=1.17e-02 s32=1.17e-02 s33=1.17e-02 s34=1.17e-02 s35=1.17e-02 s36=1.17e-02 s37=1.17e-02 s50=1.17e-02 s51=1.17e-02 s52=1.17e-02 s53=1.17e-02 s54=1.17e-02 s60=1.17e-02 s61=1.17e-02 s62=1.17e-02 s63=1.17e-02 s64=1.17e-02 s65=1.17e-02 s66=1.17e-02 s67=1.17e-02 s68=1.17e-02 s70=1.17e-02 s71=1.17e-02 s72=1.17e-02 s73=1.17e-02  bit-identical across all shapes: True
  o_proj     k=2048   n=1536   M=256   s0=1.16e-02 s1=1.16e-02 s2=1.16e-02 s3=1.16e-02 s4=1.16e-02 s5=1.16e-02 s6=1.16e-02 s7=1.16e-02 s8=1.16e-02 s9=1.16e-02 s10=1.16e-02 s11=1.16e-02 s12=1.16e-02 s13=1.16e-02 s20=1.16e-02 s21=1.16e-02 s22=1.16e-02 s23=1.16e-02 s24=1.16e-02 s25=1.16e-02 s26=1.16e-02 s27=1.16e-02 s30=1.16e-02 s31=1.16e-02 s32=1.16e-02 s33=1.16e-02 s34=1.16e-02 s35=1.16e-02 s36=1.16e-02 s37=1.16e-02 s50=1.16e-02 s51=1.16e-02 s52=1.16e-02 s53=1.16e-02 s54=1.16e-02 s60=1.16e-02 s61=1.16e-02 s62=1.16e-02 s63=1.16e-02 s64=1.16e-02 s65=1.16e-02 s66=1.16e-02 s67=1.16e-02 s68=1.16e-02 s70=1.16e-02 s71=1.16e-02 s72=1.16e-02 s73=1.16e-02  bit-identical across all shapes: True
  o_proj     k=2048   n=1536   M=1000  s0=1.18e-02 s1=1.18e-02 s2=1.18e-02 s3=1.18e-02 s4=1.18e-02 s5=1.18e-02 s6=1.18e-02 s7=1.18e-02 s8=1.18e-02 s9=1.18e-02 s10=1.18e-02 s11=1.18e-02 s12=1.18e-02 s13=1.18e-02 s20=1.18e-02 s21=1.18e-02 s22=1.18e-02 s23=1.18e-02 s24=1.18e-02 s25=1.18e-02 s26=1.18e-02 s27=1.18e-02 s30=1.18e-02 s31=1.18e-02 s32=1.18e-02 s33=1.18e-02 s34=1.18e-02 s35=1.18e-02 s36=1.18e-02 s37=1.18e-02 s50=1.18e-02 s51=1.18e-02 s52=1.18e-02 s53=1.18e-02 s54=1.18e-02 s60=1.18e-02 s61=1.18e-02 s62=1.18e-02 s63=1.18e-02 s64=1.18e-02 s65=1.18e-02 s66=1.18e-02 s67=1.18e-02 s68=1.18e-02 s70=1.18e-02 s71=1.18e-02 s72=1.18e-02 s73=1.18e-02  bit-identical across all shapes: True
  o_proj     k=2048   n=1536   M=2048  s0=1.18e-02 s1=1.18e-02 s2=1.18e-02 s3=1.18e-02 s4=1.18e-02 s5=1.18e-02 s6=1.18e-02 s7=1.18e-02 s8=1.18e-02 s9=1.18e-02 s10=1.18e-02 s11=1.18e-02 s12=1.18e-02 s13=1.18e-02 s20=1.18e-02 s21=1.18e-02 s22=1.18e-02 s23=1.18e-02 s24=1.18e-02 s25=1.18e-02 s26=1.18e-02 s27=1.18e-02 s30=1.18e-02 s31=1.18e-02 s32=1.18e-02 s33=1.18e-02 s34=1.18e-02 s35=1.18e-02 s36=1.18e-02 s37=1.18e-02 s50=1.18e-02 s51=1.18e-02 s52=1.18e-02 s53=1.18e-02 s54=1.18e-02 s60=1.18e-02 s61=1.18e-02 s62=1.18e-02 s63=1.18e-02 s64=1.18e-02 s65=1.18e-02 s66=1.18e-02 s67=1.18e-02 s68=1.18e-02 s70=1.18e-02 s71=1.18e-02 s72=1.18e-02 s73=1.18e-02  bit-identical across all shapes: True
  o_proj     k=2048   n=1536   M=8192  s0=1.18e-02 s1=1.18e-02 s2=1.18e-02 s3=1.18e-02 s4=1.18e-02 s5=1.18e-02 s6=1.18e-02 s7=1.18e-02 s8=1.18e-02 s9=1.18e-02 s10=1.18e-02 s11=1.18e-02 s12=1.18e-02 s13=1.18e-02 s20=1.18e-02 s21=1.18e-02 s22=1.18e-02 s23=1.18e-02 s24=1.18e-02 s25=1.18e-02 s26=1.18e-02 s27=1.18e-02 s30=1.18e-02 s31=1.18e-02 s32=1.18e-02 s33=1.18e-02 s34=1.18e-02 s35=1.18e-02 s36=1.18e-02 s37=1.18e-02 s50=1.18e-02 s51=1.18e-02 s52=1.18e-02 s53=1.18e-02 s54=1.18e-02 s60=1.18e-02 s61=1.18e-02 s62=1.18e-02 s63=1.18e-02 s64=1.18e-02 s65=1.18e-02 s66=1.18e-02 s67=1.18e-02 s68=1.18e-02 s70=1.18e-02 s71=1.18e-02 s72=1.18e-02 s73=1.18e-02  bit-identical across all shapes: True
  gate_proj  k=1536   n=4608   M=1     s0=1.10e-02 s1=1.10e-02 s2=1.10e-02 s3=1.10e-02 s4=1.10e-02 s5=1.10e-02 s6=1.10e-02 s7=1.10e-02 s8=1.10e-02 s9=1.10e-02 s10=1.10e-02 s11=1.10e-02 s12=1.10e-02 s13=1.10e-02 s20=1.10e-02 s21=1.10e-02 s22=1.10e-02 s23=1.10e-02 s24=1.10e-02 s25=1.10e-02 s26=1.10e-02 s27=1.10e-02 s30=1.10e-02 s31=1.10e-02 s32=1.10e-02 s33=1.10e-02 s34=1.10e-02 s35=1.10e-02 s36=1.10e-02 s37=1.10e-02 s50=1.10e-02 s51=1.10e-02 s52=1.10e-02 s53=1.10e-02 s54=1.10e-02 s60=1.10e-02 s61=1.10e-02 s62=1.10e-02 s63=1.10e-02 s64=1.10e-02 s65=1.10e-02 s66=1.10e-02 s67=1.10e-02 s68=1.10e-02 s70=1.10e-02 s71=1.10e-02 s72=1.10e-02 s73=1.10e-02  bit-identical across all shapes: True
  gate_proj  k=1536   n=4608   M=17    s0=1.13e-02 s1=1.13e-02 s2=1.13e-02 s3=1.13e-02 s4=1.13e-02 s5=1.13e-02 s6=1.13e-02 s7=1.13e-02 s8=1.13e-02 s9=1.13e-02 s10=1.13e-02 s11=1.13e-02 s12=1.13e-02 s13=1.13e-02 s20=1.13e-02 s21=1.13e-02 s22=1.13e-02 s23=1.13e-02 s24=1.13e-02 s25=1.13e-02 s26=1.13e-02 s27=1.13e-02 s30=1.13e-02 s31=1.13e-02 s32=1.13e-02 s33=1.13e-02 s34=1.13e-02 s35=1.13e-02 s36=1.13e-02 s37=1.13e-02 s50=1.13e-02 s51=1.13e-02 s52=1.13e-02 s53=1.13e-02 s54=1.13e-02 s60=1.13e-02 s61=1.13e-02 s62=1.13e-02 s63=1.13e-02 s64=1.13e-02 s65=1.13e-02 s66=1.13e-02 s67=1.13e-02 s68=1.13e-02 s70=1.13e-02 s71=1.13e-02 s72=1.13e-02 s73=1.13e-02  bit-identical across all shapes: True
  gate_proj  k=1536   n=4608   M=63    s0=1.14e-02 s1=1.14e-02 s2=1.14e-02 s3=1.14e-02 s4=1.14e-02 s5=1.14e-02 s6=1.14e-02 s7=1.14e-02 s8=1.14e-02 s9=1.14e-02 s10=1.14e-02 s11=1.14e-02 s12=1.14e-02 s13=1.14e-02 s20=1.14e-02 s21=1.14e-02 s22=1.14e-02 s23=1.14e-02 s24=1.14e-02 s25=1.14e-02 s26=1.14e-02 s27=1.14e-02 s30=1.14e-02 s31=1.14e-02 s32=1.14e-02 s33=1.14e-02 s34=1.14e-02 s35=1.14e-02 s36=1.14e-02 s37=1.14e-02 s50=1.14e-02 s51=1.14e-02 s52=1.14e-02 s53=1.14e-02 s54=1.14e-02 s60=1.14e-02 s61=1.14e-02 s62=1.14e-02 s63=1.14e-02 s64=1.14e-02 s65=1.14e-02 s66=1.14e-02 s67=1.14e-02 s68=1.14e-02 s70=1.14e-02 s71=1.14e-02 s72=1.14e-02 s73=1.14e-02  bit-identical across all shapes: True
  gate_proj  k=1536   n=4608   M=256   s0=1.13e-02 s1=1.13e-02 s2=1.13e-02 s3=1.13e-02 s4=1.13e-02 s5=1.13e-02 s6=1.13e-02 s7=1.13e-02 s8=1.13e-02 s9=1.13e-02 s10=1.13e-02 s11=1.13e-02 s12=1.13e-02 s13=1.13e-02 s20=1.13e-02 s21=1.13e-02 s22=1.13e-02 s23=1.13e-02 s24=1.13e-02 s25=1.13e-02 s26=1.13e-02 s27=1.13e-02 s30=1.13e-02 s31=1.13e-02 s32=1.13e-02 s33=1.13e-02 s34=1.13e-02 s35=1.13e-02 s36=1.13e-02 s37=1.13e-02 s50=1.13e-02 s51=1.13e-02 s52=1.13e-02 s53=1.13e-02 s54=1.13e-02 s60=1.13e-02 s61=1.13e-02 s62=1.13e-02 s63=1.13e-02 s64=1.13e-02 s65=1.13e-02 s66=1.13e-02 s67=1.13e-02 s68=1.13e-02 s70=1.13e-02 s71=1.13e-02 s72=1.13e-02 s73=1.13e-02  bit-identical across all shapes: True
  gate_proj  k=1536   n=4608   M=1000  s0=1.13e-02 s1=1.13e-02 s2=1.13e-02 s3=1.13e-02 s4=1.13e-02 s5=1.13e-02 s6=1.13e-02 s7=1.13e-02 s8=1.13e-02 s9=1.13e-02 s10=1.13e-02 s11=1.13e-02 s12=1.13e-02 s13=1.13e-02 s20=1.13e-02 s21=1.13e-02 s22=1.13e-02 s23=1.13e-02 s24=1.13e-02 s25=1.13e-02 s26=1.13e-02 s27=1.13e-02 s30=1.13e-02 s31=1.13e-02 s32=1.13e-02 s33=1.13e-02 s34=1.13e-02 s35=1.13e-02 s36=1.13e-02 s37=1.13e-02 s50=1.13e-02 s51=1.13e-02 s52=1.13e-02 s53=1.13e-02 s54=1.13e-02 s60=1.13e-02 s61=1.13e-02 s62=1.13e-02 s63=1.13e-02 s64=1.13e-02 s65=1.13e-02 s66=1.13e-02 s67=1.13e-02 s68=1.13e-02 s70=1.13e-02 s71=1.13e-02 s72=1.13e-02 s73=1.13e-02  bit-identical across all shapes: True
  gate_proj  k=1536   n=4608   M=2048  s0=1.13e-02 s1=1.13e-02 s2=1.13e-02 s3=1.13e-02 s4=1.13e-02 s5=1.13e-02 s6=1.13e-02 s7=1.13e-02 s8=1.13e-02 s9=1.13e-02 s10=1.13e-02 s11=1.13e-02 s12=1.13e-02 s13=1.13e-02 s20=1.13e-02 s21=1.13e-02 s22=1.13e-02 s23=1.13e-02 s24=1.13e-02 s25=1.13e-02 s26=1.13e-02 s27=1.13e-02 s30=1.13e-02 s31=1.13e-02 s32=1.13e-02 s33=1.13e-02 s34=1.13e-02 s35=1.13e-02 s36=1.13e-02 s37=1.13e-02 s50=1.13e-02 s51=1.13e-02 s52=1.13e-02 s53=1.13e-02 s54=1.13e-02 s60=1.13e-02 s61=1.13e-02 s62=1.13e-02 s63=1.13e-02 s64=1.13e-02 s65=1.13e-02 s66=1.13e-02 s67=1.13e-02 s68=1.13e-02 s70=1.13e-02 s71=1.13e-02 s72=1.13e-02 s73=1.13e-02  bit-identical across all shapes: True
  gate_proj  k=1536   n=4608   M=8192  s0=1.13e-02 s1=1.13e-02 s2=1.13e-02 s3=1.13e-02 s4=1.13e-02 s5=1.13e-02 s6=1.13e-02 s7=1.13e-02 s8=1.13e-02 s9=1.13e-02 s10=1.13e-02 s11=1.13e-02 s12=1.13e-02 s13=1.13e-02 s20=1.13e-02 s21=1.13e-02 s22=1.13e-02 s23=1.13e-02 s24=1.13e-02 s25=1.13e-02 s26=1.13e-02 s27=1.13e-02 s30=1.13e-02 s31=1.13e-02 s32=1.13e-02 s33=1.13e-02 s34=1.13e-02 s35=1.13e-02 s36=1.13e-02 s37=1.13e-02 s50=1.13e-02 s51=1.13e-02 s52=1.13e-02 s53=1.13e-02 s54=1.13e-02 s60=1.13e-02 s61=1.13e-02 s62=1.13e-02 s63=1.13e-02 s64=1.13e-02 s65=1.13e-02 s66=1.13e-02 s67=1.13e-02 s68=1.13e-02 s70=1.13e-02 s71=1.13e-02 s72=1.13e-02 s73=1.13e-02  bit-identical across all shapes: True
[sweep] compared 14 (linear, M) pair(s)
SHAPE SWEEP: PASS

# ---------------------------------------------------------------------------
# The codebook census of every exl3 checkpoint on /mnt/k8scache/models.
# 13 mcg, 5 mul1, 1 with NEITHER seed.
#
# Qwen3-0.6B-exl3 carries 197 trellises and no seed: exl3's DEFAULT codebook,
# which has no int8 decode and still refuses.  That is why the check does not
# go away when mcg lands -- it is a live checkpoint, not a hypothetical.
#
# Muse-Glimmer-30B-assistant appears in both lists, the same model converted
# twice, which is exactly the case a shape-and-dtype test cannot separate.
# ---------------------------------------------------------------------------
MiniCPM5-1B-exl3-4bit                                trellis=  169 mcg=  169 mul1=    0
MiniCPM5-1B-exl3-5bit                                trellis=  169 mcg=  169 mul1=    0
Muse-Glimmer-30B-assistant-exl3-6.0bpw               trellis=   36 mcg=   36 mul1=    0
Muse-Glimmer-30B-assistant-exl3-official-6.0bpw      trellis=   36 mcg=    0 mul1=   36
Muse-Glimmer-30B-exl3-4.0bpw                         trellis=  720 mcg=    0 mul1=  720
NVIDIA-Nemotron-3.5-Lightning-30B-A3B-exl3-4bpw      trellis= 6005 mcg=    0 mul1= 6005
NVIDIA-NemotronLabs-VoiceChat-11B-exl3-4bpw          trellis=   70 mcg=   70 mul1=    0
NVIDIA-NemotronLabs-VoiceChat-11B-exl3-4bpw-padded   trellis=  120 mcg=  120 mul1=    0
NVIDIA-NemotronLabs-VoiceChat-11B-exl3-6bpw          trellis=  120 mcg=  120 mul1=    0
Qwen3-0.6B-exl3                                      trellis=  197 mcg=    0 mul1=    0
Qwen3-0.6B-exl3-fresh4bit                            trellis=  197 mcg=  197 mul1=    0
Qwen3-4B-exl3-4bit                                   trellis=  253 mcg=  253 mul1=    0
Qwen3.5-0.8B-exl3-4bit                               trellis=  151 mcg=  151 mul1=    0
Qwen3.5-0.8B-exl3-4bit-mul1                          trellis=  159 mcg=    0 mul1=  159
Qwen3.8-27B-DFlash2-exl3-6.0bpw                      trellis=   47 mcg=   47 mul1=    0
Qwen3.8-27B-exl3-4.0bpw                              trellis=  409 mcg=    0 mul1=  409
Step-Audio-2-mini-exl3-4.15bpw                       trellis=  197 mcg=  197 mul1=    0
Step-Audio-2-mini-exl3-6.0bpw                        trellis=  197 mcg=  197 mul1=    0
Step-Audio-2-mini-textlm-exl3-4.15bpw                trellis=  197 mcg=  197 mul1=    0
Step-Audio-2-mini-textlm-exl3-6.0bpw                 trellis=  197 mcg=  197 mul1=    0

# ---------------------------------------------------------------------------
# Batch invariance on a real mcg checkpoint: a row's output must not depend on
# how many rows shipped with it.
#
#   I8_MODEL=/mnt/k8scache/models/MiniCPM5-1B-exl3-4bit I8_ONLY_BITS=4 \
#     I8_ONLY_CB=1 python3 row_invariance.py --linears o_proj gate_proj
#
# THE CONTROL FIRES.  Leg B is NOT bit-identical at 512 and 1000 rows
# (max|delta| 1.07e-02 on o_proj, 7.81e-03 on gate_proj) -- cuBLAS switches
# kernel with M -- so the harness can see M-dependence, and the int8 leg
# having none is a measurement rather than a tolerance.
# ---------------------------------------------------------------------------
  o_proj     rows 128   of 4096: int8 bit-identical=True   legB bit-identical=True (max|delta|=0.000e+00)
  o_proj     rows 512   of 4096: int8 bit-identical=True   legB bit-identical=False (max|delta|=1.074e-02)
  o_proj     rows 1000  of 4096: int8 bit-identical=True   legB bit-identical=False (max|delta|=1.074e-02)
  o_proj     rows 2048  of 4096: int8 bit-identical=True   legB bit-identical=True (max|delta|=0.000e+00)
  o_proj     across 6 tile shapes at M=1000: bit-identical=True
  gate_proj  rows 128   of 4096: int8 bit-identical=True   legB bit-identical=True (max|delta|=0.000e+00)
  gate_proj  rows 512   of 4096: int8 bit-identical=True   legB bit-identical=True (max|delta|=0.000e+00)
  gate_proj  rows 1000  of 4096: int8 bit-identical=True   legB bit-identical=False (max|delta|=7.812e-03)
  gate_proj  rows 2048  of 4096: int8 bit-identical=True   legB bit-identical=True (max|delta|=0.000e+00)
  gate_proj  across 6 tile shapes at M=1000: bit-identical=True
[rowinv] compared 2 linear(s)
ROW INVARIANCE: PASS
