From 69314ce4dee3b76225a434263e305cbd2cdb04ae Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Sat, 18 Jul 2026 22:36:45 -0500 Subject: =?UTF-8?q?RESULT=2047+48:=20=E6=B6=A8=E7=9A=84=E9=87=8F=3Dblocks8?= =?UTF-8?q?-11=E5=8D=95=E6=A8=A1=E6=80=81=E5=A2=9E=E7=9B=8A(=CF=81?= =?UTF-8?q?=E5=A4=8D=E5=88=BB=E6=8E=A2=E9=92=88,=E4=B8=A4=E8=A1=80?= =?UTF-8?q?=E7=BB=9F=E5=90=8C=E6=9E=84=E5=9E=8B90%=E8=B4=A8=E9=87=8F,plain?= =?UTF-8?q?=20205-210k=E8=BF=871,cent=E5=85=A8=E7=A8=8B=E5=B9=B3)+b8=20log?= =?UTF-8?q?it=E8=B7=91=E9=A3=9E70=E2=86=9294;=20BBP=E5=AE=A1=E8=AE=A1=3Dfp?= =?UTF-8?q?32=E6=A8=A1=E6=8B=9F=E5=99=A8=E6=97=A0=E5=8A=A0=E6=80=A7?= =?UTF-8?q?=E4=B8=8B=E7=95=8C(a=3D0,overlap=201.000@3e-4)=E2=86=92BBP?= =?UTF-8?q?=E6=98=AF=E7=A1=AC=E4=BB=B6=E8=AE=BE=E8=AE=A1=E6=96=B9=E7=A8=8B?= =?UTF-8?q?;=20=E4=B8=89=E6=8E=A2=E9=92=88=E5=85=A5=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/probe_specaudit.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 ep_run/probe_specaudit.py (limited to 'ep_run/probe_specaudit.py') diff --git a/ep_run/probe_specaudit.py b/ep_run/probe_specaudit.py new file mode 100644 index 0000000..041d068 --- /dev/null +++ b/ep_run/probe_specaudit.py @@ -0,0 +1,64 @@ +"""Weight-space spectral audit: which loop-gain component separates the plain vs cent +lineages before the 196k ceiling collapse? Per-block sigma of Wq/Wk/Wv/proj/w1/w3/w2 and +rms of qk-norm & post-norm gains, at matched steps. Pure CPU, reads the 5k ckpt ladders.""" +import torch, sys + +STEPS_BOTH = [150000, 170000, 185000, 190000, 195000, 200000] +STEPS_PLAIN_EXTRA = [210000, 220000, 230000] +L = 12 + +def sig(w): + return float(torch.linalg.svdvals(w.float())[0]) + +def rms(g): + return float(g.float().pow(2).mean().sqrt()) + +def audit(path): + ck = torch.load(path, map_location='cpu', weights_only=False) + b = ck['blocks'] + rows = [] + for l in range(L): + qkv = b[f'{l}.attn.qkv.weight'].float() + wq, wk, wv = qkv[0:512], qkv[512:1024], qkv[1024:1536] + rows.append(dict( + q=sig(wq), k=sig(wk), v=sig(wv), pr=sig(b[f'{l}.attn.proj.weight']), + w1=sig(b[f'{l}.ff.w1.weight']), w3=sig(b[f'{l}.ff.w3.weight']), + w2=sig(b[f'{l}.ff.w2.weight']), + qg=rms(b[f'{l}.attn.qn.g']), kg=rms(b[f'{l}.attn.kn.g']), + na=rms(b[f'{l}.na.g']), nf=rms(b[f'{l}.nf.g']))) + wout = sig(ck['wout']) + return rows, wout + +def summarize(tag, step, rows, wout): + # aggregates: mean over blocks, top-half mean (6-11), max block, plus the derived products + def agg(key, blks): + vals = [rows[l][key] for l in blks] + return sum(vals) / len(vals) + bot, top = range(0, 6), range(6, 12) + logit = [rows[l]['qg'] * rows[l]['kg'] for l in range(L)] # qk-norm logit scale + vpath = [rows[l]['v'] * rows[l]['pr'] for l in range(L)] # attn value-path gain + fpath = [max(rows[l]['w1'], rows[l]['w3']) * rows[l]['w2'] for l in range(L)] + print(f'{tag} s{step//1000:>3}k | logit bot {agg("qg",bot)*agg("kg",bot):6.3f} top {agg("qg",top)*agg("kg",top):6.3f} max {max(logit):6.3f}(b{logit.index(max(logit))}) ' + f'| vpath top {sum(vpath[6:])/6:7.2f} max {max(vpath):7.2f}(b{vpath.index(max(vpath))}) ' + f'| ffn top {sum(fpath[6:])/6:7.2f} max {max(fpath):7.2f}(b{fpath.index(max(fpath))}) ' + f'| na top {agg("na",top):5.3f} nf top {agg("nf",top):5.3f} | wout {wout:6.1f}', flush=True) + return dict(logit=logit, vpath=vpath, fpath=fpath) + +res = {} +for lineage, steps in [('plain', STEPS_BOTH + STEPS_PLAIN_EXTRA), ('cent', STEPS_BOTH + STEPS_PLAIN_EXTRA)]: + for s in steps: + p = f'runs/fw72m_{lineage}_s{s}.pt' + try: + rows, wout = audit(p) + except FileNotFoundError: + print(f'{lineage} s{s}: MISSING', flush=True); continue + res[(lineage, s)] = summarize(lineage, s, rows, wout) + +# per-block divergence table at 195k (last matched healthy step) +if ('plain', 195000) in res and ('cent', 195000) in res: + print('\nper-block plain/cent ratio at 195k (the pre-collapse fingerprint):', flush=True) + p, c = res[('plain', 195000)], res[('cent', 195000)] + print('blk logit-ratio vpath-ratio ffn-ratio') + for l in range(L): + print(f'{l:3d} {p["logit"][l]/c["logit"][l]:10.3f} {p["vpath"][l]/c["vpath"][l]:10.3f} {p["fpath"][l]/c["fpath"][l]:9.3f}') +print('AUDIT_DONE', flush=True) -- cgit v1.2.3