From 98d5ea09d1498f802f2ad06f5f67b385e228f063 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Tue, 28 Jul 2026 13:20:34 -0500 Subject: =?UTF-8?q?RESULT=2066:=20head=E8=A2=AB=E8=8A=82=E6=B5=81=E5=88=B0?= =?UTF-8?q?60%(=E4=BD=8D=E7=A7=BB=E5=BE=AE=E6=8E=A2=E9=92=88:=E5=85=A8?= =?UTF-8?q?=E6=97=8F1.00-1.02=E5=94=AFW=5Fout=200.60)=20=E2=80=94=20?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=AD=89=E4=BB=B7LR=E5=81=87=E8=AF=B4?= =?UTF-8?q?=E5=91=BD=E4=B8=AD=E5=B9=B6=E5=AE=9A=E4=BD=8D=E5=88=B0=E5=8D=95?= =?UTF-8?q?=E7=9F=A9=E9=98=B5;=20Adam=E5=B2=9Bm/=E2=88=9Av=E8=8A=82?= =?UTF-8?q?=E6=B5=81,=E5=AF=B9centered/fp32/K8=E5=85=A8=E4=B8=8D=E6=95=8F?= =?UTF-8?q?=E6=84=9F=3DEP=E5=A4=B4=E8=AF=BB=E5=85=B1=E4=BA=AB=E7=BB=93?= =?UTF-8?q?=E6=9E=84;=20--head=5Flr=5Fmult=E5=AE=9E=E8=A3=85;=20=E7=94=B5?= =?UTF-8?q?=E6=B1=A02(headmix=E8=AF=8A=E6=96=AD+hlr1.67/2.5=E8=A1=A5?= =?UTF-8?q?=E5=81=BF)=E5=9C=A8=E9=A3=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/probe_stepdisp.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ep_run/probe_stepdisp.py (limited to 'ep_run/probe_stepdisp.py') diff --git a/ep_run/probe_stepdisp.py b/ep_run/probe_stepdisp.py new file mode 100644 index 0000000..2d86867 --- /dev/null +++ b/ep_run/probe_stepdisp.py @@ -0,0 +1,24 @@ +"""Direct effective-step-size measurement (user hypothesis: high cos but shrunken equivalent +LR -> slow learning masquerading as plateau). Load two ckpts of the same tag 600 steps apart +(saved by the micro-runs), report per-family ||dW||/||W|| — the actual distance moved in +weight space. Compare EP vs BP micro-runs from the same start.""" +import sys, torch +a, b, label = sys.argv[1], sys.argv[2], sys.argv[3] +ca, cb = (torch.load(p, map_location='cpu', weights_only=False) for p in (a, b)) +fams = {'tok': None, 'wout': None} +out = {} +ta, tb = ca['tok']['weight'].float(), cb['tok']['weight'].float() +out['tok'] = float((tb-ta).norm()/ta.norm()) +wa, wb = ca['wout'].float(), cb['wout'].float() +out['wout'] = float((wb-wa).norm()/wa.norm()) +Ba, Bb = ca['blocks'], cb['blocks'] +import re +groups = {'attn_bot': [], 'attn_top': [], 'ffn_bot': [], 'ffn_top': []} +for k in Ba: + m = re.match(r'(\d+)\.(attn|ff)\.', k) + if not m or Ba[k].dim() < 2: continue + l = int(m.group(1)); fam = ('attn' if m.group(2)=='attn' else 'ffn') + ('_bot' if l < 6 else '_top') + d = float((Bb[k].float()-Ba[k].float()).norm()/Ba[k].float().norm()) + groups[fam].append(d) +for k, v in groups.items(): out[k] = sum(v)/len(v) +print(label + ': ' + ' '.join(f'{k} {v:.5f}' for k, v in out.items())) -- cgit v1.2.3