From ca756df145c132f152f40d10e1d363f34b5ed321 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Thu, 30 Jul 2026 17:30:42 -0500 Subject: =?UTF-8?q?RESULT=2077:=20=E5=9C=B0=E6=9D=BF=E7=90=86=E8=AE=BA?= =?UTF-8?q?=E4=B8=89=E7=AE=AD=E5=85=A8=E4=B8=AD(fp64=E5=A1=8C=E9=9B=B6/?= =?UTF-8?q?=CE=B2=E5=B9=B3=E7=A7=BB/=CE=B2=C3=978=E7=97=8A=E6=84=88)=20?= =?UTF-8?q?=E2=80=94=20=E7=89=A9=E7=90=86=E6=B3=84=E6=BC=8F=E6=94=B9?= =?UTF-8?q?=E5=88=A4fp32=E4=BC=AA=E5=BD=B1;=20--read=5Flin=E9=9B=B6?= =?UTF-8?q?=E6=88=90=E6=9C=AC=E4=BF=AE=E5=A4=8D=E5=AE=9E=E8=A3=85(?= =?UTF-8?q?=E8=AF=BB=E5=87=BA=E7=94=A8=E6=BB=A1=E7=B2=BE=E5=BA=A6d?= =?UTF-8?q?=E6=9B=BF=E4=BB=A3=E8=88=8D=E5=85=A5=E7=9A=84z=E2=88=92o),=20C7?= =?UTF-8?q?68=E9=AA=8C=E8=AF=81=E5=9C=A8=E9=A3=9E;=20geo=E7=94=B5=E6=B1=A0?= =?UTF-8?q?=E6=92=A4=E9=99=A4;=20dgain=E9=87=8D=E6=96=B0=E5=AE=9A=E6=80=A7?= =?UTF-8?q?=E4=B8=BA=E6=95=B0=E5=80=BCworkaround?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/casc_eq_train.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'ep_run/casc_eq_train.py') diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 76b96fd..3cb9494 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -56,6 +56,9 @@ ap.add_argument('--dgain_rand', type=float, default=0.0) ap.add_argument('--probe_dgspec', type=int, default=0) # >0: M1 spectroscopy, value = n batches; exits before training ap.add_argument('--probe_gains', default='1,2,4,8,16,32,64,128,256') ap.add_argument('--probe_f64', action='store_true') # fp64 states+model in the probe: the fp-floor decisive arm # >1: per-STEP log-uniform dgain_top in +ap.add_argument('--read_lin', action='store_true') # linear-form theta-read: cotangent = the stored d tensor (full + # precision) instead of (z - o) (an fp32-ROUNDED copy of d); + # algebraically identical via the read identity z - o = d # [1, this] (spread-spectrum probing of the # decade-spread threshold distribution) ap.add_argument('--dgain_top', type=float, default=1.0) # amplify d in STATE FORMATION for blocks @@ -892,7 +895,10 @@ if args.probe_dgspec > 0: for l in range(args.L): disp[l] += float(GOV['_last_d'][l].norm() / max(float(zp[l].norm()), 1e-12)) / NB E = 0.0 - for z, o in zip(zp, lo): E = E + 0.5 * ((z.detach().to(SDT) - o.to(SDT)) ** 2).sum() + if args.read_lin: + for dl, o in zip(GOV['_last_d'], lo): E = E - (dl.detach().to(SDT) * o.to(SDT)).sum() + else: + for z, o in zip(zp, lo): E = E + 0.5 * ((z.detach().to(SDT) - o.to(SDT)) ** 2).sum() obj = E / (NBT * bt) + obj_loss(readout(zp[-1].detach()).reshape(-1, vocab), y.reshape(-1)) gs = torch.autograd.grad(obj, all_params, allow_unused=True) for j, ix in enumerate(bix): -- cgit v1.2.3