diff options
Diffstat (limited to 'ep_run')
| -rw-r--r-- | ep_run/casc_eq_train.py | 8 |
1 files changed, 7 insertions, 1 deletions
diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 76b96fd..3cb9494 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -56,6 +56,9 @@ ap.add_argument('--dgain_rand', type=float, default=0.0) ap.add_argument('--probe_dgspec', type=int, default=0) # >0: M1 spectroscopy, value = n batches; exits before training ap.add_argument('--probe_gains', default='1,2,4,8,16,32,64,128,256') ap.add_argument('--probe_f64', action='store_true') # fp64 states+model in the probe: the fp-floor decisive arm # >1: per-STEP log-uniform dgain_top in +ap.add_argument('--read_lin', action='store_true') # linear-form theta-read: cotangent = the stored d tensor (full + # precision) instead of (z - o) (an fp32-ROUNDED copy of d); + # algebraically identical via the read identity z - o = d # [1, this] (spread-spectrum probing of the # decade-spread threshold distribution) ap.add_argument('--dgain_top', type=float, default=1.0) # amplify d in STATE FORMATION for blocks @@ -892,7 +895,10 @@ if args.probe_dgspec > 0: for l in range(args.L): disp[l] += float(GOV['_last_d'][l].norm() / max(float(zp[l].norm()), 1e-12)) / NB E = 0.0 - for z, o in zip(zp, lo): E = E + 0.5 * ((z.detach().to(SDT) - o.to(SDT)) ** 2).sum() + if args.read_lin: + for dl, o in zip(GOV['_last_d'], lo): E = E - (dl.detach().to(SDT) * o.to(SDT)).sum() + else: + for z, o in zip(zp, lo): E = E + 0.5 * ((z.detach().to(SDT) - o.to(SDT)) ** 2).sum() obj = E / (NBT * bt) + obj_loss(readout(zp[-1].detach()).reshape(-1, vocab), y.reshape(-1)) gs = torch.autograd.grad(obj, all_params, allow_unused=True) for j, ix in enumerate(bix): |