summaryrefslogtreecommitdiff
path: root/ep_run/casc_eq_train.py
diff options
context:
space:
mode:
Diffstat (limited to 'ep_run/casc_eq_train.py')
-rw-r--r--ep_run/casc_eq_train.py9
1 files changed, 8 insertions, 1 deletions
diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py
index 3cb9494..ed164ce 100644
--- a/ep_run/casc_eq_train.py
+++ b/ep_run/casc_eq_train.py
@@ -699,7 +699,14 @@ def ep_step(x, y):
gs = torch.autograd.grad(obj, all_params, allow_unused=True)
elif EST == 'single':
E = 0.0
- for z, o in zip(zp, last_outs): E = E + 0.5 * ((z.detach().float() - o.float()) ** 2).sum() # fp32 accumulation (bf16-safe; no-op in fp32)
+ if args.read_lin:
+ # FP-FLOOR FIX (RESULT 77/78): the read identity z - o = d holds algebraically but not in
+ # fp32 — (z - o) is a ROUNDED copy of d (components below eps·|o| are erased; the C768
+ # top-half leak). The d tensors themselves are full-precision; the linear form -<d,o> has
+ # the identical theta-derivative and bypasses the rounding. Probe-verified == fp64.
+ for dl, o in zip(GOV['_last_d'], last_outs): E = E - (dl.detach().float() * o.float()).sum()
+ else:
+ for z, o in zip(zp, last_outs): E = E + 0.5 * ((z.detach().float() - o.float()) ** 2).sum() # fp32 accumulation (bf16-safe; no-op in fp32)
obj = E / (NBT * beta_t) + obj_loss(readout(zp[-1].detach()).reshape(-1, vocab), y.reshape(-1))
gs = torch.autograd.grad(obj, all_params, allow_unused=True)
else: