diff options
Diffstat (limited to 'ep_run')
| -rw-r--r-- | ep_run/casc_eq_train.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index ad82b5e..91a1cca 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -40,6 +40,7 @@ ap.add_argument('--zloss', type=float, default=0.0) # z-loss coefficient on ap.add_argument('--kretry', type=int, default=0) # >0: on drift-reject, RETRY the batch once with this many fb rounds (diag B: K8 converges the marginal batches) instead of dropping it ap.add_argument('--bf_late', type=float, default=0.0) # >0: raise beta_floor to this value from step --bf_late_at (late-training SNR fix; dose-response 2026-07-10) ap.add_argument('--bf_late_at', type=int, default=25000) +ap.add_argument('--bsign_rand', action='store_true') # random-sign beta per step (KHS 'random scheme'): averages the O(beta) single-sided bias at single-phase cost ap.add_argument('--bf16', action='store_true') # cast model to bf16 (E-accumulation + tok_sigma stay fp32) — the x0.5 cost lever, GATE before production ap.add_argument('--dtop_every', type=int, default=1) # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr) ap.add_argument('--gate_every', type=int, default=200) # in-training cos(EP,BP) telemetry; <=0 = fully BP-free (no bp_gate at all) @@ -280,6 +281,7 @@ def dFdtheta(zs, x, y, beta): SIG0 = None +BGEN = torch.Generator().manual_seed(args.seed + 990) # separate RNG: sign flips must not shift the data stream GOV = {'K': None, 'bscale': 1.0, 'gema': None, 'drift': 0.0, 'gn': 0.0, 'sig': 0.0} def ep_step(x, y): """single-sided EP with a QUALITY-GOVERNED estimator: beta_t = beta0*bscale*sig0^2/sig^2, @@ -296,6 +298,7 @@ def ep_step(x, y): fl = args.beta_floor if args.bf_late > 0.0 and GOV.get('step', 0) >= args.bf_late_at: fl = args.bf_late if fl > 0.0: beta_t = max(beta_t, fl) + if args.bsign_rand and torch.rand((), generator=BGEN).item() < 0.5: beta_t = -beta_t z0, zs, ins, outs = free_states_graphed(x) zs_free = [z.clone() for z in zs] free_ce = F.cross_entropy(readout(zs_free[-1]).reshape(-1, vocab), y.reshape(-1)).item() |