summaryrefslogtreecommitdiff
path: root/ep_run/casc_eq_train.py
diff options
context:
space:
mode:
authorYuren Hao <yurenh2@illinois.edu>2026-07-19 15:28:37 -0500
committerYuren Hao <yurenh2@illinois.edu>2026-07-19 15:28:37 -0500
commitaca5faae121caf286a9fe2567b8159a0d347605c (patch)
tree07147bff530a3385466c9e48cd87a92e7064c734 /ep_run/casc_eq_train.py
parentf302c8a971064c3b22ac9cf252546967f6e62581 (diff)
CODEX审计修复×4: _legal门AND→OR+NaN非法+ddp_max(600步连续放行的逃生孔,主因); kretry救活持久化折半; up-step移到commit点后; bsimp进ckpt — codex裁决:主因=门代码缺陷,AIMD算术无错(0.01423≈ln1.01/ln2,392.37=1.01^600精确),用户'是bug'判定成立
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn
Diffstat (limited to 'ep_run/casc_eq_train.py')
-rw-r--r--ep_run/casc_eq_train.py27
1 files changed, 19 insertions, 8 deletions
diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py
index 05659f0..2ad44ac 100644
--- a/ep_run/casc_eq_train.py
+++ b/ep_run/casc_eq_train.py
@@ -286,6 +286,7 @@ if args.resume:
with torch.no_grad(): W_out.copy_(_ck['wout'].to(dev))
if _ck.get('lnf') is not None and not isinstance(ln_f, nn.Identity): ln_f.load_state_dict(_ck['lnf'])
start_step = int(_ck.get('step', 0))
+ if _ck.get('bsimp') is not None: GOV['bsimp'] = float(_ck['bsimp'])
print(f'[resume] loaded {args.resume} at step {start_step}', flush=True)
if args.bf16:
for _m in (tok, pos, blocks):
@@ -517,12 +518,15 @@ def ep_step(x, y):
# telemetry — an out-of-window beta can be attempted but can never COMMIT.
if (not math.isfinite(gd)) or (gd > dthr and not args.noguard): return False
if args.beta_sync > 0 or args.wsync > 0 or args.beta_cap_rho > 0:
- # if you measure the ceiling, illegality COUNTS — crown-3's poison entered
- # through this gate being wired only to beta_sync (18k accepted garbage steps
- # with the rho-meter screaming); the meter and the gate are now永久 connected.
- res_s = ddp_bcast_scalar(GOV.get('res', 0.0))
- rho_s = ddp_bcast_scalar(GOV.get('rho') or 0.0)
- if res_s > 0.02 and rho_s > args.beta_cap_rho: return False
+ # CODEX AUDIT FIX (2026-07-19): worst-rank rules (max, matching the drift gate,
+ # NOT rank-0 bcast), NaN-illegal, and OR — the old AND let "res huge but rho<0.9"
+ # (large-displacement converging relax) pass as legal: 600 consecutive legal
+ # verdicts while beta climbed x392 into damaging gradients (fw72m_simple 35.4-36k).
+ res_s = ddp_max_scalar(GOV.get('res', 0.0))
+ rho_s = ddp_max_scalar(GOV.get('rho') or 0.0)
+ if ((not math.isfinite(res_s)) or (not math.isfinite(rho_s)) or
+ res_s > 0.02 or rho_s > args.beta_cap_rho):
+ return False
return True
_ok0 = _legal(gdrift)
if not _ok0:
@@ -579,12 +583,14 @@ def ep_step(x, y):
drift = _drift(zp, zs_free)
gdrift = ddp_max_scalar(drift)
ok_retry = _legal(gdrift)
+ if ok_retry and args.beta_simple > 1.0:
+ # CODEX FIX: K=8 rescue ran at beta/2^beta_sync — persist those halvings too,
+ # else the next step jumps straight back to the failed beta (upward bias)
+ GOV['bsimp'] = GOV.get('bsimp', 1.0) * 0.5 ** args.beta_sync
if not ok_retry:
GOV['skd'] = GOV.get('skd', 0) + 1 # drift-guard reject (relaxation non-convergence)
for p in all_params: p.grad = None
return free_ce, beta_t, GOV.get('kuse', GOV['K']), False
- if args.beta_simple > 1.0 and _ok0:
- GOV['bsimp'] = GOV.get('bsimp', 1.0) * args.beta_simple # clean step -> probe upward
GOV['drift'] = gdrift
if args.drift_adapt > 0:
GOV['drift_ema'] = 0.95 * GOV.get('drift_ema', gdrift) + 0.05 * gdrift
@@ -686,6 +692,10 @@ def ep_step(x, y):
return free_ce, beta_t, GOV.get('kuse', GOV['K']), False
for p, g in zip(all_params, gs):
p.grad = g
+ if args.beta_simple > 1.0 and _ok0:
+ # CODEX FIX: probe upward only on steps that actually COMMIT (after the second-pass
+ # drift and gn-EMA guards) — the old placement raised beta even on later-rejected steps
+ GOV['bsimp'] = GOV.get('bsimp', 1.0) * args.beta_simple
return free_ce, beta_t, GOV.get('kuse', GOV['K']), True
def bp_gate(x, y):
@@ -864,6 +874,7 @@ for step in range(start_step, args.steps + 1):
'wout': (W_out.detach().cpu() if args.untie else None),
'lnf': (ln_f.state_dict() if not isinstance(ln_f, nn.Identity) else None),
'opt': opt.state_dict(), # full optimizer state -> exact resume for chunked HPC jobs
+ 'bsimp': GOV.get('bsimp', 1.0), # CODEX FIX: controller state survives resume
'step': step, 'val': best, 'config': vars(args)}, Path('runs') / f'{args.tag}_s{step}.pt')
if RANK == 0:
print(f'[{args.tag}] DONE best val CE {best:.4f}', flush=True)