diff options
| author | Yuren Hao <yurenh2@illinois.edu> | 2026-07-19 15:28:37 -0500 |
|---|---|---|
| committer | Yuren Hao <yurenh2@illinois.edu> | 2026-07-19 15:28:37 -0500 |
| commit | aca5faae121caf286a9fe2567b8159a0d347605c (patch) | |
| tree | 07147bff530a3385466c9e48cd87a92e7064c734 /ep_run/casc_eq_train.py | |
| parent | f302c8a971064c3b22ac9cf252546967f6e62581 (diff) | |
CODEX审计修复×4: _legal门AND→OR+NaN非法+ddp_max(600步连续放行的逃生孔,主因); kretry救活持久化折半; up-step移到commit点后; bsimp进ckpt — codex裁决:主因=门代码缺陷,AIMD算术无错(0.01423≈ln1.01/ln2,392.37=1.01^600精确),用户'是bug'判定成立
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn
Diffstat (limited to 'ep_run/casc_eq_train.py')
| -rw-r--r-- | ep_run/casc_eq_train.py | 27 |
1 files changed, 19 insertions, 8 deletions
diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 05659f0..2ad44ac 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -286,6 +286,7 @@ if args.resume: with torch.no_grad(): W_out.copy_(_ck['wout'].to(dev)) if _ck.get('lnf') is not None and not isinstance(ln_f, nn.Identity): ln_f.load_state_dict(_ck['lnf']) start_step = int(_ck.get('step', 0)) + if _ck.get('bsimp') is not None: GOV['bsimp'] = float(_ck['bsimp']) print(f'[resume] loaded {args.resume} at step {start_step}', flush=True) if args.bf16: for _m in (tok, pos, blocks): @@ -517,12 +518,15 @@ def ep_step(x, y): # telemetry — an out-of-window beta can be attempted but can never COMMIT. if (not math.isfinite(gd)) or (gd > dthr and not args.noguard): return False if args.beta_sync > 0 or args.wsync > 0 or args.beta_cap_rho > 0: - # if you measure the ceiling, illegality COUNTS — crown-3's poison entered - # through this gate being wired only to beta_sync (18k accepted garbage steps - # with the rho-meter screaming); the meter and the gate are now永久 connected. - res_s = ddp_bcast_scalar(GOV.get('res', 0.0)) - rho_s = ddp_bcast_scalar(GOV.get('rho') or 0.0) - if res_s > 0.02 and rho_s > args.beta_cap_rho: return False + # CODEX AUDIT FIX (2026-07-19): worst-rank rules (max, matching the drift gate, + # NOT rank-0 bcast), NaN-illegal, and OR — the old AND let "res huge but rho<0.9" + # (large-displacement converging relax) pass as legal: 600 consecutive legal + # verdicts while beta climbed x392 into damaging gradients (fw72m_simple 35.4-36k). + res_s = ddp_max_scalar(GOV.get('res', 0.0)) + rho_s = ddp_max_scalar(GOV.get('rho') or 0.0) + if ((not math.isfinite(res_s)) or (not math.isfinite(rho_s)) or + res_s > 0.02 or rho_s > args.beta_cap_rho): + return False return True _ok0 = _legal(gdrift) if not _ok0: @@ -579,12 +583,14 @@ def ep_step(x, y): drift = _drift(zp, zs_free) gdrift = ddp_max_scalar(drift) ok_retry = _legal(gdrift) + if ok_retry and args.beta_simple > 1.0: + # CODEX FIX: K=8 rescue ran at beta/2^beta_sync — persist those halvings too, + # else the next step jumps straight back to the failed beta (upward bias) + GOV['bsimp'] = GOV.get('bsimp', 1.0) * 0.5 ** args.beta_sync if not ok_retry: GOV['skd'] = GOV.get('skd', 0) + 1 # drift-guard reject (relaxation non-convergence) for p in all_params: p.grad = None return free_ce, beta_t, GOV.get('kuse', GOV['K']), False - if args.beta_simple > 1.0 and _ok0: - GOV['bsimp'] = GOV.get('bsimp', 1.0) * args.beta_simple # clean step -> probe upward GOV['drift'] = gdrift if args.drift_adapt > 0: GOV['drift_ema'] = 0.95 * GOV.get('drift_ema', gdrift) + 0.05 * gdrift @@ -686,6 +692,10 @@ def ep_step(x, y): return free_ce, beta_t, GOV.get('kuse', GOV['K']), False for p, g in zip(all_params, gs): p.grad = g + if args.beta_simple > 1.0 and _ok0: + # CODEX FIX: probe upward only on steps that actually COMMIT (after the second-pass + # drift and gn-EMA guards) — the old placement raised beta even on later-rejected steps + GOV['bsimp'] = GOV.get('bsimp', 1.0) * args.beta_simple return free_ce, beta_t, GOV.get('kuse', GOV['K']), True def bp_gate(x, y): @@ -864,6 +874,7 @@ for step in range(start_step, args.steps + 1): 'wout': (W_out.detach().cpu() if args.untie else None), 'lnf': (ln_f.state_dict() if not isinstance(ln_f, nn.Identity) else None), 'opt': opt.state_dict(), # full optimizer state -> exact resume for chunked HPC jobs + 'bsimp': GOV.get('bsimp', 1.0), # CODEX FIX: controller state survives resume 'step': step, 'val': best, 'config': vars(args)}, Path('runs') / f'{args.tag}_s{step}.pt') if RANK == 0: print(f'[{args.tag}] DONE best val CE {best:.4f}', flush=True) |
