From aca5faae121caf286a9fe2567b8159a0d347605c Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Sun, 19 Jul 2026 15:28:37 -0500 Subject: =?UTF-8?q?CODEX=E5=AE=A1=E8=AE=A1=E4=BF=AE=E5=A4=8D=C3=974:=20=5F?= =?UTF-8?q?legal=E9=97=A8AND=E2=86=92OR+NaN=E9=9D=9E=E6=B3=95+ddp=5Fmax(60?= =?UTF-8?q?0=E6=AD=A5=E8=BF=9E=E7=BB=AD=E6=94=BE=E8=A1=8C=E7=9A=84?= =?UTF-8?q?=E9=80=83=E7=94=9F=E5=AD=94,=E4=B8=BB=E5=9B=A0);=20kretry?= =?UTF-8?q?=E6=95=91=E6=B4=BB=E6=8C=81=E4=B9=85=E5=8C=96=E6=8A=98=E5=8D=8A?= =?UTF-8?q?;=20up-step=E7=A7=BB=E5=88=B0commit=E7=82=B9=E5=90=8E;=20bsimp?= =?UTF-8?q?=E8=BF=9Bckpt=20=E2=80=94=20codex=E8=A3=81=E5=86=B3:=E4=B8=BB?= =?UTF-8?q?=E5=9B=A0=3D=E9=97=A8=E4=BB=A3=E7=A0=81=E7=BC=BA=E9=99=B7,AIMD?= =?UTF-8?q?=E7=AE=97=E6=9C=AF=E6=97=A0=E9=94=99(0.01423=E2=89=88ln1.01/ln2?= =?UTF-8?q?,392.37=3D1.01^600=E7=B2=BE=E7=A1=AE),=E7=94=A8=E6=88=B7'?= =?UTF-8?q?=E6=98=AFbug'=E5=88=A4=E5=AE=9A=E6=88=90=E7=AB=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/casc_eq_train.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 05659f0..2ad44ac 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -286,6 +286,7 @@ if args.resume: with torch.no_grad(): W_out.copy_(_ck['wout'].to(dev)) if _ck.get('lnf') is not None and not isinstance(ln_f, nn.Identity): ln_f.load_state_dict(_ck['lnf']) start_step = int(_ck.get('step', 0)) + if _ck.get('bsimp') is not None: GOV['bsimp'] = float(_ck['bsimp']) print(f'[resume] loaded {args.resume} at step {start_step}', flush=True) if args.bf16: for _m in (tok, pos, blocks): @@ -517,12 +518,15 @@ def ep_step(x, y): # telemetry — an out-of-window beta can be attempted but can never COMMIT. if (not math.isfinite(gd)) or (gd > dthr and not args.noguard): return False if args.beta_sync > 0 or args.wsync > 0 or args.beta_cap_rho > 0: - # if you measure the ceiling, illegality COUNTS — crown-3's poison entered - # through this gate being wired only to beta_sync (18k accepted garbage steps - # with the rho-meter screaming); the meter and the gate are now永久 connected. - res_s = ddp_bcast_scalar(GOV.get('res', 0.0)) - rho_s = ddp_bcast_scalar(GOV.get('rho') or 0.0) - if res_s > 0.02 and rho_s > args.beta_cap_rho: return False + # CODEX AUDIT FIX (2026-07-19): worst-rank rules (max, matching the drift gate, + # NOT rank-0 bcast), NaN-illegal, and OR — the old AND let "res huge but rho<0.9" + # (large-displacement converging relax) pass as legal: 600 consecutive legal + # verdicts while beta climbed x392 into damaging gradients (fw72m_simple 35.4-36k). + res_s = ddp_max_scalar(GOV.get('res', 0.0)) + rho_s = ddp_max_scalar(GOV.get('rho') or 0.0) + if ((not math.isfinite(res_s)) or (not math.isfinite(rho_s)) or + res_s > 0.02 or rho_s > args.beta_cap_rho): + return False return True _ok0 = _legal(gdrift) if not _ok0: @@ -579,12 +583,14 @@ def ep_step(x, y): drift = _drift(zp, zs_free) gdrift = ddp_max_scalar(drift) ok_retry = _legal(gdrift) + if ok_retry and args.beta_simple > 1.0: + # CODEX FIX: K=8 rescue ran at beta/2^beta_sync — persist those halvings too, + # else the next step jumps straight back to the failed beta (upward bias) + GOV['bsimp'] = GOV.get('bsimp', 1.0) * 0.5 ** args.beta_sync if not ok_retry: GOV['skd'] = GOV.get('skd', 0) + 1 # drift-guard reject (relaxation non-convergence) for p in all_params: p.grad = None return free_ce, beta_t, GOV.get('kuse', GOV['K']), False - if args.beta_simple > 1.0 and _ok0: - GOV['bsimp'] = GOV.get('bsimp', 1.0) * args.beta_simple # clean step -> probe upward GOV['drift'] = gdrift if args.drift_adapt > 0: GOV['drift_ema'] = 0.95 * GOV.get('drift_ema', gdrift) + 0.05 * gdrift @@ -686,6 +692,10 @@ def ep_step(x, y): return free_ce, beta_t, GOV.get('kuse', GOV['K']), False for p, g in zip(all_params, gs): p.grad = g + if args.beta_simple > 1.0 and _ok0: + # CODEX FIX: probe upward only on steps that actually COMMIT (after the second-pass + # drift and gn-EMA guards) — the old placement raised beta even on later-rejected steps + GOV['bsimp'] = GOV.get('bsimp', 1.0) * args.beta_simple return free_ce, beta_t, GOV.get('kuse', GOV['K']), True def bp_gate(x, y): @@ -864,6 +874,7 @@ for step in range(start_step, args.steps + 1): 'wout': (W_out.detach().cpu() if args.untie else None), 'lnf': (ln_f.state_dict() if not isinstance(ln_f, nn.Identity) else None), 'opt': opt.state_dict(), # full optimizer state -> exact resume for chunked HPC jobs + 'bsimp': GOV.get('bsimp', 1.0), # CODEX FIX: controller state survives resume 'step': step, 'val': best, 'config': vars(args)}, Path('runs') / f'{args.tag}_s{step}.pt') if RANK == 0: print(f'[{args.tag}] DONE best val CE {best:.4f}', flush=True) -- cgit v1.2.3