From 7eb40a516849cbc3e93540d4fdf40366f6a8df9e Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 03:59:52 -0500 Subject: results: close context-vectorizer C2 recovery --- experiments/analyze_c2_context_validation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'experiments/analyze_c2_context_validation.py') diff --git a/experiments/analyze_c2_context_validation.py b/experiments/analyze_c2_context_validation.py index c1ba9b3..f9ed7f6 100644 --- a/experiments/analyze_c2_context_validation.py +++ b/experiments/analyze_c2_context_validation.py @@ -45,8 +45,6 @@ def audit_row(path, row): raise RuntimeError(f"protocol mismatch {path}: {mismatches}") if row["final"].get("eval_split") != "validation": raise RuntimeError(f"test-contaminated validation row: {path}") - if not math.isfinite(row["final"].get("val_loss", math.nan)): - raise RuntimeError(f"nonfinite final validation loss: {path}") if any("eval_acc" in step or "cos_r_negg" in step for step in row.get("steps", [])): raise RuntimeError(f"intermediate held-out metric/diagnostic: {path}") split = row.get("split", {}) @@ -93,6 +91,9 @@ def main(): if any(len(hashes) != 1 for hashes in split_hashes.values()): raise RuntimeError(f"methods did not share splits within tasks: {split_hashes}") + nonfinite = [key for key, row in rows.items() + if not math.isfinite(row["final"].get("val_loss", math.nan))] + print(f"commit={next(iter(commits))} rows={len(rows)} task_seeds={list(TASK_SEEDS)}") print("| method | depth | validation (%) | depth gain | lesion drop |") print("|:---|---:|---:|---:|---:|") @@ -134,7 +135,7 @@ def main(): lesion_mean = statistics.mean(lesion_drops) lesion_positive = sum(value > 0 for value in lesion_drops) comparator_ok = strongest_recovery <= 0.5 or deep_advantage >= 2.0 - passed = (bp_gain >= 5.0 and recovery >= 0.7 and positive_gains >= 10 + passed = (not nonfinite and bp_gain >= 5.0 and recovery >= 0.7 and positive_gains >= 10 and comparator_ok and lesion_mean >= 2.0 and lesion_positive >= 10) print(f"BP gain={bp_gain:+.3f}; context-SDIL gain={sdil_gain:+.3f}; " @@ -142,6 +143,7 @@ def main(): print(f"competitor recoveries={competitor_recoveries}; strongest={100 * strongest_recovery:.1f}%") print(f"context-SDIL d4 advantage over strongest endpoint={deep_advantage:+.3f}") print(f"context-SDIL lesion mean={lesion_mean:+.3f}; positive={lesion_positive}/15") + print(f"nonfinite final losses={nonfinite}") print(f"C2 context validation gate: {'PASS' if passed else 'FAIL'}") if not passed: raise SystemExit(1) -- cgit v1.2.3