summaryrefslogtreecommitdiff
path: root/experiments/analyze_c2_context_validation.py
diff options
context:
space:
mode:
Diffstat (limited to 'experiments/analyze_c2_context_validation.py')
-rw-r--r--experiments/analyze_c2_context_validation.py8
1 files changed, 5 insertions, 3 deletions
diff --git a/experiments/analyze_c2_context_validation.py b/experiments/analyze_c2_context_validation.py
index c1ba9b3..f9ed7f6 100644
--- a/experiments/analyze_c2_context_validation.py
+++ b/experiments/analyze_c2_context_validation.py
@@ -45,8 +45,6 @@ def audit_row(path, row):
raise RuntimeError(f"protocol mismatch {path}: {mismatches}")
if row["final"].get("eval_split") != "validation":
raise RuntimeError(f"test-contaminated validation row: {path}")
- if not math.isfinite(row["final"].get("val_loss", math.nan)):
- raise RuntimeError(f"nonfinite final validation loss: {path}")
if any("eval_acc" in step or "cos_r_negg" in step for step in row.get("steps", [])):
raise RuntimeError(f"intermediate held-out metric/diagnostic: {path}")
split = row.get("split", {})
@@ -93,6 +91,9 @@ def main():
if any(len(hashes) != 1 for hashes in split_hashes.values()):
raise RuntimeError(f"methods did not share splits within tasks: {split_hashes}")
+ nonfinite = [key for key, row in rows.items()
+ if not math.isfinite(row["final"].get("val_loss", math.nan))]
+
print(f"commit={next(iter(commits))} rows={len(rows)} task_seeds={list(TASK_SEEDS)}")
print("| method | depth | validation (%) | depth gain | lesion drop |")
print("|:---|---:|---:|---:|---:|")
@@ -134,7 +135,7 @@ def main():
lesion_mean = statistics.mean(lesion_drops)
lesion_positive = sum(value > 0 for value in lesion_drops)
comparator_ok = strongest_recovery <= 0.5 or deep_advantage >= 2.0
- passed = (bp_gain >= 5.0 and recovery >= 0.7 and positive_gains >= 10
+ passed = (not nonfinite and bp_gain >= 5.0 and recovery >= 0.7 and positive_gains >= 10
and comparator_ok and lesion_mean >= 2.0 and lesion_positive >= 10)
print(f"BP gain={bp_gain:+.3f}; context-SDIL gain={sdil_gain:+.3f}; "
@@ -142,6 +143,7 @@ def main():
print(f"competitor recoveries={competitor_recoveries}; strongest={100 * strongest_recovery:.1f}%")
print(f"context-SDIL d4 advantage over strongest endpoint={deep_advantage:+.3f}")
print(f"context-SDIL lesion mean={lesion_mean:+.3f}; positive={lesion_positive}/15")
+ print(f"nonfinite final losses={nonfinite}")
print(f"C2 context validation gate: {'PASS' if passed else 'FAIL'}")
if not passed:
raise SystemExit(1)