#!/usr/bin/env python3 """Audit the frozen Oral-A failure and quantify its calibration precursor.""" import argparse import json import math import os def load(path): with open(path) as handle: return json.load(handle) def finite(value): return isinstance(value, (int, float)) and math.isfinite(value) def main(): parser = argparse.ArgumentParser() parser.add_argument( "--full", default="results/oral_a_dev/sdil_full_r20_s0.json") parser.add_argument( "--short", default="results/oral_a_short/sdil_channel_gated_lr0.03.json") parser.add_argument( "--gate", default="results/oral_a_full_gate.json") parser.add_argument( "--v2_gate", default="results/oral_a_v2_calibration_gate.json") parser.add_argument( "--representation", default="results/oral_a_representation_diagnosis.json") parser.add_argument( "--v3_gate", default="results/oral_a_v3_calibration_gate.json") parser.add_argument( "--hierarchical_oracle", default="results/oral_a_hierarchical_oracle.json") parser.add_argument( "--out", default="results/oral_a_failure_diagnosis.json") args = parser.parse_args() full = load(args.full) short = load(args.short) gate = load(args.gate) v2_gate = load(args.v2_gate) representation = load(args.representation) v3_gate = load(args.v3_gate) hierarchical = load(args.hierarchical_oracle) expected = { "mode": "sdil", "depth": 20, "epochs": 200, "vectorizer_mode": "channel_gated", "a_scale": 1.0, "eta_A": 0.001, "pert_directions": 1, "pert_every": 4, } for key, value in expected.items(): if full["args"].get(key) != value: raise ValueError(f"full A3 drift: {key}={full['args'].get(key)!r}") if full["provenance"]["git_tracked_dirty"]: raise ValueError("full A3 result has tracked source edits") if gate["status"] != "failed" or gate["confirmation_test_seeds_touched"]: raise ValueError("expected a failed A3 gate with untouched confirmation") if (v2_gate["status"] != "failed" or v2_gate["confirmation_test_seeds_touched"]): raise ValueError("expected a failed v2 gate with untouched confirmation") if representation["probe"]["test_examples_touched"] != 0: raise ValueError("representation diagnosis touched test examples") if (v3_gate["status"] != "failed" or v3_gate["confirmation_test_seeds_touched"]): raise ValueError("expected a failed v3 gate with untouched confirmation") if hierarchical["probe"]["test_examples_touched"] != 0: raise ValueError("hierarchical oracle touched test examples") calibration_rows = [] first_nonfinite_epoch = None eval_rows = [] for row in full["epochs"]: loss = row["train_loss"] if first_nonfinite_epoch is None and not finite(loss): first_nonfinite_epoch = row["epoch"] if finite(row.get("eval_accuracy")): eval_rows.append({ "epoch": row["epoch"], "accuracy": row["eval_accuracy"], "loss": row["eval_loss"] if finite(row["eval_loss"]) else None, "finite": finite(row["eval_loss"]), }) calibration = row.get("calibration") if calibration and all(finite(calibration.get(key)) for key in ( "calibration_mse", "target_power", "prediction_target_cosine")): power = calibration["target_power"] calibration_rows.append({ "epoch": row["epoch"], "target_power": power, "mse_to_target_power": calibration["calibration_mse"] / power, "prediction_target_cosine": calibration["prediction_target_cosine"], }) if first_nonfinite_epoch is None or not calibration_rows: raise ValueError("A3 failure trajectory is incomplete") finite_eval_rows = [row for row in eval_rows if row["finite"]] row_by_epoch = {row["epoch"]: row for row in calibration_rows} checkpoints = [row_by_epoch[epoch] for epoch in (1, 20, 40, 60, 80, 87)] target_growth_1_to_80 = ( row_by_epoch[80]["target_power"] / row_by_epoch[1]["target_power"]) target_growth_1_to_87 = ( row_by_epoch[87]["target_power"] / row_by_epoch[1]["target_power"]) max_abs_cosine = max(abs(row["prediction_target_cosine"]) for row in calibration_rows) max_mse_power_gap = max(abs(row["mse_to_target_power"] - 1.0) for row in calibration_rows) short_calibration = short["apical_warmup"]["last"] unit = v2_gate["selected"]["unit_targets"] structured = v2_gate["selected"]["channel_subspace"] output = { "protocol": "oral_a_A3_failure_diagnosis_v3", "source_paths": { "full": args.full, "short": args.short, "gate": args.gate, "v2_gate": args.v2_gate, "representation": args.representation, "v3_gate": args.v3_gate, "hierarchical_oracle": args.hierarchical_oracle, }, "source_commits": { "full": full["provenance"]["git_commit"], "short": short["provenance"]["git_commit"], }, "frozen_outcome": { "first_nonfinite_epoch": first_nonfinite_epoch, "final_accuracy": full["final"]["accuracy"], "final_finite": full["final"]["finite"], "last_finite_validation": finite_eval_rows[-1], "last_reported_validation": eval_rows[-1], "confirmation_test_seeds_touched": False, }, "short_screen_contrast": { "accuracy": short["final"]["accuracy"], "early_third_alignment": short["diagnostics"]["early_third_mean"], "warmup_prediction_target_cosine": ( short_calibration["prediction_target_cosine"]), "warmup_mse_to_target_power": ( short_calibration["calibration_mse"] / short_calibration["target_power"]), }, "calibration_precursor": { "checkpoints": checkpoints, "target_power_growth_epoch1_to_80": target_growth_1_to_80, "target_power_growth_epoch1_to_87": target_growth_1_to_87, "max_abs_prediction_target_cosine_before_nonfinite": max_abs_cosine, "max_abs_mse_to_target_power_minus_one": max_mse_power_gap, }, "post_failure_v2_refinement": { "unit_target_early_third_alignment": ( unit["metrics"]["early_third_alignment"]), "unit_target_all_layer_alignment": ( unit["metrics"]["all_layer_alignment"]), "structured_early_third_alignment": ( structured["metrics"]["early_third_alignment"]), "structured_all_layer_alignment": ( structured["metrics"]["all_layer_alignment"]), "structured_gate_status": v2_gate["status"], }, "post_failure_representation_refinement": { "trained_structured_early_third_alignment": ( structured["metrics"]["early_third_alignment"]), "channel_gated_cv_oracle_early_third_alignment": ( representation["cosine"]["channel_gated_cv"] ["early_third_mean"]), "per_example_spatial_oracle_early_third_alignment": ( representation["cosine"]["per_example_spatial_oracle"] ["early_third_mean"]), "naive_local_context_cv_early_third_alignment": ( representation["cosine"]["local_context_cv"] ["early_third_mean"]), }, "post_failure_v3_refinement": { "vectorizer_subspace_early_third_alignment": ( v3_gate["selected_v3"]["metrics"]["early_third_alignment"]), "vectorizer_subspace_all_layer_alignment": ( v3_gate["selected_v3"]["metrics"]["all_layer_alignment"]), "matched_v2_early_third_alignment": ( v3_gate["matched_v2_reference"]["metrics"] ["early_third_alignment"]), "matched_v2_all_layer_alignment": ( v3_gate["matched_v2_reference"]["metrics"] ["all_layer_alignment"]), "vectorizer_subspace_gate_status": v3_gate["status"], }, "post_failure_hierarchical_refinement": { key: hierarchical["cosine"][key]["early_third_mean"] for key in ( "direct_output_channel_gated", "downstream_activation_context", "hierarchical_1x1_gated", "hierarchical_3x3_gated") }, "diagnosis": { "classification": ( "early_credit_limited_by_estimator_efficiency_and_feedback_capacity"), "evidence": [ "instantaneous prediction-target cosine remains effectively zero", "calibration MSE remains indistinguishable from stochastic target power", "target power grows by orders of magnitude before nonfiniteness", "matched frozen-forward unit targets later yield only 0.0011 early-layer alignment", "structured learning reaches 0.0072 versus a 0.0240 cross-validated family oracle", "unconstrained coefficients raise the same spatial basis oracle only to 0.0549", "direct A/G estimation leaves early alignment unchanged at 0.0071", "held-out gated 3x3 maps from exact child errors reach 0.9998", ], "interpretation_limit": ( "instantaneous target metrics alone do not prove zero conditional " "learning; structured v2 has low instantaneous cosine yet positive " "exact-gradient alignment"), "next_test_outcome": ( "representable-subspace perturbation improved alignment but failed; " "direct vectorizer-space estimation then improved only middle/late " "layers and also failed the frozen early-layer gate"), "next_design_constraint": ( "do not add the tested activation contexts or repeat the closed " "A/G rate grid; learn spatial hierarchical feedback without " "weight copying and audit against hierarchical FA/BurstCCN"), }, } os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w") as handle: json.dump(output, handle, indent=2, sort_keys=True) handle.write("\n") print(json.dumps(output, indent=2, sort_keys=True)) if __name__ == "__main__": main()