protocol: freeze KP mixed-traffic innovation gates

2 files changed, 238 insertions, 0 deletions

diff --git a/experiments/analyze_kp_innovation_short.py b/experiments/analyze_kp_innovation_short.py
new file mode 100644
index 0000000..4c70575
--- /dev/null
+++ b/experiments/analyze_kp_innovation_short.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Audit and gate the frozen MT-1 mixed-traffic innovation panel."""
+import argparse
+import json
+import math
+import os
+
+
+RULES = ("raw", "matched", "innovation")
+SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b"
+BP_EPOCH20_MACS = 109_487_808_000_000
+KP_SHORT_ACCURACY = 0.8266
+
+
+def mean_early(values):
+    count = max(1, len(values) // 3)
+    return sum(float(value) for value in values[:count]) / count
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_dir", default="results/kp_innovation_short")
+    parser.add_argument("--kp_full_gate", default="results/kp_full_gate.json")
+    parser.add_argument("--out", default="results/kp_innovation_short_gate.json")
+    args = parser.parse_args()
+    with open(args.kp_full_gate) as handle:
+        kp_gate = json.load(handle)
+    if (kp_gate.get("protocol") != "kolen_pollack_full_v1"
+            or kp_gate.get("status") != "passed"):
+        raise ValueError("a KP-2 pass is required")
+
+    records = {}
+    expected_common = {
+        "mode": "kp_traffic", "depth": 20, "width": 16, "seed": 0,
+        "loader_seed": 0, "batch_size": 128, "epochs": 20,
+        "train_limit": 0, "val_examples": 5000, "split_seed": 2027,
+        "eval_split": "validation", "eval_every": 0,
+        "augment_train": 1, "lr": 0.1, "output_lr": 0.1,
+        "lr_schedule": "step", "lr_milestones": "100,150",
+        "lr_gamma": 0.1, "warmup_epochs": 0, "momentum": 0.9,
+        "weight_decay": 1e-4, "normalization": "batchnorm",
+        "a_scale": 1.0, "traffic_seed": 4000, "traffic_ratio": 4.0,
+        "traffic_calibration_examples": 64, "learn_P": 1, "eta_P": 0.1,
+        "predictor_warmup_steps": 20, "predictor_every": 16,
+        "alignment_probe": 32,
+    }
+    source_commits = set()
+    for rule in RULES:
+        with open(os.path.join(args.input_dir, f"{rule}.json")) as handle:
+            record = json.load(handle)
+        records[rule] = record
+        run_args = record["args"]
+        for key, value in {**expected_common, "traffic_rule": rule}.items():
+            if run_args.get(key) != value:
+                raise ValueError(f"MT-1 {rule} {key} drift")
+        if record["provenance"]["git_tracked_dirty"]:
+            raise ValueError(f"tracked-dirty MT-1 {rule} result")
+        source_commits.add(record["provenance"]["git_commit"])
+        if record["split"]["validation_index_sha256"] != SPLIT_HASH:
+            raise ValueError(f"MT-1 {rule} split drift")
+        protocol = record["evaluation_protocol"]
+        if protocol["test_evaluations"] or protocol["test_used_for_selection"]:
+            raise ValueError(f"MT-1 {rule} touched test")
+        if record.get("calibration_metric_space") != (
+                "reciprocal_local_activity_products_with_mixed_apical_traffic"):
+            raise ValueError(f"MT-1 {rule} metric-space drift")
+    if len(source_commits) != 1:
+        raise ValueError("MT-1 conditions must share one source revision")
+
+    accuracies = {rule: float(records[rule]["final"]["accuracy"])
+                  for rule in RULES}
+    diagnostics = {rule: records[rule]["diagnostics"] for rule in RULES}
+    innovation_early = float(diagnostics["innovation"]["early_third_mean"])
+    innovation_raw_early = mean_early(
+        diagnostics["innovation"]["raw_negative_gradient_cosine"])
+    tracking = {
+        rule: [row.get("feedback_tracking") for row in records[rule]["epochs"]]
+        for rule in RULES
+    }
+    trajectory_values = []
+    for rule in RULES:
+        if len(tracking[rule]) != 20 or any(value is None for value in tracking[rule]):
+            raise ValueError(f"MT-1 {rule} tracking trajectory is incomplete")
+        for row, values in zip(records[rule]["epochs"], tracking[rule]):
+            trajectory_values.extend([
+                float(row["train_loss"]),
+                float(values["mean_feedback_forward_cosine"]),
+                float(values["mean_feedback_forward_relative_error"]),
+                float(values["min_feedback_forward_cosine"]),
+                float(values["max_feedback_forward_relative_error"]),
+            ])
+
+    all_finite = all(record["final"]["finite"] for record in records.values())
+    all_finite = all_finite and all(math.isfinite(value) for value in (
+        trajectory_values + list(accuracies.values()) + [innovation_early,
+                                                         innovation_raw_early]))
+    initial_ratio_errors = []
+    predictor_residual_ratios = []
+    total_macs = {}
+    queries = {}
+    elementwise = {}
+    for rule, record in records.items():
+        initial_ratio_errors.extend(abs(float(value) - 4.0) for value in
+            record["traffic_calibration"]["realized_traffic_instruction_rms_ratio"])
+        predictor_residual_ratios.append(float(
+            record["predictor_warmup"]["post_warmup_traffic_residual_rms_ratio"]))
+        total_macs[rule] = int(record["work"]["total_macs_estimate"])
+        queries[rule] = int(record["work"]["logical_batch_loss_queries"])
+        elementwise[rule] = int(record["work"]["elementwise_operations_estimate"])
+    matched_norm_error = float(
+        diagnostics["matched"]["max_norm_match_relative_error"])
+    final_feedback_cosine = float(
+        diagnostics["innovation"]["mean_feedback_forward_cosine"])
+    late_feedback_cosine = sum(float(value["mean_feedback_forward_cosine"])
+                               for value in tracking["innovation"][10:]) / 10
+
+    checks = {
+        "records_trajectories_and_diagnostics_finite": all_finite,
+        "innovation_accuracy_at_least_0.75": accuracies["innovation"] >= 0.75,
+        "innovation_within_5_points_of_clean_kp": (
+            accuracies["innovation"] >= KP_SHORT_ACCURACY - 0.05),
+        "innovation_gain_over_raw_at_least_0.05": (
+            accuracies["innovation"] - accuracies["raw"] >= 0.05),
+        "innovation_gain_over_matched_at_least_0.03": (
+            accuracies["innovation"] - accuracies["matched"] >= 0.03),
+        "innovation_early_alignment_at_least_0.70": innovation_early >= 0.70,
+        "same_network_alignment_gain_over_raw_at_least_0.15": (
+            innovation_early - innovation_raw_early >= 0.15),
+        "final_feedback_cosine_at_least_0.80": final_feedback_cosine >= 0.80,
+        "epoch11_to20_feedback_cosine_at_least_0.70": (
+            late_feedback_cosine >= 0.70),
+        "initial_layer_ratio_error_at_most_1e-5": (
+            max(initial_ratio_errors) <= 1e-5),
+        "post_warmup_predictor_residual_ratio_at_most_0.25": (
+            max(predictor_residual_ratios) <= 0.25),
+        "matched_norm_relative_error_at_most_1e-6": matched_norm_error <= 1e-6,
+        "zero_task_loss_queries": all(value == 0 for value in queries.values()),
+        "each_affine_mac_count_at_most_1.40x_bp": all(
+            value <= 1.40 * BP_EPOCH20_MACS for value in total_macs.values()),
+        "elementwise_cost_reported": all(value > 0 for value in elementwise.values()),
+    }
+    status = "passed" if all(checks.values()) else "failed"
+    output = {
+        "protocol": "kp_mixed_traffic_short_v1", "status": status,
+        "checks": checks,
+        "metrics": {
+            "accuracy": accuracies,
+            "clean_kp_accuracy": KP_SHORT_ACCURACY,
+            "innovation_gain_over_raw": (
+                accuracies["innovation"] - accuracies["raw"]),
+            "innovation_gain_over_matched": (
+                accuracies["innovation"] - accuracies["matched"]),
+            "innovation_early_third_alignment": innovation_early,
+            "same_innovation_network_raw_early_third_alignment": innovation_raw_early,
+            "final_feedback_forward_cosine": final_feedback_cosine,
+            "epoch11_to20_feedback_forward_cosine": late_feedback_cosine,
+            "max_initial_layer_ratio_error": max(initial_ratio_errors),
+            "max_post_warmup_predictor_residual_rms_ratio": (
+                max(predictor_residual_ratios)),
+            "matched_norm_relative_error": matched_norm_error,
+            "total_macs": total_macs,
+            "mac_ratio_to_bp": {rule: value / BP_EPOCH20_MACS
+                                for rule, value in total_macs.items()},
+            "elementwise_operations_estimate": elementwise,
+            "logical_batch_loss_queries": queries,
+            "source_commit": next(iter(source_commits)),
+        },
+        "full_validation_opened": status == "passed",
+        "confirmation_test_seeds_touched": False,
+        "review_score_before": 5, "review_score_after": 5,
+        "score_change_rule": (
+            "a one-seed short mechanism screen cannot raise the review score"),
+    }
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
+    with open(args.out, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/kp_innovation_short_development.py b/experiments/kp_innovation_short_development.py
new file mode 100644
index 0000000..a4c5645
--- /dev/null
+++ b/experiments/kp_innovation_short_development.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Run the three frozen MT-1 mixed-traffic development records."""
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+
+RULES = ("raw", "matched", "innovation")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--kp_full_gate", default="results/kp_full_gate.json")
+    parser.add_argument("--rule", choices=("all",) + RULES, default="all")
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--dry_run", action="store_true")
+    args = parser.parse_args()
+    with open(args.kp_full_gate) as handle:
+        kp_gate = json.load(handle)
+    if kp_gate.get("protocol") != "kolen_pollack_full_v1":
+        raise ValueError("unexpected KP-2 protocol")
+    if kp_gate.get("status") != "passed":
+        raise ValueError("KP-2 did not open MT-1")
+
+    rules = RULES if args.rule == "all" else (args.rule,)
+    os.makedirs("results/kp_innovation_short", exist_ok=True)
+    for rule in rules:
+        command = [
+            sys.executable, "experiments/conv_run.py",
+            "--mode", "kp_traffic", "--traffic_rule", rule,
+            "--device", args.device, "--depth", "20", "--width", "16",
+            "--seed", "0", "--loader_seed", "0", "--batch_size", "128",
+            "--epochs", "20", "--train_limit", "0",
+            "--val_examples", "5000", "--split_seed", "2027",
+            "--eval_split", "validation", "--eval_every", "0",
+            "--augment_train", "1", "--lr", "0.1", "--output_lr", "0.1",
+            "--lr_schedule", "step", "--lr_milestones", "100,150",
+            "--lr_gamma", "0.1", "--warmup_epochs", "0",
+            "--momentum", "0.9", "--weight_decay", "1e-4",
+            "--normalization", "batchnorm", "--a_scale", "1",
+            "--traffic_seed", "4000", "--traffic_ratio", "4",
+            "--traffic_calibration_examples", "64",
+            "--learn_P", "1", "--eta_P", "0.1",
+            "--predictor_warmup_steps", "20", "--predictor_every", "16",
+            "--alignment_probe", "32",
+            "--out", f"results/kp_innovation_short/{rule}.json",
+        ]
+        print(" ".join(command), flush=True)
+        if not args.dry_run:
+            subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()