#!/usr/bin/env python3 """Audit and gate the frozen KP-1 ResNet-20 development record.""" import argparse import json import math import os SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" BP_EPOCH20_ACCURACY = 0.8102 BP_EPOCH20_MACS = 109_487_808_000_000 def main(): parser = argparse.ArgumentParser() parser.add_argument("--input", default="results/kp_short/kp.json") parser.add_argument( "--bp", default="results/oral_a_dev/bp_reference_primary.json") parser.add_argument("--out", default="results/kp_short_gate.json") args = parser.parse_args() with open(args.input) as handle: record = json.load(handle) run_args = record["args"] expected = { "mode": "kp", "depth": 20, "width": 16, "seed": 0, "loader_seed": 0, "batch_size": 128, "epochs": 20, "train_limit": 0, "val_examples": 5000, "split_seed": 2027, "eval_split": "validation", "eval_every": 0, "augment_train": 1, "lr": 0.1, "output_lr": 0.1, "lr_schedule": "step", "lr_milestones": "100,150", "lr_gamma": 0.1, "warmup_epochs": 0, "momentum": 0.9, "weight_decay": 1e-4, "normalization": "batchnorm", "a_scale": 1.0, "alignment_probe": 32, } for key, value in expected.items(): if run_args.get(key) != value: raise ValueError(f"KP-1 {key} drift") if record["provenance"]["git_tracked_dirty"]: raise ValueError("tracked-dirty KP-1 result") if record["split"]["validation_index_sha256"] != SPLIT_HASH: raise ValueError("KP-1 split drift") protocol = record["evaluation_protocol"] if protocol["test_evaluations"] or protocol["test_used_for_selection"]: raise ValueError("KP-1 touched test") if record.get("calibration_metric_space") != ( "reciprocal_local_activity_products"): raise ValueError("KP-1 metric-space drift") with open(args.bp) as handle: bp = json.load(handle) bp_epoch20 = [row for row in bp["epochs"] if row["epoch"] == 20] if (len(bp_epoch20) != 1 or float(bp_epoch20[0]["eval_accuracy"]) != BP_EPOCH20_ACCURACY): raise ValueError("matched BP epoch-20 endpoint drift") if int(bp["work"]["total_macs_estimate"]) // 10 != BP_EPOCH20_MACS: raise ValueError("matched BP epoch-20 MAC reference drift") epoch_tracking = [row.get("feedback_tracking") for row in record["epochs"]] if len(epoch_tracking) != 20 or any(value is None for value in epoch_tracking): raise ValueError("KP-1 tracking trajectory is incomplete") trajectory_values = [] for row, tracking in zip(record["epochs"], epoch_tracking): trajectory_values.extend([ float(row["train_loss"]), float(tracking["mean_feedback_forward_cosine"]), float(tracking["mean_feedback_forward_relative_error"]), float(tracking["min_feedback_forward_cosine"]), float(tracking["max_feedback_forward_relative_error"]), ]) diagnostics = record["diagnostics"] accuracy = float(record["final"]["accuracy"]) loss = float(record["final"]["loss"]) early = float(diagnostics["early_third_mean"]) final_cosine = float(diagnostics["mean_feedback_forward_cosine"]) late_cosine = sum(float(value["mean_feedback_forward_cosine"]) for value in epoch_tracking[10:]) / 10 total_macs = int(record["work"]["total_macs_estimate"]) queries = int(record["work"]["logical_batch_loss_queries"]) finite = (bool(record["final"]["finite"]) and all(math.isfinite(value) for value in trajectory_values + [accuracy, loss, early, final_cosine])) checks = { "record_and_trajectory_finite": finite, "accuracy_at_least_0.70": accuracy >= 0.70, "within_15_points_of_bp_epoch20": ( accuracy >= BP_EPOCH20_ACCURACY - 0.15), "early_alignment_at_least_0.50": early >= 0.50, "final_feedback_cosine_at_least_0.80": final_cosine >= 0.80, "epoch11_to20_feedback_cosine_at_least_0.70": late_cosine >= 0.70, "zero_task_loss_queries": queries == 0, "macs_at_most_1.40x_bp": total_macs <= 1.40 * BP_EPOCH20_MACS, } status = "passed" if all(checks.values()) else "failed" metrics = { "accuracy": accuracy, "loss": loss, "bp_epoch20_accuracy": BP_EPOCH20_ACCURACY, "early_third_alignment": early, "final_mean_feedback_forward_cosine": final_cosine, "epoch11_to20_mean_feedback_forward_cosine": late_cosine, "final_mean_feedback_forward_relative_error": float( diagnostics["mean_feedback_forward_relative_error"]), "total_macs": total_macs, "bp_epoch20_total_macs": BP_EPOCH20_MACS, "mac_ratio_to_bp": total_macs / BP_EPOCH20_MACS, "logical_batch_loss_queries": queries, "peak_memory_allocated_bytes": int( record["hardware"]["peak_memory_allocated_bytes"]), "wall_s": float(record["timing"]["total_timed_wall_s"]), "source_commit": record["provenance"]["git_commit"], } output = { "protocol": "kolen_pollack_short_v1", "status": status, "checks": checks, "metrics": metrics, "full_validation_opened": status == "passed", "confirmation_test_seeds_touched": False, "review_score_before": 5, "review_score_after": 5, "score_change_rule": "inherited KP baseline cannot raise score", } os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w") as handle: json.dump(output, handle, indent=2, sort_keys=True) handle.write("\n") print(json.dumps(output, indent=2)) if __name__ == "__main__": main()