1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
#!/usr/bin/env python3
"""Audit and gate the frozen KP-2 full ResNet-20 validation baseline."""
import argparse
import json
import math
import os
SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--selection", default="results/kp_short_gate.json")
parser.add_argument(
"--bp_selection", default="results/oral_a_bp_selection.json")
parser.add_argument("--input", default="results/kp_full/kp.json")
parser.add_argument("--out", default="results/kp_full_gate.json")
args = parser.parse_args()
with open(args.selection) as handle:
selection = json.load(handle)
if selection.get("protocol") != "kolen_pollack_short_v1":
raise ValueError("unexpected KP-1 selection protocol")
if selection.get("status") != "passed":
raise ValueError("KP-1 did not open KP-2")
with open(args.input) as handle:
record = json.load(handle)
run_args = record["args"]
expected = {
"mode": "kp", "depth": 20, "width": 16, "seed": 0,
"loader_seed": 0, "batch_size": 128, "epochs": 200,
"train_limit": 0, "val_examples": 5000, "split_seed": 2027,
"eval_split": "validation", "eval_every": 0,
"augment_train": 1, "lr": 0.1, "output_lr": 0.1,
"lr_schedule": "step", "lr_milestones": "100,150",
"lr_gamma": 0.1, "warmup_epochs": 0, "momentum": 0.9,
"weight_decay": 1e-4, "normalization": "batchnorm",
"a_scale": 1.0, "alignment_probe": 32,
}
for key, value in expected.items():
if run_args.get(key) != value:
raise ValueError(f"KP-2 {key} drift")
if record["provenance"]["git_tracked_dirty"]:
raise ValueError("tracked-dirty KP-2 result")
if record["split"]["validation_index_sha256"] != SPLIT_HASH:
raise ValueError("KP-2 split drift")
protocol = record["evaluation_protocol"]
if protocol["test_evaluations"] or protocol["test_used_for_selection"]:
raise ValueError("KP-2 touched test")
if record.get("calibration_metric_space") != (
"reciprocal_local_activity_products"):
raise ValueError("KP-2 metric-space drift")
with open(args.bp_selection) as handle:
bp_selection = json.load(handle)
if bp_selection.get("status") != "passed_primary":
raise ValueError("matched full BP reference is not frozen")
with open(bp_selection["selected"]["path"]) as handle:
bp = json.load(handle)
bp_accuracy = float(bp["final"]["accuracy"])
bp_macs = int(bp["work"]["total_macs_estimate"])
tracking = [row.get("feedback_tracking") for row in record["epochs"]]
if len(tracking) != 200 or any(value is None for value in tracking):
raise ValueError("KP-2 tracking trajectory is incomplete")
trajectory_values = []
for row, values in zip(record["epochs"], tracking):
trajectory_values.extend([
float(row["train_loss"]),
float(values["mean_feedback_forward_cosine"]),
float(values["mean_feedback_forward_relative_error"]),
float(values["min_feedback_forward_cosine"]),
float(values["max_feedback_forward_relative_error"]),
])
diagnostics = record["diagnostics"]
accuracy = float(record["final"]["accuracy"])
loss = float(record["final"]["loss"])
early = float(diagnostics["early_third_mean"])
final_cosine = float(diagnostics["mean_feedback_forward_cosine"])
late_cosine = sum(float(value["mean_feedback_forward_cosine"])
for value in tracking[150:]) / 50
total_macs = int(record["work"]["total_macs_estimate"])
queries = int(record["work"]["logical_batch_loss_queries"])
finite = (bool(record["final"]["finite"])
and all(math.isfinite(value) for value in
trajectory_values + [accuracy, loss, early, final_cosine]))
checks = {
"record_and_trajectory_finite": finite,
"accuracy_at_least_0.88": accuracy >= 0.88,
"early_alignment_at_least_0.80": early >= 0.80,
"final_feedback_cosine_at_least_0.95": final_cosine >= 0.95,
"epoch151_to200_feedback_cosine_at_least_0.95": late_cosine >= 0.95,
"zero_task_loss_queries": queries == 0,
"macs_at_most_1.40x_bp": total_macs <= 1.40 * bp_macs,
}
status = "passed" if all(checks.values()) else "failed"
metrics = {
"accuracy": accuracy, "loss": loss, "bp_accuracy": bp_accuracy,
"early_third_alignment": early,
"final_mean_feedback_forward_cosine": final_cosine,
"epoch151_to200_mean_feedback_forward_cosine": late_cosine,
"final_mean_feedback_forward_relative_error": float(
diagnostics["mean_feedback_forward_relative_error"]),
"total_macs": total_macs, "bp_total_macs": bp_macs,
"mac_ratio_to_bp": total_macs / bp_macs,
"logical_batch_loss_queries": queries,
"peak_memory_allocated_bytes": int(
record["hardware"]["peak_memory_allocated_bytes"]),
"wall_s": float(record["timing"]["total_timed_wall_s"]),
"source_commit": record["provenance"]["git_commit"],
}
output = {
"protocol": "kolen_pollack_full_v1", "status": status,
"checks": checks, "metrics": metrics,
"innovation_experiment_opened": status == "passed",
"confirmation_test_seeds_touched": False,
"review_score_before": 5, "review_score_after": 5,
"score_change_rule": "inherited KP baseline cannot raise score",
}
os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
with open(args.out, "w") as handle:
json.dump(output, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps(output, indent=2))
if __name__ == "__main__":
main()
|
