1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
#!/usr/bin/env python3
"""Audit the frozen Oral-A failure and quantify its calibration precursor."""
import argparse
import json
import math
import os
def load(path):
with open(path) as handle:
return json.load(handle)
def finite(value):
return isinstance(value, (int, float)) and math.isfinite(value)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--full", default="results/oral_a_dev/sdil_full_r20_s0.json")
parser.add_argument(
"--short", default="results/oral_a_short/sdil_channel_gated_lr0.03.json")
parser.add_argument(
"--gate", default="results/oral_a_full_gate.json")
parser.add_argument(
"--v2_gate", default="results/oral_a_v2_calibration_gate.json")
parser.add_argument(
"--out", default="results/oral_a_failure_diagnosis.json")
args = parser.parse_args()
full = load(args.full)
short = load(args.short)
gate = load(args.gate)
v2_gate = load(args.v2_gate)
expected = {
"mode": "sdil", "depth": 20, "epochs": 200,
"vectorizer_mode": "channel_gated", "a_scale": 1.0,
"eta_A": 0.001, "pert_directions": 1, "pert_every": 4,
}
for key, value in expected.items():
if full["args"].get(key) != value:
raise ValueError(f"full A3 drift: {key}={full['args'].get(key)!r}")
if full["provenance"]["git_tracked_dirty"]:
raise ValueError("full A3 result has tracked source edits")
if gate["status"] != "failed" or gate["confirmation_test_seeds_touched"]:
raise ValueError("expected a failed A3 gate with untouched confirmation")
if (v2_gate["status"] != "failed"
or v2_gate["confirmation_test_seeds_touched"]):
raise ValueError("expected a failed v2 gate with untouched confirmation")
calibration_rows = []
first_nonfinite_epoch = None
eval_rows = []
for row in full["epochs"]:
loss = row["train_loss"]
if first_nonfinite_epoch is None and not finite(loss):
first_nonfinite_epoch = row["epoch"]
if finite(row.get("eval_accuracy")):
eval_rows.append({
"epoch": row["epoch"],
"accuracy": row["eval_accuracy"],
"loss": row["eval_loss"] if finite(row["eval_loss"]) else None,
"finite": finite(row["eval_loss"]),
})
calibration = row.get("calibration")
if calibration and all(finite(calibration.get(key)) for key in (
"calibration_mse", "target_power", "prediction_target_cosine")):
power = calibration["target_power"]
calibration_rows.append({
"epoch": row["epoch"],
"target_power": power,
"mse_to_target_power": calibration["calibration_mse"] / power,
"prediction_target_cosine": calibration["prediction_target_cosine"],
})
if first_nonfinite_epoch is None or not calibration_rows:
raise ValueError("A3 failure trajectory is incomplete")
finite_eval_rows = [row for row in eval_rows if row["finite"]]
row_by_epoch = {row["epoch"]: row for row in calibration_rows}
checkpoints = [row_by_epoch[epoch] for epoch in (1, 20, 40, 60, 80, 87)]
target_growth_1_to_80 = (
row_by_epoch[80]["target_power"] / row_by_epoch[1]["target_power"])
target_growth_1_to_87 = (
row_by_epoch[87]["target_power"] / row_by_epoch[1]["target_power"])
max_abs_cosine = max(abs(row["prediction_target_cosine"])
for row in calibration_rows)
max_mse_power_gap = max(abs(row["mse_to_target_power"] - 1.0)
for row in calibration_rows)
short_calibration = short["apical_warmup"]["last"]
unit = v2_gate["selected"]["unit_targets"]
structured = v2_gate["selected"]["channel_subspace"]
output = {
"protocol": "oral_a_A3_failure_diagnosis_v2",
"source_paths": {
"full": args.full, "short": args.short, "gate": args.gate,
"v2_gate": args.v2_gate,
},
"source_commits": {
"full": full["provenance"]["git_commit"],
"short": short["provenance"]["git_commit"],
},
"frozen_outcome": {
"first_nonfinite_epoch": first_nonfinite_epoch,
"final_accuracy": full["final"]["accuracy"],
"final_finite": full["final"]["finite"],
"last_finite_validation": finite_eval_rows[-1],
"last_reported_validation": eval_rows[-1],
"confirmation_test_seeds_touched": False,
},
"short_screen_contrast": {
"accuracy": short["final"]["accuracy"],
"early_third_alignment": short["diagnostics"]["early_third_mean"],
"warmup_prediction_target_cosine": (
short_calibration["prediction_target_cosine"]),
"warmup_mse_to_target_power": (
short_calibration["calibration_mse"]
/ short_calibration["target_power"]),
},
"calibration_precursor": {
"checkpoints": checkpoints,
"target_power_growth_epoch1_to_80": target_growth_1_to_80,
"target_power_growth_epoch1_to_87": target_growth_1_to_87,
"max_abs_prediction_target_cosine_before_nonfinite": max_abs_cosine,
"max_abs_mse_to_target_power_minus_one": max_mse_power_gap,
},
"post_failure_v2_refinement": {
"unit_target_early_third_alignment": (
unit["metrics"]["early_third_alignment"]),
"unit_target_all_layer_alignment": (
unit["metrics"]["all_layer_alignment"]),
"structured_early_third_alignment": (
structured["metrics"]["early_third_alignment"]),
"structured_all_layer_alignment": (
structured["metrics"]["all_layer_alignment"]),
"structured_gate_status": v2_gate["status"],
},
"diagnosis": {
"classification": "unit_target_regression_snr_inadequate_before_runaway",
"evidence": [
"instantaneous prediction-target cosine remains effectively zero",
"calibration MSE remains indistinguishable from stochastic target power",
"target power grows by orders of magnitude before nonfiniteness",
"matched frozen-forward unit targets later yield only 0.0011 early-layer alignment",
],
"interpretation_limit": (
"instantaneous target metrics alone do not prove zero conditional "
"learning; structured v2 has low instantaneous cosine yet positive "
"exact-gradient alignment"),
"next_test_outcome": (
"representable-subspace perturbation improved exact alignment but "
"failed the frozen early-layer causal-capture gate"),
},
}
os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
with open(args.out, "w") as handle:
json.dump(output, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps(output, indent=2, sort_keys=True))
if __name__ == "__main__":
main()
|
