1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
|
#!/usr/bin/env python3
"""Audit the frozen Oral-A failure and quantify its calibration precursor."""
import argparse
import json
import math
import os
def load(path):
with open(path) as handle:
return json.load(handle)
def finite(value):
return isinstance(value, (int, float)) and math.isfinite(value)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--full", default="results/oral_a_dev/sdil_full_r20_s0.json")
parser.add_argument(
"--short", default="results/oral_a_short/sdil_channel_gated_lr0.03.json")
parser.add_argument(
"--gate", default="results/oral_a_full_gate.json")
parser.add_argument(
"--v2_gate", default="results/oral_a_v2_calibration_gate.json")
parser.add_argument(
"--representation",
default="results/oral_a_representation_diagnosis.json")
parser.add_argument(
"--v3_gate", default="results/oral_a_v3_calibration_gate.json")
parser.add_argument(
"--hierarchical_oracle",
default="results/oral_a_hierarchical_oracle.json")
parser.add_argument(
"--out", default="results/oral_a_failure_diagnosis.json")
args = parser.parse_args()
full = load(args.full)
short = load(args.short)
gate = load(args.gate)
v2_gate = load(args.v2_gate)
representation = load(args.representation)
v3_gate = load(args.v3_gate)
hierarchical = load(args.hierarchical_oracle)
expected = {
"mode": "sdil", "depth": 20, "epochs": 200,
"vectorizer_mode": "channel_gated", "a_scale": 1.0,
"eta_A": 0.001, "pert_directions": 1, "pert_every": 4,
}
for key, value in expected.items():
if full["args"].get(key) != value:
raise ValueError(f"full A3 drift: {key}={full['args'].get(key)!r}")
if full["provenance"]["git_tracked_dirty"]:
raise ValueError("full A3 result has tracked source edits")
if gate["status"] != "failed" or gate["confirmation_test_seeds_touched"]:
raise ValueError("expected a failed A3 gate with untouched confirmation")
if (v2_gate["status"] != "failed"
or v2_gate["confirmation_test_seeds_touched"]):
raise ValueError("expected a failed v2 gate with untouched confirmation")
if representation["probe"]["test_examples_touched"] != 0:
raise ValueError("representation diagnosis touched test examples")
if (v3_gate["status"] != "failed"
or v3_gate["confirmation_test_seeds_touched"]):
raise ValueError("expected a failed v3 gate with untouched confirmation")
if hierarchical["probe"]["test_examples_touched"] != 0:
raise ValueError("hierarchical oracle touched test examples")
calibration_rows = []
first_nonfinite_epoch = None
eval_rows = []
for row in full["epochs"]:
loss = row["train_loss"]
if first_nonfinite_epoch is None and not finite(loss):
first_nonfinite_epoch = row["epoch"]
if finite(row.get("eval_accuracy")):
eval_rows.append({
"epoch": row["epoch"],
"accuracy": row["eval_accuracy"],
"loss": row["eval_loss"] if finite(row["eval_loss"]) else None,
"finite": finite(row["eval_loss"]),
})
calibration = row.get("calibration")
if calibration and all(finite(calibration.get(key)) for key in (
"calibration_mse", "target_power", "prediction_target_cosine")):
power = calibration["target_power"]
calibration_rows.append({
"epoch": row["epoch"],
"target_power": power,
"mse_to_target_power": calibration["calibration_mse"] / power,
"prediction_target_cosine": calibration["prediction_target_cosine"],
})
if first_nonfinite_epoch is None or not calibration_rows:
raise ValueError("A3 failure trajectory is incomplete")
finite_eval_rows = [row for row in eval_rows if row["finite"]]
row_by_epoch = {row["epoch"]: row for row in calibration_rows}
checkpoints = [row_by_epoch[epoch] for epoch in (1, 20, 40, 60, 80, 87)]
target_growth_1_to_80 = (
row_by_epoch[80]["target_power"] / row_by_epoch[1]["target_power"])
target_growth_1_to_87 = (
row_by_epoch[87]["target_power"] / row_by_epoch[1]["target_power"])
max_abs_cosine = max(abs(row["prediction_target_cosine"])
for row in calibration_rows)
max_mse_power_gap = max(abs(row["mse_to_target_power"] - 1.0)
for row in calibration_rows)
short_calibration = short["apical_warmup"]["last"]
unit = v2_gate["selected"]["unit_targets"]
structured = v2_gate["selected"]["channel_subspace"]
output = {
"protocol": "oral_a_A3_failure_diagnosis_v3",
"source_paths": {
"full": args.full, "short": args.short, "gate": args.gate,
"v2_gate": args.v2_gate, "representation": args.representation,
"v3_gate": args.v3_gate,
"hierarchical_oracle": args.hierarchical_oracle,
},
"source_commits": {
"full": full["provenance"]["git_commit"],
"short": short["provenance"]["git_commit"],
},
"frozen_outcome": {
"first_nonfinite_epoch": first_nonfinite_epoch,
"final_accuracy": full["final"]["accuracy"],
"final_finite": full["final"]["finite"],
"last_finite_validation": finite_eval_rows[-1],
"last_reported_validation": eval_rows[-1],
"confirmation_test_seeds_touched": False,
},
"short_screen_contrast": {
"accuracy": short["final"]["accuracy"],
"early_third_alignment": short["diagnostics"]["early_third_mean"],
"warmup_prediction_target_cosine": (
short_calibration["prediction_target_cosine"]),
"warmup_mse_to_target_power": (
short_calibration["calibration_mse"]
/ short_calibration["target_power"]),
},
"calibration_precursor": {
"checkpoints": checkpoints,
"target_power_growth_epoch1_to_80": target_growth_1_to_80,
"target_power_growth_epoch1_to_87": target_growth_1_to_87,
"max_abs_prediction_target_cosine_before_nonfinite": max_abs_cosine,
"max_abs_mse_to_target_power_minus_one": max_mse_power_gap,
},
"post_failure_v2_refinement": {
"unit_target_early_third_alignment": (
unit["metrics"]["early_third_alignment"]),
"unit_target_all_layer_alignment": (
unit["metrics"]["all_layer_alignment"]),
"structured_early_third_alignment": (
structured["metrics"]["early_third_alignment"]),
"structured_all_layer_alignment": (
structured["metrics"]["all_layer_alignment"]),
"structured_gate_status": v2_gate["status"],
},
"post_failure_representation_refinement": {
"trained_structured_early_third_alignment": (
structured["metrics"]["early_third_alignment"]),
"channel_gated_cv_oracle_early_third_alignment": (
representation["cosine"]["channel_gated_cv"]
["early_third_mean"]),
"per_example_spatial_oracle_early_third_alignment": (
representation["cosine"]["per_example_spatial_oracle"]
["early_third_mean"]),
"naive_local_context_cv_early_third_alignment": (
representation["cosine"]["local_context_cv"]
["early_third_mean"]),
},
"post_failure_v3_refinement": {
"vectorizer_subspace_early_third_alignment": (
v3_gate["selected_v3"]["metrics"]["early_third_alignment"]),
"vectorizer_subspace_all_layer_alignment": (
v3_gate["selected_v3"]["metrics"]["all_layer_alignment"]),
"matched_v2_early_third_alignment": (
v3_gate["matched_v2_reference"]["metrics"]
["early_third_alignment"]),
"matched_v2_all_layer_alignment": (
v3_gate["matched_v2_reference"]["metrics"]
["all_layer_alignment"]),
"vectorizer_subspace_gate_status": v3_gate["status"],
},
"post_failure_hierarchical_refinement": {
key: hierarchical["cosine"][key]["early_third_mean"]
for key in (
"direct_output_channel_gated",
"downstream_activation_context",
"hierarchical_1x1_gated",
"hierarchical_3x3_gated")
},
"diagnosis": {
"classification": (
"early_credit_limited_by_estimator_efficiency_and_feedback_capacity"),
"evidence": [
"instantaneous prediction-target cosine remains effectively zero",
"calibration MSE remains indistinguishable from stochastic target power",
"target power grows by orders of magnitude before nonfiniteness",
"matched frozen-forward unit targets later yield only 0.0011 early-layer alignment",
"structured learning reaches 0.0072 versus a 0.0240 cross-validated family oracle",
"unconstrained coefficients raise the same spatial basis oracle only to 0.0549",
"direct A/G estimation leaves early alignment unchanged at 0.0071",
"held-out gated 3x3 maps from exact child errors reach 0.9998",
],
"interpretation_limit": (
"instantaneous target metrics alone do not prove zero conditional "
"learning; structured v2 has low instantaneous cosine yet positive "
"exact-gradient alignment"),
"next_test_outcome": (
"representable-subspace perturbation improved alignment but failed; "
"direct vectorizer-space estimation then improved only middle/late "
"layers and also failed the frozen early-layer gate"),
"next_design_constraint": (
"do not add the tested activation contexts or repeat the closed "
"A/G rate grid; learn spatial hierarchical feedback without "
"weight copying and audit against hierarchical FA/BurstCCN"),
},
}
os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
with open(args.out, "w") as handle:
json.dump(output, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps(output, indent=2, sort_keys=True))
if __name__ == "__main__":
main()
|
