1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
#!/usr/bin/env python3
"""Audit calibration-split oral-B-v2 development validation."""
import argparse
import glob
import hashlib
import json
import os
from experiments.analyze_bci_v2_recovery_development import checks
from experiments.bci_v2_calibrated_run import (
CALIBRATION_EPISODES,
CHALLENGE_EPISODES,
MODEL_SEEDS,
TARGET_QUANTILES,
TASK_SEEDS,
build_recovery_config,
source_paths,
)
from experiments.bci_v2_recovery_run import FIXED_CONFIG
from experiments.bci_v2_run import CONDITIONS
def require(condition, message):
if not condition:
raise ValueError(message)
def sha256(path):
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def validate(row, path, task_seed, digests):
require(row.get("schema_version") == 4, f"{path}: schema")
require(
row.get("args", {}).get("task_seed") == task_seed
and row["args"].get("model_seed") in MODEL_SEEDS,
f"{path}: seeds",
)
protocol = row.get("protocol", {})
require(
protocol.get("name")
== "oral_b_v2_calibrated_recovery_development_v1"
and protocol.get("selection_split") == "development_validation"
and protocol.get("hyperparameter_selection") is False
and protocol.get("confirmation_seeds_touched") is False
and protocol.get("fixed_config") == FIXED_CONFIG
and protocol.get("calibration_uses_outcome_labels") is False
and protocol.get("protocol_sha256") == digests["protocol"]
and protocol.get("failed_target_gate_sha256")
== digests["failed_target_gate"],
f"{path}: protocol",
)
provenance = row.get("provenance", {})
require(
provenance.get("git_tracked_dirty") is False
and provenance.get("tracked_inputs")
and all(provenance["tracked_inputs"].values())
and set(provenance["tracked_inputs"]) == set(digests)
and provenance.get("input_sha256") == digests,
f"{path}: provenance",
)
require(
row.get("config") == vars(build_recovery_config())
and row.get("split") == "development_validation"
and row.get("finite") is True,
f"{path}: config/split",
)
require(
set(row.get("conditions", {})) == set(CONDITIONS),
f"{path}: conditions",
)
for name in CONDITIONS:
warmup = row["warmup"][name]
condition = row["conditions"][name]
cost = condition["cost"]
require(
warmup["batches"] == 100
and warmup["examples"] == 6400
and warmup["instruction_present"] is False
and warmup["role_cursor_scalar_observations"] == 12800
and warmup["predictor_max_abs_error"] <= 1e-5
and len(condition["daily_success"]) == 14
and cost["maximum_state_episode_steps"] == 25088
and cost["cursor_scalar_observations"]
== 2 * cost["role_probe_examples"]
and cost["task_loss_queries"] == 0
and cost["reverse_mode_calls"] == 0,
f"{path}: condition {name}",
)
challenge = row["assays"]["challenge"]
calibration = challenge["calibration"]
targets = challenge["targets"]
require(
calibration["episodes"] == CALIBRATION_EPISODES
and calibration["quantiles"] == list(TARGET_QUANTILES)
and calibration["targets"] == targets
and calibration["uses_outcome_labels"] is False
and calibration["seed"] == 550_000 + task_seed
and challenge["episodes_per_target"] == CHALLENGE_EPISODES
and challenge["selection_over_evaluation_outcomes"] is False
and challenge["maximum_steps_per_episode"] == 28
and len(targets) == len(TARGET_QUANTILES)
and all(
targets[index] < targets[index + 1]
for index in range(len(targets) - 1)
)
and row["signatures"]["challenge_episodes"]
== len(TARGET_QUANTILES) * CHALLENGE_EPISODES,
f"{path}: calibration",
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--results", default="results/bci_v2_calibrated_dev"
)
parser.add_argument(
"--out", default="results/bci_v2_calibrated_dev_gate.json"
)
args = parser.parse_args()
digests = {
name: sha256(path)
for name, path in source_paths().items()
}
expected = {
f"bci_v2_calibrated_t{task_seed}_m0.json"
for task_seed in TASK_SEEDS
}
observed = {
os.path.basename(path)
for path in glob.glob(os.path.join(args.results, "*.json"))
}
require(
observed == expected,
(
f"calibrated grid drift: missing={sorted(expected-observed)}, "
f"extra={sorted(observed-expected)}"
),
)
records = {}
commits = set()
source_sha256 = {}
for task_seed in TASK_SEEDS:
path = os.path.join(
args.results,
f"bci_v2_calibrated_t{task_seed}_m0.json",
)
with open(path) as handle:
row = json.load(handle)
validate(row, path, task_seed, digests)
records[task_seed] = row
commits.add(row["provenance"]["git_commit"])
source_sha256[path] = sha256(path)
require(len(commits) == 1, "calibrated source revision drift")
checks_by_seed = {
str(seed): checks(records[seed]) for seed in TASK_SEEDS
}
passed = all(
value for seed_checks in checks_by_seed.values()
for value in seed_checks.values()
)
output = {
"protocol":
"oral_b_v2_calibrated_recovery_development_v1",
"status": "passed" if passed else "failed",
"complete_grid": True,
"fixed_config": FIXED_CONFIG,
"calibration_quantiles": list(TARGET_QUANTILES),
"checks_by_task_seed": checks_by_seed,
"calibrated_targets_by_task_seed": {
str(seed): records[seed]["assays"]["challenge"]["targets"]
for seed in TASK_SEEDS
},
"intact_final_by_task_seed": [
records[seed]["conditions"]["intact"]["final_success"]
for seed in TASK_SEEDS
],
"challenge_success_fraction_by_task_seed": [
records[seed]["signatures"]["challenge_success_fraction"]
for seed in TASK_SEEDS
],
"terminal_outcome_accuracy_by_task_seed": [
records[seed]["signatures"][
"terminal_residual_outcome_balanced_acc"
]
for seed in TASK_SEEDS
],
"confirmation_seeds_touched": False,
"calibrated_confirmation_opened": passed,
"source_commit": next(iter(commits)),
"source_sha256": source_sha256,
"input_sha256": digests,
"review_score_before": 7,
"review_score_after": 7,
"score_change_rule": (
"development validation never changes the formal score"
),
}
os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
if os.path.exists(args.out):
with open(args.out) as handle:
existing = json.load(handle)
require(existing == output, "existing calibrated R1 gate differs")
else:
with open(args.out, "w") as handle:
json.dump(output, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps(output, indent=2))
if __name__ == "__main__":
main()
|
