1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
#!/usr/bin/env python3
"""Summarize the frozen PickupLoc shared-feedback endpoint."""
import argparse
import json
from pathlib import Path
import statistics
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_RESULTS = ROOT / "results" / "babyai_shared" / "pickup_p2"
DEFAULT_OUT = ROOT / "results" / "babyai_shared" / "pickup_p2_analysis.json"
SEEDS = (4101, 4102, 4103)
CONDITIONS = ("bp", "clean_kp", "raw_shared", "sdil")
def summarize(values):
return {
"values": values,
"mean": statistics.mean(values),
"sample_std": statistics.stdev(values),
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
args = parser.parse_args()
records = {}
for seed in SEEDS:
for condition in CONDITIONS:
path = args.results / f"seed{seed}_{condition}.json"
with open(path, encoding="utf-8") as handle:
records[(seed, condition)] = json.load(handle)
summaries = {}
for condition in CONDITIONS:
rows = [records[(seed, condition)] for seed in SEEDS]
summaries[condition] = {
"rollout_success_percent": summarize([
100.0 * row["rollout"]["success"] for row in rows]),
"expert_action_accuracy_percent": summarize([
100.0 * row["validation"]["accuracy"] for row in rows]),
"mission_lesion_rollout_success_percent": summarize([
100.0 * row["mission_lesion_rollout"]["success"]
for row in rows]),
"all_finite": all(row["finite"] for row in rows),
}
sdil_minus_raw = [100.0 * (
records[(seed, "sdil")]["rollout"]["success"]
- records[(seed, "raw_shared")]["rollout"]["success"])
for seed in SEEDS]
clean_minus_raw = [100.0 * (
records[(seed, "clean_kp")]["rollout"]["success"]
- records[(seed, "raw_shared")]["rollout"]["success"])
for seed in SEEDS]
predictor_r2 = [statistics.mean(
row["mean_per_cell_r2"]
for row in records[(seed, "sdil")]["predictor"])
for seed in SEEDS]
residual_ratios = [max(
row["residual_context_rms_ratio"]
for row in records[(seed, "sdil")]["predictor"])
for seed in SEEDS]
checks = {
"all_runs_finite": all(row["finite"] for row in records.values()),
"clean_kp_success_at_least_60_every_seed": all(
records[(seed, "clean_kp")]["rollout"]["success"] >= 0.6
for seed in SEEDS),
"raw_below_clean_kp_every_seed": all(
value > 0 for value in clean_minus_raw),
"sdil_above_raw_every_seed": all(
value > 0 for value in sdil_minus_raw),
"sdil_nonzero_success_every_seed": all(
records[(seed, "sdil")]["rollout"]["success"] > 0
for seed in SEEDS),
}
report = {
"stage": "babyai_pickup_p2_analysis",
"gate": "pass" if all(checks.values()) else "fail",
"checks": checks,
"conditions": summaries,
"paired_clean_kp_minus_raw_rollout_points": summarize(clean_minus_raw),
"paired_sdil_minus_raw_rollout_points": summarize(sdil_minus_raw),
"sdil_predictor": {
"mean_per_cell_r2": summarize(predictor_r2),
"maximum_layer_residual_context_rms_ratio": summarize(
residual_ratios),
"action_or_teaching_observations": 0,
},
"test_split_generated_or_read": False,
"decision": (
"Close the diagonal per-cell predictor on PickupLoc. Raw shared "
"feedback fails as hypothesized, but this SDIL implementation does "
"not recover it. Screen the originally specified population "
"predictor using neutral prediction only before another endpoint."),
}
args.out.parent.mkdir(parents=True, exist_ok=True)
with open(args.out, "w", encoding="utf-8") as handle:
json.dump(report, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps(report, indent=2, sort_keys=True))
if __name__ == "__main__":
main()
|
