1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
|
#!/usr/bin/env python3
"""Endpoint-free contract and boundary checks for the oral-A recovery."""
import copy
import os
import sys
from collections import Counter
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from experiments.analyze_oral_a_dynamic_scaling import (
DEPTHS, expected_args, oral_a_checks)
from experiments.conv_run import parse_args
from experiments.oral_a_dynamic_scaling import jobs
def check_job_contract():
cells = jobs("cuda", "results/oral_a_dynamic_scaling")
assert len(cells) == 50
assert len({(method, depth, seed)
for method, depth, seed, _, _ in cells}) == 50
assert Counter(method for method, _, _, _, _ in cells) == {
"bp": 15, "dfa": 15, "clean_kp": 10, "dynamic": 10}
assert Counter(depth for _, depth, _, _, _ in cells) == {
20: 10, 32: 20, 56: 20}
for method, depth, seed, path, command in cells:
saved = sys.argv
sys.argv = command[1:]
args = parse_args()
sys.argv = saved
actual = vars(args)
for key, value in expected_args(method, depth, seed).items():
assert actual[key] == value, (
method, depth, seed, key, actual[key], value)
assert command[-2:] == ["--out", path]
print("oral-A recovery runner/analyzer contract: exact 50 new cells")
def passing_arrays():
offsets = [-0.002, -0.001, 0.0, 0.001, 0.002]
means = {
"bp": {20: 0.915, 32: 0.925, 56: 0.935},
"dfa": {20: 0.40, 32: 0.40, 56: 0.40},
"clean_kp": {20: 0.912, 32: 0.922, 56: 0.932},
"dynamic": {20: 0.908, 32: 0.918, 56: 0.928},
}
accuracies = {method: {
depth: [mean + offset for offset in offsets]
for depth, mean in by_depth.items()}
for method, by_depth in means.items()}
alignments = {depth: [0.95 + offset for offset in offsets]
for depth in DEPTHS}
return accuracies, alignments
def check_gate_boundaries():
accuracies, alignments = passing_arrays()
checks, _ = oral_a_checks(accuracies, alignments, [])
assert all(checks.values())
collapsed = copy.deepcopy(accuracies)
for method in collapsed:
for depth in DEPTHS:
collapsed[method][depth] = [0.10] * 5
collapsed_checks, _ = oral_a_checks(collapsed, alignments, [])
assert not collapsed_checks["bp_mean_accuracy_at_least_0p90_each_depth"]
assert not collapsed_checks["every_dynamic_accuracy_at_least_0p87"]
assert collapsed_checks["dynamic_mean_within_2pt_bp_each_depth"]
flat = copy.deepcopy(accuracies)
flat["dynamic"][56] = list(flat["dynamic"][20])
flat_checks, _ = oral_a_checks(flat, alignments, [])
assert not flat_checks["dynamic_mean_d20_to_d56_gain_at_least_0p5pt"]
assert not flat_checks["at_least_four_dynamic_seeds_improve_with_depth"]
invariant_checks, _ = oral_a_checks(
accuracies, alignments, ["synthetic:cost_failure"])
assert not invariant_checks[
"all_mechanism_query_cost_memory_invariants"]
assert not invariant_checks["all_60_records_and_audited_values_finite"]
print("oral-A recovery absolute, depth-benefit, and invariant gates: falsifiable")
if __name__ == "__main__":
check_job_contract()
check_gate_boundaries()
print("ALL ORAL-A RECOVERY MECHANICS CHECKS PASSED")
|
