results: audit Rain EP bias confirmation

1 files changed, 142 insertions, 0 deletions

diff --git a/results/ep_bias/c1_gate.json b/results/ep_bias/c1_gate.json
new file mode 100644
index 0000000..1266bba
--- /dev/null
+++ b/results/ep_bias/c1_gate.json
@@ -0,0 +1,142 @@
+{
+  "author_revision": "6b253fd8a5d267535f58ab79992256ef10031ceb",
+  "checks": {
+    "clean_deficit_upper_95_below_3_points": false,
+    "finite_complete_required_conditions": true,
+    "innovation_beats_constant_every_seed": false,
+    "innovation_beats_raw_every_seed": true,
+    "innovation_constant_lower_95_at_least_5_points": false,
+    "innovation_raw_lower_95_at_least_10_points": true,
+    "innovation_residual_below_constant_every_seed": true,
+    "matched_128_neutral_observations": true,
+    "mean_clean_validation_at_least_70": true,
+    "mean_wall_overhead_at_most_15_percent": true,
+    "noise_abs_mean_upper_95_below_3_points": false,
+    "oracle_abs_mean_upper_95_below_3_points": false,
+    "protocol_fixed_and_bp_free": true,
+    "raw_damage_at_least_10_points_every_seed": true,
+    "single_author_revision": true,
+    "single_sdil_revision": true
+  },
+  "gate": "fail",
+  "mean_clean_validation_accuracy": 0.7626000000000001,
+  "num_records": 30,
+  "rows": [
+    {
+      "clean": 0.7705,
+      "clean_minus_innovation": 0.020499999999999963,
+      "constant": 0.681,
+      "innovation": 0.75,
+      "innovation_minus_constant": 0.06899999999999995,
+      "innovation_minus_oracle": -0.01749999999999996,
+      "innovation_minus_raw": 0.5385,
+      "innovation_wall_over_clean": 1.0200028100178125,
+      "noise_minus_clean": 0.026000000000000023,
+      "oracle": 0.7675,
+      "raw": 0.2115,
+      "same_rms_noise": 0.7965,
+      "seed": 1989
+    },
+    {
+      "clean": 0.716,
+      "clean_minus_innovation": -0.020500000000000074,
+      "constant": 0.7915,
+      "innovation": 0.7365,
+      "innovation_minus_constant": -0.05499999999999994,
+      "innovation_minus_oracle": -0.03849999999999998,
+      "innovation_minus_raw": 0.44850000000000007,
+      "innovation_wall_over_clean": 1.002160782870049,
+      "noise_minus_clean": 0.03949999999999998,
+      "oracle": 0.775,
+      "raw": 0.288,
+      "same_rms_noise": 0.7555,
+      "seed": 1990
+    },
+    {
+      "clean": 0.7485,
+      "clean_minus_innovation": -0.017999999999999905,
+      "constant": 0.5425,
+      "innovation": 0.7665,
+      "innovation_minus_constant": 0.22399999999999998,
+      "innovation_minus_oracle": 0.10649999999999993,
+      "innovation_minus_raw": 0.6705,
+      "innovation_wall_over_clean": 0.9799086026637867,
+      "noise_minus_clean": -0.08100000000000007,
+      "oracle": 0.66,
+      "raw": 0.096,
+      "same_rms_noise": 0.6675,
+      "seed": 1991
+    },
+    {
+      "clean": 0.8,
+      "clean_minus_innovation": 0.03500000000000003,
+      "constant": 0.7125,
+      "innovation": 0.765,
+      "innovation_minus_constant": 0.05249999999999999,
+      "innovation_minus_oracle": -0.03200000000000003,
+      "innovation_minus_raw": 0.35100000000000003,
+      "innovation_wall_over_clean": 0.9760042015047453,
+      "noise_minus_clean": -0.007500000000000062,
+      "oracle": 0.797,
+      "raw": 0.414,
+      "same_rms_noise": 0.7925,
+      "seed": 1992
+    },
+    {
+      "clean": 0.778,
+      "clean_minus_innovation": 0.07000000000000006,
+      "constant": 0.6675,
+      "innovation": 0.708,
+      "innovation_minus_constant": 0.04049999999999998,
+      "innovation_minus_oracle": -0.07050000000000001,
+      "innovation_minus_raw": 0.14849999999999997,
+      "innovation_wall_over_clean": 1.0005652695437517,
+      "noise_minus_clean": -0.018000000000000016,
+      "oracle": 0.7785,
+      "raw": 0.5595,
+      "same_rms_noise": 0.76,
+      "seed": 1993
+    }
+  ],
+  "sdil_revision": "a10a84066a6f7aa38d86b1a71ed371c9c6f22815",
+  "stage": "rain_ep_bias_c1",
+  "statistics": {
+    "clean_minus_innovation": {
+      "lower_95": -0.018827912063523824,
+      "mean": 0.017400000000000016,
+      "sem": 0.016993675294061622,
+      "upper_95": 0.05362791206352385
+    },
+    "innovation_minus_constant": {
+      "lower_95": -0.029669484192664783,
+      "mean": 0.0662,
+      "sem": 0.044970156770907516,
+      "upper_95": 0.16206948419266476
+    },
+    "innovation_minus_oracle": {
+      "lower_95": -0.0753849705784051,
+      "mean": -0.01040000000000001,
+      "sem": 0.030482946051850025,
+      "upper_95": 0.0753849705784051
+    },
+    "innovation_minus_raw": {
+      "lower_95": 0.24346576516238344,
+      "mean": 0.4314,
+      "sem": 0.08815560106992634,
+      "upper_95": 0.6193342348376165
+    },
+    "innovation_wall_over_clean": {
+      "lower_95": 0.97858499866313,
+      "mean": 0.9957283333200291,
+      "sem": 0.008041541618681968,
+      "upper_95": 1.0128716679769283
+    },
+    "noise_minus_clean": {
+      "lower_95": -0.053022807458936866,
+      "mean": -0.00820000000000003,
+      "sem": 0.02102534185215547,
+      "upper_95": 0.053022807458936866
+    }
+  },
+  "test_policy": "test_set_never_loaded; fixed epoch-3 train holdout"
+}