diff options
Diffstat (limited to 'results/babyai_shared/pickup_p2_analysis.json')
| -rw-r--r-- | results/babyai_shared/pickup_p2_analysis.json | 174 |
1 files changed, 174 insertions, 0 deletions
diff --git a/results/babyai_shared/pickup_p2_analysis.json b/results/babyai_shared/pickup_p2_analysis.json new file mode 100644 index 0000000..845e70c --- /dev/null +++ b/results/babyai_shared/pickup_p2_analysis.json @@ -0,0 +1,174 @@ +{ + "checks": { + "all_runs_finite": true, + "clean_kp_success_at_least_60_every_seed": true, + "raw_below_clean_kp_every_seed": true, + "sdil_above_raw_every_seed": false, + "sdil_nonzero_success_every_seed": false + }, + "conditions": { + "bp": { + "all_finite": true, + "expert_action_accuracy_percent": { + "mean": 87.78380713263681, + "sample_std": 0.1078533752082175, + "values": [ + 87.8108042439459, + 87.87559731108772, + 87.66501984287682 + ] + }, + "mission_lesion_rollout_success_percent": { + "mean": 49.333333333333336, + "sample_std": 1.1015141094572214, + "values": [ + 48.8, + 50.6, + 48.6 + ] + }, + "rollout_success_percent": { + "mean": 79.33333333333333, + "sample_std": 5.1588112325741635, + "values": [ + 80.80000000000001, + 83.6, + 73.6 + ] + } + }, + "clean_kp": { + "all_finite": true, + "expert_action_accuracy_percent": { + "mean": 86.43665127831322, + "sample_std": 0.3517610682148285, + "values": [ + 86.13428363165141, + 86.8227099700332, + 86.35296023325503 + ] + }, + "mission_lesion_rollout_success_percent": { + "mean": 48.86666666666667, + "sample_std": 7.245228314783008, + "values": [ + 49.8, + 55.60000000000001, + 41.199999999999996 + ] + }, + "rollout_success_percent": { + "mean": 70.86666666666667, + "sample_std": 5.34540300944029, + "values": [ + 67.2, + 77.0, + 68.4 + ] + } + }, + "raw_shared": { + "all_finite": true, + "expert_action_accuracy_percent": { + "mean": 59.642018304041464, + "sample_std": 9.767595165741607, + "values": [ + 49.89066169919818, + 69.42577144245566, + 59.609621770470554 + ] + }, + "mission_lesion_rollout_success_percent": { + "mean": 8.733333333333334, + "sample_std": 9.843441132720475, + "values": [ + 0.0, + 19.400000000000002, + 6.800000000000001 + ] + }, + "rollout_success_percent": { + "mean": 10.000000000000002, + "sample_std": 11.92644121270046, + "values": [ + 0.0, + 23.200000000000003, + 6.800000000000001 + ] + } + }, + "sdil": { + "all_finite": true, + "expert_action_accuracy_percent": { + "mean": 49.89066169919818, + "sample_std": 0.0, + "values": [ + 49.89066169919818, + 49.89066169919818, + 49.89066169919818 + ] + }, + "mission_lesion_rollout_success_percent": { + "mean": 0.0, + "sample_std": 0.0, + "values": [ + 0.0, + 0.0, + 0.0 + ] + }, + "rollout_success_percent": { + "mean": 0.0, + "sample_std": 0.0, + "values": [ + 0.0, + 0.0, + 0.0 + ] + } + } + }, + "decision": "Close the diagonal per-cell predictor on PickupLoc. Raw shared feedback fails as hypothesized, but this SDIL implementation does not recover it. Screen the originally specified population predictor using neutral prediction only before another endpoint.", + "gate": "fail", + "paired_clean_kp_minus_raw_rollout_points": { + "mean": 60.866666666666674, + "sample_std": 6.730032194078519, + "values": [ + 67.2, + 53.800000000000004, + 61.60000000000001 + ] + }, + "paired_sdil_minus_raw_rollout_points": { + "mean": -10.000000000000002, + "sample_std": 11.92644121270046, + "values": [ + 0.0, + -23.200000000000003, + -6.800000000000001 + ] + }, + "sdil_predictor": { + "action_or_teaching_observations": 0, + "maximum_layer_residual_context_rms_ratio": { + "mean": 0.614041248957316, + "sample_std": 0.011718512673796736, + "values": [ + 0.6220114827156067, + 0.6005861759185791, + 0.6195260882377625 + ] + }, + "mean_per_cell_r2": { + "mean": 0.30999962737162906, + "sample_std": 0.005533015870761776, + "values": [ + 0.3146054223179817, + 0.3038621172308922, + 0.31153134256601334 + ] + } + }, + "stage": "babyai_pickup_p2_analysis", + "test_split_generated_or_read": false +} |