results: close BabyAI population predictor rescue

1 files changed, 515 insertions, 0 deletions

diff --git a/results/babyai_shared/population_p3/seed4102.json b/results/babyai_shared/population_p3/seed4102.json
new file mode 100644
index 0000000..d6aeaf3
--- /dev/null
+++ b/results/babyai_shared/population_p3/seed4102.json
@@ -0,0 +1,515 @@
+{
+  "audits": [
+    {
+      "epoch": 0,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.975501537322998,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09886644780635834
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.9715493321418762,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1070699542760849
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9942841529846191,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.04786578193306923
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9934000372886658,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.05185777321457863
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.99674391746521,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.035994671285152435
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.9962091445922852,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.03937750309705734
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9980149269104004,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.02863112837076187
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9977172613143921,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.031041312962770462
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 1,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.8727436661720276,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2265353947877884
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.8598257303237915,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.23877772688865662
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9780939221382141,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09408465027809143
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9749774932861328,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10145710408687592
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.9838603734970093,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.08039840310811996
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.9813672304153442,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.08734050393104553
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9901689291000366,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.06410957127809525
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.988613486289978,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.06975055485963821
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 5,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.800828218460083,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2844180166721344
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.7751771807670593,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.3035990297794342
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9436342716217041,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.15104201436042786
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9357898235321045,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.16277429461479187
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.9581910967826843,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.12930455803871155
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.9521858096122742,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1400468796491623
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9799500107765198,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09153742343187332
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9768326878547668,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.0993802398443222
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 10,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.8033591508865356,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.282537043094635
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.7746613621711731,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.3040773570537567
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9248608350753784,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1742551028728485
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9133741855621338,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.18895503878593445
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.9435411691665649,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.15058007836341858
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.9352096319198608,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.16326111555099487
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9773434400558472,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09754791110754013
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9744316339492798,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10467227548360825
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 20,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.8509368896484375,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.24803291261196136
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.8269829750061035,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2686232626438141
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9160059690475464,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1840994507074356
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9035534858703613,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.19918905198574066
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.9384135007858276,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1571311354637146
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.929930567741394,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.16975943744182587
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9770718216896057,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09782316535711288
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9735088348388672,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10616537183523178
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 40,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.8065019249916077,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2811703383922577
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.7738609313964844,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.3060470223426819
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.87445068359375,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.22593224048614502
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.8567341566085815,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2437417060136795
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.908510148525238,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.192082479596138
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.8953220844268799,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.20811691880226135
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9752291440963745,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10170674324035645
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9718630313873291,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10950509458780289
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    }
+  ],
+  "calibration_examples": 4096,
+  "checks": {
+    "holdout_r2_at_least_0p8_every_epoch_and_layer": false,
+    "holdout_residual_ratio_at_most_0p25_every_epoch_and_layer": false,
+    "zero_action_or_teaching_observations": true
+  },
+  "config": {
+    "action_dim": 7,
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "input_dim": 3964,
+    "learning_rate": 0.03,
+    "mission_dim": 23,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 256
+  },
+  "epoch_clean_kp_train_loss": [
+    0.681839566669966,
+    0.5106817564838811,
+    0.4866123088410026,
+    0.46724821322842647,
+    0.45275609010144285,
+    0.444956036241431,
+    0.4342330169050317,
+    0.42846722878907856,
+    0.41873464232996893,
+    0.4157650209100623,
+    0.40508943651851853,
+    0.3999078395492152,
+    0.3982298595654337,
+    0.3850154632016232,
+    0.3766156781347174,
+    0.368439452836388,
+    0.3616289800719211,
+    0.3508304416505914,
+    0.3401417228422667,
+    0.3305343171797301,
+    0.3165550116802517,
+    0.3019398627469414,
+    0.29349281496123264,
+    0.2876167286697187,
+    0.26772761492352737,
+    0.26045800096110294,
+    0.24848401973122045,
+    0.2440636997787576,
+    0.2351069336188467,
+    0.23047932257777767,
+    0.22535064524725865,
+    0.22159839981480647,
+    0.2125453116078126,
+    0.20711057270828045,
+    0.20062021220985213,
+    0.19858844374355517,
+    0.1931002575629636,
+    0.1893660755063358,
+    0.18631849710878573,
+    0.18104866024694943
+  ],
+  "gate": "fail",
+  "holdout_examples": 4096,
+  "model_seed": 4102,
+  "provenance": {
+    "cuda_visible_devices": "1",
+    "device": "cuda:0",
+    "git_commit": "b5eab6cd911e074ae9268303389dfbe52e636e26",
+    "torch_version": "2.3.1+cu118"
+  },
+  "raw_sdil_rollout_or_test_outcomes_read": false,
+  "ridge_relative_to_mean_soma_variance": 0.001,
+  "shuffle_seed": 4102,
+  "stage": "babyai_population_predictor_capacity_screen"
+}