summaryrefslogtreecommitdiff
path: root/results/babyai_shared/population_p3/seed4101.json
diff options
context:
space:
mode:
Diffstat (limited to 'results/babyai_shared/population_p3/seed4101.json')
-rw-r--r--results/babyai_shared/population_p3/seed4101.json515
1 files changed, 515 insertions, 0 deletions
diff --git a/results/babyai_shared/population_p3/seed4101.json b/results/babyai_shared/population_p3/seed4101.json
new file mode 100644
index 0000000..75a0b0c
--- /dev/null
+++ b/results/babyai_shared/population_p3/seed4101.json
@@ -0,0 +1,515 @@
+{
+ "audits": [
+ {
+ "epoch": 0,
+ "layers": [
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.20984283089637756,
+ "mean_per_cell_r2": 0.9767748713493347,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.09602414071559906
+ },
+ "holdout": {
+ "context_rms": 0.20962461829185486,
+ "mean_per_cell_r2": 0.9725972414016724,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.1049511507153511
+ },
+ "layer": 0,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.2008315920829773,
+ "mean_per_cell_r2": 0.9949333667755127,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.04755431041121483
+ },
+ "holdout": {
+ "context_rms": 0.20081116259098053,
+ "mean_per_cell_r2": 0.9941949844360352,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.05143459141254425
+ },
+ "layer": 1,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19726455211639404,
+ "mean_per_cell_r2": 0.997251570224762,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.03510858491063118
+ },
+ "holdout": {
+ "context_rms": 0.19765536487102509,
+ "mean_per_cell_r2": 0.996864914894104,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.03769101947546005
+ },
+ "layer": 2,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19451536238193512,
+ "mean_per_cell_r2": 0.9981643557548523,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.029410211369395256
+ },
+ "holdout": {
+ "context_rms": 0.194533109664917,
+ "mean_per_cell_r2": 0.997877836227417,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.031882479786872864
+ },
+ "layer": 3,
+ "teaching_observations": 0
+ }
+ ]
+ },
+ {
+ "epoch": 1,
+ "layers": [
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.20984283089637756,
+ "mean_per_cell_r2": 0.8648810386657715,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.23362895846366882
+ },
+ "holdout": {
+ "context_rms": 0.20962461829185486,
+ "mean_per_cell_r2": 0.8518301844596863,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.24587911367416382
+ },
+ "layer": 0,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.2008315920829773,
+ "mean_per_cell_r2": 0.9841650128364563,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.08432316780090332
+ },
+ "holdout": {
+ "context_rms": 0.20081116259098053,
+ "mean_per_cell_r2": 0.9819508790969849,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.09081554412841797
+ },
+ "layer": 1,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19726455211639404,
+ "mean_per_cell_r2": 0.9874575138092041,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.07510098814964294
+ },
+ "holdout": {
+ "context_rms": 0.19765536487102509,
+ "mean_per_cell_r2": 0.9855629205703735,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.08102375268936157
+ },
+ "layer": 2,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19451536238193512,
+ "mean_per_cell_r2": 0.9913246631622314,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.06404691189527512
+ },
+ "holdout": {
+ "context_rms": 0.194533109664917,
+ "mean_per_cell_r2": 0.9899281859397888,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.06954371184110641
+ },
+ "layer": 3,
+ "teaching_observations": 0
+ }
+ ]
+ },
+ {
+ "epoch": 5,
+ "layers": [
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.20984283089637756,
+ "mean_per_cell_r2": 0.8257500529289246,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.2655547857284546
+ },
+ "holdout": {
+ "context_rms": 0.20962461829185486,
+ "mean_per_cell_r2": 0.800991415977478,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.2851858139038086
+ },
+ "layer": 0,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.2008315920829773,
+ "mean_per_cell_r2": 0.9562175273895264,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.14005239307880402
+ },
+ "holdout": {
+ "context_rms": 0.20081116259098053,
+ "mean_per_cell_r2": 0.9500809907913208,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.15084101259708405
+ },
+ "layer": 1,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19726455211639404,
+ "mean_per_cell_r2": 0.9649862051010132,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.12556372582912445
+ },
+ "holdout": {
+ "context_rms": 0.19765536487102509,
+ "mean_per_cell_r2": 0.9601048827171326,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.13468508422374725
+ },
+ "layer": 2,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19451536238193512,
+ "mean_per_cell_r2": 0.9821997880935669,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.09148777276277542
+ },
+ "holdout": {
+ "context_rms": 0.194533109664917,
+ "mean_per_cell_r2": 0.9794628024101257,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.09917064756155014
+ },
+ "layer": 3,
+ "teaching_observations": 0
+ }
+ ]
+ },
+ {
+ "epoch": 10,
+ "layers": [
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.20984283089637756,
+ "mean_per_cell_r2": 0.8274786472320557,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.26442500948905945
+ },
+ "holdout": {
+ "context_rms": 0.20962461829185486,
+ "mean_per_cell_r2": 0.8023961782455444,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.28441107273101807
+ },
+ "layer": 0,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.2008315920829773,
+ "mean_per_cell_r2": 0.9351796507835388,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.17075972259044647
+ },
+ "holdout": {
+ "context_rms": 0.20081116259098053,
+ "mean_per_cell_r2": 0.9255064725875854,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.1848365217447281
+ },
+ "layer": 1,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19726455211639404,
+ "mean_per_cell_r2": 0.9514586925506592,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.1478380709886551
+ },
+ "holdout": {
+ "context_rms": 0.19765536487102509,
+ "mean_per_cell_r2": 0.943648099899292,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.1601058840751648
+ },
+ "layer": 2,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19451536238193512,
+ "mean_per_cell_r2": 0.9786931872367859,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.10017189383506775
+ },
+ "holdout": {
+ "context_rms": 0.194533109664917,
+ "mean_per_cell_r2": 0.9755187630653381,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.10835007578134537
+ },
+ "layer": 3,
+ "teaching_observations": 0
+ }
+ ]
+ },
+ {
+ "epoch": 20,
+ "layers": [
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.20984283089637756,
+ "mean_per_cell_r2": 0.8492339253425598,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.24697861075401306
+ },
+ "holdout": {
+ "context_rms": 0.20962461829185486,
+ "mean_per_cell_r2": 0.826335072517395,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.2665298283100128
+ },
+ "layer": 0,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.2008315920829773,
+ "mean_per_cell_r2": 0.9172567129135132,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.1933620125055313
+ },
+ "holdout": {
+ "context_rms": 0.20081116259098053,
+ "mean_per_cell_r2": 0.9072909951210022,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.206399068236351
+ },
+ "layer": 1,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19726455211639404,
+ "mean_per_cell_r2": 0.9384824633598328,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.1660328060388565
+ },
+ "holdout": {
+ "context_rms": 0.19765536487102509,
+ "mean_per_cell_r2": 0.9301103353500366,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.1780754029750824
+ },
+ "layer": 2,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19451536238193512,
+ "mean_per_cell_r2": 0.9720658659934998,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.11473160237073898
+ },
+ "holdout": {
+ "context_rms": 0.194533109664917,
+ "mean_per_cell_r2": 0.9683250784873962,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.12316417694091797
+ },
+ "layer": 3,
+ "teaching_observations": 0
+ }
+ ]
+ },
+ {
+ "epoch": 40,
+ "layers": [
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.20984283089637756,
+ "mean_per_cell_r2": 0.7983269691467285,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.28594422340393066
+ },
+ "holdout": {
+ "context_rms": 0.20962461829185486,
+ "mean_per_cell_r2": 0.7654898762702942,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.310098260641098
+ },
+ "layer": 0,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.2008315920829773,
+ "mean_per_cell_r2": 0.8684720993041992,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.24367430806159973
+ },
+ "holdout": {
+ "context_rms": 0.20081116259098053,
+ "mean_per_cell_r2": 0.851003885269165,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.26162275671958923
+ },
+ "layer": 1,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19726455211639404,
+ "mean_per_cell_r2": 0.9062105417251587,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.20594285428524017
+ },
+ "holdout": {
+ "context_rms": 0.19765536487102509,
+ "mean_per_cell_r2": 0.8948484659194946,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.2192971259355545
+ },
+ "layer": 2,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "calibration": {
+ "context_rms": 0.19451536238193512,
+ "mean_per_cell_r2": 0.9705915451049805,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.11770590394735336
+ },
+ "holdout": {
+ "context_rms": 0.194533109664917,
+ "mean_per_cell_r2": 0.9667372107505798,
+ "observations": 4096,
+ "residual_context_rms_ratio": 0.12624777853488922
+ },
+ "layer": 3,
+ "teaching_observations": 0
+ }
+ ]
+ }
+ ],
+ "calibration_examples": 4096,
+ "checks": {
+ "holdout_r2_at_least_0p8_every_epoch_and_layer": false,
+ "holdout_residual_ratio_at_most_0p25_every_epoch_and_layer": false,
+ "zero_action_or_teaching_observations": true
+ },
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 3964,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "epoch_clean_kp_train_loss": [
+ 0.6793887242517973,
+ 0.5070055065029546,
+ 0.4780606180743167,
+ 0.46464681211270786,
+ 0.45651326932405173,
+ 0.4414772361203244,
+ 0.4295881901916705,
+ 0.4249380529554267,
+ 0.4178563620542225,
+ 0.40975865175849513,
+ 0.40546370631770084,
+ 0.3982252176811821,
+ 0.3858343799490678,
+ 0.379007538933503,
+ 0.3656208556576779,
+ 0.35379995308424295,
+ 0.33681938353337737,
+ 0.32208908896697197,
+ 0.30617407221543164,
+ 0.2926546273419732,
+ 0.28012341188757045,
+ 0.27170656586948194,
+ 0.2618562983211718,
+ 0.25525567559819473,
+ 0.24340780433855558,
+ 0.241100468886526,
+ 0.22936643666342685,
+ 0.22153453873960596,
+ 0.2213413413574821,
+ 0.21846851916689622,
+ 0.2104246342495868,
+ 0.20888692896617087,
+ 0.19663872825471979,
+ 0.19428577112524134,
+ 0.1947261790539089,
+ 0.18938347912148426,
+ 0.1853264942294673,
+ 0.17872829263147555,
+ 0.17811942146012658,
+ 0.1693246652264344
+ ],
+ "gate": "fail",
+ "holdout_examples": 4096,
+ "model_seed": 4101,
+ "provenance": {
+ "cuda_visible_devices": "0",
+ "device": "cuda:0",
+ "git_commit": "b5eab6cd911e074ae9268303389dfbe52e636e26",
+ "torch_version": "2.3.1+cu118"
+ },
+ "raw_sdil_rollout_or_test_outcomes_read": false,
+ "ridge_relative_to_mean_soma_variance": 0.001,
+ "shuffle_seed": 4101,
+ "stage": "babyai_population_predictor_capacity_screen"
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-16 10:49:11 +0000