summaryrefslogtreecommitdiff
path: root/results
diff options
context:
space:
mode:
Diffstat (limited to 'results')
-rw-r--r--results/babyai_shared/b1/seed4101_bp.json251
-rw-r--r--results/babyai_shared/b1/seed4101_clean_kp.json251
-rw-r--r--results/babyai_shared/b1/seed4101_raw_shared.json251
-rw-r--r--results/babyai_shared/b1/seed4101_sdil.json268
-rw-r--r--results/babyai_shared/b1/seed4102_bp.json251
-rw-r--r--results/babyai_shared/b1/seed4102_clean_kp.json251
-rw-r--r--results/babyai_shared/b1/seed4102_raw_shared.json251
-rw-r--r--results/babyai_shared/b1/seed4102_sdil.json268
-rw-r--r--results/babyai_shared/b1/seed4103_bp.json251
-rw-r--r--results/babyai_shared/b1/seed4103_clean_kp.json251
-rw-r--r--results/babyai_shared/b1/seed4103_raw_shared.json251
-rw-r--r--results/babyai_shared/b1/seed4103_sdil.json268
-rw-r--r--results/babyai_shared/b1_analysis.json198
13 files changed, 3261 insertions, 0 deletions
diff --git a/results/babyai_shared/b1/seed4101_bp.json b/results/babyai_shared/b1/seed4101_bp.json
new file mode 100644
index 0000000..8f9aa04
--- /dev/null
+++ b/results/babyai_shared/b1/seed4101_bp.json
@@ -0,0 +1,251 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.35939537301659585
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.04168066847730767
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.01859026498753916
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.015395146521993659
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.015227786766534503
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.012962078500369734
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.013404185661241751
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.01239592205690728
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.012198219955157996
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.011407826012000442
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.010740509365939281
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.01174285890576853
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.012136201211251318
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.010752951244569637
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.010369447311809794
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.01007143171355975
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.011025185833727432
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.010670005309564823
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.010151156608692624
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.010251642129891976
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.01011152534618635
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.009685466071823612
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.01069813368905505
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.010710137346018615
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.01011071698105132
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.009644118504260075
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.009702050907283344
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.010063305073708761
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.009666770600332794
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.009591050190554763
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.00987653798296709
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.00944871237702583
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.009831787788584321
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.009708568485220894
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.00941432591813447
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.009692842404315756
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.009456626759545709
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.009676380681551315
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.009368902054848149
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.009439258407060565
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 12.65,
+ "mean_return": 0.65555,
+ "success": 0.718
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.8236215985183074,
+ "loss": 0.5513675970677246
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39800832,
+ "cuda_visible_devices": "0",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.29,
+ "mean_return": 0.8905500000000001,
+ "success": 0.978
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 184.13239169120789,
+ "training_wall_seconds": 178.87876749038696
+ },
+ "validation": {
+ "accuracy": 0.9974355321270836,
+ "loss": 0.007173381226747901
+ }
+}
diff --git a/results/babyai_shared/b1/seed4101_clean_kp.json b/results/babyai_shared/b1/seed4101_clean_kp.json
new file mode 100644
index 0000000..1448a93
--- /dev/null
+++ b/results/babyai_shared/b1/seed4101_clean_kp.json
@@ -0,0 +1,251 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.42220023420724
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.042795064787973056
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.01790234833363105
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.014970682340420106
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.014731389162638648
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.012619410693137482
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.013174676087642596
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.012047174987383186
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.011912513411900198
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.011198682525110516
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.010507440414618362
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.011464962014843795
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.011738500667905266
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.010477929167999802
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.010263334540861913
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.009940880649638446
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.010815388056322593
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.010474997439569879
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.010014792824460363
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.010134345417503606
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.009996894708072598
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.009606571660101922
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.010532528261061418
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.010634511170527813
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.01003280480074781
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.00955302332104607
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.009628229297655212
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.00994972179519457
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.009593221751592038
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.009536803015190262
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.009819202917331661
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.009349258048777384
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.009778419116681272
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.009624411242019217
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.009337681661973792
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.009646011604345403
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.009394642366731371
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.009614316142397002
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.009303753527876159
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.009400151128012856
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 16.168,
+ "mean_return": 0.5565999999999999,
+ "success": 0.608
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.7578002564467873,
+ "loss": 0.7507116913133103
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39800832,
+ "cuda_visible_devices": "1",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.29,
+ "mean_return": 0.8905500000000001,
+ "success": 0.978
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 186.62301325798035,
+ "training_wall_seconds": 182.5949375629425
+ },
+ "validation": {
+ "accuracy": 0.9974355321270836,
+ "loss": 0.00720344482790899
+ }
+}
diff --git a/results/babyai_shared/b1/seed4101_raw_shared.json b/results/babyai_shared/b1/seed4101_raw_shared.json
new file mode 100644
index 0000000..91d5cb3
--- /dev/null
+++ b/results/babyai_shared/b1/seed4101_raw_shared.json
@@ -0,0 +1,251 @@
+{
+ "condition": "raw_shared",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.912005416913466
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.4067533041672273
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.17175627085295592
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.101405300850218
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.15046165440570225
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.06658889784054323
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.06327508542348038
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.06550759131258184
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.17024030198597095
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.07377955204383893
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.053011001592332664
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.06881083751943978
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.10856242247603157
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.06692509006370198
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.0633675326068293
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.11487563056024638
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.0888773624836044
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.06327656554227526
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.057851375937461855
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.07349031143229116
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.04631090391088616
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.07982008521868424
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.058804434524341066
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.09953950692645529
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.06437903018498962
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.057973786971785804
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.09784971290014007
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.04299957272003997
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.0515958441590721
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.06496320383453912
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.1328263165361502
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.04388667389750481
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.07855626332150265
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.050679853653365914
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.049945260245691646
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.051781387623738157
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.04179245889525522
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.04486359935592521
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.04852475961162286
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.05743650953878056
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 13.522,
+ "mean_return": 0.63055,
+ "success": 0.686
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.870921783729876,
+ "loss": 0.24419261227477426
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39770112,
+ "cuda_visible_devices": "2",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 14.164,
+ "mean_return": 0.6130999999999999,
+ "success": 0.672
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 184.85894656181335,
+ "training_wall_seconds": 178.62242436408997
+ },
+ "validation": {
+ "accuracy": 0.8724889585411028,
+ "loss": 0.3364062811306254
+ }
+}
diff --git a/results/babyai_shared/b1/seed4101_sdil.json b/results/babyai_shared/b1/seed4101_sdil.json
new file mode 100644
index 0000000..f5c0e1a
--- /dev/null
+++ b/results/babyai_shared/b1/seed4101_sdil.json
@@ -0,0 +1,268 @@
+{
+ "condition": "sdil",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7169609726017172
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.37284291326999663
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.17600450354543598
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.29167314236814323
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.0823398195952177
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.05786021038889885
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.04764524250545285
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.47599790914492174
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.27363136577335273
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.18569429102269086
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.1496148065545342
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.1059368799355897
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.11592144590887156
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.04884156241335652
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.044375065808946436
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.5301400216601112
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.2342631328918717
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.1805937432294542
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.15011224646459925
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.06355005253445019
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.16213352036747067
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.07662442246621305
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.06566207943992182
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.09292673818767071
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.27312749338420955
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.3949113512581045
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.25578115195035933
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.20474397618662227
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.1151967611096122
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.22775728605010293
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.11137387694282966
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.32518788596445863
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.3679282115806233
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.11811869431625713
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.25989035964012147
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.1275611624392596
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.24255421275442296
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.24983196166428653
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.10486641916361722
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.2554104011031714
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 4.29,
+ "mean_return": 0.8905500000000001,
+ "success": 0.978
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.997008120814931,
+ "loss": 0.09852080094472926
+ },
+ "predictor": [
+ {
+ "action_observations": 0,
+ "context_rms": 0.29070043563842773,
+ "mean_per_cell_r2": 0.45361873507499695,
+ "neutral_observations": 4096,
+ "residual_context_rms_ratio": 0.3701525032520294,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "context_rms": 0.2747928202152252,
+ "mean_per_cell_r2": 0.5599287748336792,
+ "neutral_observations": 4096,
+ "residual_context_rms_ratio": 0.33980196714401245,
+ "teaching_observations": 0
+ }
+ ],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 114644992,
+ "cuda_visible_devices": "3",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.298,
+ "mean_return": 0.8903500000000001,
+ "success": 0.978
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 4096,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 185.46793723106384,
+ "training_wall_seconds": 183.09582328796387
+ },
+ "validation": {
+ "accuracy": 0.994728593816783,
+ "loss": 0.1036817075384531
+ }
+}
diff --git a/results/babyai_shared/b1/seed4102_bp.json b/results/babyai_shared/b1/seed4102_bp.json
new file mode 100644
index 0000000..65b0955
--- /dev/null
+++ b/results/babyai_shared/b1/seed4102_bp.json
@@ -0,0 +1,251 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.35355447364124387
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.042185719047080385
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.019281441472809423
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.015513235063884747
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.013296923749995502
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.012671216844411736
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.014801131509646605
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.011190175573574379
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.011067502123346045
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.012195465334860439
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.011578771525104954
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.011191591258076105
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.010304119926665656
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.010280991342045706
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.010555616432949054
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.010838844217995013
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.011003393904221328
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.00977229246150025
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.010014041190416637
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.010900152776686644
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.011283926658798009
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.010686473336536438
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.009769557828972625
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.010240042670841583
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.009798416777471588
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.01003834310199388
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.00971504575424743
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.009448815450199287
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.009595394653098827
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.009837813718159769
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.009620558681940151
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.009997257824500345
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.009587076458529653
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.009427764000191184
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.009570616646458141
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.009372298921966418
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.009358159794302826
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.00933290207785004
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.009472835550338708
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.009313055751527743
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 4.61,
+ "mean_return": 0.8815500000000002,
+ "success": 0.968
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.9920216555064824,
+ "loss": 0.0308635174710113
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39800832,
+ "cuda_visible_devices": "4",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.29,
+ "mean_return": 0.8905500000000001,
+ "success": 0.978
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4102,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4102,
+ "total_wall_seconds_including_rollouts": 180.92169880867004,
+ "training_wall_seconds": 176.77257347106934
+ },
+ "validation": {
+ "accuracy": 0.9974355321270836,
+ "loss": 0.009139203621054123
+ }
+}
diff --git a/results/babyai_shared/b1/seed4102_clean_kp.json b/results/babyai_shared/b1/seed4102_clean_kp.json
new file mode 100644
index 0000000..287cd05
--- /dev/null
+++ b/results/babyai_shared/b1/seed4102_clean_kp.json
@@ -0,0 +1,251 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.4369402367960323
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.04315035553818399
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.018373423097655175
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.01484828113832257
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.01290607647673989
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.012369149463589895
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.014328753638441083
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.01090253034458411
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.010823998920280825
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.012246859658255496
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.011209721950687129
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.010971906608478589
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.010095493538643826
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.010141288362104784
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.010389165276640348
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.010640575006942857
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.010856473856732588
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.009704423175108704
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.009899068040011281
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.010538074031937867
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.011126897338439117
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.010471862061795864
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.00964502102123912
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.0100560691582293
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.009638783985389058
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.009922710771206766
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.00961751041320068
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.009335849603151225
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.009497242902130396
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.009709630153268914
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.009487844813530419
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.009836118470411747
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.009497633754826066
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.009304083153653086
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.009465594382596795
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.00925753563949415
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.009258489562833512
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.009257057890909809
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.00942517074491744
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.009241441347350535
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 7.192,
+ "mean_return": 0.8090000000000002,
+ "success": 0.888
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.9484257016669041,
+ "loss": 0.09698232739616075
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39800832,
+ "cuda_visible_devices": "5",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.29,
+ "mean_return": 0.8905500000000001,
+ "success": 0.978
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4102,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4102,
+ "total_wall_seconds_including_rollouts": 185.72890639305115,
+ "training_wall_seconds": 182.7532684803009
+ },
+ "validation": {
+ "accuracy": 0.9974355321270836,
+ "loss": 0.008958498200963987
+ }
+}
diff --git a/results/babyai_shared/b1/seed4102_raw_shared.json b/results/babyai_shared/b1/seed4102_raw_shared.json
new file mode 100644
index 0000000..c9ffea2
--- /dev/null
+++ b/results/babyai_shared/b1/seed4102_raw_shared.json
@@ -0,0 +1,251 @@
+{
+ "condition": "raw_shared",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7823279625719244
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.2521568740227006
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.12487077650698748
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.0969545353813605
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.07753775053403594
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.08754700477827679
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.06143687832084569
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.05799600293690508
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.07455560488457029
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.063334731310606
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.0776304616779089
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.04787300471555103
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.05559959182007746
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.05424102694473484
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.04626016687263142
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.07087413750588895
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.07598450397903268
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.04286422164602713
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.04325513403862715
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.04424519382417202
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.04258359877223318
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.0688056850365617
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.054192612320184706
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.05785939395089041
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.05102440967139873
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.03782162671400742
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.04176153375682506
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.05095106322656978
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.08887688144025478
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.05466046009551395
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.05866600335321643
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.03923974530602043
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.04042056799950925
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.07468748280609196
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.04026515393771909
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.04441547732800245
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.053928801034661855
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.03681060602380471
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.03838446866043589
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.06332002540203659
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 3.724,
+ "mean_return": 0.9069,
+ "success": 1.0
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.9770622595811369,
+ "loss": 0.06753120634321101
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39770112,
+ "cuda_visible_devices": "6",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 7.636,
+ "mean_return": 0.7965000000000001,
+ "success": 0.874
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4102,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4102,
+ "total_wall_seconds_including_rollouts": 183.50573301315308,
+ "training_wall_seconds": 179.4754331111908
+ },
+ "validation": {
+ "accuracy": 0.9461461746687563,
+ "loss": 0.17094452691258213
+ }
+}
diff --git a/results/babyai_shared/b1/seed4102_sdil.json b/results/babyai_shared/b1/seed4102_sdil.json
new file mode 100644
index 0000000..006a8c1
--- /dev/null
+++ b/results/babyai_shared/b1/seed4102_sdil.json
@@ -0,0 +1,268 @@
+{
+ "condition": "sdil",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7134594283320687
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.3401663822477514
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.19433609214696018
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.32963184771212667
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.08546664945103906
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.2630213478343053
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.11469778050075878
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.05664703453128988
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.04731606664982709
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.041082520572976636
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.20956628300926902
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.05752169869840145
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.03557884735817259
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.03279060132124207
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.031631635776297615
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.03048184691166336
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.3688752563975074
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.07081738247112794
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.054225066940892824
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.046492697589776734
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.047163276597857476
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.7845130688493902
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.3934751184691082
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.14599689253351905
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.3236034153672782
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.16323385662653228
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.09238848522305489
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.11827930971980095
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.15449962057850578
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.14310111538930373
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.12045336569574747
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.14461230769753455
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.08665146978064017
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.12435456068678336
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.15446093495596538
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.09183386780998924
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.06461878389120101
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.05828312648290938
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.15417793202129276
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.6200590939007022
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 8.284,
+ "mean_return": 0.7789,
+ "success": 0.86
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.8928622310870494,
+ "loss": 0.36016965703859616
+ },
+ "predictor": [
+ {
+ "action_observations": 0,
+ "context_rms": 0.2660917639732361,
+ "mean_per_cell_r2": 0.47728049755096436,
+ "neutral_observations": 4096,
+ "residual_context_rms_ratio": 0.3765350580215454,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "context_rms": 0.2768581211566925,
+ "mean_per_cell_r2": 0.6187853217124939,
+ "neutral_observations": 4096,
+ "residual_context_rms_ratio": 0.3115047812461853,
+ "teaching_observations": 0
+ }
+ ],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 114644992,
+ "cuda_visible_devices": "7",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 5.956,
+ "mean_return": 0.8441000000000002,
+ "success": 0.93
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4102,
+ "neutral_examples_per_epoch": 4096,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4102,
+ "total_wall_seconds_including_rollouts": 186.63246417045593,
+ "training_wall_seconds": 183.2198555469513
+ },
+ "validation": {
+ "accuracy": 0.9477133494799829,
+ "loss": 0.29150108231966915
+ }
+}
diff --git a/results/babyai_shared/b1/seed4103_bp.json b/results/babyai_shared/b1/seed4103_bp.json
new file mode 100644
index 0000000..9b06244
--- /dev/null
+++ b/results/babyai_shared/b1/seed4103_bp.json
@@ -0,0 +1,251 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.34741790928623895
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.04047851919450543
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.01946668891236186
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.014651047926057469
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.014299569986760616
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.013254153941842643
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.013294946311507374
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.0111591817476702
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.011023848238401114
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.01068408746695654
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.011179883105214686
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.012154127080628479
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.011421862830069254
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.010540508885486898
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.01119329729265618
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.01123202916001901
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.010137052976907316
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.010419989238448165
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.01014127544444901
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.010489503509703684
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.009867867013989864
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.0107619523731145
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.010490423445484008
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.01005430692859756
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.01007219225909053
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.00975766327761283
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.009965825283645906
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.00980187646854161
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.009484837243375791
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.009791840241946789
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.009611271926392377
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.010396038602762432
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.010572744316675447
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.009761923416411843
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.009488878117091107
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.009714971253150989
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.009911997303451327
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.009453672587088393
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.009813165926078166
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.009634912587617608
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 4.914,
+ "mean_return": 0.8731500000000001,
+ "success": 0.96
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.9611055705941017,
+ "loss": 0.07351782016561215
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39800832,
+ "cuda_visible_devices": "0",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.29,
+ "mean_return": 0.8905500000000001,
+ "success": 0.978
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4103,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4103,
+ "total_wall_seconds_including_rollouts": 83.40724444389343,
+ "training_wall_seconds": 79.6464741230011
+ },
+ "validation": {
+ "accuracy": 0.9974355321270836,
+ "loss": 0.00777159465447945
+ }
+}
diff --git a/results/babyai_shared/b1/seed4103_clean_kp.json b/results/babyai_shared/b1/seed4103_clean_kp.json
new file mode 100644
index 0000000..cbe452a
--- /dev/null
+++ b/results/babyai_shared/b1/seed4103_clean_kp.json
@@ -0,0 +1,251 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.4082638917863369
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.04009808559986678
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.01811559673398733
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.014143027541312305
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.013741936458444053
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.012966733724725517
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.013003955709493973
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.010960795615207065
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.010879007470370693
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.010519698688083074
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.010998198806138879
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.011942592064892366
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.011174577157521112
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.010476980858418921
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.01105824689401991
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.011077042488296602
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.0101143445413221
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.010370828109232455
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.010069763263983822
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.010439676970497452
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.00984062194686637
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.01072979854694991
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.010458854098529131
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.010025178403276103
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.010013739626050334
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.009695339784322476
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.00992449996506118
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.009752792384136807
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.009458325277294287
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.009779711920471692
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.009554736970140684
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.010344536146043207
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.010445120590527287
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.009722411417460535
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.009442475319056856
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.009679464375367389
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.00984952370032922
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.009403426681370052
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.009725391912612725
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.009571617891038345
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 8.17,
+ "mean_return": 0.7815500000000002,
+ "success": 0.858
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.9244906681863513,
+ "loss": 0.1711390281032677
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39800832,
+ "cuda_visible_devices": "1",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.29,
+ "mean_return": 0.8905500000000001,
+ "success": 0.978
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4103,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4103,
+ "total_wall_seconds_including_rollouts": 87.66643977165222,
+ "training_wall_seconds": 85.08785223960876
+ },
+ "validation": {
+ "accuracy": 0.9974355321270836,
+ "loss": 0.007668511618540691
+ }
+}
diff --git a/results/babyai_shared/b1/seed4103_raw_shared.json b/results/babyai_shared/b1/seed4103_raw_shared.json
new file mode 100644
index 0000000..18baea6
--- /dev/null
+++ b/results/babyai_shared/b1/seed4103_raw_shared.json
@@ -0,0 +1,251 @@
+{
+ "condition": "raw_shared",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.9230526057156649
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.3748072928732092
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.1566001031073657
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.10870673175562512
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.12177402764558792
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.08673705132170156
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.06574006106365811
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.06591689348220825
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.059585516981103204
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.05415208701382984
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.05290201323276216
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.056410093801942736
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.04764564476229928
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.049847253039479254
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.05844363085248253
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.044315730451860214
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.09560010188005187
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.057736393910917366
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.05797194173728878
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.04596193544905294
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.051635723124173555
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.04816918779164553
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.052199349152770906
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.08759990688074719
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.04206920251250267
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.04370714916085655
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.11293787405233492
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.05867061015557159
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.03914543169127269
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.046046290055594664
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.04019219841130755
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.03525011847980998
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.040651550897481764
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.036307538307525894
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.03384690742601048
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.044114207821813495
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.041040615293789995
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.05004180110991001
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.04359338021752509
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.039025707397271285
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 4.298,
+ "mean_return": 0.8903500000000001,
+ "success": 0.978
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.9967231799401625,
+ "loss": 0.024486202139989623
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 39770112,
+ "cuda_visible_devices": "2",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.5,
+ "mean_return": 0.8847,
+ "success": 0.972
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4103,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4103,
+ "total_wall_seconds_including_rollouts": 87.0457055568695,
+ "training_wall_seconds": 84.57916736602783
+ },
+ "validation": {
+ "accuracy": 0.9955834164410885,
+ "loss": 0.028918176757795108
+ }
+}
diff --git a/results/babyai_shared/b1/seed4103_sdil.json b/results/babyai_shared/b1/seed4103_sdil.json
new file mode 100644
index 0000000..fef0699
--- /dev/null
+++ b/results/babyai_shared/b1/seed4103_sdil.json
@@ -0,0 +1,268 @@
+{
+ "condition": "sdil",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 13,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 37.730633020401,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 70159,
+ "validation_episodes": 2000,
+ "validation_steps": 7019,
+ "vocabulary": [
+ "<unk>",
+ "ball",
+ "blue",
+ "box",
+ "go",
+ "green",
+ "grey",
+ "key",
+ "purple",
+ "red",
+ "the",
+ "to",
+ "yellow"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7408094794641842
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.4141200262849981
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.20642850984226574
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.2396153619072654
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.1747013689035719
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.09160391047596932
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.20036431182514539
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.05948227807879448
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.21365087948062203
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.0611378302899274
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.06920530608431859
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.047904286282983696
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.21460180244662544
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.05637756698510864
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.05161521524190903
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.3368245812234553
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.19509815630587665
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.2183473968234929
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.09419559308073737
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.07092469604177909
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.059987902383912696
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.05138562641360543
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.05321425412866202
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.2473546962236816
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.05551467421379956
+ },
+ {
+ "epoch": 26,
+ "train_loss": 0.11517106813463297
+ },
+ {
+ "epoch": 27,
+ "train_loss": 0.1695636088502678
+ },
+ {
+ "epoch": 28,
+ "train_loss": 0.04232350076464089
+ },
+ {
+ "epoch": 29,
+ "train_loss": 0.040572751760482785
+ },
+ {
+ "epoch": 30,
+ "train_loss": 0.041964036503976045
+ },
+ {
+ "epoch": 31,
+ "train_loss": 0.03777888340706175
+ },
+ {
+ "epoch": 32,
+ "train_loss": 0.043543690524318
+ },
+ {
+ "epoch": 33,
+ "train_loss": 0.041410166783766314
+ },
+ {
+ "epoch": 34,
+ "train_loss": 0.23046876887706194
+ },
+ {
+ "epoch": 35,
+ "train_loss": 0.04775922474536029
+ },
+ {
+ "epoch": 36,
+ "train_loss": 0.04496883080086925
+ },
+ {
+ "epoch": 37,
+ "train_loss": 0.038768304579637265
+ },
+ {
+ "epoch": 38,
+ "train_loss": 0.03775290479375557
+ },
+ {
+ "epoch": 39,
+ "train_loss": 0.1857465661791238
+ },
+ {
+ "epoch": 40,
+ "train_loss": 0.20626876744357023
+ }
+ ],
+ "epochs_completed": 40,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 5.742,
+ "mean_return": 0.8496500000000001,
+ "success": 0.932
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.9762074369568314,
+ "loss": 0.11846651761242086
+ },
+ "predictor": [
+ {
+ "action_observations": 0,
+ "context_rms": 0.2629075348377228,
+ "mean_per_cell_r2": 0.49027836322784424,
+ "neutral_observations": 4096,
+ "residual_context_rms_ratio": 0.3944878876209259,
+ "teaching_observations": 0
+ },
+ {
+ "action_observations": 0,
+ "context_rms": 0.2731991410255432,
+ "mean_per_cell_r2": 0.6234094500541687,
+ "neutral_observations": 4096,
+ "residual_context_rms_ratio": 0.32784295082092285,
+ "teaching_observations": 0
+ }
+ ],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 114644992,
+ "cuda_visible_devices": "3",
+ "device": "cuda:0",
+ "git_commit": "e3011f144a018f6415b3c2a636c0646e412b3ced",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 4.59,
+ "mean_return": 0.8820500000000002,
+ "success": 0.968
+ },
+ "stage": "babyai_shared_b1",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4103,
+ "neutral_examples_per_epoch": 4096,
+ "parameter_count_including_reciprocal_context_predictor": 394759,
+ "shuffle_seed": 4103,
+ "total_wall_seconds_including_rollouts": 87.60878133773804,
+ "training_wall_seconds": 85.27321362495422
+ },
+ "validation": {
+ "accuracy": 0.9844707223251176,
+ "loss": 0.11933256204455374
+ }
+}
diff --git a/results/babyai_shared/b1_analysis.json b/results/babyai_shared/b1_analysis.json
new file mode 100644
index 0000000..6c605a5
--- /dev/null
+++ b/results/babyai_shared/b1_analysis.json
@@ -0,0 +1,198 @@
+{
+ "checks": {
+ "all_runs_finite": true,
+ "bp_and_clean_kp_mean_rollout_at_least_80": true,
+ "mission_structurally_required": false,
+ "sdil_rollout_above_raw_every_seed": false
+ },
+ "conditions": {
+ "bp": {
+ "all_finite": true,
+ "expert_action_accuracy_percent": {
+ "mean": 99.74355321270836,
+ "sample_std": 0.0,
+ "values": [
+ 99.74355321270836,
+ 99.74355321270836,
+ 99.74355321270836
+ ]
+ },
+ "mission_lesion_rollout_success_percent": {
+ "mean": 88.2,
+ "sample_std": 14.208448191129108,
+ "values": [
+ 71.8,
+ 96.8,
+ 96.0
+ ]
+ },
+ "rollout_success_percent": {
+ "mean": 97.8,
+ "sample_std": 0.0,
+ "values": [
+ 97.8,
+ 97.8,
+ 97.8
+ ]
+ },
+ "training_wall_seconds": {
+ "mean": 145.09927169481912,
+ "sample_std": 56.693567058307366,
+ "values": [
+ 178.87876749038696,
+ 176.77257347106934,
+ 79.6464741230011
+ ]
+ }
+ },
+ "clean_kp": {
+ "all_finite": true,
+ "expert_action_accuracy_percent": {
+ "mean": 99.74355321270836,
+ "sample_std": 0.0,
+ "values": [
+ 99.74355321270836,
+ 99.74355321270836,
+ 99.74355321270836
+ ]
+ },
+ "mission_lesion_rollout_success_percent": {
+ "mean": 78.46666666666667,
+ "sample_std": 15.37313674346694,
+ "values": [
+ 60.8,
+ 88.8,
+ 85.8
+ ]
+ },
+ "rollout_success_percent": {
+ "mean": 97.8,
+ "sample_std": 0.0,
+ "values": [
+ 97.8,
+ 97.8,
+ 97.8
+ ]
+ },
+ "training_wall_seconds": {
+ "mean": 150.14535276095071,
+ "sample_std": 56.34150377593018,
+ "values": [
+ 182.5949375629425,
+ 182.7532684803009,
+ 85.08785223960876
+ ]
+ }
+ },
+ "raw_shared": {
+ "all_finite": true,
+ "expert_action_accuracy_percent": {
+ "mean": 93.80728498836491,
+ "sample_std": 6.19430813024119,
+ "values": [
+ 87.24889585411027,
+ 94.61461746687563,
+ 99.55834164410885
+ ]
+ },
+ "mission_lesion_rollout_success_percent": {
+ "mean": 88.8,
+ "sample_std": 17.528262891684385,
+ "values": [
+ 68.60000000000001,
+ 100.0,
+ 97.8
+ ]
+ },
+ "rollout_success_percent": {
+ "mean": 83.93333333333334,
+ "sample_std": 15.297494348204001,
+ "values": [
+ 67.2,
+ 87.4,
+ 97.2
+ ]
+ },
+ "training_wall_seconds": {
+ "mean": 147.5590082804362,
+ "sample_std": 54.54380970544501,
+ "values": [
+ 178.62242436408997,
+ 179.4754331111908,
+ 84.57916736602783
+ ]
+ }
+ },
+ "sdil": {
+ "all_finite": true,
+ "expert_action_accuracy_percent": {
+ "mean": 97.56375552072944,
+ "sample_std": 2.472098159480726,
+ "values": [
+ 99.4728593816783,
+ 94.77133494799828,
+ 98.44707223251176
+ ]
+ },
+ "mission_lesion_rollout_success_percent": {
+ "mean": 92.33333333333333,
+ "sample_std": 5.94754851458425,
+ "values": [
+ 97.8,
+ 86.0,
+ 93.2
+ ]
+ },
+ "rollout_success_percent": {
+ "mean": 95.86666666666666,
+ "sample_std": 2.5324559884296756,
+ "values": [
+ 97.8,
+ 93.0,
+ 96.8
+ ]
+ },
+ "training_wall_seconds": {
+ "mean": 150.52963081995645,
+ "sample_std": 56.5137490779585,
+ "values": [
+ 183.09582328796387,
+ 183.2198555469513,
+ 85.27321362495422
+ ]
+ }
+ }
+ },
+ "decision": "Retain as an implementation diagnostic; do not use as the paper's task-required shared-feedback result. Move to PickupLoc.",
+ "gate": "fail",
+ "paired_sdil_minus_raw": {
+ "expert_action_accuracy_points": {
+ "mean": 3.7564705323645313,
+ "sample_std": 7.360419547002228,
+ "values": [
+ 12.223963527568028,
+ 0.15671748112265904,
+ -1.1112694115970934
+ ]
+ },
+ "rollout_success_points": {
+ "mean": 11.933333333333334,
+ "sample_std": 16.44181660685136,
+ "values": [
+ 30.599999999999994,
+ 5.600000000000005,
+ -0.40000000000000036
+ ]
+ }
+ },
+ "stage": "babyai_shared_b1_analysis",
+ "task_structure_audit": {
+ "audited_episode_seeds": 256,
+ "env_id": "BabyAI-GoToObjS6-v1",
+ "interpretation": "The sole object is the target, so the mission does not select among alternatives. Mission lesion is an input ablation, not evidence that language is required by the task.",
+ "maximum_task_object_count": 1,
+ "minimum_task_object_count": 1,
+ "mission_structurally_required_to_identify_the_only_object": false
+ },
+ "test_split_generated_or_read": false
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-15 08:46:04 +0000