From e3011f144a018f6415b3c2a636c0646e412b3ced Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Mon, 10 Aug 2026 10:18:26 -0500 Subject: results: select BabyAI clean configuration --- results/babyai_shared/b0/d2_lr0.01_bp.json | 151 +++++++++++++++++++++++ results/babyai_shared/b0/d2_lr0.01_clean_kp.json | 151 +++++++++++++++++++++++ results/babyai_shared/b0/d2_lr0.03_bp.json | 151 +++++++++++++++++++++++ results/babyai_shared/b0/d2_lr0.03_clean_kp.json | 151 +++++++++++++++++++++++ results/babyai_shared/b0/d4_lr0.01_bp.json | 151 +++++++++++++++++++++++ results/babyai_shared/b0/d4_lr0.01_clean_kp.json | 151 +++++++++++++++++++++++ results/babyai_shared/b0/d4_lr0.03_bp.json | 151 +++++++++++++++++++++++ results/babyai_shared/b0/d4_lr0.03_clean_kp.json | 151 +++++++++++++++++++++++ results/babyai_shared/b0_selector.json | 105 ++++++++++++++++ 9 files changed, 1313 insertions(+) create mode 100644 results/babyai_shared/b0/d2_lr0.01_bp.json create mode 100644 results/babyai_shared/b0/d2_lr0.01_clean_kp.json create mode 100644 results/babyai_shared/b0/d2_lr0.03_bp.json create mode 100644 results/babyai_shared/b0/d2_lr0.03_clean_kp.json create mode 100644 results/babyai_shared/b0/d4_lr0.01_bp.json create mode 100644 results/babyai_shared/b0/d4_lr0.01_clean_kp.json create mode 100644 results/babyai_shared/b0/d4_lr0.03_bp.json create mode 100644 results/babyai_shared/b0/d4_lr0.03_clean_kp.json create mode 100644 results/babyai_shared/b0_selector.json (limited to 'results/babyai_shared') diff --git a/results/babyai_shared/b0/d2_lr0.01_bp.json b/results/babyai_shared/b0/d2_lr0.01_bp.json new file mode 100644 index 0000000..1e60df3 --- /dev/null +++ b/results/babyai_shared/b0/d2_lr0.01_bp.json @@ -0,0 +1,151 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 2, + "input_dim": 984, + "learning_rate": 0.01, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.01, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.35939537301659585 + }, + { + "epoch": 2, + "train_loss": 0.04168066847730767 + }, + { + "epoch": 3, + "train_loss": 0.01859026498753916 + }, + { + "epoch": 4, + "train_loss": 0.015395146521993659 + }, + { + "epoch": 5, + "train_loss": 0.015227786766534503 + }, + { + "epoch": 6, + "train_loss": 0.012962078500369734 + }, + { + "epoch": 7, + "train_loss": 0.013404185661241751 + }, + { + "epoch": 8, + "train_loss": 0.01239592205690728 + }, + { + "epoch": 9, + "train_loss": 0.012198219955157996 + }, + { + "epoch": 10, + "train_loss": 0.011407826012000442 + }, + { + "epoch": 11, + "train_loss": 0.010740509365939281 + }, + { + "epoch": 12, + "train_loss": 0.01174285890576853 + }, + { + "epoch": 13, + "train_loss": 0.012136201211251318 + }, + { + "epoch": 14, + "train_loss": 0.010752951244569637 + }, + { + "epoch": 15, + "train_loss": 0.010369447311809794 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 13.1, + "mean_return": 0.6429, + "success": 0.704 + }, + "mission_lesion_validation": { + "accuracy": 0.8139336087761789, + "loss": 0.5671851808553681 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 39800832, + "cuda_visible_devices": "0", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 394759, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 72.15492010116577, + "training_wall_seconds": 68.66148805618286 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.010374719048008347 + } +} diff --git a/results/babyai_shared/b0/d2_lr0.01_clean_kp.json b/results/babyai_shared/b0/d2_lr0.01_clean_kp.json new file mode 100644 index 0000000..c63545b --- /dev/null +++ b/results/babyai_shared/b0/d2_lr0.01_clean_kp.json @@ -0,0 +1,151 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 2, + "input_dim": 984, + "learning_rate": 0.01, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.01, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.42220023420724 + }, + { + "epoch": 2, + "train_loss": 0.042795064787973056 + }, + { + "epoch": 3, + "train_loss": 0.01790234833363105 + }, + { + "epoch": 4, + "train_loss": 0.014970682340420106 + }, + { + "epoch": 5, + "train_loss": 0.014731389162638648 + }, + { + "epoch": 6, + "train_loss": 0.012619410693137482 + }, + { + "epoch": 7, + "train_loss": 0.013174676087642596 + }, + { + "epoch": 8, + "train_loss": 0.012047174987383186 + }, + { + "epoch": 9, + "train_loss": 0.011912513411900198 + }, + { + "epoch": 10, + "train_loss": 0.011198682525110516 + }, + { + "epoch": 11, + "train_loss": 0.010507440414618362 + }, + { + "epoch": 12, + "train_loss": 0.011464962014843795 + }, + { + "epoch": 13, + "train_loss": 0.011738500667905266 + }, + { + "epoch": 14, + "train_loss": 0.010477929167999802 + }, + { + "epoch": 15, + "train_loss": 0.010263334540861913 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 16.362, + "mean_return": 0.5511499999999999, + "success": 0.602 + }, + "mission_lesion_validation": { + "accuracy": 0.7602222538823195, + "loss": 0.7746106897906575 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 39800832, + "cuda_visible_devices": "1", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 394759, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 72.49147534370422, + "training_wall_seconds": 68.89583253860474 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.010123796248337468 + } +} diff --git a/results/babyai_shared/b0/d2_lr0.03_bp.json b/results/babyai_shared/b0/d2_lr0.03_bp.json new file mode 100644 index 0000000..53c0cd2 --- /dev/null +++ b/results/babyai_shared/b0/d2_lr0.03_bp.json @@ -0,0 +1,151 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 2, + "input_dim": 984, + "learning_rate": 0.03, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.32622500834817236 + }, + { + "epoch": 2, + "train_loss": 0.022967058995907955 + }, + { + "epoch": 3, + "train_loss": 0.013182067105292596 + }, + { + "epoch": 4, + "train_loss": 0.12631320300893012 + }, + { + "epoch": 5, + "train_loss": 0.03265200278869915 + }, + { + "epoch": 6, + "train_loss": 0.01237200420434502 + }, + { + "epoch": 7, + "train_loss": 0.012444034099012655 + }, + { + "epoch": 8, + "train_loss": 0.010830031347418712 + }, + { + "epoch": 9, + "train_loss": 0.010618501521848057 + }, + { + "epoch": 10, + "train_loss": 0.010414972044315867 + }, + { + "epoch": 11, + "train_loss": 0.009751175390768118 + }, + { + "epoch": 12, + "train_loss": 0.01019353245279159 + }, + { + "epoch": 13, + "train_loss": 0.010311937543394213 + }, + { + "epoch": 14, + "train_loss": 0.00966560988549397 + }, + { + "epoch": 15, + "train_loss": 0.009808915172311986 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 6.608, + "mean_return": 0.8254000000000001, + "success": 0.906 + }, + "mission_lesion_validation": { + "accuracy": 0.9393075936743126, + "loss": 0.15966886549462006 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 39800832, + "cuda_visible_devices": "2", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 394759, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 71.35738158226013, + "training_wall_seconds": 68.60933923721313 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.008090585890548625 + } +} diff --git a/results/babyai_shared/b0/d2_lr0.03_clean_kp.json b/results/babyai_shared/b0/d2_lr0.03_clean_kp.json new file mode 100644 index 0000000..767f65d --- /dev/null +++ b/results/babyai_shared/b0/d2_lr0.03_clean_kp.json @@ -0,0 +1,151 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 2, + "input_dim": 984, + "learning_rate": 0.03, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.4284296942739324 + }, + { + "epoch": 2, + "train_loss": 0.021993979699909686 + }, + { + "epoch": 3, + "train_loss": 0.012347337700511244 + }, + { + "epoch": 4, + "train_loss": 0.11120144955475222 + }, + { + "epoch": 5, + "train_loss": 0.02621290213089775 + }, + { + "epoch": 6, + "train_loss": 0.011413470651869748 + }, + { + "epoch": 7, + "train_loss": 0.011871577144404661 + }, + { + "epoch": 8, + "train_loss": 0.010163592805302787 + }, + { + "epoch": 9, + "train_loss": 0.009837346972728317 + }, + { + "epoch": 10, + "train_loss": 0.01011049080259082 + }, + { + "epoch": 11, + "train_loss": 0.009574851958419789 + }, + { + "epoch": 12, + "train_loss": 0.009629912125899202 + }, + { + "epoch": 13, + "train_loss": 0.009786839098352092 + }, + { + "epoch": 14, + "train_loss": 0.009409503974602558 + }, + { + "epoch": 15, + "train_loss": 0.009612279576884413 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 5.856, + "mean_return": 0.8466, + "success": 0.93 + }, + "mission_lesion_validation": { + "accuracy": 0.9532696965379683, + "loss": 0.12229915629222635 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 39800832, + "cuda_visible_devices": "3", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 394759, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 71.55634665489197, + "training_wall_seconds": 68.94226169586182 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.007343835429191453 + } +} diff --git a/results/babyai_shared/b0/d4_lr0.01_bp.json b/results/babyai_shared/b0/d4_lr0.01_bp.json new file mode 100644 index 0000000..ce30242 --- /dev/null +++ b/results/babyai_shared/b0/d4_lr0.01_bp.json @@ -0,0 +1,151 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 984, + "learning_rate": 0.01, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.01, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.3253778777474707 + }, + { + "epoch": 2, + "train_loss": 0.019206965685060078 + }, + { + "epoch": 3, + "train_loss": 0.012563892805271528 + }, + { + "epoch": 4, + "train_loss": 0.013036079497330568 + }, + { + "epoch": 5, + "train_loss": 0.014585806006692688 + }, + { + "epoch": 6, + "train_loss": 0.011469631964256141 + }, + { + "epoch": 7, + "train_loss": 0.012590554792894346 + }, + { + "epoch": 8, + "train_loss": 0.010635647804857316 + }, + { + "epoch": 9, + "train_loss": 0.01044608271402963 + }, + { + "epoch": 10, + "train_loss": 0.01032080053330653 + }, + { + "epoch": 11, + "train_loss": 0.009672622701830485 + }, + { + "epoch": 12, + "train_loss": 0.010132916131353175 + }, + { + "epoch": 13, + "train_loss": 0.010232909059956332 + }, + { + "epoch": 14, + "train_loss": 0.009590742462103001 + }, + { + "epoch": 15, + "train_loss": 0.009766413487642157 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 18.046, + "mean_return": 0.50385, + "success": 0.55 + }, + "mission_lesion_validation": { + "accuracy": 0.7350049864653084, + "loss": 1.0839614618162532 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 42582016, + "cuda_visible_devices": "4", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 665095, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 69.9526629447937, + "training_wall_seconds": 63.36152243614197 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.007651511259713494 + } +} diff --git a/results/babyai_shared/b0/d4_lr0.01_clean_kp.json b/results/babyai_shared/b0/d4_lr0.01_clean_kp.json new file mode 100644 index 0000000..72a132c --- /dev/null +++ b/results/babyai_shared/b0/d4_lr0.01_clean_kp.json @@ -0,0 +1,151 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 984, + "learning_rate": 0.01, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.01, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.4667839081585407 + }, + { + "epoch": 2, + "train_loss": 0.04674517553976991 + }, + { + "epoch": 3, + "train_loss": 0.01326043784279715 + }, + { + "epoch": 4, + "train_loss": 0.061285746785896746 + }, + { + "epoch": 5, + "train_loss": 0.026032158348878676 + }, + { + "epoch": 6, + "train_loss": 0.013545795751904899 + }, + { + "epoch": 7, + "train_loss": 0.014207788907065564 + }, + { + "epoch": 8, + "train_loss": 0.011456064975143156 + }, + { + "epoch": 9, + "train_loss": 0.011107088818402685 + }, + { + "epoch": 10, + "train_loss": 0.010615441516486251 + }, + { + "epoch": 11, + "train_loss": 0.009885513414840468 + }, + { + "epoch": 12, + "train_loss": 0.01040836228650402 + }, + { + "epoch": 13, + "train_loss": 0.010366036330180411 + }, + { + "epoch": 14, + "train_loss": 0.009639627484350719 + }, + { + "epoch": 15, + "train_loss": 0.009855725453976033 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 4.292, + "mean_return": 0.8905000000000001, + "success": 0.978 + }, + "mission_lesion_validation": { + "accuracy": 0.9955834164410885, + "loss": 0.01790462883301042 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 42582016, + "cuda_visible_devices": "5", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 665095, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 68.48475790023804, + "training_wall_seconds": 64.21018767356873 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.007562428802009797 + } +} diff --git a/results/babyai_shared/b0/d4_lr0.03_bp.json b/results/babyai_shared/b0/d4_lr0.03_bp.json new file mode 100644 index 0000000..33f8418 --- /dev/null +++ b/results/babyai_shared/b0/d4_lr0.03_bp.json @@ -0,0 +1,151 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 984, + "learning_rate": 0.03, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.5505747761509635 + }, + { + "epoch": 2, + "train_loss": 0.3316267966343598 + }, + { + "epoch": 3, + "train_loss": 0.09943052434563553 + }, + { + "epoch": 4, + "train_loss": 0.04950741482962092 + }, + { + "epoch": 5, + "train_loss": 0.013397140696906718 + }, + { + "epoch": 6, + "train_loss": 0.010083013286560097 + }, + { + "epoch": 7, + "train_loss": 0.011028704402918017 + }, + { + "epoch": 8, + "train_loss": 0.009480555155737834 + }, + { + "epoch": 9, + "train_loss": 0.009246192916672788 + }, + { + "epoch": 10, + "train_loss": 0.009409724382448158 + }, + { + "epoch": 11, + "train_loss": 0.00913212353257801 + }, + { + "epoch": 12, + "train_loss": 0.00898012409410016 + }, + { + "epoch": 13, + "train_loss": 0.009049554331206971 + }, + { + "epoch": 14, + "train_loss": 0.00897477681505155 + }, + { + "epoch": 15, + "train_loss": 0.009097340858795426 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "mission_lesion_validation": { + "accuracy": 0.9974355321270836, + "loss": 0.009770111168767503 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 42582016, + "cuda_visible_devices": "6", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 665095, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 67.55784368515015, + "training_wall_seconds": 63.23557496070862 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.00700099701928284 + } +} diff --git a/results/babyai_shared/b0/d4_lr0.03_clean_kp.json b/results/babyai_shared/b0/d4_lr0.03_clean_kp.json new file mode 100644 index 0000000..b0c853d --- /dev/null +++ b/results/babyai_shared/b0/d4_lr0.03_clean_kp.json @@ -0,0 +1,151 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 984, + "learning_rate": 0.03, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.560454016964544 + }, + { + "epoch": 2, + "train_loss": 0.11413068584420465 + }, + { + "epoch": 3, + "train_loss": 0.01235340728620279 + }, + { + "epoch": 4, + "train_loss": 0.010933047032872723 + }, + { + "epoch": 5, + "train_loss": 0.010575457136702342 + }, + { + "epoch": 6, + "train_loss": 0.00985285716623449 + }, + { + "epoch": 7, + "train_loss": 0.010482580381583168 + }, + { + "epoch": 8, + "train_loss": 0.009581763261918572 + }, + { + "epoch": 9, + "train_loss": 0.009271365735324005 + }, + { + "epoch": 10, + "train_loss": 0.00958018275731857 + }, + { + "epoch": 11, + "train_loss": 0.009324251855247316 + }, + { + "epoch": 12, + "train_loss": 0.00918881041791544 + }, + { + "epoch": 13, + "train_loss": 0.009378983245802705 + }, + { + "epoch": 14, + "train_loss": 0.009324983275675384 + }, + { + "epoch": 15, + "train_loss": 0.009338655431241601 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 4.306, + "mean_return": 0.8901500000000001, + "success": 0.978 + }, + "mission_lesion_validation": { + "accuracy": 0.9921641259438666, + "loss": 0.029844395743083507 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 42582016, + "cuda_visible_devices": "7", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 665095, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 72.05698609352112, + "training_wall_seconds": 69.7202775478363 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.006933236702977295 + } +} diff --git a/results/babyai_shared/b0_selector.json b/results/babyai_shared/b0_selector.json new file mode 100644 index 0000000..4368e44 --- /dev/null +++ b/results/babyai_shared/b0_selector.json @@ -0,0 +1,105 @@ +{ + "candidates": [ + { + "bp_action_accuracy": 0.9974355321270836, + "bp_mission_lesion_drop": 0.274, + "bp_mission_lesion_success": 0.704, + "bp_rollout_success": 0.978, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_0p2": true, + "bp_success_at_least_0p8": true, + "clean_kp_success_at_least_0p8": true + }, + "clean_kp_action_accuracy": 0.9974355321270836, + "clean_kp_rollout_success": 0.978, + "eligible": true, + "hidden_layers": 2, + "learning_rate": 0.01, + "source_files": [ + "results/babyai_shared/b0/d2_lr0.01_bp.json", + "results/babyai_shared/b0/d2_lr0.01_clean_kp.json" + ] + }, + { + "bp_action_accuracy": 0.9974355321270836, + "bp_mission_lesion_drop": 0.07199999999999995, + "bp_mission_lesion_success": 0.906, + "bp_rollout_success": 0.978, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_0p2": false, + "bp_success_at_least_0p8": true, + "clean_kp_success_at_least_0p8": true + }, + "clean_kp_action_accuracy": 0.9974355321270836, + "clean_kp_rollout_success": 0.978, + "eligible": false, + "hidden_layers": 2, + "learning_rate": 0.03, + "source_files": [ + "results/babyai_shared/b0/d2_lr0.03_bp.json", + "results/babyai_shared/b0/d2_lr0.03_clean_kp.json" + ] + }, + { + "bp_action_accuracy": 0.9974355321270836, + "bp_mission_lesion_drop": 0.42799999999999994, + "bp_mission_lesion_success": 0.55, + "bp_rollout_success": 0.978, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_0p2": true, + "bp_success_at_least_0p8": true, + "clean_kp_success_at_least_0p8": true + }, + "clean_kp_action_accuracy": 0.9974355321270836, + "clean_kp_rollout_success": 0.978, + "eligible": true, + "hidden_layers": 4, + "learning_rate": 0.01, + "source_files": [ + "results/babyai_shared/b0/d4_lr0.01_bp.json", + "results/babyai_shared/b0/d4_lr0.01_clean_kp.json" + ] + }, + { + "bp_action_accuracy": 0.9974355321270836, + "bp_mission_lesion_drop": 0.0, + "bp_mission_lesion_success": 0.978, + "bp_rollout_success": 0.978, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_0p2": false, + "bp_success_at_least_0p8": true, + "clean_kp_success_at_least_0p8": true + }, + "clean_kp_action_accuracy": 0.9974355321270836, + "clean_kp_rollout_success": 0.978, + "eligible": false, + "hidden_layers": 4, + "learning_rate": 0.03, + "source_files": [ + "results/babyai_shared/b0/d4_lr0.03_bp.json", + "results/babyai_shared/b0/d4_lr0.03_clean_kp.json" + ] + } + ], + "gate": "pass", + "raw_or_sdil_results_read": false, + "selected": { + "b1_epochs": 40, + "b1_model_and_shuffle_seeds": [ + 4101, + 4102, + 4103 + ], + "context_gain": 1.0, + "hidden_layers": 2, + "learning_rate": 0.01, + "width": 256 + }, + "selection_rule": "highest clean-KP rollout success, then action accuracy, then fewer layers, then smaller learning rate, among eligible rows", + "stage": "babyai_shared_b0_selector", + "test_split_generated_or_read": false +} -- cgit v1.2.3