diff options
Diffstat (limited to 'results/resnet_crossover/r2_audit.json')
| -rw-r--r-- | results/resnet_crossover/r2_audit.json | 1001 |
1 files changed, 1001 insertions, 0 deletions
diff --git a/results/resnet_crossover/r2_audit.json b/results/resnet_crossover/r2_audit.json new file mode 100644 index 0000000..1014e96 --- /dev/null +++ b/results/resnet_crossover/r2_audit.json @@ -0,0 +1,1001 @@ +{ + "audit_status": "passed", + "complete_grid": true, + "failed_or_incomplete_cells": [ + "resnet56::dualprop" + ], + "failure_retaining": true, + "hardware_policy": { + "allowed_device_names": [ + "NVIDIA RTX A6000" + ], + "allowed_physical_gpu_indices": [ + 0 + ], + "profile": "a6000", + "single_gpu_per_process": true + }, + "launch_lock": { + "path": "results/resnet_crossover/r2_launch.json", + "registry_sha256": "bc205e39b0403bb2e80c289467a1fb3f746ef970bfe0ee8993a20ebab08713ba", + "sha256": "2d30076208dc9eb69f28e3d0b617b09760a9fea61436e7fb83037eb2379c5e5d" + }, + "num_audited_cells": 27, + "num_expected_cells": 27, + "num_finite_cells": 26, + "records": [ + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.1, + "cell_id": "resnet56::ep", + "complete_trajectory": true, + "depth": 56, + "driver_wall_seconds": 50669.00653362274, + "epochs_completed": 100, + "final_validation_accuracy": 0.1, + "final_validation_loss": 0.09450665283203125, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 853018, + "history_training_examples": 4500000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-ep-d56-lr0p0003.json.manifest.json", + "method": "ep", + "ordinary_training_examples": 4500000, + "ordinary_validation_examples": 505000, + "output_sha256": "f582283d07e8d090b09b62270031ef5fe81d0e79c3da4b1ff4e23084265ec957", + "peak_memory_allocated_bytes": 11683346432, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 50663.67151045799, + "training_example_presentations": 4500000, + "work": { + "candidate_label_evaluation_presentations": 0, + "feedforward_example_passes": 5005000, + "forward_macs_per_example": 125485696, + "local_vjp_example_evaluations": 118100000, + "relaxation_example_passes": 118100000 + } + }, + { + "best_validation_accuracy": null, + "cell_id": "resnet56::dualprop", + "complete_trajectory": false, + "depth": 56, + "driver_wall_seconds": 172800.46599197388, + "epochs_completed": 0, + "final_validation_accuracy": null, + "final_validation_loss": null, + "finite": false, + "first_nonfinite_step": null, + "forward_parameter_count": null, + "logical_task_loss_queries": null, + "manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d56-lr0p05.json.manifest.json", + "method": "dualprop", + "ordinary_training_examples": null, + "ordinary_validation_examples": null, + "output_sha256": null, + "peak_memory_allocated_bytes": null, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5", + "rate": 0.05, + "status": "timeout", + "total_wall_seconds": null, + "training_example_presentations": null, + "work": null + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.1122, + "cell_id": "resnet56::ff", + "complete_trajectory": true, + "depth": 56, + "driver_wall_seconds": 23686.061631202698, + "epochs_completed": 2240, + "final_validation_accuracy": 0.1122, + "final_validation_loss": null, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 853018, + "history_training_examples": 100800000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-ff-d56-lr0p03.json.manifest.json", + "method": "ff", + "ordinary_training_examples": 100800000, + "ordinary_validation_examples": 5000, + "output_sha256": "9b7724301a9ea9ca9b938355594c03d1f5d06bd7e77410c5b27ecac4344445b7", + "peak_memory_allocated_bytes": 13807850496, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5", + "rate": 0.03, + "status": "completed", + "total_wall_seconds": 23673.216548919678, + "training_example_presentations": 201600000, + "work": { + "candidate_label_evaluation_presentations": 50000, + "feedforward_example_passes": 201600000, + "forward_macs_per_example": 125485696, + "local_vjp_example_evaluations": 0, + "relaxation_example_passes": 0 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.1, + "cell_id": "resnet32::ep", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 24671.18036866188, + "epochs_completed": 100, + "final_validation_accuracy": 0.1, + "final_validation_loss": 0.09370457153320312, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 4500000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-ep-d32-lr0p0003.json.manifest.json", + "method": "ep", + "ordinary_training_examples": 4500000, + "ordinary_validation_examples": 505000, + "output_sha256": "421c661bd6c13d7df5840943805ae1f72ccfd629b8d41b994fe9ca5e2ed65771", + "peak_memory_allocated_bytes": 6988224512, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-2ddb8c2b-c2d4-96ac-fdd7-f2ded951f824", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 24661.099477529526, + "training_example_presentations": 4500000, + "work": { + "candidate_label_evaluation_presentations": 0, + "feedforward_example_passes": 5005000, + "forward_macs_per_example": 68862592, + "local_vjp_example_evaluations": 118100000, + "relaxation_example_passes": 118100000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.871, + "cell_id": "resnet32::dualprop", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 111870.60429954529, + "epochs_completed": 200, + "final_validation_accuracy": 0.8654, + "final_validation_loss": 0.6110852416992187, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d32-lr0p05.json.manifest.json", + "method": "dualprop", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "d45682de145f1650b61be527d44ba9e38495ef5203112166a19b72ec7d8f0917", + "peak_memory_allocated_bytes": 2322300928, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-9cbd0c6c-f3fb-633d-be44-101a79992530", + "rate": 0.05, + "status": "completed", + "total_wall_seconds": 111860.19822311401, + "training_example_presentations": 9000000, + "work": { + "candidate_label_evaluation_presentations": 0, + "feedforward_example_passes": 10005000, + "forward_macs_per_example": 68862592, + "local_vjp_example_evaluations": 144000000, + "relaxation_example_passes": 144000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.1228, + "cell_id": "resnet32::ff", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 7484.067112207413, + "epochs_completed": 1280, + "final_validation_accuracy": 0.1228, + "final_validation_loss": null, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 57600000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-ff-d32-lr0p03.json.manifest.json", + "method": "ff", + "ordinary_training_examples": 57600000, + "ordinary_validation_examples": 5000, + "output_sha256": "d48f2e4651502170feb4afc4665bebb9625f077335db9f59c129748c4fdbf731", + "peak_memory_allocated_bytes": 8251766272, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.03, + "status": "completed", + "total_wall_seconds": 7469.690595626831, + "training_example_presentations": 115200000, + "work": { + "candidate_label_evaluation_presentations": 50000, + "feedforward_example_passes": 115200000, + "forward_macs_per_example": 68862592, + "local_vjp_example_evaluations": 0, + "relaxation_example_passes": 0 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.1, + "cell_id": "resnet20::ep", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 15028.455446720123, + "epochs_completed": 100, + "final_validation_accuracy": 0.1, + "final_validation_loss": 0.09437472686767578, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 4500000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-ep-d20-lr0p0003.json.manifest.json", + "method": "ep", + "ordinary_training_examples": 4500000, + "ordinary_validation_examples": 505000, + "output_sha256": "2be303594e67846889489474ed9bde767bf0abcdea660716ddf421bfc455060f", + "peak_memory_allocated_bytes": 4845611008, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.0003, + "status": "completed", + "total_wall_seconds": 15025.534816026688, + "training_example_presentations": 4500000, + "work": { + "candidate_label_evaluation_presentations": 0, + "feedforward_example_passes": 5005000, + "forward_macs_per_example": 40551040, + "local_vjp_example_evaluations": 118100000, + "relaxation_example_passes": 118100000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.9112, + "cell_id": "resnet20::dualprop", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 49289.51700615883, + "epochs_completed": 200, + "final_validation_accuracy": 0.9104, + "final_validation_loss": 0.43468799438476563, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d20-lr0p05.json.manifest.json", + "method": "dualprop", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "62576fdd2cce8c465f80416e8eba0f449a025fb0c04e3d13b6aacf6ef5699cd3", + "peak_memory_allocated_bytes": 2008608768, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.05, + "status": "completed", + "total_wall_seconds": 49286.44217157364, + "training_example_presentations": 9000000, + "work": { + "candidate_label_evaluation_presentations": 0, + "feedforward_example_passes": 10005000, + "forward_macs_per_example": 40551040, + "local_vjp_example_evaluations": 144000000, + "relaxation_example_passes": 144000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.1174, + "cell_id": "resnet20::ff", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 3187.272476911545, + "epochs_completed": 800, + "final_validation_accuracy": 0.1174, + "final_validation_loss": null, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 36000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-ff-d20-lr0p03.json.manifest.json", + "method": "ff", + "ordinary_training_examples": 36000000, + "ordinary_validation_examples": 5000, + "output_sha256": "e6cde27ffd50512b43afb18b6ddccdf1a49b2185001bf6740c96ec84ee7a89ec", + "peak_memory_allocated_bytes": 5474375168, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.03, + "status": "completed", + "total_wall_seconds": 3180.144710302353, + "training_example_presentations": 72000000, + "work": { + "candidate_label_evaluation_presentations": 50000, + "feedforward_example_passes": 72000000, + "forward_macs_per_example": 40551040, + "local_vjp_example_evaluations": 0, + "relaxation_example_passes": 0 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.2354, + "cell_id": "resnet56::pepita", + "complete_trajectory": true, + "depth": 56, + "driver_wall_seconds": 1602.2039358615875, + "epochs_completed": 100, + "final_validation_accuracy": 0.228, + "final_validation_loss": 2.124036181640625, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 853018, + "history_training_examples": 4500000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d56-lr0p003.json.manifest.json", + "method": "pepita", + "ordinary_training_examples": 4500000, + "ordinary_validation_examples": 505000, + "output_sha256": "dcebb1518d54fbefbe9bd0cb2a3b63a18591187b4dcad3cebeb3e93b1748c6e6", + "peak_memory_allocated_bytes": 3249364480, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.003, + "status": "completed", + "total_wall_seconds": 1598.730625152588, + "training_example_presentations": 9000000, + "work": { + "candidate_label_evaluation_presentations": 0, + "feedforward_example_passes": 14005000, + "forward_macs_per_example": 125485696, + "local_vjp_example_evaluations": 0, + "relaxation_example_passes": 0 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.935, + "cell_id": "resnet56::sdil", + "complete_trajectory": true, + "depth": 56, + "driver_wall_seconds": 8350.019015789032, + "epochs_completed": 200, + "final_validation_accuracy": 0.9322, + "final_validation_loss": 0.3675199523925781, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 853018, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d56-lr0p1.json.manifest.json", + "method": "sdil", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "84f1139c05218524c1be2807d0e2bacb54bc0a0bc0d0526a0dcf7d8a2de8f5f6", + "peak_memory_allocated_bytes": 3262279168, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 8344.301742315292, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 115016770519040, + "forward_macs_per_example": 125485696, + "total_forward_equivalent_examples": 9000064, + "total_macs_estimate": 4509538465857536 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.9308, + "cell_id": "resnet56::clean_kp", + "complete_trajectory": true, + "depth": 56, + "driver_wall_seconds": 5064.757298946381, + "epochs_completed": 200, + "final_validation_accuracy": 0.9274, + "final_validation_loss": 0.3740764099121094, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 853018, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d56-lr0p1.json.manifest.json", + "method": "clean_kp", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "69d1c59511cfe98a72af9c1be079d4fa581096630b2e93f4ad028a4a935bc9fa", + "peak_memory_allocated_bytes": 3256025088, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 5054.333840370178, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 125485696, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 4509522432000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.6674, + "cell_id": "resnet56::fa", + "complete_trajectory": true, + "depth": 56, + "driver_wall_seconds": 3838.2244839668274, + "epochs_completed": 200, + "final_validation_accuracy": 0.6436, + "final_validation_loss": 1.0704212158203126, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 853018, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-fa-d56-lr0p1.json.manifest.json", + "method": "fa", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "db985851db69598966506f4067c2febdd411b2fa61b02d330263f6a9ed90a776", + "peak_memory_allocated_bytes": 3252628992, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 3827.3347294330597, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 125485696, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 3384132480000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.4056, + "cell_id": "resnet56::dfa", + "complete_trajectory": true, + "depth": 56, + "driver_wall_seconds": 3246.3810822963715, + "epochs_completed": 200, + "final_validation_accuracy": 0.284, + "final_validation_loss": 56.89802421875, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 853018, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d56-lr0p01.json.manifest.json", + "method": "dfa", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "f646a557aca62c9dbfb2672fedd765f0cd51d11256c50995933d43d90a18c8f9", + "peak_memory_allocated_bytes": 3276921856, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", + "rate": 0.01, + "status": "completed", + "total_wall_seconds": 3241.7450346946716, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 125485696, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 2306665728000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.9308, + "cell_id": "resnet56::bp", + "complete_trajectory": true, + "depth": 56, + "driver_wall_seconds": 3326.933441400528, + "epochs_completed": 200, + "final_validation_accuracy": 0.93, + "final_validation_loss": 0.3817935302734375, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 853018, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-bp-d56-lr0p1.json.manifest.json", + "method": "bp", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "e4876181a6d1d8ddbda3b439319a758b9c9e42170add0a78f72df5378abd4088", + "peak_memory_allocated_bytes": 3257588736, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 3323.5966560840607, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 125485696, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 3388113792000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.2682, + "cell_id": "resnet32::pepita", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 909.2571918964386, + "epochs_completed": 100, + "final_validation_accuracy": 0.2618, + "final_validation_loss": 2.029934326171875, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 4500000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d32-lr0p003.json.manifest.json", + "method": "pepita", + "ordinary_training_examples": 4500000, + "ordinary_validation_examples": 505000, + "output_sha256": "461ff5f189160d80f86da8972f3742e260ebfb4acc720aab961f403f047ba4f1", + "peak_memory_allocated_bytes": 2322423808, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", + "rate": 0.003, + "status": "completed", + "total_wall_seconds": 896.4814369678497, + "training_example_presentations": 9000000, + "work": { + "candidate_label_evaluation_presentations": 0, + "feedforward_example_passes": 14005000, + "forward_macs_per_example": 68862592, + "local_vjp_example_evaluations": 0, + "relaxation_example_passes": 0 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.9254, + "cell_id": "resnet32::sdil", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 4740.1479415893555, + "epochs_completed": 200, + "final_validation_accuracy": 0.9236, + "final_validation_loss": 0.3822314697265625, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d32-lr0p1.json.manifest.json", + "method": "sdil", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "516a62577e7b024b7686e816ebb55bb95f9de07204117af1905ade3b67f14f09", + "peak_memory_allocated_bytes": 2329526272, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 4736.380233287811, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 65471084756992, + "forward_macs_per_example": 68862592, + "total_forward_equivalent_examples": 9000064, + "total_macs_estimate": 2471099474100224 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.9262, + "cell_id": "resnet32::clean_kp", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 2880.570287704468, + "epochs_completed": 200, + "final_validation_accuracy": 0.9238, + "final_validation_loss": 0.39594262084960935, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d32-lr0p1.json.manifest.json", + "method": "clean_kp", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "c8170145b2374f71f7bfdc161acfba10610b8ceb3c561b00b20a6182c369349c", + "peak_memory_allocated_bytes": 2325996544, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 2877.0025794506073, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 68862592, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 2471090688000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.6762, + "cell_id": "resnet32::fa", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 2180.355216741562, + "epochs_completed": 200, + "final_validation_accuracy": 0.6548, + "final_validation_loss": 0.9980604614257812, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-fa-d32-lr0p1.json.manifest.json", + "method": "fa", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "0a5f7017c2ab6fb991e98cf2b9c8e1438685f2575a2eaa41e80bdfdaa4162cf6", + "peak_memory_allocated_bytes": 2324148736, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 2176.771654367447, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 68862592, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 1855308672000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.4546, + "cell_id": "resnet32::dfa", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 1879.2923183441162, + "epochs_completed": 200, + "final_validation_accuracy": 0.3874, + "final_validation_loss": 6.95157197265625, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d32-lr0p01.json.manifest.json", + "method": "dfa", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "fdc45fa5f2e27326288de0b06db43af4ff8b0d46164473daf9f1b9a114dcb474", + "peak_memory_allocated_bytes": 2338062336, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", + "rate": 0.01, + "status": "completed", + "total_wall_seconds": 1875.3221385478973, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 68862592, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 1266806016000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.928, + "cell_id": "resnet32::bp", + "complete_trajectory": true, + "depth": 32, + "driver_wall_seconds": 1930.4826476573944, + "epochs_completed": 200, + "final_validation_accuracy": 0.928, + "final_validation_loss": 0.38430252685546873, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 464154, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-bp-d32-lr0p1.json.manifest.json", + "method": "bp", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "2450755a60aeebffe5376d58ac95a5a17b1b4af03202b8a09e71ca6672555cef", + "peak_memory_allocated_bytes": 2330820608, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 1926.7095592021942, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 68862592, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 1859289984000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.2782, + "cell_id": "resnet20::pepita", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 565.1650466918945, + "epochs_completed": 100, + "final_validation_accuracy": 0.2726, + "final_validation_loss": 1.9752171875, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 4500000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d20-lr0p003.json.manifest.json", + "method": "pepita", + "ordinary_training_examples": 4500000, + "ordinary_validation_examples": 505000, + "output_sha256": "b9120fa19a170ae650e432067cbf3df388be3f1606f18489180e646b65ac70e2", + "peak_memory_allocated_bytes": 2008731648, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", + "rate": 0.003, + "status": "completed", + "total_wall_seconds": 561.5238916873932, + "training_example_presentations": 9000000, + "work": { + "candidate_label_evaluation_presentations": 0, + "feedforward_example_passes": 14005000, + "forward_macs_per_example": 40551040, + "local_vjp_example_evaluations": 0, + "relaxation_example_passes": 0 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.9158, + "cell_id": "resnet20::sdil", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 3006.6630001068115, + "epochs_completed": 200, + "final_validation_accuracy": 0.9132, + "final_validation_loss": 0.4233872314453125, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d20-lr0p1.json.manifest.json", + "method": "sdil", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "b0be7a2e395429c4bb29209bdce5d250dccd484048551fde04102553ed2ad272", + "peak_memory_allocated_bytes": 2013018112, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 3003.0224483013153, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 40698241875968, + "forward_macs_per_example": 40551040, + "total_forward_equivalent_examples": 9000064, + "total_macs_estimate": 1451879978221568 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.9174, + "cell_id": "resnet20::clean_kp", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 1784.3792207241058, + "epochs_completed": 200, + "final_validation_accuracy": 0.9144, + "final_validation_loss": 0.4333634704589844, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d20-lr0p1.json.manifest.json", + "method": "clean_kp", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "87914e8c8674f9fc2d136f6dbf11ee9512c7e797d2d757b675ea7610ba6db795", + "peak_memory_allocated_bytes": 2010747392, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 1780.5374817848206, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 40551040, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 1451874816000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.7066, + "cell_id": "resnet20::fa", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 1365.4268357753754, + "epochs_completed": 200, + "final_validation_accuracy": 0.6854, + "final_validation_loss": 0.9467566040039063, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-fa-d20-lr0p1.json.manifest.json", + "method": "fa", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "4214a5ec034977d0fe8f927f9de040e282833008270204936eb81b419f16c399", + "peak_memory_allocated_bytes": 2009673728, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 1361.8625671863556, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 40551040, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 1090896768000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.4502, + "cell_id": "resnet20::dfa", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 1187.1560876369476, + "epochs_completed": 200, + "final_validation_accuracy": 0.3492, + "final_validation_loss": 3.6838392578125, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d20-lr0p01.json.manifest.json", + "method": "dfa", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "638ace440ba8224c18e4e0f09f2dbac291ec32e4f54bc9efadbbff58ddf2ac45", + "peak_memory_allocated_bytes": 2018397696, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", + "rate": 0.01, + "status": "completed", + "total_wall_seconds": 1183.4196124076843, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 40551040, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 746876160000000 + } + }, + { + "all_training_losses_finite": true, + "best_validation_accuracy": 0.9196, + "cell_id": "resnet20::bp", + "complete_trajectory": true, + "depth": 20, + "driver_wall_seconds": 1228.3254659175873, + "epochs_completed": 200, + "final_validation_accuracy": 0.9176, + "final_validation_loss": 0.39629163818359375, + "finite": true, + "first_nonfinite_step": null, + "forward_parameter_count": 269722, + "history_training_examples": 9000000, + "logical_task_loss_queries": 0, + "manifest": "results/resnet_crossover/r2/resnet-r2-bp-d20-lr0p1.json.manifest.json", + "method": "bp", + "ordinary_training_examples": 9000000, + "ordinary_validation_examples": 1005000, + "output_sha256": "5cf3b2928b4d27a0968fb42d633d5c23d91dad233b24a25daffb0fa9cd2fe7ee", + "peak_memory_allocated_bytes": 2017119744, + "physical_gpu_index": 0, + "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", + "rate": 0.1, + "status": "completed", + "total_wall_seconds": 1224.873178243637, + "training_example_presentations": 9000000, + "work": { + "elementwise_operations_estimate": 0, + "forward_macs_per_example": 40551040, + "total_forward_equivalent_examples": 9000000, + "total_macs_estimate": 1094878080000000 + } + } + ], + "selector": { + "p1_source": { + "git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc", + "protocol_path": "/scratch/yurenh2/sdil-runs/resnet-p1/RESNET_CROSSOVER.md", + "protocol_sha256": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d", + "python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3", + "tracked_files": { + "RESNET_CROSSOVER.md": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d", + "experiments/analyze_resnet_crossover_p1.py": "6ff485a3fecc7b814dedf55d04abdb770a6ce904cdeb9e06a7dddcf26f996d5b", + "experiments/conv_run.py": "9fa2daea597d63981a78248fafb154da39775ccfae57aac9889e78d83dc4eb44", + "experiments/resnet_crossover_grid.py": "669c84846bc3115816ae3c9036be8722f21577612dc9d56a689d3321e837ca95", + "experiments/resnet_crossover_native.py": "c5e54324b000c37c53d495f7d3eb0d4a85dcf15e83ea3aab125a097ce61eff87", + "sdil/conv.py": "09f87eef636aa5397b39a2a7a66909b8861640624507eabbdfaffe98577a5d2c", + "sdil/conv_crossover.py": "3c1373aa3555e77cce37f093dec7b7aaedca475f608b092d13706bfff1dbe1a8", + "sdil/data.py": "262031353b5952fb4d1b0e3c3d1e9b3186ae42b88601ccb89e1748ffd36ed36d" + } + }, + "path": "results/resnet_crossover/p1_selector.json", + "selected_rates": { + "bp": 0.1, + "clean_kp": 0.1, + "dfa": 0.01, + "dualprop": 0.05, + "ep": 0.0003, + "fa": 0.1, + "ff": 0.03, + "pepita": 0.003, + "sdil": 0.1 + }, + "sha256": "0f0fcdfcd0056b034a9cd95e57970782bb2c33785832ab1cbe27e5dbe212111b" + }, + "source": { + "git_commit": "f55872beaee658ff76ac15fd421b73017820c320", + "protocol_path": "/orion/u/oscarwan/scratch/sdil-work/sdil-a6000-full-rerun/RESNET_CROSSOVER.md", + "protocol_sha256": "16d110298f88d383f528b9c9b81f20ba11920c89596534ebb4a5c1e057f97223", + "python": "/orion/u/oscarwan/scratch/miniconda3/envs/sdil-main/bin/python", + "tracked_files": { + "RESNET_CROSSOVER.md": "16d110298f88d383f528b9c9b81f20ba11920c89596534ebb4a5c1e057f97223", + "experiments/analyze_resnet_crossover_r2.py": "ddad2a41b45ef41b7f26ddde0d6fdb9255592307714351d65b98e109b73ca6d1", + "experiments/conv_run.py": "9fa2daea597d63981a78248fafb154da39775ccfae57aac9889e78d83dc4eb44", + "experiments/crossover_hardware.py": "f7fd5ab940d6790d1f60c1bf0ec2115db8db7163a4ee034b536908ef72ac025f", + "experiments/resnet_crossover_native.py": "c04a5311c2aa4f7820f0f6883aca6e678b1c3227085c5d88fef46c83ac65e83d", + "experiments/resnet_crossover_r2.py": "8fe2844211a0220873a11ad801d672db8e2869fcb914bc972396bfd24adb040f", + "results/resnet_crossover/p1_selector.json": "0f0fcdfcd0056b034a9cd95e57970782bb2c33785832ab1cbe27e5dbe212111b", + "sdil/conv.py": "09f87eef636aa5397b39a2a7a66909b8861640624507eabbdfaffe98577a5d2c", + "sdil/conv_crossover.py": "3c1373aa3555e77cce37f093dec7b7aaedca475f608b092d13706bfff1dbe1a8", + "sdil/data.py": "0e8496c3bbcaa090c280afc3dbdd8b4c9490eeaca3306afff8aea081dccde254" + } + }, + "stage": "resnet_crossover_r2", + "test_policy": "none" +} |