{ "audit_status": "passed", "complete_grid": true, "failed_or_incomplete_cells": [ "resnet56::dualprop" ], "failure_retaining": true, "hardware_policy": { "allowed_device_names": [ "NVIDIA RTX A6000" ], "allowed_physical_gpu_indices": [ 0 ], "profile": "a6000", "single_gpu_per_process": true }, "launch_lock": { "path": "results/resnet_crossover/r2_launch.json", "registry_sha256": "bc205e39b0403bb2e80c289467a1fb3f746ef970bfe0ee8993a20ebab08713ba", "sha256": "2d30076208dc9eb69f28e3d0b617b09760a9fea61436e7fb83037eb2379c5e5d" }, "num_audited_cells": 27, "num_expected_cells": 27, "num_finite_cells": 26, "records": [ { "all_training_losses_finite": true, "best_validation_accuracy": 0.1, "cell_id": "resnet56::ep", "complete_trajectory": true, "depth": 56, "driver_wall_seconds": 50669.00653362274, "epochs_completed": 100, "final_validation_accuracy": 0.1, "final_validation_loss": 0.09450665283203125, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 853018, "history_training_examples": 4500000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-ep-d56-lr0p0003.json.manifest.json", "method": "ep", "ordinary_training_examples": 4500000, "ordinary_validation_examples": 505000, "output_sha256": "f582283d07e8d090b09b62270031ef5fe81d0e79c3da4b1ff4e23084265ec957", "peak_memory_allocated_bytes": 11683346432, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5", "rate": 0.0003, "status": "completed", "total_wall_seconds": 50663.67151045799, "training_example_presentations": 4500000, "work": { "candidate_label_evaluation_presentations": 0, "feedforward_example_passes": 5005000, "forward_macs_per_example": 125485696, "local_vjp_example_evaluations": 118100000, "relaxation_example_passes": 118100000 } }, { "best_validation_accuracy": null, "cell_id": "resnet56::dualprop", "complete_trajectory": false, "depth": 56, "driver_wall_seconds": 172800.46599197388, "epochs_completed": 0, "final_validation_accuracy": null, "final_validation_loss": null, "finite": false, "first_nonfinite_step": null, "forward_parameter_count": null, "logical_task_loss_queries": null, "manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d56-lr0p05.json.manifest.json", "method": "dualprop", "ordinary_training_examples": null, "ordinary_validation_examples": null, "output_sha256": null, "peak_memory_allocated_bytes": null, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5", "rate": 0.05, "status": "timeout", "total_wall_seconds": null, "training_example_presentations": null, "work": null }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.1122, "cell_id": "resnet56::ff", "complete_trajectory": true, "depth": 56, "driver_wall_seconds": 23686.061631202698, "epochs_completed": 2240, "final_validation_accuracy": 0.1122, "final_validation_loss": null, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 853018, "history_training_examples": 100800000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-ff-d56-lr0p03.json.manifest.json", "method": "ff", "ordinary_training_examples": 100800000, "ordinary_validation_examples": 5000, "output_sha256": "9b7724301a9ea9ca9b938355594c03d1f5d06bd7e77410c5b27ecac4344445b7", "peak_memory_allocated_bytes": 13807850496, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-b520421c-b120-70f0-eae8-1ff79ad97bf5", "rate": 0.03, "status": "completed", "total_wall_seconds": 23673.216548919678, "training_example_presentations": 201600000, "work": { "candidate_label_evaluation_presentations": 50000, "feedforward_example_passes": 201600000, "forward_macs_per_example": 125485696, "local_vjp_example_evaluations": 0, "relaxation_example_passes": 0 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.1, "cell_id": "resnet32::ep", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 24671.18036866188, "epochs_completed": 100, "final_validation_accuracy": 0.1, "final_validation_loss": 0.09370457153320312, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 4500000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-ep-d32-lr0p0003.json.manifest.json", "method": "ep", "ordinary_training_examples": 4500000, "ordinary_validation_examples": 505000, "output_sha256": "421c661bd6c13d7df5840943805ae1f72ccfd629b8d41b994fe9ca5e2ed65771", "peak_memory_allocated_bytes": 6988224512, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-2ddb8c2b-c2d4-96ac-fdd7-f2ded951f824", "rate": 0.0003, "status": "completed", "total_wall_seconds": 24661.099477529526, "training_example_presentations": 4500000, "work": { "candidate_label_evaluation_presentations": 0, "feedforward_example_passes": 5005000, "forward_macs_per_example": 68862592, "local_vjp_example_evaluations": 118100000, "relaxation_example_passes": 118100000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.871, "cell_id": "resnet32::dualprop", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 111870.60429954529, "epochs_completed": 200, "final_validation_accuracy": 0.8654, "final_validation_loss": 0.6110852416992187, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d32-lr0p05.json.manifest.json", "method": "dualprop", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "d45682de145f1650b61be527d44ba9e38495ef5203112166a19b72ec7d8f0917", "peak_memory_allocated_bytes": 2322300928, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-9cbd0c6c-f3fb-633d-be44-101a79992530", "rate": 0.05, "status": "completed", "total_wall_seconds": 111860.19822311401, "training_example_presentations": 9000000, "work": { "candidate_label_evaluation_presentations": 0, "feedforward_example_passes": 10005000, "forward_macs_per_example": 68862592, "local_vjp_example_evaluations": 144000000, "relaxation_example_passes": 144000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.1228, "cell_id": "resnet32::ff", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 7484.067112207413, "epochs_completed": 1280, "final_validation_accuracy": 0.1228, "final_validation_loss": null, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 57600000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-ff-d32-lr0p03.json.manifest.json", "method": "ff", "ordinary_training_examples": 57600000, "ordinary_validation_examples": 5000, "output_sha256": "d48f2e4651502170feb4afc4665bebb9625f077335db9f59c129748c4fdbf731", "peak_memory_allocated_bytes": 8251766272, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.03, "status": "completed", "total_wall_seconds": 7469.690595626831, "training_example_presentations": 115200000, "work": { "candidate_label_evaluation_presentations": 50000, "feedforward_example_passes": 115200000, "forward_macs_per_example": 68862592, "local_vjp_example_evaluations": 0, "relaxation_example_passes": 0 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.1, "cell_id": "resnet20::ep", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 15028.455446720123, "epochs_completed": 100, "final_validation_accuracy": 0.1, "final_validation_loss": 0.09437472686767578, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 4500000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-ep-d20-lr0p0003.json.manifest.json", "method": "ep", "ordinary_training_examples": 4500000, "ordinary_validation_examples": 505000, "output_sha256": "2be303594e67846889489474ed9bde767bf0abcdea660716ddf421bfc455060f", "peak_memory_allocated_bytes": 4845611008, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.0003, "status": "completed", "total_wall_seconds": 15025.534816026688, "training_example_presentations": 4500000, "work": { "candidate_label_evaluation_presentations": 0, "feedforward_example_passes": 5005000, "forward_macs_per_example": 40551040, "local_vjp_example_evaluations": 118100000, "relaxation_example_passes": 118100000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.9112, "cell_id": "resnet20::dualprop", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 49289.51700615883, "epochs_completed": 200, "final_validation_accuracy": 0.9104, "final_validation_loss": 0.43468799438476563, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-dualprop-d20-lr0p05.json.manifest.json", "method": "dualprop", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "62576fdd2cce8c465f80416e8eba0f449a025fb0c04e3d13b6aacf6ef5699cd3", "peak_memory_allocated_bytes": 2008608768, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.05, "status": "completed", "total_wall_seconds": 49286.44217157364, "training_example_presentations": 9000000, "work": { "candidate_label_evaluation_presentations": 0, "feedforward_example_passes": 10005000, "forward_macs_per_example": 40551040, "local_vjp_example_evaluations": 144000000, "relaxation_example_passes": 144000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.1174, "cell_id": "resnet20::ff", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 3187.272476911545, "epochs_completed": 800, "final_validation_accuracy": 0.1174, "final_validation_loss": null, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 36000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-ff-d20-lr0p03.json.manifest.json", "method": "ff", "ordinary_training_examples": 36000000, "ordinary_validation_examples": 5000, "output_sha256": "e6cde27ffd50512b43afb18b6ddccdf1a49b2185001bf6740c96ec84ee7a89ec", "peak_memory_allocated_bytes": 5474375168, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.03, "status": "completed", "total_wall_seconds": 3180.144710302353, "training_example_presentations": 72000000, "work": { "candidate_label_evaluation_presentations": 50000, "feedforward_example_passes": 72000000, "forward_macs_per_example": 40551040, "local_vjp_example_evaluations": 0, "relaxation_example_passes": 0 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.2354, "cell_id": "resnet56::pepita", "complete_trajectory": true, "depth": 56, "driver_wall_seconds": 1602.2039358615875, "epochs_completed": 100, "final_validation_accuracy": 0.228, "final_validation_loss": 2.124036181640625, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 853018, "history_training_examples": 4500000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d56-lr0p003.json.manifest.json", "method": "pepita", "ordinary_training_examples": 4500000, "ordinary_validation_examples": 505000, "output_sha256": "dcebb1518d54fbefbe9bd0cb2a3b63a18591187b4dcad3cebeb3e93b1748c6e6", "peak_memory_allocated_bytes": 3249364480, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.003, "status": "completed", "total_wall_seconds": 1598.730625152588, "training_example_presentations": 9000000, "work": { "candidate_label_evaluation_presentations": 0, "feedforward_example_passes": 14005000, "forward_macs_per_example": 125485696, "local_vjp_example_evaluations": 0, "relaxation_example_passes": 0 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.935, "cell_id": "resnet56::sdil", "complete_trajectory": true, "depth": 56, "driver_wall_seconds": 8350.019015789032, "epochs_completed": 200, "final_validation_accuracy": 0.9322, "final_validation_loss": 0.3675199523925781, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 853018, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d56-lr0p1.json.manifest.json", "method": "sdil", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "84f1139c05218524c1be2807d0e2bacb54bc0a0bc0d0526a0dcf7d8a2de8f5f6", "peak_memory_allocated_bytes": 3262279168, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 8344.301742315292, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 115016770519040, "forward_macs_per_example": 125485696, "total_forward_equivalent_examples": 9000064, "total_macs_estimate": 4509538465857536 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.9308, "cell_id": "resnet56::clean_kp", "complete_trajectory": true, "depth": 56, "driver_wall_seconds": 5064.757298946381, "epochs_completed": 200, "final_validation_accuracy": 0.9274, "final_validation_loss": 0.3740764099121094, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 853018, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d56-lr0p1.json.manifest.json", "method": "clean_kp", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "69d1c59511cfe98a72af9c1be079d4fa581096630b2e93f4ad028a4a935bc9fa", "peak_memory_allocated_bytes": 3256025088, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 5054.333840370178, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 125485696, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 4509522432000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.6674, "cell_id": "resnet56::fa", "complete_trajectory": true, "depth": 56, "driver_wall_seconds": 3838.2244839668274, "epochs_completed": 200, "final_validation_accuracy": 0.6436, "final_validation_loss": 1.0704212158203126, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 853018, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-fa-d56-lr0p1.json.manifest.json", "method": "fa", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "db985851db69598966506f4067c2febdd411b2fa61b02d330263f6a9ed90a776", "peak_memory_allocated_bytes": 3252628992, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", "rate": 0.1, "status": "completed", "total_wall_seconds": 3827.3347294330597, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 125485696, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 3384132480000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.4056, "cell_id": "resnet56::dfa", "complete_trajectory": true, "depth": 56, "driver_wall_seconds": 3246.3810822963715, "epochs_completed": 200, "final_validation_accuracy": 0.284, "final_validation_loss": 56.89802421875, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 853018, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d56-lr0p01.json.manifest.json", "method": "dfa", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "f646a557aca62c9dbfb2672fedd765f0cd51d11256c50995933d43d90a18c8f9", "peak_memory_allocated_bytes": 3276921856, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", "rate": 0.01, "status": "completed", "total_wall_seconds": 3241.7450346946716, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 125485696, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 2306665728000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.9308, "cell_id": "resnet56::bp", "complete_trajectory": true, "depth": 56, "driver_wall_seconds": 3326.933441400528, "epochs_completed": 200, "final_validation_accuracy": 0.93, "final_validation_loss": 0.3817935302734375, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 853018, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-bp-d56-lr0p1.json.manifest.json", "method": "bp", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "e4876181a6d1d8ddbda3b439319a758b9c9e42170add0a78f72df5378abd4088", "peak_memory_allocated_bytes": 3257588736, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 3323.5966560840607, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 125485696, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 3388113792000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.2682, "cell_id": "resnet32::pepita", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 909.2571918964386, "epochs_completed": 100, "final_validation_accuracy": 0.2618, "final_validation_loss": 2.029934326171875, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 4500000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d32-lr0p003.json.manifest.json", "method": "pepita", "ordinary_training_examples": 4500000, "ordinary_validation_examples": 505000, "output_sha256": "461ff5f189160d80f86da8972f3742e260ebfb4acc720aab961f403f047ba4f1", "peak_memory_allocated_bytes": 2322423808, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", "rate": 0.003, "status": "completed", "total_wall_seconds": 896.4814369678497, "training_example_presentations": 9000000, "work": { "candidate_label_evaluation_presentations": 0, "feedforward_example_passes": 14005000, "forward_macs_per_example": 68862592, "local_vjp_example_evaluations": 0, "relaxation_example_passes": 0 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.9254, "cell_id": "resnet32::sdil", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 4740.1479415893555, "epochs_completed": 200, "final_validation_accuracy": 0.9236, "final_validation_loss": 0.3822314697265625, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d32-lr0p1.json.manifest.json", "method": "sdil", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "516a62577e7b024b7686e816ebb55bb95f9de07204117af1905ade3b67f14f09", "peak_memory_allocated_bytes": 2329526272, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", "rate": 0.1, "status": "completed", "total_wall_seconds": 4736.380233287811, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 65471084756992, "forward_macs_per_example": 68862592, "total_forward_equivalent_examples": 9000064, "total_macs_estimate": 2471099474100224 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.9262, "cell_id": "resnet32::clean_kp", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 2880.570287704468, "epochs_completed": 200, "final_validation_accuracy": 0.9238, "final_validation_loss": 0.39594262084960935, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d32-lr0p1.json.manifest.json", "method": "clean_kp", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "c8170145b2374f71f7bfdc161acfba10610b8ceb3c561b00b20a6182c369349c", "peak_memory_allocated_bytes": 2325996544, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 2877.0025794506073, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 68862592, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 2471090688000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.6762, "cell_id": "resnet32::fa", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 2180.355216741562, "epochs_completed": 200, "final_validation_accuracy": 0.6548, "final_validation_loss": 0.9980604614257812, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-fa-d32-lr0p1.json.manifest.json", "method": "fa", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "0a5f7017c2ab6fb991e98cf2b9c8e1438685f2575a2eaa41e80bdfdaa4162cf6", "peak_memory_allocated_bytes": 2324148736, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 2176.771654367447, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 68862592, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 1855308672000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.4546, "cell_id": "resnet32::dfa", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 1879.2923183441162, "epochs_completed": 200, "final_validation_accuracy": 0.3874, "final_validation_loss": 6.95157197265625, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d32-lr0p01.json.manifest.json", "method": "dfa", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "fdc45fa5f2e27326288de0b06db43af4ff8b0d46164473daf9f1b9a114dcb474", "peak_memory_allocated_bytes": 2338062336, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", "rate": 0.01, "status": "completed", "total_wall_seconds": 1875.3221385478973, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 68862592, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 1266806016000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.928, "cell_id": "resnet32::bp", "complete_trajectory": true, "depth": 32, "driver_wall_seconds": 1930.4826476573944, "epochs_completed": 200, "final_validation_accuracy": 0.928, "final_validation_loss": 0.38430252685546873, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 464154, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-bp-d32-lr0p1.json.manifest.json", "method": "bp", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "2450755a60aeebffe5376d58ac95a5a17b1b4af03202b8a09e71ca6672555cef", "peak_memory_allocated_bytes": 2330820608, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 1926.7095592021942, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 68862592, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 1859289984000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.2782, "cell_id": "resnet20::pepita", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 565.1650466918945, "epochs_completed": 100, "final_validation_accuracy": 0.2726, "final_validation_loss": 1.9752171875, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 4500000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-pepita-d20-lr0p003.json.manifest.json", "method": "pepita", "ordinary_training_examples": 4500000, "ordinary_validation_examples": 505000, "output_sha256": "b9120fa19a170ae650e432067cbf3df388be3f1606f18489180e646b65ac70e2", "peak_memory_allocated_bytes": 2008731648, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", "rate": 0.003, "status": "completed", "total_wall_seconds": 561.5238916873932, "training_example_presentations": 9000000, "work": { "candidate_label_evaluation_presentations": 0, "feedforward_example_passes": 14005000, "forward_macs_per_example": 40551040, "local_vjp_example_evaluations": 0, "relaxation_example_passes": 0 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.9158, "cell_id": "resnet20::sdil", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 3006.6630001068115, "epochs_completed": 200, "final_validation_accuracy": 0.9132, "final_validation_loss": 0.4233872314453125, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-sdil-d20-lr0p1.json.manifest.json", "method": "sdil", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "b0be7a2e395429c4bb29209bdce5d250dccd484048551fde04102553ed2ad272", "peak_memory_allocated_bytes": 2013018112, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", "rate": 0.1, "status": "completed", "total_wall_seconds": 3003.0224483013153, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 40698241875968, "forward_macs_per_example": 40551040, "total_forward_equivalent_examples": 9000064, "total_macs_estimate": 1451879978221568 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.9174, "cell_id": "resnet20::clean_kp", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 1784.3792207241058, "epochs_completed": 200, "final_validation_accuracy": 0.9144, "final_validation_loss": 0.4333634704589844, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-clean_kp-d20-lr0p1.json.manifest.json", "method": "clean_kp", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "87914e8c8674f9fc2d136f6dbf11ee9512c7e797d2d757b675ea7610ba6db795", "peak_memory_allocated_bytes": 2010747392, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 1780.5374817848206, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 40551040, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 1451874816000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.7066, "cell_id": "resnet20::fa", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 1365.4268357753754, "epochs_completed": 200, "final_validation_accuracy": 0.6854, "final_validation_loss": 0.9467566040039063, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-fa-d20-lr0p1.json.manifest.json", "method": "fa", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "4214a5ec034977d0fe8f927f9de040e282833008270204936eb81b419f16c399", "peak_memory_allocated_bytes": 2009673728, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 1361.8625671863556, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 40551040, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 1090896768000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.4502, "cell_id": "resnet20::dfa", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 1187.1560876369476, "epochs_completed": 200, "final_validation_accuracy": 0.3492, "final_validation_loss": 3.6838392578125, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-dfa-d20-lr0p01.json.manifest.json", "method": "dfa", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "638ace440ba8224c18e4e0f09f2dbac291ec32e4f54bc9efadbbff58ddf2ac45", "peak_memory_allocated_bytes": 2018397696, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-8c47bc34-8284-ba08-eb44-8f381122ed6d", "rate": 0.01, "status": "completed", "total_wall_seconds": 1183.4196124076843, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 40551040, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 746876160000000 } }, { "all_training_losses_finite": true, "best_validation_accuracy": 0.9196, "cell_id": "resnet20::bp", "complete_trajectory": true, "depth": 20, "driver_wall_seconds": 1228.3254659175873, "epochs_completed": 200, "final_validation_accuracy": 0.9176, "final_validation_loss": 0.39629163818359375, "finite": true, "first_nonfinite_step": null, "forward_parameter_count": 269722, "history_training_examples": 9000000, "logical_task_loss_queries": 0, "manifest": "results/resnet_crossover/r2/resnet-r2-bp-d20-lr0p1.json.manifest.json", "method": "bp", "ordinary_training_examples": 9000000, "ordinary_validation_examples": 1005000, "output_sha256": "5cf3b2928b4d27a0968fb42d633d5c23d91dad233b24a25daffb0fa9cd2fe7ee", "peak_memory_allocated_bytes": 2017119744, "physical_gpu_index": 0, "physical_gpu_uuid": "GPU-71e7b0a5-bc0f-e6c0-6965-8a3e2ab76b0b", "rate": 0.1, "status": "completed", "total_wall_seconds": 1224.873178243637, "training_example_presentations": 9000000, "work": { "elementwise_operations_estimate": 0, "forward_macs_per_example": 40551040, "total_forward_equivalent_examples": 9000000, "total_macs_estimate": 1094878080000000 } } ], "selector": { "p1_source": { "git_commit": "19f3966fc2d15b1ed18862a779d91ee16ed56bfc", "protocol_path": "/scratch/yurenh2/sdil-runs/resnet-p1/RESNET_CROSSOVER.md", "protocol_sha256": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d", "python": "/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3", "tracked_files": { "RESNET_CROSSOVER.md": "048dbaf06866bdd3959a637ba827c06063205f6d1cc693d89aa8fc9c1d24975d", "experiments/analyze_resnet_crossover_p1.py": "6ff485a3fecc7b814dedf55d04abdb770a6ce904cdeb9e06a7dddcf26f996d5b", "experiments/conv_run.py": "9fa2daea597d63981a78248fafb154da39775ccfae57aac9889e78d83dc4eb44", "experiments/resnet_crossover_grid.py": "669c84846bc3115816ae3c9036be8722f21577612dc9d56a689d3321e837ca95", "experiments/resnet_crossover_native.py": "c5e54324b000c37c53d495f7d3eb0d4a85dcf15e83ea3aab125a097ce61eff87", "sdil/conv.py": "09f87eef636aa5397b39a2a7a66909b8861640624507eabbdfaffe98577a5d2c", "sdil/conv_crossover.py": "3c1373aa3555e77cce37f093dec7b7aaedca475f608b092d13706bfff1dbe1a8", "sdil/data.py": "262031353b5952fb4d1b0e3c3d1e9b3186ae42b88601ccb89e1748ffd36ed36d" } }, "path": "results/resnet_crossover/p1_selector.json", "selected_rates": { "bp": 0.1, "clean_kp": 0.1, "dfa": 0.01, "dualprop": 0.05, "ep": 0.0003, "fa": 0.1, "ff": 0.03, "pepita": 0.003, "sdil": 0.1 }, "sha256": "0f0fcdfcd0056b034a9cd95e57970782bb2c33785832ab1cbe27e5dbe212111b" }, "source": { "git_commit": "f55872beaee658ff76ac15fd421b73017820c320", "protocol_path": "/orion/u/oscarwan/scratch/sdil-work/sdil-a6000-full-rerun/RESNET_CROSSOVER.md", "protocol_sha256": "16d110298f88d383f528b9c9b81f20ba11920c89596534ebb4a5c1e057f97223", "python": "/orion/u/oscarwan/scratch/miniconda3/envs/sdil-main/bin/python", "tracked_files": { "RESNET_CROSSOVER.md": "16d110298f88d383f528b9c9b81f20ba11920c89596534ebb4a5c1e057f97223", "experiments/analyze_resnet_crossover_r2.py": "ddad2a41b45ef41b7f26ddde0d6fdb9255592307714351d65b98e109b73ca6d1", "experiments/conv_run.py": "9fa2daea597d63981a78248fafb154da39775ccfae57aac9889e78d83dc4eb44", "experiments/crossover_hardware.py": "f7fd5ab940d6790d1f60c1bf0ec2115db8db7163a4ee034b536908ef72ac025f", "experiments/resnet_crossover_native.py": "c04a5311c2aa4f7820f0f6883aca6e678b1c3227085c5d88fef46c83ac65e83d", "experiments/resnet_crossover_r2.py": "8fe2844211a0220873a11ad801d672db8e2869fcb914bc972396bfd24adb040f", "results/resnet_crossover/p1_selector.json": "0f0fcdfcd0056b034a9cd95e57970782bb2c33785832ab1cbe27e5dbe212111b", "sdil/conv.py": "09f87eef636aa5397b39a2a7a66909b8861640624507eabbdfaffe98577a5d2c", "sdil/conv_crossover.py": "3c1373aa3555e77cce37f093dec7b7aaedca475f608b092d13706bfff1dbe1a8", "sdil/data.py": "0e8496c3bbcaa090c280afc3dbdd8b4c9490eeaca3306afff8aea081dccde254" } }, "stage": "resnet_crossover_r2", "test_policy": "none" }