1 files changed, 37 insertions, 10 deletions
diff --git a/experiments/analyze_resnet_crossover_p1.py b/experiments/analyze_resnet_crossover_p1.py
index 6fe34c9..72a46e6 100644
--- a/experiments/analyze_resnet_crossover_p1.py
+++ b/experiments/analyze_resnet_crossover_p1.py
@@ -91,7 +91,30 @@ def audit_job(job, expected_source):
     hardware = manifest["hardware_lock"]
     assert hardware["physical_gpu_index"] in (5, 7)
     assert hardware["physical_gpu_uuid"]
-    assert manifest["status"] == "completed"
+    common = {
+        "method": job["method"],
+        "rate": job["rate"],
+        "experiment_name": job["experiment_name"],
+        "manifest": os.path.relpath(manifest_path, ROOT),
+        "status": manifest["status"],
+        "output_sha256": manifest["output_sha256"],
+        "driver_wall_seconds": manifest["driver_wall_seconds"],
+        "physical_gpu_index": hardware["physical_gpu_index"],
+        "physical_gpu_uuid": hardware["physical_gpu_uuid"],
+    }
+    if manifest["status"] != "completed":
+        assert manifest["status"] in {
+            "timeout", "nonzero_exit", "missing_output"}
+        return {
+            **common,
+            "finite": False,
+            "best_validation_accuracy": None,
+            "final_validation_accuracy": None,
+            "final_validation_loss": None,
+            "best_epoch": None,
+            "epochs_completed": None,
+            "all_training_losses_finite": False,
+        }
     assert manifest["output_exists"] is True
     assert manifest["output_sha256"] == sha256(job["output"])
     record = read_json(job["output"])
@@ -115,21 +138,25 @@ def audit_job(job, expected_source):
             "sdil": "kp_traffic",
         }[job["method"]]
         assert record["args"]["mode"] == expected_mode
+    metrics = history_metrics(record, job["method"])
     return {
-        "method": job["method"],
-        "rate": job["rate"],
-        "experiment_name": job["experiment_name"],
-        "manifest": os.path.relpath(manifest_path, ROOT),
-        "output_sha256": manifest["output_sha256"],
-        "driver_wall_seconds": manifest["driver_wall_seconds"],
-        **history_metrics(record, job["method"]),
+        **common,
+        "finite": (
+            metrics["all_training_losses_finite"]
+            and math.isfinite(metrics["final_validation_loss"])),
+        **metrics,
     }
 
 
 def choose(candidates):
     return max(candidates, key=lambda row: (
-        row["best_validation_accuracy"],
-        int(math.isfinite(row["final_validation_loss"])),
+        int(row["finite"]),
+        (
+            row["best_validation_accuracy"]
+            if row["best_validation_accuracy"] is not None else -math.inf),
+        int(
+            row["final_validation_loss"] is not None
+            and math.isfinite(row["final_validation_loss"])),
         -row["rate"],
     ))