experiment: harden remote crossover handoffs

5 files changed, 178 insertions, 59 deletions

diff --git a/experiments/crossover_artifact_validation_smoke.py b/experiments/crossover_artifact_validation_smoke.py
new file mode 100644
index 0000000..e267832
--- /dev/null
+++ b/experiments/crossover_artifact_validation_smoke.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Deterministic checks for dependency-free handoff validation."""
+import copy
+
+from validate_crossover_artifact import validate
+
+
+def must_fail(report, kind, expected_commit=None):
+    try:
+        validate(report, kind, expected_commit)
+    except (AssertionError, KeyError, ValueError):
+        return
+    raise AssertionError(f"invalid {kind} artifact was accepted")
+
+
+def selector(stage, count):
+    return {
+        "gate": "pass",
+        "stage": stage,
+        "num_expected_records": count,
+        "num_audited_records": count,
+        "missing_experiments": [],
+        "selected": {str(index): {} for index in range(9)},
+        "test_policy": "none",
+    }
+
+
+def audit(stage, count):
+    return {
+        "audit_status": "passed",
+        "stage": stage,
+        "complete_grid": True,
+        "failure_retaining": True,
+        "num_audited_cells": count,
+        "test_policy": "none",
+    }
+
+
+def main():
+    launch = {
+        "stage": "p1",
+        "source": {"git_commit": "abc"},
+        "num_jobs": 19,
+        "allowed_physical_gpus": [5, 7],
+    }
+    validate(launch, "resnet_p1_launch", "abc")
+    bad_launch = copy.deepcopy(launch)
+    bad_launch["allowed_physical_gpus"] = [0, 1]
+    must_fail(bad_launch, "resnet_p1_launch", "abc")
+
+    resnet_selector = selector("resnet_crossover_p1", 19)
+    transformer_selector = selector("transformer_crossover_p1", 27)
+    validate(resnet_selector, "resnet_p1_selector")
+    validate(transformer_selector, "transformer_p1_selector")
+    bad_selector = copy.deepcopy(transformer_selector)
+    bad_selector["selected"].pop("0")
+    must_fail(bad_selector, "transformer_p1_selector")
+
+    resnet = audit("resnet_crossover_r2", 27)
+    transformer = audit("transformer_crossover_t2", 27)
+    crossover = audit("crossover_x1_81", 81)
+    validate(resnet, "resnet_r2_audit")
+    validate(transformer, "transformer_t2_audit")
+    validate(crossover, "crossover_81_audit")
+    bad_audit = copy.deepcopy(crossover)
+    bad_audit["num_audited_cells"] = 80
+    must_fail(bad_audit, "crossover_81_audit")
+    print("crossover artifact validation smoke: all checks passed")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/finalize_accept.sh b/experiments/finalize_accept.sh
index fa9fe97..1af94a7 100755
--- a/experiments/finalize_accept.sh
+++ b/experiments/finalize_accept.sh
@@ -29,6 +29,8 @@ experiments/finalize_claims.sh
   experiments/transformer_crossover_t2_smoke.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
   experiments/crossover_81_smoke.py
+/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
+  experiments/crossover_artifact_validation_smoke.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 -m py_compile \
   experiments/bci_td_run.py experiments/analyze_bci_td_development.py \
   experiments/bci_td_confirmation.py \
@@ -49,6 +51,7 @@ experiments/finalize_claims.sh
   experiments/transformer_crossover_t2.py \
   experiments/analyze_transformer_crossover_t2.py \
   experiments/analyze_crossover_81.py \
+  experiments/validate_crossover_artifact.py \
   experiments/plot_resnet_confirmation.py \
   experiments/plot_oral_a_scaling.py \
   experiments/plot_bci_v2_confirmation.py \
diff --git a/experiments/handoff_resnet_p1_to_transformer_p1.sh b/experiments/handoff_resnet_p1_to_transformer_p1.sh
index 123f5c9..3ad0f56 100755
--- a/experiments/handoff_resnet_p1_to_transformer_p1.sh
+++ b/experiments/handoff_resnet_p1_to_transformer_p1.sh
@@ -15,6 +15,7 @@ expected_transformer_commit="$2"
 resnet_root="/home/yurenh2/sdil-resnet-p1"
 transformer_root="/home/yurenh2/sdil-transformer-p1"
 python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3"
+validator="/home/yurenh2/sdil/experiments/validate_crossover_artifact.py"
 gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74"
 gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b"
 resnet_launch="$resnet_root/results/resnet_crossover/p1_launch.json"
@@ -75,12 +76,10 @@ echo "waiting for frozen ResNet P1 launch at $(date --iso-8601=seconds)"
 while [ ! -f "$resnet_launch" ]; do
   sleep 60
 done
-jq -e --arg commit "$expected_resnet_commit" '
-  .stage == "p1" and
-  .source.git_commit == $commit and
-  .num_jobs == 19 and
-  .allowed_physical_gpus == [5, 7]
-' "$resnet_launch" >/dev/null
+"$python_bin" "$validator" \
+  --path "$resnet_launch" \
+  --kind resnet_p1_launch \
+  --expected-commit "$expected_resnet_commit"
 
 echo "ResNet P1 launch observed; waiting for both shards"
 while tmux has-session -t resnet_p1_gpu5 2>/dev/null ||
@@ -95,13 +94,9 @@ require_source "$resnet_root" "$expected_resnet_commit"
   "$python_bin" experiments/analyze_resnet_crossover_p1.py \
     --out results/resnet_crossover/p1_selector.json
 ) >"$resnet_root/results/resnet_crossover/p1_selector.log" 2>&1
-jq -e '
-  .gate == "pass" and
-  .num_audited_records == 19 and
-  .missing_experiments == [] and
-  (.selected | length) == 9 and
-  .test_policy == "none"
-' "$resnet_root/results/resnet_crossover/p1_selector.json" >/dev/null
+"$python_bin" "$validator" \
+  --path "$resnet_root/results/resnet_crossover/p1_selector.json" \
+  --kind resnet_p1_selector
 echo "ResNet P1 selector audited at $(date --iso-8601=seconds)"
 
 wait_for_idle_gpus
@@ -136,12 +131,7 @@ require_source "$transformer_root" "$expected_transformer_commit"
   "$python_bin" experiments/analyze_transformer_crossover_p1.py \
     --out results/transformer_crossover/p1_selector.json
 ) >"$transformer_root/results/transformer_crossover/p1_selector.log" 2>&1
-jq -e '
-  .gate == "pass" and
-  .num_audited_records == 27 and
-  .missing_experiments == [] and
-  (.selected | length) == 9 and
-  .test_policy == "none"
-' "$transformer_root/results/transformer_crossover/p1_selector.json" \
-  >/dev/null
+"$python_bin" "$validator" \
+  --path "$transformer_root/results/transformer_crossover/p1_selector.json" \
+  --kind transformer_p1_selector
 echo "Transformer P1 selector audited at $(date --iso-8601=seconds)"
diff --git a/experiments/handoff_selectors_to_full_crossover.sh b/experiments/handoff_selectors_to_full_crossover.sh
index 04d1b5d..61d0f07 100755
--- a/experiments/handoff_selectors_to_full_crossover.sh
+++ b/experiments/handoff_selectors_to_full_crossover.sh
@@ -18,6 +18,8 @@ resnet_r2_root="/home/yurenh2/sdil-resnet-r2"
 transformer_t2_root="/home/yurenh2/sdil-transformer-t2"
 plain_root="/scratch/yurenh2/sdil-crossover/dualprop_vgg16"
 python_bin="/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3"
+plain_python="/scratch/yurenh2/venvs/dualprop/bin/python"
+validator="/home/yurenh2/sdil/experiments/validate_crossover_artifact.py"
 gpu5_uuid="GPU-f4aed756-228e-7522-94e7-fb543c02cc74"
 gpu7_uuid="GPU-df3d5933-c3e9-9ea1-d595-9d2a3b68c14b"
 
@@ -73,20 +75,12 @@ wait_for_file() {
 
 validate_selector() {
   local path="$1"
-  local stage="$2"
-  local count="$3"
-  jq -e --arg stage "$stage" --argjson count "$count" '
-    .gate == "pass" and
-    .stage == $stage and
-    .num_expected_records == $count and
-    .num_audited_records == $count and
-    .missing_experiments == [] and
-    (.selected | length) == 9 and
-    .test_policy == "none"
-  ' "$path" >/dev/null
+  local kind="$2"
+  "$python_bin" "$validator" --path "$path" --kind "$kind"
 }
 
 test -x "$python_bin"
+test -x "$plain_python"
 test "$(gpu_uuid 5)" = "$gpu5_uuid"
 test "$(gpu_uuid 7)" = "$gpu7_uuid"
 require_source "$resnet_p1_root" "$expected_p1_commit"
@@ -99,9 +93,9 @@ transformer_p1_selector="$transformer_p1_root/results/transformer_crossover/p1_s
 echo "waiting for both audited P1 selectors at $(date --iso-8601=seconds)"
 wait_for_file "$resnet_p1_selector"
 wait_for_file "$transformer_p1_selector"
-validate_selector "$resnet_p1_selector" "resnet_crossover_p1" 19
+validate_selector "$resnet_p1_selector" "resnet_p1_selector"
 validate_selector \
-  "$transformer_p1_selector" "transformer_crossover_p1" 27
+  "$transformer_p1_selector" "transformer_p1_selector"
 
 resnet_r2_selector="$resnet_r2_root/results/resnet_crossover/p1_selector.json"
 transformer_t2_selector="$transformer_t2_root/results/transformer_crossover/p1_selector.json"
@@ -119,9 +113,9 @@ mkdir -p "$(dirname "$transformer_t2_selector")"
   "$python_bin" experiments/analyze_transformer_crossover_p1.py \
     --out "$transformer_t2_selector"
 ) >"$transformer_t2_root/results/transformer_crossover/p1_import.log" 2>&1
-validate_selector "$resnet_r2_selector" "resnet_crossover_p1" 19
+validate_selector "$resnet_r2_selector" "resnet_p1_selector"
 validate_selector \
-  "$transformer_t2_selector" "transformer_crossover_p1" 27
+  "$transformer_t2_selector" "transformer_p1_selector"
 git -C "$resnet_r2_root" add -f \
   results/resnet_crossover/p1_selector.json
 git -C "$resnet_r2_root" commit -m \
@@ -160,13 +154,9 @@ require_source "$resnet_r2_root" "$resnet_r2_commit"
   "$python_bin" experiments/analyze_resnet_crossover_r2.py \
     --out results/resnet_crossover/r2_audit.json
 ) >"$resnet_r2_root/results/resnet_crossover/r2_audit.log" 2>&1
-jq -e '
-  .audit_status == "passed" and
-  .complete_grid == true and
-  .failure_retaining == true and
-  .num_audited_cells == 27 and
-  .test_policy == "none"
-' "$resnet_r2_root/results/resnet_crossover/r2_audit.json" >/dev/null
+"$python_bin" "$validator" \
+  --path "$resnet_r2_root/results/resnet_crossover/r2_audit.json" \
+  --kind resnet_r2_audit
 git -C "$resnet_r2_root" add -f results/resnet_crossover
 git -C "$resnet_r2_root" commit -m \
   "results: record complete ResNet crossover"
@@ -197,19 +187,14 @@ require_source "$transformer_t2_root" "$transformer_t2_commit"
   "$python_bin" experiments/analyze_transformer_crossover_t2.py \
     --out results/transformer_crossover/t2_audit.json
 ) >"$transformer_t2_root/results/transformer_crossover/t2_audit.log" 2>&1
-jq -e '
-  .audit_status == "passed" and
-  .complete_grid == true and
-  .failure_retaining == true and
-  .num_audited_cells == 27 and
-  .test_policy == "none"
-' "$transformer_t2_root/results/transformer_crossover/t2_audit.json" \
-  >/dev/null
+"$python_bin" "$validator" \
+  --path "$transformer_t2_root/results/transformer_crossover/t2_audit.json" \
+  --kind transformer_t2_audit
 
 plain_audit="$transformer_t2_root/results/plain_cnn_p2_audit.json"
 (
   cd "$plain_root"
-  "$python_bin" analyze_p2.py --output "$plain_audit"
+  "$plain_python" analyze_p2.py --output "$plain_audit"
 )
 "$python_bin" "$transformer_t2_root/experiments/analyze_crossover_81.py" \
   --plain "$plain_audit" \
@@ -217,13 +202,9 @@ plain_audit="$transformer_t2_root/results/plain_cnn_p2_audit.json"
   --transformer \
     "$transformer_t2_root/results/transformer_crossover/t2_audit.json" \
   --out "$transformer_t2_root/results/crossover_81_audit.json"
-jq -e '
-  .audit_status == "passed" and
-  .complete_grid == true and
-  .failure_retaining == true and
-  .num_audited_cells == 81 and
-  .test_policy == "none"
-' "$transformer_t2_root/results/crossover_81_audit.json" >/dev/null
+"$python_bin" "$validator" \
+  --path "$transformer_t2_root/results/crossover_81_audit.json" \
+  --kind crossover_81_audit
 git -C "$transformer_t2_root" add -f \
   results/transformer_crossover \
   results/plain_cnn_p2_audit.json \
diff --git a/experiments/validate_crossover_artifact.py b/experiments/validate_crossover_artifact.py
new file mode 100644
index 0000000..8b82ec4
--- /dev/null
+++ b/experiments/validate_crossover_artifact.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""Small dependency-free validator used by remote crossover handoffs."""
+import argparse
+import json
+
+
+def validate_selector(report, stage, count):
+    assert report["gate"] == "pass"
+    assert report["stage"] == stage
+    assert report["num_expected_records"] == count
+    assert report["num_audited_records"] == count
+    assert report["missing_experiments"] == []
+    assert len(report["selected"]) == 9
+    assert report["test_policy"] == "none"
+
+
+def validate_audit(report, count):
+    assert report["audit_status"] == "passed"
+    assert report["complete_grid"] is True
+    assert report["failure_retaining"] is True
+    assert report["num_audited_cells"] == count
+    assert report["test_policy"] == "none"
+
+
+def validate(report, kind, expected_commit=None):
+    if kind == "resnet_p1_launch":
+        assert expected_commit
+        assert report["stage"] == "p1"
+        assert report["source"]["git_commit"] == expected_commit
+        assert report["num_jobs"] == 19
+        assert report["allowed_physical_gpus"] == [5, 7]
+    elif kind == "resnet_p1_selector":
+        validate_selector(report, "resnet_crossover_p1", 19)
+    elif kind == "transformer_p1_selector":
+        validate_selector(report, "transformer_crossover_p1", 27)
+    elif kind == "resnet_r2_audit":
+        assert report["stage"] == "resnet_crossover_r2"
+        validate_audit(report, 27)
+    elif kind == "transformer_t2_audit":
+        assert report["stage"] == "transformer_crossover_t2"
+        validate_audit(report, 27)
+    elif kind == "crossover_81_audit":
+        assert report["stage"] == "crossover_x1_81"
+        validate_audit(report, 81)
+    else:
+        raise ValueError(f"unknown artifact kind: {kind}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", required=True)
+    parser.add_argument(
+        "--kind",
+        required=True,
+        choices=(
+            "resnet_p1_launch",
+            "resnet_p1_selector",
+            "transformer_p1_selector",
+            "resnet_r2_audit",
+            "transformer_t2_audit",
+            "crossover_81_audit",
+        ),
+    )
+    parser.add_argument("--expected-commit")
+    args = parser.parse_args()
+    with open(args.path, encoding="utf-8") as handle:
+        report = json.load(handle)
+    validate(report, args.kind, args.expected_commit)
+    print(f"validated {args.kind}: {args.path}")
+
+
+if __name__ == "__main__":
+    main()