results: validate SDIL on device imperfection model

1 files changed, 36 insertions, 1 deletions

diff --git a/experiments/physical_imperfection_p3.py b/experiments/physical_imperfection_p3.py
index f573edd..f3157d9 100644
--- a/experiments/physical_imperfection_p3.py
+++ b/experiments/physical_imperfection_p3.py
@@ -145,16 +145,31 @@ def minimum_passing_record(
     return candidates[0] if candidates else None
 
 
-def summarize_pair(devices: list[dict], threshold: float) -> dict:
+def summarize_pair(
+    devices: list[dict], threshold: float, fixed_budget_eta: float = 0.025
+) -> dict:
+    available_etas = {
+        record["overclamp_eta"]
+        for device in devices
+        for record in device["overclamp_sweep"]
+    }
+    if fixed_budget_eta not in available_etas:
+        fixed_budget_eta = min(
+            available_etas, key=lambda value: abs(value - fixed_budget_eta))
     eta_ratios = []
     exposure_ratios = []
     paired_successes = 0
     combination_no_larger = 0
     standard_by_method = {method: [] for method in STANDARD_METHODS}
+    fixed_budget_by_method = {method: [] for method in OVERCLAMP_METHODS}
     for device in devices:
         for record in device["standard_clamping"]:
             standard_by_method[record["method"]].append(
                 record["mean_combined_error"])
+        for record in device["overclamp_sweep"]:
+            if record["overclamp_eta"] == fixed_budget_eta:
+                fixed_budget_by_method[record["method"]].append(
+                    record["mean_combined_error"])
         baseline = minimum_passing_record(
             device["overclamp_sweep"], "overclamp", threshold)
         combined = minimum_passing_record(
@@ -181,6 +196,21 @@ def summarize_pair(devices: list[dict], threshold: float) -> dict:
             }
             for method, values in standard_by_method.items()
         },
+        "standard_clamping_threshold_success_fraction": {
+            method: float(np.mean(np.asarray(values) <= threshold))
+            for method, values in standard_by_method.items()
+        },
+        "fixed_overclamp_budget": {
+            "eta": fixed_budget_eta,
+            "threshold_success_fraction": {
+                method: float(np.mean(np.asarray(values) <= threshold))
+                for method, values in fixed_budget_by_method.items()
+            },
+            "median_error": {
+                method: float(np.median(values))
+                for method, values in fixed_budget_by_method.items()
+            },
+        },
         "paired_devices_reaching_threshold": paired_successes,
         "combination_no_larger_eta_fraction": (
             combination_no_larger / len(devices)),
@@ -379,6 +409,11 @@ def main() -> None:
                 "median_clamp_l2_exposure_reduction_at_passing_endpoint"]
             for name, pair in report["pairs"].items()
         },
+        "fixed_eta_0p025_success_fraction_by_pair": {
+            name: pair["summary"]["fixed_overclamp_budget"][
+                "threshold_success_fraction"]
+            for name, pair in report["pairs"].items()
+        },
     }
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(json.dumps(report, indent=2) + "\n")