analysis: compare three CLLN scaling endpoints

1 files changed, 91 insertions, 39 deletions

diff --git a/experiments/analyze_coupled_ladder_scaling.py b/experiments/analyze_coupled_ladder_scaling.py
index bfa31f0..4db593e 100644
--- a/experiments/analyze_coupled_ladder_scaling.py
+++ b/experiments/analyze_coupled_ladder_scaling.py
@@ -198,20 +198,26 @@ def task_size_matrix(
     return task_indices, matrix
 
 
-def slope_bootstrap(
+def metric_slope_bootstrap(
     records: list[dict],
     sizes: list[int],
     edges: np.ndarray,
     *,
+    metric: str,
+    y_axis: str,
     replicates: int,
     seed: int,
 ) -> dict:
     _, clean = task_size_matrix(
-        records, sizes, "clean", "classification_error")
+        records, sizes, "clean", metric)
     _, constant = task_size_matrix(
-        records, sizes, "constant", "classification_error")
+        records, sizes, "constant", metric)
     _, sdil = task_size_matrix(
-        records, sizes, "sdil", "classification_error")
+        records, sizes, "sdil", metric)
+    if metric == "reached_stable_zero_error":
+        clean = 1.0 - clean
+        constant = 1.0 - constant
+        sdil = 1.0 - sdil
     x = np.log10(edges)
 
     def slope(matrix: np.ndarray) -> float:
@@ -250,7 +256,8 @@ def slope_bootstrap(
 
     return {
         "x_axis": "log10 learnable edges",
-        "y_axis": "classification error excess over clean",
+        "metric": metric,
+        "y_axis": y_axis,
         "static_calibration_slope": point_constant,
         "static_calibration_slope_95ci": interval(constant_slopes),
         "sdil_slope": point_sdil,
@@ -371,13 +378,33 @@ def build_analysis(
         },
         "sizes": sizes,
         "summaries": summaries,
-        "excess_error_scaling": slope_bootstrap(
+        "excess_error_scaling": metric_slope_bootstrap(
             records,
             sizes,
             edges,
+            metric="classification_error",
+            y_axis="final classification error excess over clean",
             replicates=replicates,
             seed=seed + 1,
         ),
+        "excess_error_auc_scaling": metric_slope_bootstrap(
+            records,
+            sizes,
+            edges,
+            metric="classification_error_auc",
+            y_axis="classification error AUC excess over clean",
+            replicates=replicates,
+            seed=seed + 2,
+        ),
+        "excess_stable_failure_scaling": metric_slope_bootstrap(
+            records,
+            sizes,
+            edges,
+            metric="reached_stable_zero_error",
+            y_axis="stable failure fraction excess over clean",
+            replicates=replicates,
+            seed=seed + 3,
+        ),
         "largest_grid_learning_curves": {
             method: trace_summary(records, sizes[-1], method)
             for method in METHOD_ORDER
@@ -462,64 +489,86 @@ def plot_figure(path: Path, analysis: dict) -> None:
     axes[0].set_ylim(-3, 70)
     axes[0].set_xlabel("Learnable edges")
     axes[0].set_ylabel("Final classification error (%)")
-    reduction = analysis["excess_error_scaling"][
-        "relative_slope_reduction_percent"]
-    axes[0].set_title(
-        f"(a) Error-growth slope reduced {reduction:.0f}% vs static calibration")
+    def scaling_title(letter: str, label: str, key: str) -> str:
+        scaling = analysis[key]
+        reduction = scaling["relative_slope_reduction_percent"]
+        difference_low = scaling["paired_slope_difference_95ci"][0]
+        if reduction is not None and difference_low > 0.0:
+            return (
+                f"({letter}) {label}\n{reduction:.0f}% lower slope vs static calibration")
+        return f"({letter}) {label}\nNo resolved slope reduction"
+
+    axes[0].set_title(scaling_title(
+        "a", "Final-error growth", "excess_error_scaling"))
 
     for method in METHOD_ORDER:
         means = np.asarray([
             analysis["summaries"][str(side)]["methods"][method][
-                "reached_stable_zero_error"
-            ]["mean"] * 100.0
+                "classification_error_auc"
+            ]["mean"]
+            for side in sizes
+        ])
+        intervals = np.asarray([
+            analysis["summaries"][str(side)]["methods"][method][
+                "classification_error_auc"
+            ]["task_bootstrap_95ci"]
             for side in sizes
         ])
-        axes[1].plot(
+        axes[1].errorbar(
             edges,
             means,
+            yerr=np.vstack((means - intervals[:, 0], intervals[:, 1] - means)),
             linewidth=1.7 if method == "sdil" else 1.15,
             markersize=4.5,
+            capsize=2.0,
             label=DISPLAY[method],
             **STYLE[method],
         )
     axes[1].set_xscale("log", base=2)
     axes[1].set_xticks(edges, edge_labels)
-    axes[1].set_ylim(-5, 105)
+    axes[1].set_ylim(-0.03, 0.72)
     axes[1].set_xlabel("Learnable edges")
-    axes[1].set_ylabel("Stable zero-error runs (%)")
-    axes[1].set_title("(b) Recovery remains reliable")
+    axes[1].set_ylabel("Classification-error AUC")
+    axes[1].set_title(scaling_title(
+        "b", "Error-AUC growth", "excess_error_auc_scaling"))
 
     for method in METHOD_ORDER:
-        trace = analysis["largest_grid_learning_curves"][method]
-        epochs = np.asarray([record["epoch"] for record in trace])
         means = np.asarray([
-            record["mean_classification_error"] * 100.0
-            for record in trace
+            (1.0 - analysis["summaries"][str(side)]["methods"][method][
+                "reached_stable_zero_error"
+            ]["mean"]) * 100.0
+            for side in sizes
         ])
-        standard_errors = np.asarray([
-            record["standard_error"] * 100.0 for record in trace
+        success_intervals = np.asarray([
+            analysis["summaries"][str(side)]["methods"][method][
+                "reached_stable_zero_error"
+            ]["task_bootstrap_95ci"]
+            for side in sizes
         ])
-        axes[2].plot(
-            epochs,
+        failure_intervals = np.column_stack((
+            1.0 - success_intervals[:, 1],
+            1.0 - success_intervals[:, 0],
+        )) * 100.0
+        axes[2].errorbar(
+            edges,
             means,
+            yerr=np.vstack((
+                means - failure_intervals[:, 0],
+                failure_intervals[:, 1] - means,
+            )),
             linewidth=1.7 if method == "sdil" else 1.15,
+            markersize=4.5,
+            capsize=2.0,
             label=DISPLAY[method],
-            **{key: value for key, value in STYLE[method].items()
-               if key != "marker"},
-        )
-        axes[2].fill_between(
-            epochs,
-            np.maximum(0.0, means - standard_errors),
-            means + standard_errors,
-            color=STYLE[method]["color"],
-            alpha=0.08,
-            linewidth=0,
+            **STYLE[method],
         )
-    axes[2].set_xlim(0, 600)
-    axes[2].set_ylim(-3, 70)
-    axes[2].set_xlabel("Training epoch")
-    axes[2].set_ylabel("Classification error (%)")
-    axes[2].set_title("(c) Learning at 2,048 edges")
+    axes[2].set_xscale("log", base=2)
+    axes[2].set_xticks(edges, edge_labels)
+    axes[2].set_ylim(-5, 105)
+    axes[2].set_xlabel("Learnable edges")
+    axes[2].set_ylabel("Stable failure fraction (%)")
+    axes[2].set_title(scaling_title(
+        "c", "Stable-failure growth", "excess_stable_failure_scaling"))
 
     for axis in axes:
         axis.grid(axis="y", color="#D9D9D9", linewidth=0.55, alpha=0.8)
@@ -574,6 +623,9 @@ def main() -> None:
     plot_figure(args.output_figure, analysis)
     print(json.dumps({
         "excess_error_scaling": analysis["excess_error_scaling"],
+        "excess_error_auc_scaling": analysis["excess_error_auc_scaling"],
+        "excess_stable_failure_scaling": analysis[
+            "excess_stable_failure_scaling"],
         "largest_side": analysis["sizes"][-1],
         "largest_side_methods": analysis["summaries"][
             str(analysis["sizes"][-1])