From 3856daa3124b8dfce6bd0285646e5a92f0ff96aa Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Sat, 29 Aug 2026 18:45:18 -0500 Subject: analysis: compare three CLLN scaling endpoints --- experiments/analyze_coupled_ladder_scaling.py | 130 ++++++++++++++++++-------- 1 file changed, 91 insertions(+), 39 deletions(-) (limited to 'experiments') diff --git a/experiments/analyze_coupled_ladder_scaling.py b/experiments/analyze_coupled_ladder_scaling.py index bfa31f0..4db593e 100644 --- a/experiments/analyze_coupled_ladder_scaling.py +++ b/experiments/analyze_coupled_ladder_scaling.py @@ -198,20 +198,26 @@ def task_size_matrix( return task_indices, matrix -def slope_bootstrap( +def metric_slope_bootstrap( records: list[dict], sizes: list[int], edges: np.ndarray, *, + metric: str, + y_axis: str, replicates: int, seed: int, ) -> dict: _, clean = task_size_matrix( - records, sizes, "clean", "classification_error") + records, sizes, "clean", metric) _, constant = task_size_matrix( - records, sizes, "constant", "classification_error") + records, sizes, "constant", metric) _, sdil = task_size_matrix( - records, sizes, "sdil", "classification_error") + records, sizes, "sdil", metric) + if metric == "reached_stable_zero_error": + clean = 1.0 - clean + constant = 1.0 - constant + sdil = 1.0 - sdil x = np.log10(edges) def slope(matrix: np.ndarray) -> float: @@ -250,7 +256,8 @@ def slope_bootstrap( return { "x_axis": "log10 learnable edges", - "y_axis": "classification error excess over clean", + "metric": metric, + "y_axis": y_axis, "static_calibration_slope": point_constant, "static_calibration_slope_95ci": interval(constant_slopes), "sdil_slope": point_sdil, @@ -371,13 +378,33 @@ def build_analysis( }, "sizes": sizes, "summaries": summaries, - "excess_error_scaling": slope_bootstrap( + "excess_error_scaling": metric_slope_bootstrap( records, sizes, edges, + metric="classification_error", + y_axis="final classification error excess over clean", replicates=replicates, seed=seed + 1, ), + "excess_error_auc_scaling": metric_slope_bootstrap( + records, + sizes, + edges, + metric="classification_error_auc", + y_axis="classification error AUC excess over clean", + replicates=replicates, + seed=seed + 2, + ), + "excess_stable_failure_scaling": metric_slope_bootstrap( + records, + sizes, + edges, + metric="reached_stable_zero_error", + y_axis="stable failure fraction excess over clean", + replicates=replicates, + seed=seed + 3, + ), "largest_grid_learning_curves": { method: trace_summary(records, sizes[-1], method) for method in METHOD_ORDER @@ -462,64 +489,86 @@ def plot_figure(path: Path, analysis: dict) -> None: axes[0].set_ylim(-3, 70) axes[0].set_xlabel("Learnable edges") axes[0].set_ylabel("Final classification error (%)") - reduction = analysis["excess_error_scaling"][ - "relative_slope_reduction_percent"] - axes[0].set_title( - f"(a) Error-growth slope reduced {reduction:.0f}% vs static calibration") + def scaling_title(letter: str, label: str, key: str) -> str: + scaling = analysis[key] + reduction = scaling["relative_slope_reduction_percent"] + difference_low = scaling["paired_slope_difference_95ci"][0] + if reduction is not None and difference_low > 0.0: + return ( + f"({letter}) {label}\n{reduction:.0f}% lower slope vs static calibration") + return f"({letter}) {label}\nNo resolved slope reduction" + + axes[0].set_title(scaling_title( + "a", "Final-error growth", "excess_error_scaling")) for method in METHOD_ORDER: means = np.asarray([ analysis["summaries"][str(side)]["methods"][method][ - "reached_stable_zero_error" - ]["mean"] * 100.0 + "classification_error_auc" + ]["mean"] + for side in sizes + ]) + intervals = np.asarray([ + analysis["summaries"][str(side)]["methods"][method][ + "classification_error_auc" + ]["task_bootstrap_95ci"] for side in sizes ]) - axes[1].plot( + axes[1].errorbar( edges, means, + yerr=np.vstack((means - intervals[:, 0], intervals[:, 1] - means)), linewidth=1.7 if method == "sdil" else 1.15, markersize=4.5, + capsize=2.0, label=DISPLAY[method], **STYLE[method], ) axes[1].set_xscale("log", base=2) axes[1].set_xticks(edges, edge_labels) - axes[1].set_ylim(-5, 105) + axes[1].set_ylim(-0.03, 0.72) axes[1].set_xlabel("Learnable edges") - axes[1].set_ylabel("Stable zero-error runs (%)") - axes[1].set_title("(b) Recovery remains reliable") + axes[1].set_ylabel("Classification-error AUC") + axes[1].set_title(scaling_title( + "b", "Error-AUC growth", "excess_error_auc_scaling")) for method in METHOD_ORDER: - trace = analysis["largest_grid_learning_curves"][method] - epochs = np.asarray([record["epoch"] for record in trace]) means = np.asarray([ - record["mean_classification_error"] * 100.0 - for record in trace + (1.0 - analysis["summaries"][str(side)]["methods"][method][ + "reached_stable_zero_error" + ]["mean"]) * 100.0 + for side in sizes ]) - standard_errors = np.asarray([ - record["standard_error"] * 100.0 for record in trace + success_intervals = np.asarray([ + analysis["summaries"][str(side)]["methods"][method][ + "reached_stable_zero_error" + ]["task_bootstrap_95ci"] + for side in sizes ]) - axes[2].plot( - epochs, + failure_intervals = np.column_stack(( + 1.0 - success_intervals[:, 1], + 1.0 - success_intervals[:, 0], + )) * 100.0 + axes[2].errorbar( + edges, means, + yerr=np.vstack(( + means - failure_intervals[:, 0], + failure_intervals[:, 1] - means, + )), linewidth=1.7 if method == "sdil" else 1.15, + markersize=4.5, + capsize=2.0, label=DISPLAY[method], - **{key: value for key, value in STYLE[method].items() - if key != "marker"}, - ) - axes[2].fill_between( - epochs, - np.maximum(0.0, means - standard_errors), - means + standard_errors, - color=STYLE[method]["color"], - alpha=0.08, - linewidth=0, + **STYLE[method], ) - axes[2].set_xlim(0, 600) - axes[2].set_ylim(-3, 70) - axes[2].set_xlabel("Training epoch") - axes[2].set_ylabel("Classification error (%)") - axes[2].set_title("(c) Learning at 2,048 edges") + axes[2].set_xscale("log", base=2) + axes[2].set_xticks(edges, edge_labels) + axes[2].set_ylim(-5, 105) + axes[2].set_xlabel("Learnable edges") + axes[2].set_ylabel("Stable failure fraction (%)") + axes[2].set_title(scaling_title( + "c", "Stable-failure growth", "excess_stable_failure_scaling")) for axis in axes: axis.grid(axis="y", color="#D9D9D9", linewidth=0.55, alpha=0.8) @@ -574,6 +623,9 @@ def main() -> None: plot_figure(args.output_figure, analysis) print(json.dumps({ "excess_error_scaling": analysis["excess_error_scaling"], + "excess_error_auc_scaling": analysis["excess_error_auc_scaling"], + "excess_stable_failure_scaling": analysis[ + "excess_stable_failure_scaling"], "largest_side": analysis["sizes"][-1], "largest_side_methods": analysis["summaries"][ str(analysis["sizes"][-1]) -- cgit v1.2.3