From 3856daa3124b8dfce6bd0285646e5a92f0ff96aa Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Sat, 29 Aug 2026 18:45:18 -0500 Subject: analysis: compare three CLLN scaling endpoints --- experiments/analyze_coupled_ladder_scaling.py | 130 +- results/coupled_ladder/p1_scaling_analysis.json | 56 +- results/figs/figure_clln_scaling_pilot.pdf | Bin 24101 -> 23584 bytes results/figs/figure_clln_scaling_pilot.png | Bin 243640 -> 209252 bytes results/figs/figure_clln_scaling_pilot.svg | 2937 +++++++++-------------- visual-composer/coupled-ladder-scaling.md | 12 +- visual-composer/qa-ledger.md | 1 + 7 files changed, 1308 insertions(+), 1828 deletions(-) diff --git a/experiments/analyze_coupled_ladder_scaling.py b/experiments/analyze_coupled_ladder_scaling.py index bfa31f0..4db593e 100644 --- a/experiments/analyze_coupled_ladder_scaling.py +++ b/experiments/analyze_coupled_ladder_scaling.py @@ -198,20 +198,26 @@ def task_size_matrix( return task_indices, matrix -def slope_bootstrap( +def metric_slope_bootstrap( records: list[dict], sizes: list[int], edges: np.ndarray, *, + metric: str, + y_axis: str, replicates: int, seed: int, ) -> dict: _, clean = task_size_matrix( - records, sizes, "clean", "classification_error") + records, sizes, "clean", metric) _, constant = task_size_matrix( - records, sizes, "constant", "classification_error") + records, sizes, "constant", metric) _, sdil = task_size_matrix( - records, sizes, "sdil", "classification_error") + records, sizes, "sdil", metric) + if metric == "reached_stable_zero_error": + clean = 1.0 - clean + constant = 1.0 - constant + sdil = 1.0 - sdil x = np.log10(edges) def slope(matrix: np.ndarray) -> float: @@ -250,7 +256,8 @@ def slope_bootstrap( return { "x_axis": "log10 learnable edges", - "y_axis": "classification error excess over clean", + "metric": metric, + "y_axis": y_axis, "static_calibration_slope": point_constant, "static_calibration_slope_95ci": interval(constant_slopes), "sdil_slope": point_sdil, @@ -371,13 +378,33 @@ def build_analysis( }, "sizes": sizes, "summaries": summaries, - "excess_error_scaling": slope_bootstrap( + "excess_error_scaling": metric_slope_bootstrap( records, sizes, edges, + metric="classification_error", + y_axis="final classification error excess over clean", replicates=replicates, seed=seed + 1, ), + "excess_error_auc_scaling": metric_slope_bootstrap( + records, + sizes, + edges, + metric="classification_error_auc", + y_axis="classification error AUC excess over clean", + replicates=replicates, + seed=seed + 2, + ), + "excess_stable_failure_scaling": metric_slope_bootstrap( + records, + sizes, + edges, + metric="reached_stable_zero_error", + y_axis="stable failure fraction excess over clean", + replicates=replicates, + seed=seed + 3, + ), "largest_grid_learning_curves": { method: trace_summary(records, sizes[-1], method) for method in METHOD_ORDER @@ -462,64 +489,86 @@ def plot_figure(path: Path, analysis: dict) -> None: axes[0].set_ylim(-3, 70) axes[0].set_xlabel("Learnable edges") axes[0].set_ylabel("Final classification error (%)") - reduction = analysis["excess_error_scaling"][ - "relative_slope_reduction_percent"] - axes[0].set_title( - f"(a) Error-growth slope reduced {reduction:.0f}% vs static calibration") + def scaling_title(letter: str, label: str, key: str) -> str: + scaling = analysis[key] + reduction = scaling["relative_slope_reduction_percent"] + difference_low = scaling["paired_slope_difference_95ci"][0] + if reduction is not None and difference_low > 0.0: + return ( + f"({letter}) {label}\n{reduction:.0f}% lower slope vs static calibration") + return f"({letter}) {label}\nNo resolved slope reduction" + + axes[0].set_title(scaling_title( + "a", "Final-error growth", "excess_error_scaling")) for method in METHOD_ORDER: means = np.asarray([ analysis["summaries"][str(side)]["methods"][method][ - "reached_stable_zero_error" - ]["mean"] * 100.0 + "classification_error_auc" + ]["mean"] + for side in sizes + ]) + intervals = np.asarray([ + analysis["summaries"][str(side)]["methods"][method][ + "classification_error_auc" + ]["task_bootstrap_95ci"] for side in sizes ]) - axes[1].plot( + axes[1].errorbar( edges, means, + yerr=np.vstack((means - intervals[:, 0], intervals[:, 1] - means)), linewidth=1.7 if method == "sdil" else 1.15, markersize=4.5, + capsize=2.0, label=DISPLAY[method], **STYLE[method], ) axes[1].set_xscale("log", base=2) axes[1].set_xticks(edges, edge_labels) - axes[1].set_ylim(-5, 105) + axes[1].set_ylim(-0.03, 0.72) axes[1].set_xlabel("Learnable edges") - axes[1].set_ylabel("Stable zero-error runs (%)") - axes[1].set_title("(b) Recovery remains reliable") + axes[1].set_ylabel("Classification-error AUC") + axes[1].set_title(scaling_title( + "b", "Error-AUC growth", "excess_error_auc_scaling")) for method in METHOD_ORDER: - trace = analysis["largest_grid_learning_curves"][method] - epochs = np.asarray([record["epoch"] for record in trace]) means = np.asarray([ - record["mean_classification_error"] * 100.0 - for record in trace + (1.0 - analysis["summaries"][str(side)]["methods"][method][ + "reached_stable_zero_error" + ]["mean"]) * 100.0 + for side in sizes ]) - standard_errors = np.asarray([ - record["standard_error"] * 100.0 for record in trace + success_intervals = np.asarray([ + analysis["summaries"][str(side)]["methods"][method][ + "reached_stable_zero_error" + ]["task_bootstrap_95ci"] + for side in sizes ]) - axes[2].plot( - epochs, + failure_intervals = np.column_stack(( + 1.0 - success_intervals[:, 1], + 1.0 - success_intervals[:, 0], + )) * 100.0 + axes[2].errorbar( + edges, means, + yerr=np.vstack(( + means - failure_intervals[:, 0], + failure_intervals[:, 1] - means, + )), linewidth=1.7 if method == "sdil" else 1.15, + markersize=4.5, + capsize=2.0, label=DISPLAY[method], - **{key: value for key, value in STYLE[method].items() - if key != "marker"}, - ) - axes[2].fill_between( - epochs, - np.maximum(0.0, means - standard_errors), - means + standard_errors, - color=STYLE[method]["color"], - alpha=0.08, - linewidth=0, + **STYLE[method], ) - axes[2].set_xlim(0, 600) - axes[2].set_ylim(-3, 70) - axes[2].set_xlabel("Training epoch") - axes[2].set_ylabel("Classification error (%)") - axes[2].set_title("(c) Learning at 2,048 edges") + axes[2].set_xscale("log", base=2) + axes[2].set_xticks(edges, edge_labels) + axes[2].set_ylim(-5, 105) + axes[2].set_xlabel("Learnable edges") + axes[2].set_ylabel("Stable failure fraction (%)") + axes[2].set_title(scaling_title( + "c", "Stable-failure growth", "excess_stable_failure_scaling")) for axis in axes: axis.grid(axis="y", color="#D9D9D9", linewidth=0.55, alpha=0.8) @@ -574,6 +623,9 @@ def main() -> None: plot_figure(args.output_figure, analysis) print(json.dumps({ "excess_error_scaling": analysis["excess_error_scaling"], + "excess_error_auc_scaling": analysis["excess_error_auc_scaling"], + "excess_stable_failure_scaling": analysis[ + "excess_stable_failure_scaling"], "largest_side": analysis["sizes"][-1], "largest_side_methods": analysis["summaries"][ str(analysis["sizes"][-1]) diff --git a/results/coupled_ladder/p1_scaling_analysis.json b/results/coupled_ladder/p1_scaling_analysis.json index 4d4866a..d06c232 100644 --- a/results/coupled_ladder/p1_scaling_analysis.json +++ b/results/coupled_ladder/p1_scaling_analysis.json @@ -4,6 +4,7 @@ "bootstrap": { "unit": "task; component draws averaged within task", "task_clusters": 5, + "component_draws_per_task_size": 1, "replicates": 20000, "seed": 20260829, "interval": "percentile 95%" @@ -746,7 +747,8 @@ }, "excess_error_scaling": { "x_axis": "log10 learnable edges", - "y_axis": "classification error excess over clean", + "metric": "classification_error", + "y_axis": "final classification error excess over clean", "static_calibration_slope": 0.10415531736549301, "static_calibration_slope_95ci": [ 0.04974245841148715, @@ -769,6 +771,58 @@ ], "relative_reduction_valid_bootstrap_fraction": 1.0 }, + "excess_error_auc_scaling": { + "x_axis": "log10 learnable edges", + "metric": "classification_error_auc", + "y_axis": "classification error AUC excess over clean", + "static_calibration_slope": 0.02723285591782782, + "static_calibration_slope_95ci": [ + -0.018469611777622092, + 0.06799878937945103 + ], + "sdil_slope": 0.0050630936021534975, + "sdil_slope_95ci": [ + 0.0009725048424469787, + 0.009883683346965023 + ], + "paired_slope_difference": 0.022169762315674323, + "paired_slope_difference_95ci": [ + -0.021041329819854505, + 0.0613401506085447 + ], + "relative_slope_reduction_percent": 81.40814309953083, + "relative_slope_reduction_95ci": [ + -11.782270478082541, + 94.49463684558907 + ], + "relative_reduction_valid_bootstrap_fraction": 0.88435 + }, + "excess_stable_failure_scaling": { + "x_axis": "log10 learnable edges", + "metric": "reached_stable_zero_error", + "y_axis": "stable failure fraction excess over clean", + "static_calibration_slope": 0.17485078862811346, + "static_calibration_slope_95ci": [ + -0.15978252936756204, + 0.39793966729189717 + ], + "sdil_slope": 0.04963588122401385, + "sdil_slope_95ci": [ + -0.16871445018618145, + 0.24817940612006928 + ], + "paired_slope_difference": 0.1252149074040996, + "paired_slope_difference_95ci": [ + -0.08086274712223154, + 0.33129256193043083 + ], + "relative_slope_reduction_percent": 71.6124350290559, + "relative_slope_reduction_95ci": [ + -50.860158714747875, + 597.0884428984873 + ], + "relative_reduction_valid_bootstrap_fraction": 0.87555 + }, "largest_grid_learning_curves": { "clean": [ { diff --git a/results/figs/figure_clln_scaling_pilot.pdf b/results/figs/figure_clln_scaling_pilot.pdf index 07e1e1b..0a2c184 100644 Binary files a/results/figs/figure_clln_scaling_pilot.pdf and b/results/figs/figure_clln_scaling_pilot.pdf differ diff --git a/results/figs/figure_clln_scaling_pilot.png b/results/figs/figure_clln_scaling_pilot.png index df77b47..b5cfa05 100644 Binary files a/results/figs/figure_clln_scaling_pilot.png and b/results/figs/figure_clln_scaling_pilot.png differ diff --git a/results/figs/figure_clln_scaling_pilot.svg b/results/figs/figure_clln_scaling_pilot.svg index c1c1a18..f7b3440 100644 --- a/results/figs/figure_clln_scaling_pilot.svg +++ b/results/figs/figure_clln_scaling_pilot.svg @@ -1,12 +1,12 @@ - + - 2026-08-29T18:08:56.229221 + 2026-08-29T18:44:35.223930 image/svg+xml @@ -21,19 +21,19 @@ - - @@ -41,2209 +41,1582 @@ z - - + - 32 + 32 - + - 128 + 128 - + - 288 + 288 - + - 512 + 512 - + - 1.2k + 1.2k - + - 2.0k + 2.0k - Learnable edges + Learnable edges - + - - + - 0 + 0 - + - + - 10 + 20 - + - + - 20 + 40 - + - + - 30 + 60 - - - - - - - - - - - 40 - - - - - - - - - - - - - 50 - - - - - - - - - - - - - 60 - - - - - - - - - - - - - 70 - - - - Final classification error (%) + + Final classification error (%) - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - + + + + + + + - - - - (a) Error-growth slope reduced 88% vs static calibration + + (a) Final-error growth + 88% lower slope vs static calibration - - + - + - - 32 + + 32 - + - + - - 128 + + 128 - + - + - - 288 + + 288 - + - + - - 512 + + 512 - + - + - - 1.2k + + 1.2k - + - + - - 2.0k + + 2.0k - - Learnable edges + + Learnable edges - - - + + + - + - + - - 0 + + 0.0 - - - + + + - + - + - - 20 + + 0.2 - - - + + + - + - + - - 40 + + 0.4 - - - + + + - + - + - - 60 + + 0.6 - - - - - - - - - - - 80 - + + Classification-error AUC - - - - - - - - - - - 100 - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - Stable zero-error runs (%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + - - - - (b) Recovery remains reliable + + (b) Error-AUC growth + No resolved slope reduction - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - 0 + + 32 - + - + - - 100 + + 128 - + - + - - 200 + + 288 - + - + - - 300 + + 512 - + - + - - 400 + + 1.2k - + - + - - 500 + + 2.0k - - - - - - - - 600 - - - - Training epoch + + Learnable edges - - - - - - - - - - - 0 - - - - - - - - - - - - - 10 - - - - - + + + - + - + - - 20 + + 0 - - - + + + - + - + - - 30 + + 20 - - - + + + - + - + - - 40 + + 40 - - - + + + - + - + - - 50 + + 60 - - - + + + - + - + - - 60 + + 80 - - - + + + - + - + - - 70 + + 100 - - Classification error (%) + + Stable failure fraction (%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + - + + + + + + + + - + + + + + + + + - + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - (c) Learning at 2,048 edges + + (c) Stable-failure growth + No resolved slope reduction - - Exploratory pilot: 5 tasks, 1 component draw per task and size + + Exploratory pilot: 5 tasks, 1 component draws per task and size - - + - + - + - + - + - - + - + - + - - Clean + + Clean - - + - + - + - + - + - - + - + - + - - Same-RMS noise + + Same-RMS noise - - + - + - + - + - + - - + - + - + - - Raw imperfection + + Raw imperfection - - + - + - + - + - + - - + - + - + - - Static calibration + + Static calibration - - + - + - + - + - + - - + - + - + - - SDIL + + SDIL - - + + - - + + - - + + diff --git a/visual-composer/coupled-ladder-scaling.md b/visual-composer/coupled-ladder-scaling.md index c339551..896c02b 100644 --- a/visual-composer/coupled-ladder-scaling.md +++ b/visual-composer/coupled-ladder-scaling.md @@ -17,12 +17,13 @@ - **Figure prototype:** coordinated small-multiple lines. - **Panel map:** - (a) final classification error against learnable edge count; - - (b) fraction of runs that reach zero error and remain there through the - training horizon; - - (c) mean classification error over epochs at 2,048 edges. + - (b) classification-error area over the full training trajectory against + learnable edge count; + - (c) fraction of runs that fail to reach zero error and remain there through + the training horizon against learnable edge count. - **Exact label inventory:** Clean, same-RMS noise, raw imperfection, static - calibration, SDIL, learnable edges, classification error, stable zero-error - runs, training epoch. + calibration, SDIL, learnable edges, classification error, classification- + error AUC, stable failure fraction. - **Caption role:** state the paired protocol, distinguish fixed bias from matched noise, and label the evidence as a five-task pilot. - **Manuscript placement:** Part 2, immediately after the digital CLLN and @@ -34,4 +35,3 @@ - **Constraint:** the figure reports an error-growth slope comparison only when the static-calibration slope is positive. Raw imperfection is allowed to show a high non-monotonic floor rather than a forced power law. - diff --git a/visual-composer/qa-ledger.md b/visual-composer/qa-ledger.md index f601f93..ecb7660 100644 --- a/visual-composer/qa-ledger.md +++ b/visual-composer/qa-ledger.md @@ -6,6 +6,7 @@ | Pilot could be mistaken for confirmation | `figure_clln_scaling_pilot` | High | Added an explicit five-task, one-draw exploratory label | Resolved | | Slope title did not name its comparator | Panel (a) | Medium | Named static calibration in the title | Resolved | | Color-only distinctions could fail in print | All panels | Medium | Added distinct markers and line styles; inspected the rendered PNG | Resolved | +| Point estimates could be presented as resolved scaling laws | Panels (b) and (c) | High | Claim a percent reduction only when the paired bootstrap slope-difference interval is above zero | Resolved | | Vector editability and font embedding | SVG and PDF | Medium | Kept live SVG text and verified embedded DejaVu Sans in the PDF | Resolved | Rendered inspection: no clipped labels, legend overlap, panel overlap, or -- cgit v1.2.3