From 3856daa3124b8dfce6bd0285646e5a92f0ff96aa Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Sat, 29 Aug 2026 18:45:18 -0500 Subject: analysis: compare three CLLN scaling endpoints --- results/coupled_ladder/p1_scaling_analysis.json | 56 +- results/figs/figure_clln_scaling_pilot.pdf | Bin 24101 -> 23584 bytes results/figs/figure_clln_scaling_pilot.png | Bin 243640 -> 209252 bytes results/figs/figure_clln_scaling_pilot.svg | 2937 +++++++++-------------- 4 files changed, 1210 insertions(+), 1783 deletions(-) (limited to 'results') diff --git a/results/coupled_ladder/p1_scaling_analysis.json b/results/coupled_ladder/p1_scaling_analysis.json index 4d4866a..d06c232 100644 --- a/results/coupled_ladder/p1_scaling_analysis.json +++ b/results/coupled_ladder/p1_scaling_analysis.json @@ -4,6 +4,7 @@ "bootstrap": { "unit": "task; component draws averaged within task", "task_clusters": 5, + "component_draws_per_task_size": 1, "replicates": 20000, "seed": 20260829, "interval": "percentile 95%" @@ -746,7 +747,8 @@ }, "excess_error_scaling": { "x_axis": "log10 learnable edges", - "y_axis": "classification error excess over clean", + "metric": "classification_error", + "y_axis": "final classification error excess over clean", "static_calibration_slope": 0.10415531736549301, "static_calibration_slope_95ci": [ 0.04974245841148715, @@ -769,6 +771,58 @@ ], "relative_reduction_valid_bootstrap_fraction": 1.0 }, + "excess_error_auc_scaling": { + "x_axis": "log10 learnable edges", + "metric": "classification_error_auc", + "y_axis": "classification error AUC excess over clean", + "static_calibration_slope": 0.02723285591782782, + "static_calibration_slope_95ci": [ + -0.018469611777622092, + 0.06799878937945103 + ], + "sdil_slope": 0.0050630936021534975, + "sdil_slope_95ci": [ + 0.0009725048424469787, + 0.009883683346965023 + ], + "paired_slope_difference": 0.022169762315674323, + "paired_slope_difference_95ci": [ + -0.021041329819854505, + 0.0613401506085447 + ], + "relative_slope_reduction_percent": 81.40814309953083, + "relative_slope_reduction_95ci": [ + -11.782270478082541, + 94.49463684558907 + ], + "relative_reduction_valid_bootstrap_fraction": 0.88435 + }, + "excess_stable_failure_scaling": { + "x_axis": "log10 learnable edges", + "metric": "reached_stable_zero_error", + "y_axis": "stable failure fraction excess over clean", + "static_calibration_slope": 0.17485078862811346, + "static_calibration_slope_95ci": [ + -0.15978252936756204, + 0.39793966729189717 + ], + "sdil_slope": 0.04963588122401385, + "sdil_slope_95ci": [ + -0.16871445018618145, + 0.24817940612006928 + ], + "paired_slope_difference": 0.1252149074040996, + "paired_slope_difference_95ci": [ + -0.08086274712223154, + 0.33129256193043083 + ], + "relative_slope_reduction_percent": 71.6124350290559, + "relative_slope_reduction_95ci": [ + -50.860158714747875, + 597.0884428984873 + ], + "relative_reduction_valid_bootstrap_fraction": 0.87555 + }, "largest_grid_learning_curves": { "clean": [ { diff --git a/results/figs/figure_clln_scaling_pilot.pdf b/results/figs/figure_clln_scaling_pilot.pdf index 07e1e1b..0a2c184 100644 Binary files a/results/figs/figure_clln_scaling_pilot.pdf and b/results/figs/figure_clln_scaling_pilot.pdf differ diff --git a/results/figs/figure_clln_scaling_pilot.png b/results/figs/figure_clln_scaling_pilot.png index df77b47..b5cfa05 100644 Binary files a/results/figs/figure_clln_scaling_pilot.png and b/results/figs/figure_clln_scaling_pilot.png differ diff --git a/results/figs/figure_clln_scaling_pilot.svg b/results/figs/figure_clln_scaling_pilot.svg index c1c1a18..f7b3440 100644 --- a/results/figs/figure_clln_scaling_pilot.svg +++ b/results/figs/figure_clln_scaling_pilot.svg @@ -1,12 +1,12 @@ - + - 2026-08-29T18:08:56.229221 + 2026-08-29T18:44:35.223930 image/svg+xml @@ -21,19 +21,19 @@ - - @@ -41,2209 +41,1582 @@ z - - + - 32 + 32 - + - 128 + 128 - + - 288 + 288 - + - 512 + 512 - + - 1.2k + 1.2k - + - 2.0k + 2.0k - Learnable edges + Learnable edges - + - - + - 0 + 0 - + - + - 10 + 20 - + - + - 20 + 40 - + - + - 30 + 60 - - - - - - - - - - - 40 - - - - - - - - - - - - - 50 - - - - - - - - - - - - - 60 - - - - - - - - - - - - - 70 - - - - Final classification error (%) + + Final classification error (%) - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - + + + + + + + - - - - (a) Error-growth slope reduced 88% vs static calibration + + (a) Final-error growth + 88% lower slope vs static calibration - - + - + - - 32 + + 32 - + - + - - 128 + + 128 - + - + - - 288 + + 288 - + - + - - 512 + + 512 - + - + - - 1.2k + + 1.2k - + - + - - 2.0k + + 2.0k - - Learnable edges + + Learnable edges - - - + + + - + - + - - 0 + + 0.0 - - - + + + - + - + - - 20 + + 0.2 - - - + + + - + - + - - 40 + + 0.4 - - - + + + - + - + - - 60 + + 0.6 - - - - - - - - - - - 80 - + + Classification-error AUC - - - - - - - - - - - 100 - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - Stable zero-error runs (%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + - - - - (b) Recovery remains reliable + + (b) Error-AUC growth + No resolved slope reduction - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - 0 + + 32 - + - + - - 100 + + 128 - + - + - - 200 + + 288 - + - + - - 300 + + 512 - + - + - - 400 + + 1.2k - + - + - - 500 + + 2.0k - - - - - - - - 600 - - - - Training epoch + + Learnable edges - - - - - - - - - - - 0 - - - - - - - - - - - - - 10 - - - - - + + + - + - + - - 20 + + 0 - - - + + + - + - + - - 30 + + 20 - - - + + + - + - + - - 40 + + 40 - - - + + + - + - + - - 50 + + 60 - - - + + + - + - + - - 60 + + 80 - - - + + + - + - + - - 70 + + 100 - - Classification error (%) + + Stable failure fraction (%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + - + + + + + + + + - + + + + + + + + - + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - (c) Learning at 2,048 edges + + (c) Stable-failure growth + No resolved slope reduction - - Exploratory pilot: 5 tasks, 1 component draw per task and size + + Exploratory pilot: 5 tasks, 1 component draws per task and size - - + - + - + - + - + - - + - + - + - - Clean + + Clean - - + - + - + - + - + - - + - + - + - - Same-RMS noise + + Same-RMS noise - - + - + - + - + - + - - + - + - + - - Raw imperfection + + Raw imperfection - - + - + - + - + - + - - + - + - + - - Static calibration + + Static calibration - - + - + - + - + - + - - + - + - + - - SDIL + + SDIL - - + + - - + + - - + + -- cgit v1.2.3