baseline: add residual response mirroring

1 files changed, 39 insertions, 0 deletions

diff --git a/experiments/conv_local_smoke.py b/experiments/conv_local_smoke.py
index 7d24154..4e388a0 100644
--- a/experiments/conv_local_smoke.py
+++ b/experiments/conv_local_smoke.py
@@ -14,6 +14,7 @@ from sdil.conv import (CIFARHierarchicalFAResNet, CIFARLocalResNet,
                        conv_hierarchical_step, conv_local_step,
                        hierarchical_mirror_observations,
                        hierarchical_parameter_subspace_calibration,
+                       normalized_residual_mirror_update,
                        normalized_response_mirror_update,
                        simultaneous_conv_node_perturbation,
                        vectorizer_subspace_apical_calibration)
@@ -794,6 +795,38 @@ def normalized_response_mirror_checks():
     independence_error = max(float((a - b).abs().max()) for a, b in zip(
         left.Q[1:] + [left.R_out], right.Q[1:] + [right.R_out]))
     assert independence_error == 0.0
+
+    # Residual-response LMS has a per-observation exact fixed point: its update
+    # is zero, not merely zero in expectation, when Q/R match the forward maps.
+    fixed = CIFARHierarchicalFAResNet(
+        depth=8, base_width=2, seed=125, dtype=torch.float64)
+    fixed.Q = [value.clone() for value in fixed.W]
+    fixed.R_out.copy_(-fixed.W_out.t())
+    fixed_observations = hierarchical_mirror_observations(
+        fixed, batch_size=2, generator=torch.Generator().manual_seed(126))
+    fixed_metrics = normalized_residual_mirror_update(
+        fixed, fixed_observations, eta=1.0)
+    assert fixed_metrics["mirror_update_rms"] < 1e-14
+    assert fixed_metrics["mirror_response_residual_fraction"] < 1e-14
+
+    residual_left = CIFARHierarchicalFAResNet(
+        depth=8, base_width=2, seed=127, dtype=torch.float64)
+    residual_right = CIFARHierarchicalFAResNet(
+        depth=8, base_width=2, seed=127, dtype=torch.float64)
+    residual_observations = hierarchical_mirror_observations(
+        residual_left, batch_size=2,
+        generator=torch.Generator().manual_seed(128))
+    for value in residual_right.W + [residual_right.W_out]:
+        value.normal_(generator=torch.Generator().manual_seed(value.numel() + 1))
+    normalized_residual_mirror_update(
+        residual_left, residual_observations, eta=0.2)
+    normalized_residual_mirror_update(
+        residual_right, residual_observations, eta=0.2)
+    residual_independence_error = max(float((a - b).abs().max())
+                                      for a, b in zip(
+        residual_left.Q[1:] + [residual_left.R_out],
+        residual_right.Q[1:] + [residual_right.R_out]))
+    assert residual_independence_error == 0.0
     return {
         "mirror_estimate_mean_forward_cosine": sum(cosines) / len(cosines),
         "mirror_estimate_min_forward_cosine": min(cosines),
@@ -801,6 +834,12 @@ def normalized_response_mirror_checks():
         "mirror_estimate_max_norm_ratio": max(norm_ratios),
         "mirror_update_forward_parameter_independence_error": independence_error,
         "mirror_update_rms": metrics["mirror_update_rms"],
+        "residual_mirror_exact_fixed_point_update_rms": fixed_metrics[
+            "mirror_update_rms"],
+        "residual_mirror_exact_fixed_point_fraction": fixed_metrics[
+            "mirror_response_residual_fraction"],
+        "residual_mirror_forward_parameter_independence_error": (
+            residual_independence_error),
     }