From 1b24c872575e80e6da8aa2c6a241aacdffcc78bb Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 06:43:49 -0500 Subject: theory: generalize innovation as conditional projection --- THEORY.md | 58 +++++++++++++++++++++++++++++++++++++++++--- experiments/verify_theory.py | 53 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/THEORY.md b/THEORY.md index 25d27ef..da3e77a 100644 --- a/THEORY.md +++ b/THEORY.md @@ -120,6 +120,57 @@ single-step loss change rather than reporting hidden cosine alone. ## 3. Somato-dendritic innovation as conditional nuisance removal +The regression residual has a stronger interpretation than the linear model +alone suggests. Let `F_h` be the information available to a soma-only +predictor in a neutral period, let `n` denote ordinary apical traffic, and set + +```text +m(h) = E[n | F_h]. +``` + +For every square-integrable soma-measurable predictor `g(h)`, conditional +expectation gives the orthogonality and Pythagorean identities + +```text +E = 0, + +E ||n - g(h)||^2 + = E ||n - m(h)||^2 + E ||m(h) - g(h)||^2. +``` + +Thus `n-m(h)` is the unique minimum-power innovation after subtracting any +function of the available somatic statistic. This is an `L2` projection +statement, not a Gaussian or linear assumption. If task-period apical activity +is `a=s+n` and the neutral traffic law is invariant, neutral fitting yields + +```text +r = a - m(h) = s + n - m(h). +``` + +It removes the largest soma-predictable nuisance component without projecting +out any part of the task-only instruction `s`. The implemented diagonal affine +predictor is the corresponding optimal projection only within each cell's +restricted span `{1,h_i}`; it cannot remove traffic predictable only from +population context or unobserved state. This restriction is consistent with +the failed broad endogenous-traffic gate and prevents the theorem from being +used to claim arbitrary contextual denoising. + +The norm-matched raw control isolates direction from magnitude. For every +positive per-example scalar `alpha`, + +```text +cos(alpha a, -g) = cos(a, -g). +``` + +Choosing `alpha=||r||/||a||` gives raw feedback exactly the innovation's norm +but cannot rotate it toward the descent direction at the same network state. +Across training, rescaling can of course change the subsequent trajectory; +this is why the matched control is still trained end to end rather than being +replaced by the algebraic observation. + +The finite-dimensional special case used for the main experiment writes +neutral traffic as a linear somatic component plus unpredictable noise. + Write apical activity during a task period as ```text @@ -217,6 +268,7 @@ python experiments/verify_theory.py ``` The verifier checks the exact Rademacher MSE across depth, width, and K; the -quadratic finite-sigma bias law; the smooth-descent step threshold; neutral -predictor convergence across `eta_P` and update counts; and the task-fit -teaching-signal absorption predicted above. +quadratic finite-sigma bias law; the smooth-descent step threshold; the +conditional-projection Pythagorean identity and norm-matching direction +invariance; neutral predictor convergence across `eta_P` and update counts; +and the task-fit teaching-signal absorption predicted above. diff --git a/experiments/verify_theory.py b/experiments/verify_theory.py index 7ee87bc..ba07487 100644 --- a/experiments/verify_theory.py +++ b/experiments/verify_theory.py @@ -108,6 +108,58 @@ def check_predictor_timescale(): assert abs(empirical - theoretical) < 1e-12 +def check_conditional_projection(): + """Finite-sample L2 projection realizes the innovation identities exactly.""" + rng = np.random.default_rng(351) + examples = 4096 + h = rng.uniform(-1.5, 1.5, size=(examples, 2)) + basis = np.column_stack(( + np.ones(examples), h[:, 0], h[:, 1], np.square(h[:, 0]), + np.square(h[:, 1]), h[:, 0] * h[:, 1], np.sin(h[:, 0]), + np.cos(h[:, 1]))) + coefficients = rng.normal(size=(basis.shape[1], 3)) + conditional_mean = basis @ coefficients + innovation = rng.normal(scale=0.4, size=conditional_mean.shape) + # Project the finite-sample noise off every soma-measurable basis vector. + innovation -= basis @ np.linalg.lstsq(basis, innovation, rcond=None)[0] + nuisance = conditional_mean + innovation + + linear_basis = basis[:, :3] + restricted = linear_basis @ np.linalg.lstsq( + linear_basis, nuisance, rcond=None)[0] + lhs = np.square(nuisance - restricted).mean() + irreducible = np.square(innovation).mean() + approximation = np.square(conditional_mean - restricted).mean() + orthogonality = np.abs(basis.T @ innovation / examples).max() + + teaching = rng.normal(size=conditional_mean.shape) + raw = teaching + nuisance + residual = teaching + innovation + alpha = (np.linalg.norm(residual, axis=1) + / np.linalg.norm(raw, axis=1).clip(min=1e-12)) + matched = alpha[:, None] * raw + + def row_cosine(left, right): + numerator = np.sum(left * right, axis=1) + denominator = (np.linalg.norm(left, axis=1) + * np.linalg.norm(right, axis=1)).clip(min=1e-12) + return numerator / denominator + + direction_difference = np.abs( + row_cosine(raw, teaching) - row_cosine(matched, teaching)).max() + norm_difference = np.abs( + np.linalg.norm(matched, axis=1) - np.linalg.norm(residual, axis=1)).max() + print("\nCONDITIONAL INNOVATION PROJECTION") + print(f"orthogonality={orthogonality:.3e} pythagorean_error=" + f"{abs(lhs - irreducible - approximation):.3e}") + print(f"norm_match_error={norm_difference:.3e} " + f"direction_change={direction_difference:.3e}") + assert orthogonality < 2e-14 + assert abs(lhs - irreducible - approximation) < 2e-14 + assert norm_difference < 2e-14 + assert direction_difference < 2e-14 + + def squared_cosine(x, y): return float((x.ravel() @ y.ravel()) ** 2 / ((x.ravel() @ x.ravel()) * (y.ravel() @ y.ravel()))) @@ -148,6 +200,7 @@ def main(): check_simultaneous_variance() check_sigma_bias() check_descent_threshold() + check_conditional_projection() check_predictor_timescale() check_innovation_identification() print("\nALL THEORY CHECKS PASSED") -- cgit v1.2.3