diff options
| -rw-r--r-- | NOVELTY.md | 6 | ||||
| -rw-r--r-- | README.md | 7 | ||||
| -rw-r--r-- | THEORY.md | 61 | ||||
| -rw-r--r-- | experiments/conv_local_smoke.py | 84 | ||||
| -rw-r--r-- | experiments/conv_run.py | 63 | ||||
| -rw-r--r-- | sdil/conv.py | 195 |
6 files changed, 400 insertions, 16 deletions
@@ -59,6 +59,12 @@ feedback alignment/weight-mirroring methods, and BurstCCN. The Harnett-specific claim remains the neutral-period somato-dendritic innovation operation under mixed traffic. +The implemented feedback-parameter perturbation rule is task-causal rather +than a copied-weight or forward-reconstruction rule, but this distinction does +not by itself make recursive learned feedback novel. It is treated as a +credit-path engineering component unless the complete innovation model shows a +load-bearing residualization result beyond those mandatory comparators. + ## Candidate SDIL contribution The candidate contribution is the combination @@ -123,6 +123,13 @@ parameter update to `2.98e-8`; actual HFA never performs that copy. `HFA_BASELINE.md` freezes its matched ResNet-20 validation screen before any HFA accuracy endpoint is observed. +The learned-hierarchy development path perturbs the feedback convolution +parameter spaces themselves. One antithetic pair estimates every edge's causal +target moment simultaneously, while the current prediction moment is a local +child/parent correlation. Its BatchNorm-coupled JVP and symmetric-limit delta +rule are executable smoke tests; no accuracy claim is attached until its own +causal-capture protocol is frozen and passed. + The subsequent V3 mechanism estimates the required A/G matrix statistics directly by perturbing the vectorizer parameter subspace. It remains forward-only and uses two causal loss queries per direction. Its exact and @@ -188,6 +188,67 @@ mean has cosine `0.96928` and norm ratio `1.04888` to the exact A/G target. Those mechanics do not establish task utility; a separately frozen gate is still required. +### Perturbing the hierarchical feedback-parameter subspace + +The hierarchical oracle shows that an early field needs the spatial child +teaching map rather than only the output-error vector. For a residual-DAG edge +`e`, let `s_e` be the locally gated and BatchNorm-transformed child field and +let the feedback contribution at its parent be + +```text +p_e = conv_transpose(s_e, Q_e). +``` + +The complete predicted parent field `p` also includes the exact +parameter-free shortcut contribution. Draw an independent Rademacher tensor +`Xi_e` with the shape of every feedback convolution and intervene at each +edge's parent along + +```text +d_e = conv_transpose(s_e, Xi_e). +``` + +The readout-feedback matrix is treated identically, with +`d_out=(c Xi_out^T)/S` at the final spatial map. All edge fields are injected +in one antithetic pair. If `D` is the derivative of the summed minibatch loss, +independence across parameter tensors gives + +```text +E[-D Xi_e / (B S_parent)] + = corr(q_parent, s_e) / (B S_parent), +q_parent = -d ell / d h_parent. +``` + +All other edges contribute zero-mean cross terms. The locally available +prediction moment is + +```text +corr(p_parent, s_e) / (B S_parent), +``` + +where `p_parent` includes both convolutional and shortcut feedback. Therefore + +```text +Delta Q_e = eta_A [target_moment_e - prediction_moment_e] +``` + +is an unbiased stochastic delta rule for the normalized squared parent-field +error. It uses two task-loss queries per direction, no copied forward tensor, +and no reverse differentiation. Unlike V2/V3, its perturbation family already +contains the spatial child teaching context. Unlike weight mirroring, its +target is task-causal credit rather than reconstruction of the forward +convolution. Recursive feedback calibration and feedback convolutions are not +claimed as novel by themselves; this is an engineering prerequisite for a +hierarchical innovation model and requires direct HFA, learned-FA/mirroring, +and BurstCCN comparisons. + +The executable audit checks the complete BatchNorm-coupled antithetic JVP to +`6.2e-10` relative error. Under an audit-only symmetric feedback copy, the +predicted Q/R moments equal the exact causal delta-rule targets to `2.3e-16` +relative error. Actual training never performs that copy. These identities +establish mechanics only; a frozen-forward causal-capture gate must pass before +any task-scale endpoint is opened. + ## 2. What is sufficient for a descent step Flatten all forward parameters into `theta`, let `p = grad L(theta)`, and let diff --git a/experiments/conv_local_smoke.py b/experiments/conv_local_smoke.py index 9207e04..894f9d3 100644 --- a/experiments/conv_local_smoke.py +++ b/experiments/conv_local_smoke.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Prove the convolutional local eligibility matches exact BP when instructed.""" +import math import os import sys @@ -11,6 +12,7 @@ from sdil.conv import (CIFARHierarchicalFAResNet, CIFARLocalResNet, CIFARSDILResNet, ConvSDILConfig, channel_subspace_apical_calibration, conv_hierarchical_step, conv_local_step, + hierarchical_parameter_subspace_calibration, simultaneous_conv_node_perturbation, vectorizer_subspace_apical_calibration) @@ -673,6 +675,87 @@ def hierarchical_feedback_checks(): } +def hierarchical_parameter_calibration_checks(): + """Audit the causal JVP and the exact local Q/R delta-rule moments.""" + torch.manual_seed(109) + net = CIFARHierarchicalFAResNet( + depth=8, base_width=2, seed=110, dtype=torch.float64, + normalization="batchnorm", residual_scale=1.0) + x = torch.randn(3, 3, 32, 32, dtype=torch.float64) + y = torch.tensor([0, 4, 7]) + parameters = net.W + net.gamma + net.beta + [net.W_out, net.b_out] + for parameter in parameters: + parameter.requires_grad_(True) + forward = net.forward( + x, return_cache=True, training=True, update_stats=False) + loss = F.cross_entropy(forward["logits"], y) + hidden_gradients = torch.autograd.grad(loss, forward["hiddens"]) + output_signal = (torch.softmax(forward["logits"].detach(), dim=1) + - F.one_hot(y, 10).to(torch.float64)) + _, diagnostic = hierarchical_parameter_subspace_calibration( + net, x, y, forward, output_signal, sigma=1e-5, + n_directions=1, eta=0.0, + generator=torch.Generator().manual_seed(111), + return_diagnostics=True) + directions = diagnostic["directions"][0]["hidden"] + exact_directional = x.shape[0] * sum( + (gradient * direction).sum() + for gradient, direction in zip(hidden_gradients, directions)) + estimated_directional = diagnostic[ + "directional_derivatives"][0]["scaled_directional"] + jvp_relative = float((estimated_directional - exact_directional).abs() + / exact_directional.abs().clamp_min(1e-30)) + assert jvp_relative < 3e-7 + for parameter in parameters: + parameter.requires_grad_(False) + + # Under an audit-only symmetric copy, the hierarchical field is the exact + # negative gradient. Consequently every local Q/R predicted moment equals + # its exact causal regression target, including option-A shortcut terms. + exact = CIFARHierarchicalFAResNet( + depth=8, base_width=2, seed=112, dtype=torch.float64, + normalization="batchnorm", residual_scale=1.0) + exact.Q = [value.clone() for value in exact.W] + exact.R_out.copy_(-exact.W_out.t()) + for parameter in exact.W + exact.gamma + exact.beta + [ + exact.W_out, exact.b_out]: + parameter.requires_grad_(True) + clean = exact.forward( + x, return_cache=True, training=True, update_stats=False) + gradients = torch.autograd.grad( + F.cross_entropy(clean["logits"], y), clean["hiddens"]) + negative = [-x.shape[0] * value.detach() for value in gradients] + signal = (torch.softmax(clean["logits"].detach(), dim=1) + - F.one_hot(y, 10).to(torch.float64)) + teaching, contexts, recipients = exact.hierarchical_teaching( + signal, clean, return_edge_contexts=True) + numerator = 0.0 + denominator = 0.0 + for index in range(1, len(exact.Q)): + recipient = recipients[index] + spec = exact.layer_specs[index] + spatial = (negative[recipient].shape[2] + * negative[recipient].shape[3]) + target = torch.nn.grad.conv2d_weight( + negative[recipient], exact.Q[index].shape, contexts[index], + stride=spec.stride, padding=spec.padding) / (x.shape[0] * spatial) + prediction = torch.nn.grad.conv2d_weight( + teaching[recipient], exact.Q[index].shape, contexts[index], + stride=spec.stride, padding=spec.padding) / (x.shape[0] * spatial) + numerator += float((target - prediction).square().sum()) + denominator += float(target.square().sum()) + target_r = negative[-1].mean(dim=(2, 3)).t() @ signal / x.shape[0] + prediction_r = teaching[-1].mean(dim=(2, 3)).t() @ signal / x.shape[0] + numerator += float((target_r - prediction_r).square().sum()) + denominator += float(target_r.square().sum()) + delta_rule_relative = math.sqrt(numerator / max(denominator, 1e-300)) + assert delta_rule_relative < 2e-12 + return { + "hierarchical_parameter_subspace_jvp_relative_error": jvp_relative, + "hierarchical_parameter_delta_rule_relative_error": delta_rule_relative, + } + + def apical_learning_checks(): torch.manual_seed(11) net = CIFARSDILResNet(depth=8, base_width=2, seed=6) @@ -779,6 +862,7 @@ def main(): report.update(channel_subspace_estimator_check()) report.update(vectorizer_subspace_estimator_check()) report.update(hierarchical_feedback_checks()) + report.update(hierarchical_parameter_calibration_checks()) report.update(apical_learning_checks()) print(report) print("ALL CONVOLUTIONAL LOCAL-ELIGIBILITY CHECKS PASSED") diff --git a/experiments/conv_run.py b/experiments/conv_run.py index 162a8be..fb759b4 100644 --- a/experiments/conv_run.py +++ b/experiments/conv_run.py @@ -16,7 +16,9 @@ from sdil.conv import (CIFARHierarchicalFAResNet, CIFARLocalResNet, CIFARSDILResNet, ConvSDILConfig, conv_alignment_report, conv_apical_calibration_step, conv_hierarchical_alignment_report, - conv_hierarchical_step, conv_local_step, evaluate_conv) + conv_hierarchical_step, + conv_learned_hierarchical_step, conv_local_step, + evaluate_conv, hierarchical_parameter_subspace_calibration) from sdil.data import DATA_DIR, get_cifar_image_splits @@ -103,14 +105,18 @@ def build(args): bn_momentum=args.bn_momentum, bn_eps=args.bn_eps) if args.mode == "bp": return CIFARLocalResNet(**common), None - if args.mode == "hfa": + if args.mode in ("hfa", "lhfa"): net = CIFARHierarchicalFAResNet( **common, feedback_seed=args.apical_seed, feedback_scale=args.a_scale) config = ConvSDILConfig( - eta=args.lr, eta_output=args.output_lr, eta_A=0.0, + eta=args.lr, eta_output=args.output_lr, + eta_A=(args.eta_A if args.mode == "lhfa" else 0.0), momentum=args.momentum, weight_decay=args.weight_decay, - learn_A=False, learn_P=False) + learn_A=args.mode == "lhfa", learn_P=False, + pert_sigma=args.pert_sigma, pert_every=args.pert_every, + pert_directions=args.pert_directions, + apical_calibration_mode="hierarchical_parameter_subspace") config.validate() return net, config net = CIFARSDILResNet( @@ -146,7 +152,10 @@ def work_report(net, mode, counters): local_correlation = counters["ordinary_examples"] * forward_macs apical_inference = ((counters["ordinary_examples"] + counters["apical_warmup_examples"]) * apical_macs) - apical_regression = counters["calibration_event_examples"] * apical_macs + regression_multiplier = 2 if mode == "lhfa" else 1 + apical_regression = (regression_multiplier + * counters["calibration_event_examples"] + * apical_macs) components = { "ordinary_forward_macs": normal_forward, "warmup_clean_forward_macs": warmup_forward, @@ -182,8 +191,11 @@ def work_report(net, mode, counters): def run(args): if args.eval_split == "test" and args.eval_every: raise ValueError("test protocols must use --eval_every 0 (one final evaluation)") - if args.mode != "sdil" and (args.a_warmup_steps or args.learn_P): + if args.mode not in ("sdil", "lhfa") and ( + args.a_warmup_steps or args.learn_P): raise ValueError("apical/predictor warmup is restricted to SDIL") + if args.mode == "lhfa" and args.learn_P: + raise ValueError("predictor learning is not defined for learned HFA") torch.manual_seed(args.seed) if str(args.device).startswith("cuda"): if not torch.cuda.is_available(): @@ -225,7 +237,8 @@ def run(args): "schema_version": 1, "protocol_family": "oral_a_cifar_local_resnet_development", "calibration_metric_space": ( - None if config is None or args.mode == "hfa" else { + None if config is None or args.mode == "hfa" else + "hierarchical_feedback_parameters" if args.mode == "lhfa" else { "unit_targets": "full_hidden_field", "channel_subspace": "channel_basis_moments", "vectorizer_subspace": "vectorizer_parameter_gradients", @@ -250,8 +263,12 @@ def run(args): "vectorizer_mode": getattr(net, "vectorizer_mode", None), "fixed_traffic_coefficients": getattr( net, "n_fixed_traffic_coefficients", 0), - "fixed_feedback_parameters": getattr( - net, "n_fixed_feedback_parameters", 0), + "fixed_feedback_parameters": ( + getattr(net, "n_fixed_feedback_parameters", 0) + if args.mode == "hfa" else 0), + "adaptive_feedback_parameters": ( + getattr(net, "n_fixed_feedback_parameters", 0) + if args.mode == "lhfa" else 0), }, "epochs": [], } @@ -290,8 +307,21 @@ def run(args): except StopIteration: iterator = iter(train) x, y = next(iterator) - _, metric = conv_apical_calibration_step( - net, x, y, config, generator=warmup_generator) + if args.mode == "lhfa": + forward = net.forward( + x, return_cache=True, training=True, update_stats=False) + output_signal = ( + torch.softmax(forward["logits"], dim=1) + - torch.nn.functional.one_hot( + y, net.n_classes).to(forward["logits"].dtype)) + metric = hierarchical_parameter_subspace_calibration( + net, x, y, forward, output_signal, + sigma=config.pert_sigma, + n_directions=config.pert_directions, eta=config.eta_A, + generator=warmup_generator) + else: + _, metric = conv_apical_calibration_step( + net, x, y, config, generator=warmup_generator) warmup_metrics.append(metric) batch = x.shape[0] counters["apical_warmup_examples"] += batch @@ -346,6 +376,13 @@ def run(args): result = conv_hierarchical_step(net, x, y, config) loss = result["loss"] did_perturb = False + elif args.mode == "lhfa": + result = conv_learned_hierarchical_step( + net, x, y, config, step, generator=perturb_generator) + loss = result["loss"] + did_perturb = result["did_perturb"] + if result["calibration"] is not None: + calibration_metrics.append(result["calibration"]) else: result = conv_local_step( net, x, y, config, step, generator=perturb_generator) @@ -424,7 +461,7 @@ def run(args): diagnostic_start = time.time() diagnostics = (conv_hierarchical_alignment_report( net, train.x[:probe], train.y[:probe]) - if args.mode == "hfa" else + if args.mode in ("hfa", "lhfa") else conv_alignment_report( net, train.x[:probe], train.y[:probe], config)) sync(args.device) @@ -471,7 +508,7 @@ def run(args): def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( - "--mode", choices=("bp", "dfa", "hfa", "sdil", "nodepert"), + "--mode", choices=("bp", "dfa", "hfa", "lhfa", "sdil", "nodepert"), required=True) parser.add_argument("--out", required=True) parser.add_argument("--device", default="cpu") diff --git a/sdil/conv.py b/sdil/conv.py index f0592a6..55cdc62 100644 --- a/sdil/conv.py +++ b/sdil/conv.py @@ -473,8 +473,16 @@ class CIFARHierarchicalFAResNet(CIFARLocalResNet): return result @torch.no_grad() - def hierarchical_teaching(self, output_signal, forward): - """Propagate teaching fields through independent feedback convolutions.""" + def hierarchical_teaching(self, output_signal, forward, + return_edge_contexts=False): + """Propagate teaching fields through independent feedback convolutions. + + When requested, ``edge_contexts[l]`` is the local child field consumed + by ``Q[l]`` and ``recipients[l]`` is the hidden population receiving + that transposed-convolution contribution. These values expose the + sufficient statistics for causal feedback calibration without reading + any forward tensor. + """ caches = forward.get("caches") hiddens = forward.get("hiddens") if caches is None or hiddens is None: @@ -482,6 +490,8 @@ class CIFARHierarchicalFAResNet(CIFARLocalResNet): if len(hiddens) != self.n_hidden: raise ValueError("hierarchical hidden population mismatch") teaching = [torch.zeros_like(value) for value in hiddens] + edge_contexts = [None for _ in self.Q] + recipients = [None for _ in self.Q] spatial = hiddens[-1].shape[2] * hiddens[-1].shape[3] teaching[-1].copy_( (output_signal @ self.R_out.t())[:, :, None, None] / spatial) @@ -495,6 +505,8 @@ class CIFARHierarchicalFAResNet(CIFARLocalResNet): branch_delta, _, _ = self._normalization_backward( second, second_delta * self.residual_scale, caches[second]["normalization"]) + edge_contexts[second] = branch_delta + recipients[second] = first teaching[first].add_(F.conv_transpose2d( branch_delta, self.Q[second], stride=1, padding=1)) teaching[parent].add_(self._option_a_shortcut_transpose( @@ -505,12 +517,150 @@ class CIFARHierarchicalFAResNet(CIFARLocalResNet): first_delta = teaching[first] * first_gate first_delta, _, _ = self._normalization_backward( first, first_delta, caches[first]["normalization"]) + edge_contexts[first] = first_delta + recipients[first] = parent teaching[parent].add_(F.conv_transpose2d( first_delta, self.Q[first], stride=block["stride"], padding=1, output_padding=block["stride"] - 1)) + if return_edge_contexts: + if any(value is None for value in edge_contexts[1:]): + raise AssertionError("hierarchical edge context is incomplete") + return teaching, edge_contexts, recipients return teaching +@torch.no_grad() +def hierarchical_parameter_subspace_calibration( + net, x, y, clean_forward, output_signal, sigma=1e-2, + n_directions=1, eta=1e-3, generator=None, return_diagnostics=False): + """Calibrate the residual-DAG feedback maps with two causal queries. + + A Rademacher tensor is drawn in every Q/R parameter space. Each tensor is + applied only to its locally available child field, producing one candidate + intervention at the edge's parent population. All independent candidate + fields are injected in the same antithetic pair. Multiplying the scalar + loss derivative back into each random tensor gives an unbiased estimate of + that edge's causal target moment; subtracting the current predicted moment + is the exact local squared-field delta rule. No forward weight or reverse + differentiation is used. + """ + if not isinstance(net, CIFARHierarchicalFAResNet): + raise TypeError("hierarchical calibration requires a hierarchical net") + if sigma <= 0 or n_directions < 1 or eta < 0: + raise ValueError("invalid hierarchical calibration hyperparameters") + if generator is None: + generator = torch.Generator(device=x.device).manual_seed(0) + hiddens = clean_forward.get("hiddens") + if hiddens is None or len(hiddens) != net.n_hidden: + raise ValueError("clean forward does not match hierarchical populations") + teaching, contexts, recipients = net.hierarchical_teaching( + output_signal, clean_forward, return_edge_contexts=True) + batch = x.shape[0] + target_q = [torch.zeros_like(value) if index else None + for index, value in enumerate(net.Q)] + target_r = torch.zeros_like(net.R_out) + diagnostic_directions = [] + diagnostic_derivatives = [] + for _ in range(n_directions): + random_q = [None] + random_q.extend([ + torch.empty_like(value).bernoulli_( + 0.5, generator=generator).mul_(2).sub_(1) + for value in net.Q[1:] + ]) + random_r = torch.empty_like(net.R_out).bernoulli_( + 0.5, generator=generator).mul_(2).sub_(1) + interventions = [torch.zeros_like(value) for value in hiddens] + spatial_out = hiddens[-1].shape[2] * hiddens[-1].shape[3] + interventions[-1].add_( + (output_signal @ random_r.t())[:, :, None, None] / spatial_out) + for index in range(1, len(net.Q)): + spec = net.layer_specs[index] + recipient = recipients[index] + contribution = F.conv_transpose2d( + contexts[index], random_q[index], stride=spec.stride, + padding=spec.padding, output_padding=spec.stride - 1) + if contribution.shape != interventions[recipient].shape: + raise AssertionError("hierarchical intervention shape mismatch") + interventions[recipient].add_(contribution) + plus = F.cross_entropy(net.forward( + x, perturbations=[sigma * value for value in interventions], + training=True, update_stats=False)["logits"], y) + minus = F.cross_entropy(net.forward( + x, perturbations=[-sigma * value for value in interventions], + training=True, update_stats=False)["logits"], y) + # The parameter maps are batch-shared. Scale the mean-loss derivative + # into a summed-loss hidden signal, then average its moment over B and + # recipient spatial sites to keep one eta meaningful across stages. + directional = (plus - minus) * batch / (2.0 * sigma) + target_r.add_(random_r, alpha=-float(directional) / ( + batch * n_directions)) + for index in range(1, len(net.Q)): + recipient = recipients[index] + spatial = (hiddens[recipient].shape[2] + * hiddens[recipient].shape[3]) + target_q[index].add_( + random_q[index], alpha=-float(directional) / ( + batch * spatial * n_directions)) + if return_diagnostics: + diagnostic_directions.append({ + "hidden": interventions, "Q": random_q, "R": random_r}) + diagnostic_derivatives.append({ + "scaled_directional": directional, + "coupling": "summed_batch_objective", + }) + + prediction_q = [None] + for index in range(1, len(net.Q)): + recipient = recipients[index] + spec = net.layer_specs[index] + spatial = (hiddens[recipient].shape[2] + * hiddens[recipient].shape[3]) + prediction_q.append(torch.nn.grad.conv2d_weight( + teaching[recipient], net.Q[index].shape, contexts[index], + stride=spec.stride, padding=spec.padding) / (batch * spatial)) + prediction_r = (teaching[-1].mean(dim=(2, 3)).t() + @ output_signal) / batch + errors_q = [None] + errors_q.extend([ + target_q[index] - prediction_q[index] + for index in range(1, len(net.Q)) + ]) + error_r = target_r - prediction_r + for index in range(1, len(net.Q)): + net.Q[index].add_(errors_q[index], alpha=eta) + net.R_out.add_(error_r, alpha=eta) + + target_power = float(target_r.square().sum()) + prediction_power = float(prediction_r.square().sum()) + error_power = float(error_r.square().sum()) + dot = float((target_r * prediction_r).sum()) + parameters = target_r.numel() + for index in range(1, len(net.Q)): + target_power += float(target_q[index].square().sum()) + prediction_power += float(prediction_q[index].square().sum()) + error_power += float(errors_q[index].square().sum()) + dot += float((target_q[index] * prediction_q[index]).sum()) + parameters += target_q[index].numel() + denominator = math.sqrt(target_power * prediction_power) + calibration = { + "calibration_mse": error_power / parameters, + "target_power": target_power / parameters, + "prediction_target_cosine": dot / denominator if denominator else 0.0, + "parameter_update_rms": math.sqrt(error_power / parameters), + } + if return_diagnostics: + return calibration, { + "directions": diagnostic_directions, + "directional_derivatives": diagnostic_derivatives, + "teaching": teaching, "contexts": contexts, + "recipients": recipients, "target_Q": target_q, + "prediction_Q": prediction_q, "target_R": target_r, + "prediction_R": prediction_r, + } + return calibration + + def conv_hierarchical_step(net, x, y, config): """One hierarchical-FA update using no reverse-mode graph or weight transport.""" config.validate() @@ -541,6 +691,44 @@ def conv_hierarchical_step(net, x, y, config): } +def conv_learned_hierarchical_step(net, x, y, config, step, generator=None): + """One task update with optional causal calibration of hierarchical Q/R.""" + config.validate() + with torch.no_grad(): + forward = net.forward( + x, return_cache=True, training=True, update_stats=True) + logits = forward["logits"] + loss = F.cross_entropy(logits, y) + output_error = (torch.softmax(logits, dim=1) + - F.one_hot(y, net.n_classes).to(logits.dtype)) + teaching = net.hierarchical_teaching(output_error, forward) + total_units = sum(value.numel() for value in teaching) + teaching_rms = math.sqrt( + sum(float(value.square().sum()) for value in teaching) / total_units) + did_perturb = config.learn_A and step % config.pert_every == 0 + calibration = None + if did_perturb: + calibration = hierarchical_parameter_subspace_calibration( + net, x, y, forward, output_error, sigma=config.pert_sigma, + n_directions=config.pert_directions, eta=config.eta_A, + generator=generator) + (directions, gamma_directions, beta_directions, + output_weight, output_bias) = net.local_ascent_directions( + teaching, output_error, forward) + net.apply_ascent( + directions, output_weight, output_bias, + eta_hidden=config.eta, eta_output=config.eta_output, + momentum=config.momentum, weight_decay=config.weight_decay, + gamma_directions=gamma_directions, + beta_directions=beta_directions) + return { + "loss": float(loss), "did_perturb": did_perturb, + "calibration": calibration, "predictor_mse": None, + "teaching_rms": teaching_rms, "raw_apical_rms": teaching_rms, + "innovation_rms": teaching_rms, + } + + def conv_hierarchical_alignment_report(net, x, y): """Audit hierarchical teaching against exact hidden gradients.""" parameters = net.W + net.gamma + net.beta + [net.W_out, net.b_out] @@ -1081,7 +1269,8 @@ class ConvSDILConfig: if self.pert_every < 1 or self.pert_directions < 1: raise ValueError("perturbation cadence/directions must be positive") if self.apical_calibration_mode not in ( - "unit_targets", "channel_subspace", "vectorizer_subspace"): + "unit_targets", "channel_subspace", "vectorizer_subspace", + "hierarchical_parameter_subspace"): raise ValueError("unknown apical calibration mode") if self.direct_node_perturbation and self.pert_every != 1: raise ValueError("direct node perturbation requires a target every step") |