diff options
Diffstat (limited to 'sdil')
| -rw-r--r-- | sdil/core.py | 17 |
1 files changed, 12 insertions, 5 deletions
diff --git a/sdil/core.py b/sdil/core.py index 6137171..ce42650 100644 --- a/sdil/core.py +++ b/sdil/core.py @@ -214,8 +214,8 @@ def node_perturbation_targets(net, x, y, sigma=1e-2, antithetic=True, n_dirs=1): (B, n_l) tensors. No autograd. We perturb h_l -> h_l + sigma*xi, propagate forward THROUGH THE REST of the - net (weights only), read the loss change dL, and form - q_l = -(dL / sigma^2) * xi (spec convention; scale absorbed by eta_A) + net (weights only), read the loss change dL, and form the antithetic estimate + q_l = -((L(+sigma*xi) - L(-sigma*xi)) / (2*sigma)) * xi Antithetic pairs (+xi, -xi) cancel the O(sigma^2) even term. Averaging over n_dirs independent directions cuts the estimator variance ~1/n_dirs, which the predecessor project's failures flagged as the main risk of learning a @@ -287,9 +287,9 @@ class SDILConfig: self.feedback = feedback # "error" | "error_deriv" self.p_update_on_neutral = p_update_on_neutral # node perturbation estimates the descent DIRECTION well but its - # magnitude (~ -dL/sigma^2) is not ||grad||, so A learns a mis-scaled - # per-layer step. Normalising each layer's delta to unit RMS decouples - # step size (set by eta) from that miscalibration -- like normalized SGD. + # magnitude is noisy, so A can learn a mis-scaled per-layer step. + # Normalising each layer's delta to unit RMS decouples step size (set by + # eta) from that noise -- like normalized SGD. self.normalize_delta = normalize_delta @@ -327,6 +327,13 @@ def sdil_step(net, x, y, y_onehot, cfg, step, prev_error=None): delta = r_list[l] * gain # (B, n_l) if cfg.normalize_delta: delta = delta / (delta.pow(2).mean().sqrt() + 1e-8) + # For an interior residual block h'=h+alpha*phi(Wh), the local + # Jacobian dh'/dW contains alpha. Omitting it preserves direction + # but makes the effective step grow as 1/alpha=sqrt(depth), which + # confounds depth-scaling comparisons. The input projection (l=0) + # is plain rather than residual and therefore has scale 1. + if net.residual and l >= 1: + delta = net.res_alpha * delta dW = delta.t() @ h[l] / B # (n_l, n_{l-1}) db = delta.mean(0) _apply(net, l, dW, db, cfg) |