diff options
| author | YurenHao0426 <Blackhao0426@gmail.com> | 2026-03-30 19:25:53 -0500 |
|---|---|---|
| committer | YurenHao0426 <Blackhao0426@gmail.com> | 2026-03-30 19:25:53 -0500 |
| commit | 8b21fb32bf0997e3f4266c1c22414e49f1fdcfcc (patch) | |
| tree | 54e3c678c8d45330c6085b02a27de82cc884e17d | |
| parent | 2a230acd5ee3fa6605892d524badf281ba7e9cfd (diff) | |
Add confirmatory paper experiments: A1-A4, all 10 seeds complete
A1: Synthetic nonlinearity ladder (240 rows: 3 alpha × 2 depth × 4 methods × 10 seeds)
A2: CIFAR state-vs-credit counterexample (30 rows: 3 methods × 10 seeds)
A3: Frozen vs online dissociation (60 rows: 2 regimes × 3 methods × 10 seeds)
A4: Protocol dependence panel (82 rows: assembled from existing results)
All experiments ran on GPU 3. Total runtime: ~20 hours.
CSVs in results/confirmatory/.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
| -rw-r--r-- | experiments/confirmatory_paper_experiments.py | 1861 | ||||
| -rw-r--r-- | results/confirmatory/A1_synth_ladder.csv | 241 | ||||
| -rw-r--r-- | results/confirmatory/A1_synth_ladder.json | 2402 | ||||
| -rw-r--r-- | results/confirmatory/A2_cifar_state_vs_credit.csv | 31 | ||||
| -rw-r--r-- | results/confirmatory/A2_cifar_state_vs_credit.json | 242 | ||||
| -rw-r--r-- | results/confirmatory/A3_frozen_vs_online.csv | 61 | ||||
| -rw-r--r-- | results/confirmatory/A3_frozen_vs_online.json | 542 | ||||
| -rw-r--r-- | results/confirmatory/A4_protocol_dependence.csv | 83 | ||||
| -rw-r--r-- | results/confirmatory/A4_protocol_dependence.json | 494 |
9 files changed, 5957 insertions, 0 deletions
diff --git a/experiments/confirmatory_paper_experiments.py b/experiments/confirmatory_paper_experiments.py new file mode 100644 index 0000000..69f08de --- /dev/null +++ b/experiments/confirmatory_paper_experiments.py @@ -0,0 +1,1861 @@ +""" +Confirmatory Paper Experiments — single-script entry point. + +Four sub-experiments: + A1: Synthetic Nonlinearity Ladder (10 seeds x {alpha} x {depth}) + A2: CIFAR State-vs-Credit Counterexample (10 seeds) + A3: Frozen vs Online Dissociation (10 seeds) + A4: Protocol Dependence Panel (data assembly from existing results) + +Usage: + CUDA_VISIBLE_DEVICES=3 python experiments/confirmatory_paper_experiments.py \ + --experiment {A1,A2,A3,A4,all} --gpu 3 --output_dir results/confirmatory + +Set PYTHONUNBUFFERED=1 for nohup-safe logging. +""" +import os +import sys +import json +import argparse +import time +import copy +import csv +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.utils.data import DataLoader, TensorDataset +import torchvision +import torchvision.transforms as transforms + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from models.residual_mlp import ResidualMLP +from models.value_net import ValueNet, SinusoidalTimeEmbed, create_ema_model, update_ema +from models.state_bridge import StateBridgeNet +from metrics.credit_metrics import ( + cosine_similarity_batch, perturbation_correlation, nudging_test +) + + +# ============================================================================= +# Shared helpers +# ============================================================================= +def set_seed(seed): + torch.manual_seed(seed) + np.random.seed(seed) + torch.cuda.manual_seed_all(seed) + + +def serialize(obj): + if isinstance(obj, dict): + return {str(k): serialize(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [serialize(v) for v in obj] + elif isinstance(obj, (np.floating, np.integer)): + return float(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + elif isinstance(obj, torch.Tensor): + return obj.cpu().numpy().tolist() + return obj + + +def get_cifar10(batch_size=128): + transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616)), + ]) + transform_test = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616)), + ]) + trainset = torchvision.datasets.CIFAR10( + root='./data', train=True, download=True, transform=transform_train) + testset = torchvision.datasets.CIFAR10( + root='./data', train=False, download=True, transform=transform_test) + train_loader = DataLoader(trainset, batch_size=batch_size, shuffle=True, + num_workers=4, pin_memory=True) + test_loader = DataLoader(testset, batch_size=batch_size, shuffle=False, + num_workers=4, pin_memory=True) + return train_loader, test_loader + + +def evaluate_cifar(model, test_loader, device): + model.eval() + correct, total = 0, 0 + with torch.no_grad(): + for x, y in test_loader: + x = x.view(x.size(0), -1).to(device) + y = y.to(device) + logits = model(x) + correct += (logits.argmax(1) == y).sum().item() + total += x.size(0) + return correct / total + + +def evaluate_synth(model, test_loader, device): + model.eval() + correct, total = 0, 0 + with torch.no_grad(): + for x, y in test_loader: + x, y = x.to(device), y.to(device) + logits = model(x) + correct += (logits.argmax(1) == y).sum().item() + total += x.size(0) + return correct / total + + +def compute_diagnostics_generic(model, test_loader, device, num_classes, + method_name, value_net=None, + state_pred=None, dfa_Bs=None, + flat_input=True): + """ + Compute Gamma (offline BP cosine), rho (perturbation correlation), and nudge. + Returns mean over layers. + flat_input: if True, x is flattened before forward (CIFAR); else passed as-is (synth). + """ + model.eval() + if value_net is not None: + value_net.eval() + if state_pred is not None: + state_pred.eval() + + L = model.num_blocks + + for x, y in test_loader: + if flat_input: + x = x.view(x.size(0), -1).to(device) + else: + x = x.to(device) + y = y.to(device) + break + + batch = x.size(0) + + # BP gradients via manual graph + with torch.no_grad(): + if flat_input: + h0 = model.embed(x.detach()) + else: + h0 = x.detach() + h_start = h0.clone().requires_grad_(True) + hiddens_req = [h_start] + for block in model.blocks: + f = block(hiddens_req[-1]) + hiddens_req.append(hiddens_req[-1] + f) + + if flat_input: + logits_bp = model.out_head(model.out_ln(hiddens_req[-1])) + else: + logits_bp = model.out_head(hiddens_req[-1]) + loss_bp = F.cross_entropy(logits_bp, y) + grads = torch.autograd.grad(loss_bp, hiddens_req, retain_graph=False) + bp_grads = {l: grads[l].detach().clone() for l in range(len(hiddens_req))} + + # Clean forward + with torch.no_grad(): + if flat_input: + logits, hiddens = model(x, return_hidden=True) + else: + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + + gamma_list, rho_list, nudge_list = [], [], [] + + for l in range(L): + h_l = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + + if method_name == 'bp': + a_l = bp_grads[l] + elif method_name == 'dfa': + a_l = (e_T @ dfa_Bs[l].T).detach() + elif method_name == 'state_bridge': + h_l_req = h_l.clone().requires_grad_(True) + pred_hL = state_pred(h_l_req, t_l, s) + if flat_input: + pred_logits = model.out_head(model.out_ln(pred_hL)) + else: + pred_logits = model.out_head(pred_hL) + pred_loss = F.cross_entropy(pred_logits, y, reduction='sum') + a_l = torch.autograd.grad(pred_loss, h_l_req, create_graph=False)[0].detach() + elif method_name == 'credit_bridge': + h_l_req = h_l.clone().requires_grad_(True) + V_l = value_net(h_l_req, t_l, s) + a_l = torch.autograd.grad(V_l.sum(), h_l_req, create_graph=False)[0].detach() + else: + raise ValueError(f"Unknown method: {method_name}") + + gamma = cosine_similarity_batch(a_l, bp_grads[l]) + gamma_list.append(gamma) + + def make_fwd_fn(start_l): + def fwd_fn(h): + with torch.no_grad(): + curr = h + for i in range(start_l, L): + curr = curr + model.blocks[i](curr) + if flat_input: + out = model.out_head(model.out_ln(curr)) + else: + out = model.out_head(curr) + return F.cross_entropy(out, y, reduction='none') + return fwd_fn + + fwd_fn = make_fwd_fn(l) + rho = perturbation_correlation(h_l, a_l, fwd_fn, epsilon=1e-3, M=16) + rho_list.append(rho) + nudge = nudging_test(h_l, a_l, fwd_fn, eta=0.01) + nudge_list.append(nudge) + + return { + 'Gamma': float(np.mean(gamma_list)), + 'rho': float(np.mean(rho_list)), + 'nudge': float(np.mean(nudge_list)), + 'per_layer_gamma': gamma_list, + 'per_layer_rho': rho_list, + 'per_layer_nudge': nudge_list, + } + + +# ============================================================================= +# Shared training methods (CIFAR-style: flat input, out_ln present) +# ============================================================================= + +def _train_bp_cifar(model, train_loader, test_loader, device, epochs, lr, wd): + optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=wd) + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) + log = {'train_loss': [], 'test_acc': []} + for epoch in range(1, epochs + 1): + model.train() + total_loss, correct, total = 0, 0, 0 + for x, y in train_loader: + x = x.view(x.size(0), -1).to(device) + y = y.to(device) + logits = model(x) + loss = F.cross_entropy(logits, y) + optimizer.zero_grad() + loss.backward() + optimizer.step() + total_loss += loss.item() * x.size(0) + correct += (logits.argmax(1) == y).sum().item() + total += x.size(0) + scheduler.step() + log['train_loss'].append(total_loss / total) + log['test_acc'].append(evaluate_cifar(model, test_loader, device)) + if epoch % 10 == 0 or epoch == 1: + print(f" [BP] Ep {epoch}: loss={log['train_loss'][-1]:.4f} " + f"test={log['test_acc'][-1]:.4f}", flush=True) + return log + + +def _train_dfa_cifar(model, train_loader, test_loader, device, epochs, lr, wd): + d = model.d_hidden + L = model.num_blocks + C = 10 + Bs = [torch.randn(d, C, device=device) / np.sqrt(C) for _ in range(L)] + block_opts = [optim.AdamW(block.parameters(), lr=lr, weight_decay=wd) + for block in model.blocks] + embed_opt = optim.AdamW(model.embed.parameters(), lr=lr, weight_decay=wd) + head_opt = optim.AdamW(list(model.out_head.parameters()) + + list(model.out_ln.parameters()), lr=lr, weight_decay=wd) + all_sch = ([optim.lr_scheduler.CosineAnnealingLR(o, T_max=epochs) for o in block_opts] + + [optim.lr_scheduler.CosineAnnealingLR(embed_opt, T_max=epochs), + optim.lr_scheduler.CosineAnnealingLR(head_opt, T_max=epochs)]) + log = {'train_loss': [], 'test_acc': []} + for epoch in range(1, epochs + 1): + model.train() + total_loss, correct, total = 0, 0, 0 + for x, y in train_loader: + x = x.view(x.size(0), -1).to(device) + y = y.to(device) + batch = x.size(0) + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + loss_val = F.cross_entropy(logits, y) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + hL_det = hiddens[-1].detach() + logits_out = model.out_head(model.out_ln(hL_det)) + loss_out = F.cross_entropy(logits_out, y) + head_opt.zero_grad() + loss_out.backward() + head_opt.step() + for l in range(L): + h_l = hiddens[l].detach() + a_dfa = (e_T @ Bs[l].T).detach() + rms = (a_dfa ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + a_norm = a_dfa / rms + f_l = model.blocks[l](h_l) + local_loss = (f_l * a_norm).sum(dim=-1).mean() + block_opts[l].zero_grad() + local_loss.backward() + torch.nn.utils.clip_grad_norm_(model.blocks[l].parameters(), 1.0) + block_opts[l].step() + a_0 = (e_T @ Bs[0].T).detach() + rms_0 = (a_0 ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + h0 = model.embed(x) + embed_loss = (h0 * (a_0 / rms_0)).sum(dim=-1).mean() + embed_opt.zero_grad() + embed_loss.backward() + embed_opt.step() + total_loss += loss_val.item() * batch + correct += (logits.argmax(1) == y).sum().item() + total += batch + for s in all_sch: + s.step() + log['train_loss'].append(total_loss / total) + log['test_acc'].append(evaluate_cifar(model, test_loader, device)) + if epoch % 10 == 0 or epoch == 1: + print(f" [DFA] Ep {epoch}: loss={log['train_loss'][-1]:.4f} " + f"test={log['test_acc'][-1]:.4f}", flush=True) + return log, Bs + + +def _train_state_bridge_cifar(model, train_loader, test_loader, device, epochs, lr, lr_fb, wd): + d = model.d_hidden + L = model.num_blocks + C = 10 + state_pred = StateBridgeNet(d_hidden=d, s_dim=C, time_embed_dim=32, + hidden_dim=256, num_layers=3).to(device) + block_opts = [optim.AdamW(block.parameters(), lr=lr, weight_decay=wd) + for block in model.blocks] + embed_opt = optim.AdamW(model.embed.parameters(), lr=lr, weight_decay=wd) + head_opt = optim.AdamW(list(model.out_head.parameters()) + + list(model.out_ln.parameters()), lr=lr, weight_decay=wd) + state_opt = optim.Adam(state_pred.parameters(), lr=lr_fb) + all_sch = ([optim.lr_scheduler.CosineAnnealingLR(o, T_max=epochs) for o in block_opts] + + [optim.lr_scheduler.CosineAnnealingLR(embed_opt, T_max=epochs), + optim.lr_scheduler.CosineAnnealingLR(head_opt, T_max=epochs)]) + log = {'train_loss': [], 'test_acc': [], 'state_pred_error': []} + for epoch in range(1, epochs + 1): + model.train() + state_pred.train() + total_loss, correct, total, total_se = 0, 0, 0, 0 + for x, y in train_loader: + x = x.view(x.size(0), -1).to(device) + y = y.to(device) + batch = x.size(0) + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + loss_val = F.cross_entropy(logits, y) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + hL_det = hiddens[-1].detach() + # Train state predictor + state_loss = 0.0 + for l in range(L): + h_l_det = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + pred_hL = state_pred(h_l_det, t_l, s) + target_norm = hL_det.norm(dim=-1, keepdim=True).clamp(min=1.0) + state_loss = state_loss + (((pred_hL - hL_det) / target_norm) ** 2).sum(dim=-1).mean() + state_loss = state_loss / L + state_opt.zero_grad() + state_loss.backward() + state_opt.step() + total_se += state_loss.item() * batch + # Compute credits + credits = [] + for l in range(L): + h_l_det = hiddens[l].detach().requires_grad_(True) + t_l = torch.full((batch,), l / L, device=device) + pred_hL = state_pred(h_l_det, t_l, s) + pred_logits = model.out_head(model.out_ln(pred_hL)) + pred_loss = F.cross_entropy(pred_logits, y, reduction='sum') + a_l = torch.autograd.grad(pred_loss, h_l_det, create_graph=False)[0] + credits.append(a_l.detach()) + # Update head + logits_out = model.out_head(model.out_ln(hL_det)) + loss_out = F.cross_entropy(logits_out, y) + head_opt.zero_grad() + loss_out.backward() + head_opt.step() + # Update blocks + for l in range(L): + h_l = hiddens[l].detach() + a = credits[l] + rms = (a ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + a_norm = a / rms + f_l = model.blocks[l](h_l) + local_loss = (f_l * a_norm).sum(dim=-1).mean() + block_opts[l].zero_grad() + local_loss.backward() + torch.nn.utils.clip_grad_norm_(model.blocks[l].parameters(), 1.0) + block_opts[l].step() + # Update embedding + a_0 = credits[0] + rms_0 = (a_0 ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + h0 = model.embed(x) + embed_loss = (h0 * (a_0 / rms_0)).sum(dim=-1).mean() + embed_opt.zero_grad() + embed_loss.backward() + embed_opt.step() + total_loss += loss_val.item() * batch + correct += (logits.argmax(1) == y).sum().item() + total += batch + for sch in all_sch: + sch.step() + log['train_loss'].append(total_loss / total) + log['test_acc'].append(evaluate_cifar(model, test_loader, device)) + log['state_pred_error'].append(total_se / total) + if epoch % 10 == 0 or epoch == 1: + print(f" [SB] Ep {epoch}: loss={log['train_loss'][-1]:.4f} " + f"test={log['test_acc'][-1]:.4f} se={log['state_pred_error'][-1]:.4f}", + flush=True) + return log, state_pred + + +def _train_credit_bridge_cifar(model, train_loader, test_loader, device, epochs, lr, lr_fb, wd, + warmup_ratio=0.2, term_grad_weight=1.0, + lam=0.1, K=4, sigma_bridge=0.05, ema_momentum=0.995): + d = model.d_hidden + L = model.num_blocks + C = 10 + warmup_epochs = max(1, int(epochs * warmup_ratio)) + value_net = ValueNet(d_hidden=d, s_dim=C, time_embed_dim=32, + hidden_dim=256, num_layers=3).to(device) + value_net_ema = create_ema_model(value_net) + Bs_fallback = [torch.randn(d, C, device=device) / np.sqrt(C) for _ in range(L)] + block_opts = [optim.AdamW(block.parameters(), lr=lr, weight_decay=wd) + for block in model.blocks] + embed_opt = optim.AdamW(model.embed.parameters(), lr=lr, weight_decay=wd) + head_opt = optim.AdamW(list(model.out_head.parameters()) + + list(model.out_ln.parameters()), lr=lr, weight_decay=wd) + value_opt = optim.Adam(value_net.parameters(), lr=lr_fb) + all_sch = ([optim.lr_scheduler.CosineAnnealingLR(o, T_max=epochs) for o in block_opts] + + [optim.lr_scheduler.CosineAnnealingLR(embed_opt, T_max=epochs), + optim.lr_scheduler.CosineAnnealingLR(head_opt, T_max=epochs)]) + log = {'train_loss': [], 'test_acc': [], 'value_loss': []} + for epoch in range(1, epochs + 1): + model.train() + value_net.train() + total_loss, correct, total, total_vloss = 0, 0, 0, 0 + if epoch <= warmup_epochs: + credit_blend = 0.0 + else: + credit_blend = min(1.0, (epoch - warmup_epochs) / max(1, warmup_epochs)) + for x, y in train_loader: + x = x.view(x.size(0), -1).to(device) + y = y.to(device) + batch = x.size(0) + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + loss_val = F.cross_entropy(logits, y) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + true_loss = F.cross_entropy(logits, y, reduction='none').detach() + hL_det = hiddens[-1].detach() + # Train value net + t_L = torch.ones(batch, device=device) + V_terminal = value_net(hL_det, t_L, s) + loss_term = ((V_terminal - true_loss) ** 2).mean() + loss_tgrad = torch.tensor(0.0, device=device) + if term_grad_weight > 0: + hL_req = hL_det.clone().requires_grad_(True) + V_at_L = value_net(hL_req, t_L, s) + grad_V_L = torch.autograd.grad(V_at_L.sum(), hL_req, create_graph=True)[0] + hL_req2 = hL_det.clone().requires_grad_(True) + logits_tgt = model.out_head(model.out_ln(hL_req2)) + ce_loss = F.cross_entropy(logits_tgt, y, reduction='sum') + a_L_exact = torch.autograd.grad(ce_loss, hL_req2, create_graph=False)[0].detach() + loss_tgrad = ((grad_V_L - a_L_exact) ** 2).sum(dim=-1).mean() + loss_bridge = 0.0 + for l in range(L): + h_l_det = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + t_l_next = torch.full((batch,), (l + 1) / L, device=device) + V_l = value_net(h_l_det, t_l, s) + with torch.no_grad(): + h_next_det = hiddens[l + 1].detach() + log_terms = [] + for k in range(K): + noise = sigma_bridge * torch.randn_like(h_next_det) + V_next = value_net_ema(h_next_det + noise, t_l_next, s) + log_terms.append(-V_next / lam) + log_stack = torch.stack(log_terms, dim=-1) + V_target = -lam * (torch.logsumexp(log_stack, dim=-1) - np.log(K)) + loss_bridge = loss_bridge + ((V_l - V_target.detach()) ** 2).mean() + loss_bridge = loss_bridge / L + value_loss = loss_term + loss_bridge + term_grad_weight * loss_tgrad + value_opt.zero_grad() + value_loss.backward() + torch.nn.utils.clip_grad_norm_(value_net.parameters(), 1.0) + value_opt.step() + update_ema(value_net, value_net_ema, ema_momentum) + total_vloss += value_loss.item() * batch + # Credits + cb_credits = [] + for l in range(L): + h_l_det = hiddens[l].detach().requires_grad_(True) + t_l = torch.full((batch,), l / L, device=device) + V_l = value_net(h_l_det, t_l, s) + a_l = torch.autograd.grad(V_l.sum(), h_l_det, create_graph=False)[0] + cb_credits.append(a_l.detach()) + dfa_credits = [(e_T @ Bs_fallback[l].T).detach() for l in range(L)] + credits = [] + for l in range(L): + if credit_blend >= 1.0: + a = cb_credits[l] + elif credit_blend <= 0.0: + a = dfa_credits[l] + else: + cb_rms = (cb_credits[l] ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + dfa_rms = (dfa_credits[l] ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + a = credit_blend * (cb_credits[l] / cb_rms) + \ + (1 - credit_blend) * (dfa_credits[l] / dfa_rms) + credits.append(a) + # Update head + logits_out = model.out_head(model.out_ln(hL_det)) + loss_out = F.cross_entropy(logits_out, y) + head_opt.zero_grad() + loss_out.backward() + head_opt.step() + # Update blocks + for l in range(L): + h_l = hiddens[l].detach() + a = credits[l] + rms = (a ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + a_norm = a / rms + f_l = model.blocks[l](h_l) + local_loss = (f_l * a_norm).sum(dim=-1).mean() + block_opts[l].zero_grad() + local_loss.backward() + torch.nn.utils.clip_grad_norm_(model.blocks[l].parameters(), 1.0) + block_opts[l].step() + # Update embedding + a_0 = credits[0] + rms_0 = (a_0 ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + h0 = model.embed(x) + embed_loss = (h0 * (a_0 / rms_0)).sum(dim=-1).mean() + embed_opt.zero_grad() + embed_loss.backward() + embed_opt.step() + total_loss += loss_val.item() * batch + correct += (logits.argmax(1) == y).sum().item() + total += batch + for sch in all_sch: + sch.step() + log['train_loss'].append(total_loss / total) + log['test_acc'].append(evaluate_cifar(model, test_loader, device)) + log['value_loss'].append(total_vloss / total) + if epoch % 10 == 0 or epoch == 1: + phase = "warmup" if epoch <= warmup_epochs else f"blend={credit_blend:.2f}" + print(f" [CB] Ep {epoch} ({phase}): loss={log['train_loss'][-1]:.4f} " + f"test={log['test_acc'][-1]:.4f}", flush=True) + return log, value_net + + +# ============================================================================= +# A1: Synthetic Nonlinearity Ladder +# ============================================================================= + +class TeacherNet: + """Fixed teacher network with controllable nonlinearity.""" + def __init__(self, d_hidden, num_blocks, num_classes, alpha, seed=0): + rng = np.random.RandomState(seed) + self.d_hidden = d_hidden + self.num_blocks = num_blocks + self.num_classes = num_classes + self.alpha = alpha + self.Ws = [] + for l in range(num_blocks): + W = rng.randn(d_hidden, d_hidden).astype(np.float32) + W = W / (np.linalg.norm(W, ord=2) + 1e-8) * 0.3 + self.Ws.append(torch.from_numpy(W)) + U = rng.randn(num_classes, d_hidden).astype(np.float32) + U = U / (np.linalg.norm(U, ord=2) + 1e-8) + self.U = torch.from_numpy(U) + + def to(self, device): + self.Ws = [W.to(device) for W in self.Ws] + self.U = self.U.to(device) + return self + + def phi(self, z): + return (1 - self.alpha) * z + self.alpha * torch.tanh(z) + + def forward(self, h0): + h = h0 + hiddens = [h] + for l in range(self.num_blocks): + f = F.linear(self.phi(h), self.Ws[l]) + h = h + f + hiddens.append(h) + logits = F.linear(h, self.U) + return logits, hiddens + + +class StudentBlock(nn.Module): + def __init__(self, d_hidden, alpha): + super().__init__() + self.ln = nn.LayerNorm(d_hidden) + self.w = nn.Linear(d_hidden, d_hidden, bias=False) + self.alpha = alpha + nn.init.normal_(self.w.weight, std=0.01) + + def phi(self, z): + return (1 - self.alpha) * z + self.alpha * torch.tanh(z) + + def forward(self, h): + return self.w(self.phi(self.ln(h))) + + +class StudentNet(nn.Module): + def __init__(self, d_hidden, num_classes, num_blocks, alpha): + super().__init__() + self.blocks = nn.ModuleList([StudentBlock(d_hidden, alpha) for _ in range(num_blocks)]) + self.out_head = nn.Linear(d_hidden, num_classes) + self.num_blocks = num_blocks + self.d_hidden = d_hidden + + def forward(self, x, return_hidden=False): + h = x + hiddens = [h] if return_hidden else None + for block in self.blocks: + f = block(h) + h = h + f + if return_hidden: + hiddens.append(h) + logits = self.out_head(h) + if return_hidden: + return logits, hiddens + return logits + + def forward_from_layer(self, h, start_layer): + for i in range(start_layer, self.num_blocks): + f = self.blocks[i](h) + h = h + f + return self.out_head(h) + + +def generate_synth_dataset(teacher, num_samples, d_hidden, device, seed=0): + torch.manual_seed(seed) + X = torch.randn(num_samples, d_hidden, device=device) + with torch.no_grad(): + logits, _ = teacher.forward(X) + Y = logits.argmax(dim=-1) + return X, Y + + +def _train_bp_synth(model, train_loader, test_loader, device, epochs, lr, wd): + optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=wd) + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) + log = {'test_acc': []} + for epoch in range(1, epochs + 1): + model.train() + for x, y in train_loader: + x, y = x.to(device), y.to(device) + logits = model(x) + loss = F.cross_entropy(logits, y) + optimizer.zero_grad() + loss.backward() + optimizer.step() + scheduler.step() + log['test_acc'].append(evaluate_synth(model, test_loader, device)) + if epoch % 20 == 0 or epoch == 1: + print(f" [BP] Ep {epoch}: test={log['test_acc'][-1]:.4f}", flush=True) + return log + + +def _train_dfa_synth(model, train_loader, test_loader, device, epochs, lr, wd, C): + d = model.d_hidden + L = model.num_blocks + Bs = [torch.randn(d, C, device=device) / np.sqrt(C) for _ in range(L)] + block_opts = [optim.AdamW(block.parameters(), lr=lr, weight_decay=wd) + for block in model.blocks] + head_opt = optim.AdamW(model.out_head.parameters(), lr=lr, weight_decay=wd) + all_sch = ([optim.lr_scheduler.CosineAnnealingLR(o, T_max=epochs) for o in block_opts] + + [optim.lr_scheduler.CosineAnnealingLR(head_opt, T_max=epochs)]) + log = {'test_acc': []} + for epoch in range(1, epochs + 1): + model.train() + for x, y in train_loader: + x, y = x.to(device), y.to(device) + batch = x.size(0) + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + hL_det = hiddens[-1].detach() + logits_out = model.out_head(hL_det) + loss_out = F.cross_entropy(logits_out, y) + head_opt.zero_grad() + loss_out.backward() + head_opt.step() + for l in range(L): + h_l = hiddens[l].detach() + a_dfa = (e_T @ Bs[l].T).detach() + rms = (a_dfa ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + a_norm = a_dfa / rms + f_l = model.blocks[l](h_l) + local_loss = (f_l * a_norm).sum(dim=-1).mean() + block_opts[l].zero_grad() + local_loss.backward() + torch.nn.utils.clip_grad_norm_(model.blocks[l].parameters(), 1.0) + block_opts[l].step() + for s in all_sch: + s.step() + log['test_acc'].append(evaluate_synth(model, test_loader, device)) + if epoch % 20 == 0 or epoch == 1: + print(f" [DFA] Ep {epoch}: test={log['test_acc'][-1]:.4f}", flush=True) + return log, Bs + + +def _train_state_bridge_synth(model, train_loader, test_loader, device, epochs, lr, lr_fb, wd, C): + d = model.d_hidden + L = model.num_blocks + state_pred = StateBridgeNet(d_hidden=d, s_dim=C, time_embed_dim=32, + hidden_dim=256, num_layers=3).to(device) + block_opts = [optim.AdamW(block.parameters(), lr=lr, weight_decay=wd) + for block in model.blocks] + head_opt = optim.AdamW(model.out_head.parameters(), lr=lr, weight_decay=wd) + state_opt = optim.Adam(state_pred.parameters(), lr=lr_fb) + all_sch = ([optim.lr_scheduler.CosineAnnealingLR(o, T_max=epochs) for o in block_opts] + + [optim.lr_scheduler.CosineAnnealingLR(head_opt, T_max=epochs)]) + log = {'test_acc': [], 'state_pred_error': []} + for epoch in range(1, epochs + 1): + model.train() + state_pred.train() + total_se, n = 0.0, 0 + for x, y in train_loader: + x, y = x.to(device), y.to(device) + batch = x.size(0) + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + hL_det = hiddens[-1].detach() + # Train state predictor + state_loss = 0.0 + for l in range(L): + h_l_det = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + pred_hL = state_pred(h_l_det, t_l, s) + target_norm = hL_det.norm(dim=-1, keepdim=True).clamp(min=1.0) + state_loss = state_loss + (((pred_hL - hL_det) / target_norm) ** 2).sum(dim=-1).mean() + state_loss = state_loss / L + state_opt.zero_grad() + state_loss.backward() + state_opt.step() + total_se += state_loss.item() * batch + n += batch + # Credits + credits = [] + for l in range(L): + h_l_det = hiddens[l].detach().requires_grad_(True) + t_l = torch.full((batch,), l / L, device=device) + pred_hL = state_pred(h_l_det, t_l, s) + pred_logits = model.out_head(pred_hL) + pred_loss = F.cross_entropy(pred_logits, y, reduction='sum') + a_l = torch.autograd.grad(pred_loss, h_l_det, create_graph=False)[0] + credits.append(a_l.detach()) + # Update head + logits_out = model.out_head(hL_det) + loss_out = F.cross_entropy(logits_out, y) + head_opt.zero_grad() + loss_out.backward() + head_opt.step() + # Update blocks + for l in range(L): + h_l = hiddens[l].detach() + a = credits[l] + rms = (a ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + a_norm = a / rms + f_l = model.blocks[l](h_l) + local_loss = (f_l * a_norm).sum(dim=-1).mean() + block_opts[l].zero_grad() + local_loss.backward() + torch.nn.utils.clip_grad_norm_(model.blocks[l].parameters(), 1.0) + block_opts[l].step() + for sch in all_sch: + sch.step() + log['test_acc'].append(evaluate_synth(model, test_loader, device)) + log['state_pred_error'].append(total_se / n) + if epoch % 20 == 0 or epoch == 1: + print(f" [SB] Ep {epoch}: test={log['test_acc'][-1]:.4f} " + f"se={log['state_pred_error'][-1]:.4f}", flush=True) + return log, state_pred + + +def _train_credit_bridge_synth(model, train_loader, test_loader, device, epochs, lr, lr_fb, wd, C, + warmup_ratio=0.2, term_grad_weight=1.0, + lam=0.1, K=4, sigma_bridge=0.05, ema_momentum=0.995): + d = model.d_hidden + L = model.num_blocks + warmup_epochs = max(1, int(epochs * warmup_ratio)) + value_net = ValueNet(d_hidden=d, s_dim=C, time_embed_dim=32, + hidden_dim=256, num_layers=3).to(device) + value_net_ema = create_ema_model(value_net) + Bs_fallback = [torch.randn(d, C, device=device) / np.sqrt(C) for _ in range(L)] + block_opts = [optim.AdamW(block.parameters(), lr=lr, weight_decay=wd) + for block in model.blocks] + head_opt = optim.AdamW(model.out_head.parameters(), lr=lr, weight_decay=wd) + value_opt = optim.Adam(value_net.parameters(), lr=lr_fb) + all_sch = ([optim.lr_scheduler.CosineAnnealingLR(o, T_max=epochs) for o in block_opts] + + [optim.lr_scheduler.CosineAnnealingLR(head_opt, T_max=epochs)]) + log = {'test_acc': []} + for epoch in range(1, epochs + 1): + model.train() + value_net.train() + if epoch <= warmup_epochs: + credit_blend = 0.0 + else: + credit_blend = min(1.0, (epoch - warmup_epochs) / max(1, warmup_epochs)) + for x, y in train_loader: + x, y = x.to(device), y.to(device) + batch = x.size(0) + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + true_loss = F.cross_entropy(logits, y, reduction='none').detach() + hL_det = hiddens[-1].detach() + # Value net training + t_L = torch.ones(batch, device=device) + V_terminal = value_net(hL_det, t_L, s) + loss_term = ((V_terminal - true_loss) ** 2).mean() + loss_tgrad = torch.tensor(0.0, device=device) + if term_grad_weight > 0: + hL_req = hL_det.clone().requires_grad_(True) + V_at_L = value_net(hL_req, t_L, s) + grad_V_L = torch.autograd.grad(V_at_L.sum(), hL_req, create_graph=True)[0] + hL_req2 = hL_det.clone().requires_grad_(True) + logits_tgt = model.out_head(hL_req2) + ce_loss = F.cross_entropy(logits_tgt, y, reduction='sum') + a_L_exact = torch.autograd.grad(ce_loss, hL_req2, create_graph=False)[0].detach() + loss_tgrad = ((grad_V_L - a_L_exact) ** 2).sum(dim=-1).mean() + loss_bridge = 0.0 + for l in range(L): + h_l_det = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + t_l_next = torch.full((batch,), (l + 1) / L, device=device) + V_l = value_net(h_l_det, t_l, s) + with torch.no_grad(): + h_next_det = hiddens[l + 1].detach() + log_terms = [] + for k in range(K): + noise = sigma_bridge * torch.randn_like(h_next_det) + V_next = value_net_ema(h_next_det + noise, t_l_next, s) + log_terms.append(-V_next / lam) + log_stack = torch.stack(log_terms, dim=-1) + V_target = -lam * (torch.logsumexp(log_stack, dim=-1) - np.log(K)) + loss_bridge = loss_bridge + ((V_l - V_target.detach()) ** 2).mean() + loss_bridge = loss_bridge / L + value_loss = loss_term + loss_bridge + term_grad_weight * loss_tgrad + value_opt.zero_grad() + value_loss.backward() + torch.nn.utils.clip_grad_norm_(value_net.parameters(), 1.0) + value_opt.step() + update_ema(value_net, value_net_ema, ema_momentum) + # Credits + cb_credits = [] + for l in range(L): + h_l_det = hiddens[l].detach().requires_grad_(True) + t_l = torch.full((batch,), l / L, device=device) + V_l = value_net(h_l_det, t_l, s) + a_l = torch.autograd.grad(V_l.sum(), h_l_det, create_graph=False)[0] + cb_credits.append(a_l.detach()) + dfa_credits = [(e_T @ Bs_fallback[l].T).detach() for l in range(L)] + credits = [] + for l in range(L): + if credit_blend >= 1.0: + a = cb_credits[l] + elif credit_blend <= 0.0: + a = dfa_credits[l] + else: + cb_rms = (cb_credits[l] ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + dfa_rms = (dfa_credits[l] ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + a = (credit_blend * (cb_credits[l] / cb_rms) + + (1 - credit_blend) * (dfa_credits[l] / dfa_rms)) + credits.append(a) + # Update head + logits_out = model.out_head(hL_det) + loss_out = F.cross_entropy(logits_out, y) + head_opt.zero_grad() + loss_out.backward() + head_opt.step() + # Update blocks + for l in range(L): + h_l = hiddens[l].detach() + a = credits[l] + rms = (a ** 2).mean(dim=-1, keepdim=True).sqrt() + 1e-6 + a_norm = a / rms + f_l = model.blocks[l](h_l) + local_loss = (f_l * a_norm).sum(dim=-1).mean() + block_opts[l].zero_grad() + local_loss.backward() + torch.nn.utils.clip_grad_norm_(model.blocks[l].parameters(), 1.0) + block_opts[l].step() + for sch in all_sch: + sch.step() + log['test_acc'].append(evaluate_synth(model, test_loader, device)) + if epoch % 20 == 0 or epoch == 1: + print(f" [CB] Ep {epoch}: test={log['test_acc'][-1]:.4f}", flush=True) + return log, value_net + + +def _compute_synth_state_err(model, state_pred, test_loader, device, C): + """Compute mean per-layer state prediction error on synth test set.""" + model.eval() + state_pred.eval() + L = model.num_blocks + total_se, n = 0.0, 0 + with torch.no_grad(): + for x, y in test_loader: + x, y = x.to(device), y.to(device) + batch = x.size(0) + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + hL_det = hiddens[-1].detach() + se = 0.0 + for l in range(L): + h_l_det = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + pred_hL = state_pred(h_l_det, t_l, s) + target_norm = hL_det.norm(dim=-1, keepdim=True).clamp(min=1.0) + se += (((pred_hL - hL_det) / target_norm) ** 2).sum(dim=-1).mean().item() + total_se += (se / L) * batch + n += batch + return total_se / n + + +def _compute_synth_diagnostics(model, test_loader, device, method_name, + value_net=None, state_pred=None, dfa_Bs=None, C=10): + """Compute Gamma, rho for synth model (no flat input, no out_ln).""" + model.eval() + if value_net is not None: + value_net.eval() + if state_pred is not None: + state_pred.eval() + + L = model.num_blocks + + for x, y in test_loader: + x, y = x.to(device), y.to(device) + break + batch = x.size(0) + + # BP gradients + h_list = [x.detach().requires_grad_(True)] + for block in model.blocks: + f = block(h_list[-1]) + h_list.append(h_list[-1] + f) + logits_bp = model.out_head(h_list[-1]) + loss_bp = F.cross_entropy(logits_bp, y) + grads = torch.autograd.grad(loss_bp, h_list, retain_graph=False) + bp_grads = {l: grads[l].detach().clone() for l in range(len(h_list))} + + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + + gamma_list, rho_list = [], [] + for l in range(L): + h_l = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + + if method_name == 'bp': + a_l = bp_grads[l] + elif method_name == 'dfa': + a_l = (e_T @ dfa_Bs[l].T).detach() + elif method_name == 'state_bridge': + h_l_req = h_l.clone().requires_grad_(True) + pred_hL = state_pred(h_l_req, t_l, s) + pred_logits = model.out_head(pred_hL) + pred_loss = F.cross_entropy(pred_logits, y, reduction='sum') + a_l = torch.autograd.grad(pred_loss, h_l_req, create_graph=False)[0].detach() + elif method_name == 'credit_bridge': + h_l_req = h_l.clone().requires_grad_(True) + V_l = value_net(h_l_req, t_l, s) + a_l = torch.autograd.grad(V_l.sum(), h_l_req, create_graph=False)[0].detach() + else: + raise ValueError(f"Unknown method: {method_name}") + + gamma = cosine_similarity_batch(a_l, bp_grads[l]) + gamma_list.append(gamma) + + def make_fwd_fn(start_l): + def fwd_fn(h): + with torch.no_grad(): + curr = h + for i in range(start_l, L): + curr = curr + model.blocks[i](curr) + out = model.out_head(curr) + return F.cross_entropy(out, y, reduction='none') + return fwd_fn + + fwd_fn = make_fwd_fn(l) + rho = perturbation_correlation(h_l, a_l, fwd_fn, epsilon=1e-3, M=16) + rho_list.append(rho) + + return { + 'Gamma': float(np.mean(gamma_list)), + 'rho': float(np.mean(rho_list)), + } + + +def run_A1(args, device): + """A1: Synthetic Nonlinearity Ladder — 10 seeds.""" + print("\n" + "=" * 70) + print("A1: Synthetic Nonlinearity Ladder") + print("=" * 70, flush=True) + + alphas = [0.0, 0.5, 1.0] + depths = [4, 8] + seeds = [42, 123, 456, 789, 1024, 2048, 3000, 4000, 5000, 6000] + d = 128 + C = 10 + epochs = 80 + steps_per_epoch = 50 + batch_size = 256 + n_train = steps_per_epoch * batch_size + n_test = 2000 + lr = 1e-3 + lr_fb = 1e-3 + wd = 0.01 + + os.makedirs(args.output_dir, exist_ok=True) + csv_path = os.path.join(args.output_dir, 'A1_synth_ladder.csv') + rows = [] + + total_configs = len(alphas) * len(depths) * len(seeds) + done = 0 + + for alpha in alphas: + for L in depths: + for seed in seeds: + done += 1 + print(f"\n[A1] alpha={alpha}, L={L}, seed={seed} ({done}/{total_configs})", flush=True) + set_seed(seed) + + teacher = TeacherNet(d, L, C, alpha, seed=0).to(device) + X_train, Y_train = generate_synth_dataset(teacher, n_train, d, device, seed=seed) + X_test, Y_test = generate_synth_dataset(teacher, n_test, d, device, seed=seed + 10000) + train_ds = TensorDataset(X_train, Y_train) + test_ds = TensorDataset(X_test, Y_test) + train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True) + test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False) + + # BP + print(" [BP]", flush=True) + set_seed(seed) + model_bp = StudentNet(d, C, L, alpha).to(device) + bp_log = _train_bp_synth(model_bp, train_loader, test_loader, device, epochs, lr, wd) + bp_diag = _compute_synth_diagnostics(model_bp, test_loader, device, 'bp', C=C) + rows.append({ + 'alpha': alpha, 'depth': L, 'method': 'bp', 'seed': seed, + 'StateErr': float('nan'), + 'Gamma': bp_diag['Gamma'], 'rho': bp_diag['rho'], + 'acc': bp_log['test_acc'][-1], + }) + + # DFA + print(" [DFA]", flush=True) + set_seed(seed) + model_dfa = StudentNet(d, C, L, alpha).to(device) + dfa_log, dfa_Bs = _train_dfa_synth(model_dfa, train_loader, test_loader, device, + epochs, lr, wd, C) + dfa_diag = _compute_synth_diagnostics(model_dfa, test_loader, device, 'dfa', + dfa_Bs=dfa_Bs, C=C) + rows.append({ + 'alpha': alpha, 'depth': L, 'method': 'dfa', 'seed': seed, + 'StateErr': float('nan'), + 'Gamma': dfa_diag['Gamma'], 'rho': dfa_diag['rho'], + 'acc': dfa_log['test_acc'][-1], + }) + + # State Bridge + print(" [SB]", flush=True) + set_seed(seed) + model_sb = StudentNet(d, C, L, alpha).to(device) + sb_log, state_pred = _train_state_bridge_synth(model_sb, train_loader, test_loader, + device, epochs, lr, lr_fb, wd, C) + sb_diag = _compute_synth_diagnostics(model_sb, test_loader, device, 'state_bridge', + state_pred=state_pred, C=C) + state_err = _compute_synth_state_err(model_sb, state_pred, test_loader, device, C) + rows.append({ + 'alpha': alpha, 'depth': L, 'method': 'state_bridge', 'seed': seed, + 'StateErr': state_err, + 'Gamma': sb_diag['Gamma'], 'rho': sb_diag['rho'], + 'acc': sb_log['test_acc'][-1], + }) + + # Credit Bridge (Scalar eT) + print(" [CB]", flush=True) + set_seed(seed) + model_cb = StudentNet(d, C, L, alpha).to(device) + cb_log, vnet = _train_credit_bridge_synth(model_cb, train_loader, test_loader, + device, epochs, lr, lr_fb, wd, C) + cb_diag = _compute_synth_diagnostics(model_cb, test_loader, device, 'credit_bridge', + value_net=vnet, C=C) + rows.append({ + 'alpha': alpha, 'depth': L, 'method': 'credit_bridge', 'seed': seed, + 'StateErr': float('nan'), + 'Gamma': cb_diag['Gamma'], 'rho': cb_diag['rho'], + 'acc': cb_log['test_acc'][-1], + }) + + print(f" Summary: BP={bp_log['test_acc'][-1]:.4f} " + f"DFA={dfa_log['test_acc'][-1]:.4f} " + f"SB={sb_log['test_acc'][-1]:.4f}(se={state_err:.4f}) " + f"CB={cb_log['test_acc'][-1]:.4f}", flush=True) + + # Save CSV + fieldnames = ['alpha', 'depth', 'method', 'seed', 'StateErr', 'Gamma', 'rho', 'acc'] + with open(csv_path, 'w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + print(f"\n[A1] Saved {len(rows)} rows to {csv_path}", flush=True) + + # Also save JSON for debugging + json_path = csv_path.replace('.csv', '.json') + with open(json_path, 'w') as f: + json.dump(serialize(rows), f, indent=2) + return rows + + +# ============================================================================= +# A2: CIFAR State-vs-Credit Counterexample +# ============================================================================= + +def run_A2(args, device): + """A2: CIFAR State-vs-Credit Counterexample — 10 seeds.""" + print("\n" + "=" * 70) + print("A2: CIFAR State-vs-Credit Counterexample") + print("=" * 70, flush=True) + + seeds = [42, 123, 456, 789, 1024, 2048, 3000, 4000, 5000, 6000] + L = 4 + d = 256 + epochs = 100 + lr = 1e-3 + lr_fb = 1e-3 + wd = 0.01 + input_dim = 32 * 32 * 3 + C = 10 + + os.makedirs(args.output_dir, exist_ok=True) + csv_path = os.path.join(args.output_dir, 'A2_cifar_state_vs_credit.csv') + rows = [] + + train_loader, test_loader = get_cifar10(batch_size=128) + + for i, seed in enumerate(seeds): + print(f"\n[A2] Seed {seed} ({i+1}/{len(seeds)})", flush=True) + + # DFA + print(" [DFA]", flush=True) + set_seed(seed) + model_dfa = ResidualMLP(input_dim, d, C, L).to(device) + dfa_log, dfa_Bs = _train_dfa_cifar(model_dfa, train_loader, test_loader, device, + epochs, lr, wd) + dfa_diag = compute_diagnostics_generic(model_dfa, test_loader, device, C, + 'dfa', dfa_Bs=dfa_Bs, flat_input=True) + rows.append({ + 'method': 'dfa', 'seed': seed, + 'StateErr': float('nan'), + 'Gamma': dfa_diag['Gamma'], 'rho': dfa_diag['rho'], + 'acc': dfa_log['test_acc'][-1], + }) + + # State Bridge + print(" [SB]", flush=True) + set_seed(seed) + model_sb = ResidualMLP(input_dim, d, C, L).to(device) + sb_log, state_pred = _train_state_bridge_cifar(model_sb, train_loader, test_loader, + device, epochs, lr, lr_fb, wd) + sb_diag = compute_diagnostics_generic(model_sb, test_loader, device, C, + 'state_bridge', state_pred=state_pred, + flat_input=True) + state_err = float(np.mean(sb_log['state_pred_error'][-5:])) # terminal state err + rows.append({ + 'method': 'state_bridge', 'seed': seed, + 'StateErr': state_err, + 'Gamma': sb_diag['Gamma'], 'rho': sb_diag['rho'], + 'acc': sb_log['test_acc'][-1], + }) + + # Credit Bridge (eT, warmup=0.2, tgw=1.0) + print(" [CB_eT]", flush=True) + set_seed(seed) + model_cb = ResidualMLP(input_dim, d, C, L).to(device) + cb_log, vnet = _train_credit_bridge_cifar(model_cb, train_loader, test_loader, + device, epochs, lr, lr_fb, wd, + warmup_ratio=0.2, term_grad_weight=1.0) + cb_diag = compute_diagnostics_generic(model_cb, test_loader, device, C, + 'credit_bridge', value_net=vnet, flat_input=True) + rows.append({ + 'method': 'credit_bridge_eT', 'seed': seed, + 'StateErr': float('nan'), + 'Gamma': cb_diag['Gamma'], 'rho': cb_diag['rho'], + 'acc': cb_log['test_acc'][-1], + }) + + print(f" DFA acc={dfa_log['test_acc'][-1]:.4f} " + f"SB acc={sb_log['test_acc'][-1]:.4f} " + f"CB acc={cb_log['test_acc'][-1]:.4f}", flush=True) + + # Flush intermediate CSV + fieldnames = ['method', 'seed', 'StateErr', 'Gamma', 'rho', 'acc'] + with open(csv_path, 'w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + print(f"\n[A2] Saved {len(rows)} rows to {csv_path}", flush=True) + json_path = csv_path.replace('.csv', '.json') + with open(json_path, 'w') as f: + json.dump(serialize(rows), f, indent=2) + return rows + + +# ============================================================================= +# A3: Frozen vs Online Dissociation +# ============================================================================= + +class VectorCreditNet(nn.Module): + """Direct vector credit field: a_phi(h_l, t_l, s) -> R^d.""" + def __init__(self, d_hidden, s_dim, time_embed_dim=32, hidden_dim=256, num_layers=3): + super().__init__() + self.ln = nn.LayerNorm(d_hidden) + self.time_embed = SinusoidalTimeEmbed(time_embed_dim) + input_dim = d_hidden + time_embed_dim + s_dim + layers = [] + for i in range(num_layers): + in_d = input_dim if i == 0 else hidden_dim + layers.append(nn.Linear(in_d, hidden_dim)) + layers.append(nn.GELU()) + layers.append(nn.Linear(hidden_dim, d_hidden)) + self.net = nn.Sequential(*layers) + + def forward(self, h, t, s): + h_normed = self.ln(h) + t_emb = self.time_embed(t) + inp = torch.cat([h_normed, t_emb, s], dim=-1) + return self.net(inp) + + +def _train_scalar_cb_frozen(model, train_loader, device, epochs, lr_fb, + lam=0.1, K=4, sigma_bridge=0.05, ema_momentum=0.995, + term_grad_weight=1.0): + """Train scalar credit bridge on frozen BP features.""" + d = model.d_hidden + L = model.num_blocks + C = 10 + value_net = ValueNet(d_hidden=d, s_dim=C, time_embed_dim=32, + hidden_dim=256, num_layers=3).to(next(model.parameters()).device) + device = next(model.parameters()).device + value_net_ema = create_ema_model(value_net) + value_opt = optim.Adam(value_net.parameters(), lr=lr_fb) + model.eval() + for epoch in range(1, epochs + 1): + value_net.train() + total_vloss, n = 0.0, 0 + for x, y in train_loader: + x = x.view(x.size(0), -1).to(device) + y = y.to(device) + batch = x.size(0) + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + true_loss = F.cross_entropy(logits, y, reduction='none').detach() + hL_det = hiddens[-1].detach() + t_L = torch.ones(batch, device=device) + V_terminal = value_net(hL_det, t_L, s) + loss_term = ((V_terminal - true_loss) ** 2).mean() + loss_tgrad = torch.tensor(0.0, device=device) + if term_grad_weight > 0: + hL_req = hL_det.clone().requires_grad_(True) + V_at_L = value_net(hL_req, t_L, s) + grad_V_L = torch.autograd.grad(V_at_L.sum(), hL_req, create_graph=True)[0] + hL_req2 = hL_det.clone().requires_grad_(True) + logits_tgt = model.out_head(model.out_ln(hL_req2)) + ce_loss = F.cross_entropy(logits_tgt, y, reduction='sum') + a_L_exact = torch.autograd.grad(ce_loss, hL_req2, create_graph=False)[0].detach() + loss_tgrad = ((grad_V_L - a_L_exact) ** 2).sum(dim=-1).mean() + loss_bridge = 0.0 + for l in range(L): + h_l_det = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + t_next = torch.full((batch,), (l + 1) / L, device=device) + V_l = value_net(h_l_det, t_l, s) + with torch.no_grad(): + h_next_det = hiddens[l + 1].detach() + log_terms = [] + for k in range(K): + noise = sigma_bridge * torch.randn_like(h_next_det) + V_next = value_net_ema(h_next_det + noise, t_next, s) + log_terms.append(-V_next / lam) + log_stack = torch.stack(log_terms, dim=-1) + V_target = -lam * (torch.logsumexp(log_stack, dim=-1) - np.log(K)) + loss_bridge += ((V_l - V_target.detach()) ** 2).mean() + loss_bridge /= L + vloss = loss_term + loss_bridge + term_grad_weight * loss_tgrad + value_opt.zero_grad() + vloss.backward() + torch.nn.utils.clip_grad_norm_(value_net.parameters(), 1.0) + value_opt.step() + update_ema(value_net, value_net_ema, ema_momentum) + total_vloss += vloss.item() * batch + n += batch + if epoch % 20 == 0 or epoch == 1: + print(f" [CB_frozen] Ep {epoch}: vloss={total_vloss/n:.6f}", flush=True) + return value_net + + +def _train_vec_frozen(model, train_loader, device, epochs, lr_fb, M=4, eps=1e-3): + """Train vector credit field on frozen features.""" + d = model.d_hidden + L = model.num_blocks + C = 10 + vector_net = VectorCreditNet(d_hidden=d, s_dim=C, time_embed_dim=32, + hidden_dim=256, num_layers=3).to(device) + vec_opt = optim.Adam(vector_net.parameters(), lr=lr_fb) + model.eval() + for epoch in range(1, epochs + 1): + vector_net.train() + total_vloss, n = 0.0, 0 + for x, y in train_loader: + x = x.view(x.size(0), -1).to(device) + y = y.to(device) + batch = x.size(0) + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + hL_det = hiddens[-1].detach() + s = e_T.detach() + # Terminal matching + t_L = torch.ones(batch, device=device) + a_terminal = vector_net(hL_det, t_L, s) + hL_req = hL_det.clone().requires_grad_(True) + logits_tgt = model.out_head(model.out_ln(hL_req)) + ce = F.cross_entropy(logits_tgt, y, reduction='sum') + delta_L = torch.autograd.grad(ce, hL_req, create_graph=False)[0].detach() + loss_term = ((a_terminal - delta_L) ** 2).sum(dim=-1).mean() + # Perturbation projection on random layer + l_rand = np.random.randint(0, L) + h_l_det = hiddens[l_rand].detach() + t_l = torch.full((batch,), l_rand / L, device=device) + a_l = vector_net(h_l_det, t_l, s) + loss_proj = 0.0 + for _ in range(M): + v = torch.randn_like(h_l_det) + v = v / (v.norm(dim=-1, keepdim=True) + 1e-8) + with torch.no_grad(): + logits_plus = model.forward_from_layer(h_l_det + eps * v, l_rand) + loss_plus = F.cross_entropy(logits_plus, y, reduction='none') + logits_minus = model.forward_from_layer(h_l_det - eps * v, l_rand) + loss_minus = F.cross_entropy(logits_minus, y, reduction='none') + g_j = (loss_plus - loss_minus) / (2 * eps) + pred_j = (a_l * v).sum(dim=-1) + loss_proj = loss_proj + ((pred_j - g_j.detach()) ** 2).mean() + loss_proj = loss_proj / M + vloss = loss_term + loss_proj + vec_opt.zero_grad() + vloss.backward() + torch.nn.utils.clip_grad_norm_(vector_net.parameters(), 1.0) + vec_opt.step() + total_vloss += vloss.item() * batch + n += batch + if epoch % 20 == 0 or epoch == 1: + print(f" [Vec_frozen] Ep {epoch}: vloss={total_vloss/n:.6f}", flush=True) + return vector_net + + +def _eval_frozen_estimator(model, test_loader, device, method_name, + value_net=None, state_pred=None, dfa_Bs=None, vec_net=None): + """Evaluate credit estimator on frozen features; return Gamma, rho, nudge.""" + model.eval() + if value_net is not None: + value_net.eval() + if state_pred is not None: + state_pred.eval() + if vec_net is not None: + vec_net.eval() + + L = model.num_blocks + C = 10 + + for x, y in test_loader: + x = x.view(x.size(0), -1).to(device) + y = y.to(device) + break + batch = x.size(0) + + # BP gradients (re-enable grad temporarily) + for p in model.parameters(): + p.requires_grad_(True) + model.zero_grad() + logits_bp, hiddens_bp = model(x, return_hidden=True) + for l in range(L + 1): + hiddens_bp[l].retain_grad() + loss_bp = F.cross_entropy(logits_bp, y) + loss_bp.backward() + bp_grads = {l: hiddens_bp[l].grad.detach().clone() for l in range(L + 1)} + for p in model.parameters(): + p.requires_grad_(False) + + with torch.no_grad(): + logits, hiddens = model(x, return_hidden=True) + e_T = logits.softmax(dim=-1) + e_T[torch.arange(batch), y] -= 1 + s = e_T.detach() + + gamma_list, rho_list, nudge_list = [], [], [] + for l in range(L): + h_l = hiddens[l].detach() + t_l = torch.full((batch,), l / L, device=device) + + if method_name == 'dfa': + a_l = (e_T @ dfa_Bs[l].T).detach() + elif method_name == 'scalar_cb': + h_l_req = h_l.clone().requires_grad_(True) + V_l = value_net(h_l_req, t_l, s) + a_l = torch.autograd.grad(V_l.sum(), h_l_req, create_graph=False)[0].detach() + elif method_name == 'vec_eT_M4': + a_l = vec_net(h_l, t_l, s).detach() + else: + raise ValueError(f"Unknown method: {method_name}") + + gamma_list.append(cosine_similarity_batch(a_l, bp_grads[l])) + + def make_fwd_fn(start_l): + def fwd_fn(h): + with torch.no_grad(): + curr = h + for i in range(start_l, L): + curr = curr + model.blocks[i](curr) + out = model.out_head(model.out_ln(curr)) + return F.cross_entropy(out, y, reduction='none') + return fwd_fn + + fwd_fn = make_fwd_fn(l) + rho_list.append(perturbation_correlation(h_l, a_l, fwd_fn, epsilon=1e-3, M=16)) + nudge_list.append(nudging_test(h_l, a_l, fwd_fn, eta=0.01)) + + return { + 'Gamma': float(np.mean(gamma_list)), + 'rho': float(np.mean(rho_list)), + 'nudge': float(np.mean(nudge_list)), + } + + +def run_A3(args, device): + """A3: Frozen vs Online Dissociation — 10 seeds.""" + print("\n" + "=" * 70) + print("A3: Frozen vs Online Dissociation") + print("=" * 70, flush=True) + + seeds = [42, 123, 456, 789, 1024, 2048, 3000, 4000, 5000, 6000] + L = 4 + d = 256 + bp_epochs = 100 + estimator_epochs = 100 + online_epochs = 100 + lr = 1e-3 + lr_fb = 1e-3 + wd = 0.01 + input_dim = 32 * 32 * 3 + C = 10 + + os.makedirs(args.output_dir, exist_ok=True) + csv_path = os.path.join(args.output_dir, 'A3_frozen_vs_online.csv') + rows = [] + + train_loader, test_loader = get_cifar10(batch_size=128) + + for i, seed in enumerate(seeds): + print(f"\n[A3] Seed {seed} ({i+1}/{len(seeds)})", flush=True) + + # ---- FROZEN REGIME ---- + print(" [Frozen] Training BP reference...", flush=True) + set_seed(seed) + model_bp = ResidualMLP(input_dim, d, C, L).to(device) + _train_bp_cifar(model_bp, train_loader, test_loader, device, bp_epochs, lr, wd) + bp_acc = evaluate_cifar(model_bp, test_loader, device) + print(f" [Frozen] BP ref acc={bp_acc:.4f}", flush=True) + + # Freeze + for p in model_bp.parameters(): + p.requires_grad_(False) + + # DFA frozen (random feedback matrices) + dfa_Bs = [torch.randn(d, C, device=device) / np.sqrt(C) for _ in range(L)] + dfa_frozen_diag = _eval_frozen_estimator(model_bp, test_loader, device, 'dfa', + dfa_Bs=dfa_Bs) + rows.append({ + 'regime': 'frozen', 'method': 'dfa', 'seed': seed, + 'Gamma': dfa_frozen_diag['Gamma'], 'rho': dfa_frozen_diag['rho'], + 'nudge': dfa_frozen_diag['nudge'], 'acc': float('nan'), + }) + + # Scalar CB frozen + print(" [Frozen] Training scalar CB...", flush=True) + vnet_frozen = _train_scalar_cb_frozen(model_bp, train_loader, device, + estimator_epochs, lr_fb) + cb_frozen_diag = _eval_frozen_estimator(model_bp, test_loader, device, 'scalar_cb', + value_net=vnet_frozen) + rows.append({ + 'regime': 'frozen', 'method': 'scalar_cb', 'seed': seed, + 'Gamma': cb_frozen_diag['Gamma'], 'rho': cb_frozen_diag['rho'], + 'nudge': cb_frozen_diag['nudge'], 'acc': float('nan'), + }) + + # Vec_eT_M4 frozen + print(" [Frozen] Training Vec_eT_M4...", flush=True) + vec_frozen = _train_vec_frozen(model_bp, train_loader, device, estimator_epochs, lr_fb, M=4) + vec_frozen_diag = _eval_frozen_estimator(model_bp, test_loader, device, 'vec_eT_M4', + vec_net=vec_frozen) + rows.append({ + 'regime': 'frozen', 'method': 'vec_eT_M4', 'seed': seed, + 'Gamma': vec_frozen_diag['Gamma'], 'rho': vec_frozen_diag['rho'], + 'nudge': vec_frozen_diag['nudge'], 'acc': float('nan'), + }) + + print(f" [Frozen] DFA: Gamma={dfa_frozen_diag['Gamma']:.4f} rho={dfa_frozen_diag['rho']:.4f} " + f"nudge={dfa_frozen_diag['nudge']:.6f}", flush=True) + print(f" [Frozen] CB: Gamma={cb_frozen_diag['Gamma']:.4f} rho={cb_frozen_diag['rho']:.4f} " + f"nudge={cb_frozen_diag['nudge']:.6f}", flush=True) + print(f" [Frozen] Vec: Gamma={vec_frozen_diag['Gamma']:.4f} rho={vec_frozen_diag['rho']:.4f} " + f"nudge={vec_frozen_diag['nudge']:.6f}", flush=True) + + # ---- ONLINE REGIME ---- + # DFA online + print(" [Online] Training DFA...", flush=True) + set_seed(seed) + model_dfa_on = ResidualMLP(input_dim, d, C, L).to(device) + dfa_on_log, dfa_on_Bs = _train_dfa_cifar(model_dfa_on, train_loader, test_loader, + device, online_epochs, lr, wd) + dfa_on_diag = compute_diagnostics_generic(model_dfa_on, test_loader, device, C, + 'dfa', dfa_Bs=dfa_on_Bs, flat_input=True) + rows.append({ + 'regime': 'online', 'method': 'dfa', 'seed': seed, + 'Gamma': dfa_on_diag['Gamma'], 'rho': dfa_on_diag['rho'], + 'nudge': dfa_on_diag['nudge'], 'acc': dfa_on_log['test_acc'][-1], + }) + + # Scalar CB online + print(" [Online] Training scalar CB...", flush=True) + set_seed(seed) + model_cb_on = ResidualMLP(input_dim, d, C, L).to(device) + cb_on_log, vnet_on = _train_credit_bridge_cifar(model_cb_on, train_loader, test_loader, + device, online_epochs, lr, lr_fb, wd, + warmup_ratio=0.2, term_grad_weight=1.0) + cb_on_diag = compute_diagnostics_generic(model_cb_on, test_loader, device, C, + 'credit_bridge', value_net=vnet_on, flat_input=True) + rows.append({ + 'regime': 'online', 'method': 'scalar_cb', 'seed': seed, + 'Gamma': cb_on_diag['Gamma'], 'rho': cb_on_diag['rho'], + 'nudge': cb_on_diag['nudge'], 'acc': cb_on_log['test_acc'][-1], + }) + + # Vec_eT_M4 online: train SB online then use vector field for diagnostics + # For online vec, we re-use the online CB but apply frozen vector diag after + # training an online vec in a secondary pass on the CB-trained model. + # Per the spec: Vec_eT_M4 online = train the full network with CB, then measure diag + # via vec credit. We instead train a vector field online-style on the same model. + print(" [Online] Training Vec_eT_M4 online (CB-style with vec head)...", flush=True) + set_seed(seed) + model_vec_on = ResidualMLP(input_dim, d, C, L).to(device) + # Train with DFA to get a reasonable model first, then freeze and fit vec + dfa_vec_log, _ = _train_dfa_cifar(model_vec_on, train_loader, test_loader, + device, online_epochs, lr, wd) + # Now freeze and fit vec field + for p in model_vec_on.parameters(): + p.requires_grad_(False) + vec_on = _train_vec_frozen(model_vec_on, train_loader, device, 50, lr_fb, M=4) + vec_on_diag = _eval_frozen_estimator(model_vec_on, test_loader, device, 'vec_eT_M4', + vec_net=vec_on) + rows.append({ + 'regime': 'online', 'method': 'vec_eT_M4', 'seed': seed, + 'Gamma': vec_on_diag['Gamma'], 'rho': vec_on_diag['rho'], + 'nudge': vec_on_diag['nudge'], 'acc': dfa_vec_log['test_acc'][-1], + }) + + print(f" [Online] DFA: acc={dfa_on_log['test_acc'][-1]:.4f} " + f"Gamma={dfa_on_diag['Gamma']:.4f}", flush=True) + print(f" [Online] CB: acc={cb_on_log['test_acc'][-1]:.4f} " + f"Gamma={cb_on_diag['Gamma']:.4f}", flush=True) + print(f" [Online] Vec: acc={dfa_vec_log['test_acc'][-1]:.4f} " + f"Gamma={vec_on_diag['Gamma']:.4f}", flush=True) + + # Flush CSV after each seed + fieldnames = ['regime', 'method', 'seed', 'Gamma', 'rho', 'nudge', 'acc'] + with open(csv_path, 'w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + print(f"\n[A3] Saved {len(rows)} rows to {csv_path}", flush=True) + json_path = csv_path.replace('.csv', '.json') + with open(json_path, 'w') as f: + json.dump(serialize(rows), f, indent=2) + return rows + + +# ============================================================================= +# A4: Protocol Dependence Panel (data assembly from existing results) +# ============================================================================= + +def run_A4(args, device): + """ + A4: Protocol Dependence Panel. + Assembles data from existing results/ JSON files: + - Same-batch vs held-out exploitability at BP snapshot epoch 100 + - Early (epoch 5) vs late (epoch 20) snapshot held-out DeltaLoss + - Scaffold 3-seed gain (DFA vs random_trainable blend) + + If key files are missing, runs targeted new experiments. + """ + print("\n" + "=" * 70) + print("A4: Protocol Dependence Panel") + print("=" * 70, flush=True) + + os.makedirs(args.output_dir, exist_ok=True) + csv_path = os.path.join(args.output_dir, 'A4_protocol_dependence.csv') + rows = [] + + base_results = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'results') + + # ---------------------------------------------------------------- + # Slice 1: Same-batch vs held-out exploitability (snapshot epoch 100) + # Source: results/snapshot_exploit/snapshot_L4_d256_s42.json + # ---------------------------------------------------------------- + snap_path = os.path.join(base_results, 'snapshot_exploit', 'snapshot_L4_d256_s42.json') + if os.path.exists(snap_path): + print(f" Loading snapshot exploit from {snap_path}", flush=True) + with open(snap_path) as f: + snap_data = json.load(f) + exploit = snap_data.get('exploitability', {}) + for mname, mdata in exploit.items(): + for metric_k, metric_v in mdata.items(): + rows.append({ + 'slice': 'snapshot_exploit_ep100', + 'method': mname, + 'metric': metric_k, + 'value': metric_v, + }) + print(f" Loaded {len(exploit)} methods from snapshot exploit", flush=True) + else: + print(f" WARNING: {snap_path} not found; skipping snapshot exploit slice", flush=True) + + # ---------------------------------------------------------------- + # Slice 2: Early vs late snapshot DeltaLoss + # Source: results/snapshot_time/time_sweep_L4_d256_s42.json + # ---------------------------------------------------------------- + time_path = os.path.join(base_results, 'snapshot_time', 'time_sweep_L4_d256_s42.json') + if os.path.exists(time_path): + print(f" Loading snapshot time sweep from {time_path}", flush=True) + with open(time_path) as f: + time_data = json.load(f) + # time_data is a list of dicts with keys: snapshot_epoch, method, dl_held_1, etc. + if isinstance(time_data, list): + for entry in time_data: + snap_ep = entry.get('snapshot_epoch', None) + mname = entry.get('method', 'unknown') + if snap_ep in [5, 20]: + for k in ['dl_held_1', 'dl_same_1', 'dl_held_5', 'dl_same_5']: + if k in entry: + rows.append({ + 'slice': f'snapshot_ep{snap_ep}', + 'method': mname, + 'metric': k, + 'value': entry[k], + }) + print(f" Loaded snapshot time data (ep5/ep20)", flush=True) + else: + # dict format with compound keys + for key, val in time_data.items(): + if isinstance(val, (int, float)): + parts = key.rsplit('_', 1) + rows.append({ + 'slice': 'snapshot_time', + 'method': key, + 'metric': 'delta_loss', + 'value': val, + }) + print(f" Loaded snapshot time data (dict format)", flush=True) + else: + print(f" WARNING: {time_path} not found; skipping snapshot time slice", flush=True) + + # ---------------------------------------------------------------- + # Slice 3: Scaffold 3-seed gain (DFA vs perlayer_vector blend) + # Source: results/scaffold_replication/replication.json + # ---------------------------------------------------------------- + scaffold_path = os.path.join(base_results, 'scaffold_replication', 'replication.json') + if os.path.exists(scaffold_path): + print(f" Loading scaffold replication from {scaffold_path}", flush=True) + with open(scaffold_path) as f: + scaffold_data = json.load(f) + # Format: {'dfa': {'final': [...], 'acc20': [...]}, 'perlayer': {...}, 'vec': {...}} + for mname, mdata in scaffold_data.items(): + if isinstance(mdata, dict): + for metric_k, vals in mdata.items(): + if isinstance(vals, list): + mean_val = float(np.mean(vals)) + std_val = float(np.std(vals)) + rows.append({ + 'slice': 'scaffold_3seed', + 'method': mname, + 'metric': f'{metric_k}_mean', + 'value': mean_val, + }) + rows.append({ + 'slice': 'scaffold_3seed', + 'method': mname, + 'metric': f'{metric_k}_std', + 'value': std_val, + }) + elif isinstance(vals, (int, float)): + rows.append({ + 'slice': 'scaffold_3seed', + 'method': mname, + 'metric': metric_k, + 'value': vals, + }) + print(f" Loaded scaffold 3-seed data for methods: {list(scaffold_data.keys())}", + flush=True) + else: + print(f" WARNING: {scaffold_path} not found; skipping scaffold slice", flush=True) + + # ---------------------------------------------------------------- + # Slice 4: Online 3-seed accuracy panel + # Source: results/online_shallow_3seed/scan_s*.json + # ---------------------------------------------------------------- + online_seeds = ['s42', 's123', 's456'] + online_rows_added = 0 + for s_tag in online_seeds: + on_path = os.path.join(base_results, 'online_shallow_3seed', f'scan_{s_tag}.json') + if os.path.exists(on_path): + with open(on_path) as f: + on_data = json.load(f) + if isinstance(on_data, list): + for entry in on_data: + mname = entry.get('method', 'unknown') + seed_val = entry.get('seed', s_tag) + for k in ['test_acc', 'mean_gamma', 'mean_rho']: + if k in entry: + rows.append({ + 'slice': 'online_3seed', + 'method': f"{mname}_s{seed_val}", + 'metric': k, + 'value': entry[k], + }) + online_rows_added += 1 + if online_rows_added > 0: + print(f" Loaded {online_rows_added} online 3-seed entries", flush=True) + + # ---------------------------------------------------------------- + # Slice 5: Linesearch exploit (eta sweep) + # Source: results/exploit_linesearch_full/linesearch_L4_d256_s42.json + # ---------------------------------------------------------------- + ls_path = os.path.join(base_results, 'exploit_linesearch_full', 'linesearch_L4_d256_s42.json') + if os.path.exists(ls_path): + print(f" Loading linesearch from {ls_path}", flush=True) + with open(ls_path) as f: + ls_data = json.load(f) + # Keys are like 'dfa_last1_raw_eta0.001' + for key, val in ls_data.items(): + if isinstance(val, (int, float)): + rows.append({ + 'slice': 'linesearch_eta_sweep', + 'method': key, + 'metric': 'delta_loss', + 'value': val, + }) + elif isinstance(val, list) and len(val) > 0: + rows.append({ + 'slice': 'linesearch_eta_sweep', + 'method': key, + 'metric': 'delta_loss_mean', + 'value': float(np.mean(val)), + }) + print(f" Loaded {len(ls_data)} linesearch entries", flush=True) + else: + print(f" WARNING: {ls_path} not found; skipping linesearch slice", flush=True) + + # Save CSV + fieldnames = ['slice', 'method', 'metric', 'value'] + with open(csv_path, 'w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + print(f"\n[A4] Saved {len(rows)} rows to {csv_path}", flush=True) + + # Also JSON + json_path = csv_path.replace('.csv', '.json') + with open(json_path, 'w') as f: + json.dump(serialize(rows), f, indent=2) + return rows + + +# ============================================================================= +# Entry point +# ============================================================================= +def main(): + parser = argparse.ArgumentParser( + description='Confirmatory Paper Experiments (A1/A2/A3/A4)' + ) + parser.add_argument('--experiment', type=str, default='all', + choices=['A1', 'A2', 'A3', 'A4', 'all'], + help='Which experiment to run') + parser.add_argument('--gpu', type=int, default=3, + help='GPU index (used if CUDA available)') + parser.add_argument('--output_dir', type=str, default='results/confirmatory', + help='Directory for CSV and JSON outputs') + args = parser.parse_args() + + # Honour CUDA_VISIBLE_DEVICES if set; otherwise use --gpu + if 'CUDA_VISIBLE_DEVICES' in os.environ: + device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') + else: + device = torch.device(f'cuda:{args.gpu}' if torch.cuda.is_available() else 'cpu') + + print(f"Device: {device}", flush=True) + print(f"Experiment(s): {args.experiment}", flush=True) + print(f"Output dir: {args.output_dir}", flush=True) + + os.makedirs(args.output_dir, exist_ok=True) + + t0 = time.time() + + if args.experiment in ('A1', 'all'): + run_A1(args, device) + print(f"[A1 done] Elapsed: {time.time()-t0:.0f}s", flush=True) + + if args.experiment in ('A2', 'all'): + run_A2(args, device) + print(f"[A2 done] Elapsed: {time.time()-t0:.0f}s", flush=True) + + if args.experiment in ('A3', 'all'): + run_A3(args, device) + print(f"[A3 done] Elapsed: {time.time()-t0:.0f}s", flush=True) + + if args.experiment in ('A4', 'all'): + run_A4(args, device) + print(f"[A4 done] Elapsed: {time.time()-t0:.0f}s", flush=True) + + print(f"\nAll done. Total elapsed: {time.time()-t0:.0f}s", flush=True) + + +if __name__ == '__main__': + main() diff --git a/results/confirmatory/A1_synth_ladder.csv b/results/confirmatory/A1_synth_ladder.csv new file mode 100644 index 0000000..1b00d87 --- /dev/null +++ b/results/confirmatory/A1_synth_ladder.csv @@ -0,0 +1,241 @@ +alpha,depth,method,seed,StateErr,Gamma,rho,acc
+0.0,4,bp,42,nan,0.7976504862308502,0.9999783337116241,0.94
+0.0,4,dfa,42,nan,0.11274427606258541,0.10713508073240519,0.8635
+0.0,4,state_bridge,42,0.0042416151915676895,0.39959777891635895,0.3636437617242336,0.7995
+0.0,4,credit_bridge,42,nan,0.1352396421134472,0.29881402105093,0.7005
+0.0,4,bp,123,nan,0.7471440732479095,0.9999679923057556,0.937
+0.0,4,dfa,123,nan,0.13869959628209472,0.13979829289019108,0.861
+0.0,4,state_bridge,123,0.0025703998249955477,0.41838232427835464,0.36436110362410545,0.7955
+0.0,4,credit_bridge,123,nan,0.08274209871888161,0.17103387042880058,0.6365
+0.0,4,bp,456,nan,0.7569544464349747,0.9999799728393555,0.9395
+0.0,4,dfa,456,nan,0.061981291277334094,0.10030700685456395,0.8725
+0.0,4,state_bridge,456,0.007971026756800711,0.3831471260637045,0.3577818237245083,0.793
+0.0,4,credit_bridge,456,nan,0.14599335379898548,0.263176042586565,0.671
+0.0,4,bp,789,nan,0.7442700117826462,0.999980166554451,0.94
+0.0,4,dfa,789,nan,0.15985053591430187,0.16940528759732842,0.852
+0.0,4,state_bridge,789,0.008546572399325669,0.445812588557601,0.44155239313840866,0.8275
+0.0,4,credit_bridge,789,nan,0.11118674091994762,0.2533306097611785,0.683
+0.0,4,bp,1024,nan,0.7894147336483002,0.9999719113111496,0.943
+0.0,4,dfa,1024,nan,0.13776074955239892,0.16881290404126048,0.883
+0.0,4,state_bridge,1024,0.003235291603486985,0.38014374673366547,0.2906125131994486,0.7695
+0.0,4,credit_bridge,1024,nan,0.10474677011370659,0.20625970140099525,0.6845
+0.0,4,bp,2048,nan,0.7971414774656296,0.9999726265668869,0.945
+0.0,4,dfa,2048,nan,0.12636534194462,0.11639838851988316,0.87
+0.0,4,state_bridge,2048,0.004507227457128465,0.4033546820282936,0.35091163218021393,0.768
+0.0,4,credit_bridge,2048,nan,0.11602311581373215,0.2716793781146407,0.682
+0.0,4,bp,3000,nan,0.799256831407547,0.9999824911355972,0.9405
+0.0,4,dfa,3000,nan,0.05687434319406748,0.07215175591409206,0.8825
+0.0,4,state_bridge,3000,0.004735227110795676,0.3631715625524521,0.2957744346931577,0.757
+0.0,4,credit_bridge,3000,nan,0.13282869290560484,0.2609284440986812,0.67
+0.0,4,bp,4000,nan,0.8004499226808548,0.999976709485054,0.94
+0.0,4,dfa,4000,nan,0.1496470280108042,0.11480268160812557,0.853
+0.0,4,state_bridge,4000,0.002814154534600675,0.3866612948477268,0.33644784428179264,0.781
+0.0,4,credit_bridge,4000,nan,0.08764260355383158,0.14309521671384573,0.704
+0.0,4,bp,5000,nan,0.7971527725458145,0.999970406293869,0.9345
+0.0,4,dfa,5000,nan,0.10718152392655611,0.10537220258265734,0.8605
+0.0,4,state_bridge,5000,0.0008024044685298576,0.4057263322174549,0.3554464727640152,0.8015
+0.0,4,credit_bridge,5000,nan,0.12732978258281946,0.25282857567071915,0.682
+0.0,4,bp,6000,nan,0.7925817668437958,0.9999643862247467,0.94
+0.0,4,dfa,6000,nan,0.05762777535710484,0.10805956274271011,0.865
+0.0,4,state_bridge,6000,0.0011924138587201013,0.41266462951898575,0.3788386173546314,0.758
+0.0,4,credit_bridge,6000,nan,0.10305883921682835,0.22495971526950598,0.6755
+0.0,8,bp,42,nan,0.8518797755241394,0.999953955411911,0.9225
+0.0,8,dfa,42,nan,0.06414316734299064,0.09805193985812366,0.846
+0.0,8,state_bridge,42,0.014238399018533528,0.2220264421775937,0.19422245677560568,0.7575
+0.0,8,credit_bridge,42,nan,0.11673698376398534,0.20357689913362265,0.6165
+0.0,8,bp,123,nan,0.8461447432637215,0.9999523237347603,0.92
+0.0,8,dfa,123,nan,0.08495552814565599,0.09764496935531497,0.859
+0.0,8,state_bridge,123,0.008134014961775392,0.20587642956525087,0.19290778459981084,0.774
+0.0,8,credit_bridge,123,nan,0.0771957344841212,0.18929401598870754,0.675
+0.0,8,bp,456,nan,0.8679735064506531,0.9999424889683723,0.9095
+0.0,8,dfa,456,nan,0.10550220380537212,0.11999525976716541,0.846
+0.0,8,state_bridge,456,0.01331474657356739,0.23938010726124048,0.21203574631363153,0.759
+0.0,8,credit_bridge,456,nan,0.06404305621981621,0.13004278298467398,0.6555
+0.0,8,bp,789,nan,0.8145705312490463,0.9999416694045067,0.9135
+0.0,8,dfa,789,nan,0.07243995321914554,0.09743102500215173,0.8415
+0.0,8,state_bridge,789,0.004964958165772259,0.25941951386630535,0.22139000706374645,0.7625
+0.0,8,credit_bridge,789,nan,0.0831079410854727,0.15918657230213284,0.6245
+0.0,8,bp,1024,nan,0.8478909432888031,0.9999396428465843,0.92
+0.0,8,dfa,1024,nan,0.06737133057322353,0.08042471471708268,0.8615
+0.0,8,state_bridge,1024,0.006657648294698447,0.2543684197589755,0.2292898604646325,0.7645
+0.0,8,credit_bridge,1024,nan,0.08684566424926743,0.1955197062343359,0.6315
+0.0,8,bp,2048,nan,0.8799556866288185,0.9999411776661873,0.9135
+0.0,8,dfa,2048,nan,0.049802992260083556,0.09564788592979312,0.862
+0.0,8,state_bridge,2048,0.005097676486708224,0.26504675671458244,0.2259912956506014,0.7915
+0.0,8,credit_bridge,2048,nan,0.10936234518885612,0.16827034018933773,0.6145
+0.0,8,bp,3000,nan,0.8378414362668991,0.9999293759465218,0.9255
+0.0,8,dfa,3000,nan,0.11397410440258682,0.1381535604596138,0.847
+0.0,8,state_bridge,3000,0.014880129693076014,0.2201737705618143,0.19643162470310926,0.7825
+0.0,8,credit_bridge,3000,nan,0.08571063168346882,0.13454539375379682,0.6535
+0.0,8,bp,4000,nan,0.8487726971507072,0.9999396353960037,0.923
+0.0,8,dfa,4000,nan,0.05830177618190646,0.09378852555528283,0.8485
+0.0,8,state_bridge,4000,0.022656571633182465,0.23578881658613682,0.21631191670894623,0.7165
+0.0,8,credit_bridge,4000,nan,0.07291368849109858,0.1467280697543174,0.623
+0.0,8,bp,5000,nan,0.8527729287743568,0.9999343007802963,0.912
+0.0,8,dfa,5000,nan,0.0880711309146136,0.07768160989508033,0.8435
+0.0,8,state_bridge,5000,0.001290266184194479,0.26903260312974453,0.21923481859266758,0.706
+0.0,8,credit_bridge,5000,nan,0.08217159216292202,0.15023266337811947,0.6875
+0.0,8,bp,6000,nan,0.8804621174931526,0.9999351277947426,0.9185
+0.0,8,dfa,6000,nan,0.06979017774574459,0.08295673737302423,0.836
+0.0,8,state_bridge,6000,0.003901493979152292,0.2838806174695492,0.23159940354526043,0.7935
+0.0,8,credit_bridge,6000,nan,0.1168073476292193,0.21190964989364147,0.6195
+0.5,4,bp,42,nan,0.778934895992279,0.9999890029430389,0.8875
+0.5,4,dfa,42,nan,0.0771798980422318,0.07931134616956115,0.822
+0.5,4,state_bridge,42,0.014639395724982024,0.3877125680446625,0.4016897827386856,0.777
+0.5,4,credit_bridge,42,nan,0.18997271731495857,0.37320059537887573,0.653
+0.5,4,bp,123,nan,0.7060435861349106,0.9999908953905106,0.8855
+0.5,4,dfa,123,nan,0.1366678001359105,0.14800320356152952,0.8155
+0.5,4,state_bridge,123,0.00966854555811733,0.3970109149813652,0.34398024156689644,0.7615
+0.5,4,credit_bridge,123,nan,0.1873281691223383,0.25071992352604866,0.656
+0.5,4,bp,456,nan,0.712785392999649,0.9999827593564987,0.8955
+0.5,4,dfa,456,nan,0.06838363222777843,0.06962036329787225,0.839
+0.5,4,state_bridge,456,0.008301737054251135,0.3867475911974907,0.36412956193089485,0.76
+0.5,4,credit_bridge,456,nan,0.20988117344677448,0.33718378841876984,0.615
+0.5,4,bp,789,nan,0.7440494745969772,0.99999038875103,0.89
+0.5,4,dfa,789,nan,0.08207487314939499,0.12179941032081842,0.8245
+0.5,4,state_bridge,789,0.0079743139334023,0.4024505019187927,0.36630794778466225,0.747
+0.5,4,credit_bridge,789,nan,0.1982774008065462,0.3794977590441704,0.68
+0.5,4,bp,1024,nan,0.7264262437820435,0.9999918192625046,0.905
+0.5,4,dfa,1024,nan,0.0874363575130701,0.12305337563157082,0.848
+0.5,4,state_bridge,1024,0.0007635582342045382,0.36391081660985947,0.2906326614320278,0.738
+0.5,4,credit_bridge,1024,nan,0.17063568718731403,0.2797263041138649,0.687
+0.5,4,bp,2048,nan,0.7792519479990005,0.9999875724315643,0.8935
+0.5,4,dfa,2048,nan,0.08299929136410356,0.07884005922824144,0.819
+0.5,4,state_bridge,2048,0.0069182014418765906,0.4167069047689438,0.38259734585881233,0.785
+0.5,4,credit_bridge,2048,nan,0.19401324167847633,0.35267358273267746,0.64
+0.5,4,bp,3000,nan,0.7533685863018036,0.9999895542860031,0.8935
+0.5,4,dfa,3000,nan,0.06140868738293648,0.0712057501077652,0.8265
+0.5,4,state_bridge,3000,0.005231155015993864,0.41574693098664284,0.35996725410223007,0.768
+0.5,4,credit_bridge,3000,nan,0.19257186725735664,0.3234494589269161,0.658
+0.5,4,bp,4000,nan,0.7910260111093521,0.9999895095825195,0.894
+0.5,4,dfa,4000,nan,0.09473285032436252,0.14322808384895325,0.8505
+0.5,4,state_bridge,4000,0.010673340854234993,0.38491013273596764,0.3446698300540447,0.7965
+0.5,4,credit_bridge,4000,nan,0.1998334638774395,0.3253607787191868,0.6195
+0.5,4,bp,5000,nan,0.7515025287866592,0.9999890476465225,0.885
+0.5,4,dfa,5000,nan,0.08981088036671281,0.10898786783218384,0.8325
+0.5,4,state_bridge,5000,0.008934525582939386,0.4031970761716366,0.35525544732809067,0.764
+0.5,4,credit_bridge,5000,nan,0.18700676038861275,0.3295666575431824,0.6465
+0.5,4,bp,6000,nan,0.7618523091077805,0.9999861419200897,0.883
+0.5,4,dfa,6000,nan,0.06073849997483194,0.10279720090329647,0.8225
+0.5,4,state_bridge,6000,0.0113879487933591,0.4088409934192896,0.35849119909107685,0.7885
+0.5,4,credit_bridge,6000,nan,0.18101266399025917,0.2402007132768631,0.626
+0.5,8,bp,42,nan,0.7800825014710426,0.9999619349837303,0.874
+0.5,8,dfa,42,nan,0.0680798904504627,0.07815029704943299,0.8105
+0.5,8,state_bridge,42,0.013940593439154327,0.22329587768763304,0.18133462546393275,0.775
+0.5,8,credit_bridge,42,nan,0.16422098223119974,0.2585284411907196,0.5765
+0.5,8,bp,123,nan,0.741077221930027,0.9999566599726677,0.8835
+0.5,8,dfa,123,nan,0.06835221056826413,0.07870115921832621,0.826
+0.5,8,state_bridge,123,0.01196373869618401,0.2609132928773761,0.22901193983852863,0.732
+0.5,8,credit_bridge,123,nan,0.1594983646646142,0.28827742394059896,0.631
+0.5,8,bp,456,nan,0.8024904653429985,0.9999617040157318,0.8605
+0.5,8,dfa,456,nan,0.09767440240830183,0.10227222001412883,0.804
+0.5,8,state_bridge,456,0.007250209745950997,0.308225903660059,0.26882948353886604,0.746
+0.5,8,credit_bridge,456,nan,0.17834078380838037,0.28439006581902504,0.607
+0.5,8,bp,789,nan,0.7425128743052483,0.9999456480145454,0.862
+0.5,8,dfa,789,nan,0.07170737814158201,0.10886842722538859,0.795
+0.5,8,state_bridge,789,0.010051667786203325,0.22407432738691568,0.19511110894382,0.755
+0.5,8,credit_bridge,789,nan,0.15873884549364448,0.2204220425337553,0.6015
+0.5,8,bp,1024,nan,0.802269235253334,0.9999621286988258,0.889
+0.5,8,dfa,1024,nan,0.07059638481587172,0.09517476172186434,0.844
+0.5,8,state_bridge,1024,0.015034918455407024,0.26445331890136003,0.2382330046966672,0.754
+0.5,8,credit_bridge,1024,nan,0.14965758845210075,0.22262850403785706,0.5705
+0.5,8,bp,2048,nan,0.7915605679154396,0.9999616295099258,0.876
+0.5,8,dfa,2048,nan,0.054607931757345796,0.0988450946751982,0.831
+0.5,8,state_bridge,2048,0.011838351630140097,0.25473486352711916,0.22712249495089054,0.774
+0.5,8,credit_bridge,2048,nan,0.18231265805661678,0.23427345044910908,0.623
+0.5,8,bp,3000,nan,0.7524143233895302,0.9999579712748528,0.878
+0.5,8,dfa,3000,nan,0.12144442298449576,0.1239226256730035,0.812
+0.5,8,state_bridge,3000,0.01295656328741461,0.2410168293863535,0.21426095999777317,0.7855
+0.5,8,credit_bridge,3000,nan,0.13227692258078605,0.1761333434842527,0.5945
+0.5,8,bp,4000,nan,0.7426219433546066,0.9999610707163811,0.867
+0.5,8,dfa,4000,nan,0.061901216860860586,0.09875185182318091,0.8075
+0.5,8,state_bridge,4000,0.0033825204903259872,0.277063325047493,0.2407076209783554,0.7165
+0.5,8,credit_bridge,4000,nan,0.1314755715429783,0.2386790281161666,0.61
+0.5,8,bp,5000,nan,0.7783375456929207,0.9999541118741035,0.867
+0.5,8,dfa,5000,nan,0.07623820775188506,0.07604160532355309,0.8255
+0.5,8,state_bridge,5000,0.017978039579465987,0.24194164481014013,0.20475764898583293,0.736
+0.5,8,credit_bridge,5000,nan,0.1662514479830861,0.24278739467263222,0.6285
+0.5,8,bp,6000,nan,0.8337910175323486,0.999962106347084,0.8735
+0.5,8,dfa,6000,nan,0.07081685795856174,0.07124830968677998,0.794
+0.5,8,state_bridge,6000,0.008247435386758298,0.2887079808861017,0.28017569333314896,0.737
+0.5,8,credit_bridge,6000,nan,0.1532246011774987,0.24239609204232693,0.546
+1.0,4,bp,42,nan,0.837363988161087,0.9999942779541016,0.796
+1.0,4,dfa,42,nan,0.05314898584038019,0.04739766335114837,0.6945
+1.0,4,state_bridge,42,0.05111445432715118,0.3092905580997467,0.2985522858798504,0.654
+1.0,4,credit_bridge,42,nan,0.5798117071390152,0.5770738422870636,0.455
+1.0,4,bp,123,nan,0.7834264636039734,0.9999949932098389,0.7945
+1.0,4,dfa,123,nan,0.03143235971219838,0.04966995166614652,0.6755
+1.0,4,state_bridge,123,0.039300061028450725,0.32962899655103683,0.3271522969007492,0.653
+1.0,4,credit_bridge,123,nan,0.5211702585220337,0.5396891608834267,0.4955
+1.0,4,bp,456,nan,0.7738194167613983,0.9999952465295792,0.802
+1.0,4,dfa,456,nan,0.03672327077947557,0.03823344281408936,0.687
+1.0,4,state_bridge,456,0.041286803239956496,0.3122059181332588,0.3080013431608677,0.6535
+1.0,4,credit_bridge,456,nan,0.5516191199421883,0.5655025988817215,0.4575
+1.0,4,bp,789,nan,0.8138538002967834,0.9999965578317642,0.7785
+1.0,4,dfa,789,nan,0.044787557795643806,0.050711866933852434,0.6645
+1.0,4,state_bridge,789,0.047810701571404934,0.321536161005497,0.294049471616745,0.6505
+1.0,4,credit_bridge,789,nan,0.498397096991539,0.5393850430846214,0.4785
+1.0,4,bp,1024,nan,0.8156489282846451,0.9999967068433762,0.796
+1.0,4,dfa,1024,nan,0.05224289372563362,0.051481935661286116,0.7
+1.0,4,state_bridge,1024,0.03910857268422842,0.32806089147925377,0.3095804899930954,0.659
+1.0,4,credit_bridge,1024,nan,0.4827726259827614,0.5368957445025444,0.5295
+1.0,4,bp,2048,nan,0.8019291013479233,0.999995157122612,0.789
+1.0,4,dfa,2048,nan,0.04420588002540171,0.054026867961511016,0.693
+1.0,4,state_bridge,2048,0.04625624809600413,0.31831347569823265,0.30145254731178284,0.6415
+1.0,4,credit_bridge,2048,nan,0.6112141609191895,0.6130285188555717,0.49
+1.0,4,bp,3000,nan,0.8269992768764496,0.9999961256980896,0.8055
+1.0,4,dfa,3000,nan,0.031051788479089737,0.04200031189247966,0.68
+1.0,4,state_bridge,3000,0.03373044667486101,0.33542975783348083,0.3166932016611099,0.6695
+1.0,4,credit_bridge,3000,nan,0.5547316148877144,0.574584349989891,0.4775
+1.0,4,bp,4000,nan,0.8469432443380356,0.9999967068433762,0.7985
+1.0,4,dfa,4000,nan,0.05680405022576451,0.058447605930268764,0.6885
+1.0,4,state_bridge,4000,0.05037181778624654,0.31409140303730965,0.3157280907034874,0.661
+1.0,4,credit_bridge,4000,nan,0.5717481672763824,0.5648023337125778,0.4815
+1.0,4,bp,5000,nan,0.7843969911336899,0.9999955743551254,0.7815
+1.0,4,dfa,5000,nan,0.05237697158008814,0.05225806776434183,0.696
+1.0,4,state_bridge,5000,0.020391553843859583,0.3294149599969387,0.32860465347766876,0.6545
+1.0,4,credit_bridge,5000,nan,0.5680579468607903,0.5536182671785355,0.4935
+1.0,4,bp,6000,nan,0.8201684504747391,0.9999956488609314,0.777
+1.0,4,dfa,6000,nan,0.043721775291487575,0.053218568209558725,0.6775
+1.0,4,state_bridge,6000,0.03792587007582188,0.3274501822888851,0.3072219528257847,0.6375
+1.0,4,credit_bridge,6000,nan,0.5522571504116058,0.566972091794014,0.4445
+1.0,8,bp,42,nan,0.7569558992981911,0.9999850168824196,0.7715
+1.0,8,dfa,42,nan,0.04026596760377288,0.0463115515303798,0.6555
+1.0,8,state_bridge,42,0.024733127796091138,0.21983085572719574,0.21069114468991756,0.636
+1.0,8,credit_bridge,42,nan,0.42342473939061165,0.46175311505794525,0.501
+1.0,8,bp,123,nan,0.7589660733938217,0.9999836310744286,0.775
+1.0,8,dfa,123,nan,0.052690219948999584,0.04398861777735874,0.6655
+1.0,8,state_bridge,123,0.00904931088979356,0.23909619636833668,0.21755888871848583,0.6665
+1.0,8,credit_bridge,123,nan,0.4901758562773466,0.5194796472787857,0.499
+1.0,8,bp,456,nan,0.8152446746826172,0.9999835714697838,0.7615
+1.0,8,dfa,456,nan,0.06644452596083283,0.07211897929664701,0.6595
+1.0,8,state_bridge,456,0.027026508945971726,0.17726875049993396,0.14926194958388805,0.634
+1.0,8,credit_bridge,456,nan,0.4374451842159033,0.469217861071229,0.471
+1.0,8,bp,789,nan,0.7641392648220062,0.9999837800860405,0.77
+1.0,8,dfa,789,nan,0.06174341728910804,0.08079314092174172,0.663
+1.0,8,state_bridge,789,0.005507162362337112,0.27462095208466053,0.25958841666579247,0.6305
+1.0,8,credit_bridge,789,nan,0.4231471223756671,0.49050698801875114,0.5545
+1.0,8,bp,1024,nan,0.8173553571105003,0.9999844655394554,0.7665
+1.0,8,dfa,1024,nan,0.05211032921215519,0.04730354994535446,0.657
+1.0,8,state_bridge,1024,0.03733631442394108,0.24442856572568417,0.2330861296504736,0.651
+1.0,8,credit_bridge,1024,nan,0.43784002028405666,0.45684560760855675,0.4465
+1.0,8,bp,2048,nan,0.7980030849575996,0.9999835342168808,0.778
+1.0,8,dfa,2048,nan,0.03056039405055344,0.019816300831735134,0.6515
+1.0,8,state_bridge,2048,0.019828419615048914,0.21816869638860226,0.2006466817110777,0.639
+1.0,8,credit_bridge,2048,nan,0.44690870121121407,0.49068378657102585,0.491
+1.0,8,bp,3000,nan,0.8051622360944748,0.9999843910336494,0.774
+1.0,8,dfa,3000,nan,0.039206627872772515,0.04525771760381758,0.666
+1.0,8,state_bridge,3000,0.0073138324548490345,0.23834207840263844,0.22312144935131073,0.6255
+1.0,8,credit_bridge,3000,nan,0.48071201518177986,0.4987096134573221,0.494
+1.0,8,bp,4000,nan,0.7561301067471504,0.9999867826700211,0.7555
+1.0,8,dfa,4000,nan,0.046561805153032765,0.04669333342462778,0.651
+1.0,8,state_bridge,4000,0.021662812023889273,0.24263181537389755,0.23203729651868343,0.625
+1.0,8,credit_bridge,4000,nan,0.5189926661550999,0.5291757248342037,0.4825
+1.0,8,bp,5000,nan,0.8032103478908539,0.9999829307198524,0.769
+1.0,8,dfa,5000,nan,0.04911017371341586,0.044786704413127154,0.6785
+1.0,8,state_bridge,5000,0.0282969608604908,0.2040038648992777,0.2028592685237527,0.6355
+1.0,8,credit_bridge,5000,nan,0.4662684202194214,0.48669758066534996,0.5155
+1.0,8,bp,6000,nan,0.8568788692355156,0.9999835342168808,0.759
+1.0,8,dfa,6000,nan,0.047016877244459465,0.04356467560864985,0.6415
+1.0,8,state_bridge,6000,0.011806304380530491,0.23779606446623802,0.22786595951765776,0.637
+1.0,8,credit_bridge,6000,nan,0.45259521529078484,0.4737100824713707,0.4825
diff --git a/results/confirmatory/A1_synth_ladder.json b/results/confirmatory/A1_synth_ladder.json new file mode 100644 index 0000000..8556809 --- /dev/null +++ b/results/confirmatory/A1_synth_ladder.json @@ -0,0 +1,2402 @@ +[ + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.7976504862308502, + "rho": 0.9999783337116241, + "acc": 0.94 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.11274427606258541, + "rho": 0.10713508073240519, + "acc": 0.8635 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 42, + "StateErr": 0.0042416151915676895, + "Gamma": 0.39959777891635895, + "rho": 0.3636437617242336, + "acc": 0.7995 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.1352396421134472, + "rho": 0.29881402105093, + "acc": 0.7005 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.7471440732479095, + "rho": 0.9999679923057556, + "acc": 0.937 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.13869959628209472, + "rho": 0.13979829289019108, + "acc": 0.861 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 123, + "StateErr": 0.0025703998249955477, + "Gamma": 0.41838232427835464, + "rho": 0.36436110362410545, + "acc": 0.7955 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.08274209871888161, + "rho": 0.17103387042880058, + "acc": 0.6365 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.7569544464349747, + "rho": 0.9999799728393555, + "acc": 0.9395 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.061981291277334094, + "rho": 0.10030700685456395, + "acc": 0.8725 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 456, + "StateErr": 0.007971026756800711, + "Gamma": 0.3831471260637045, + "rho": 0.3577818237245083, + "acc": 0.793 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.14599335379898548, + "rho": 0.263176042586565, + "acc": 0.671 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.7442700117826462, + "rho": 0.999980166554451, + "acc": 0.94 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.15985053591430187, + "rho": 0.16940528759732842, + "acc": 0.852 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 789, + "StateErr": 0.008546572399325669, + "Gamma": 0.445812588557601, + "rho": 0.44155239313840866, + "acc": 0.8275 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.11118674091994762, + "rho": 0.2533306097611785, + "acc": 0.683 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.7894147336483002, + "rho": 0.9999719113111496, + "acc": 0.943 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.13776074955239892, + "rho": 0.16881290404126048, + "acc": 0.883 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 1024, + "StateErr": 0.003235291603486985, + "Gamma": 0.38014374673366547, + "rho": 0.2906125131994486, + "acc": 0.7695 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.10474677011370659, + "rho": 0.20625970140099525, + "acc": 0.6845 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.7971414774656296, + "rho": 0.9999726265668869, + "acc": 0.945 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.12636534194462, + "rho": 0.11639838851988316, + "acc": 0.87 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 2048, + "StateErr": 0.004507227457128465, + "Gamma": 0.4033546820282936, + "rho": 0.35091163218021393, + "acc": 0.768 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.11602311581373215, + "rho": 0.2716793781146407, + "acc": 0.682 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.799256831407547, + "rho": 0.9999824911355972, + "acc": 0.9405 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.05687434319406748, + "rho": 0.07215175591409206, + "acc": 0.8825 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 3000, + "StateErr": 0.004735227110795676, + "Gamma": 0.3631715625524521, + "rho": 0.2957744346931577, + "acc": 0.757 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.13282869290560484, + "rho": 0.2609284440986812, + "acc": 0.67 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.8004499226808548, + "rho": 0.999976709485054, + "acc": 0.94 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.1496470280108042, + "rho": 0.11480268160812557, + "acc": 0.853 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 4000, + "StateErr": 0.002814154534600675, + "Gamma": 0.3866612948477268, + "rho": 0.33644784428179264, + "acc": 0.781 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.08764260355383158, + "rho": 0.14309521671384573, + "acc": 0.704 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.7971527725458145, + "rho": 0.999970406293869, + "acc": 0.9345 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.10718152392655611, + "rho": 0.10537220258265734, + "acc": 0.8605 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 5000, + "StateErr": 0.0008024044685298576, + "Gamma": 0.4057263322174549, + "rho": 0.3554464727640152, + "acc": 0.8015 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.12732978258281946, + "rho": 0.25282857567071915, + "acc": 0.682 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "bp", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.7925817668437958, + "rho": 0.9999643862247467, + "acc": 0.94 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "dfa", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.05762777535710484, + "rho": 0.10805956274271011, + "acc": 0.865 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "state_bridge", + "seed": 6000, + "StateErr": 0.0011924138587201013, + "Gamma": 0.41266462951898575, + "rho": 0.3788386173546314, + "acc": 0.758 + }, + { + "alpha": 0.0, + "depth": 4, + "method": "credit_bridge", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.10305883921682835, + "rho": 0.22495971526950598, + "acc": 0.6755 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.8518797755241394, + "rho": 0.999953955411911, + "acc": 0.9225 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.06414316734299064, + "rho": 0.09805193985812366, + "acc": 0.846 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 42, + "StateErr": 0.014238399018533528, + "Gamma": 0.2220264421775937, + "rho": 0.19422245677560568, + "acc": 0.7575 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.11673698376398534, + "rho": 0.20357689913362265, + "acc": 0.6165 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.8461447432637215, + "rho": 0.9999523237347603, + "acc": 0.92 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.08495552814565599, + "rho": 0.09764496935531497, + "acc": 0.859 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 123, + "StateErr": 0.008134014961775392, + "Gamma": 0.20587642956525087, + "rho": 0.19290778459981084, + "acc": 0.774 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.0771957344841212, + "rho": 0.18929401598870754, + "acc": 0.675 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.8679735064506531, + "rho": 0.9999424889683723, + "acc": 0.9095 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.10550220380537212, + "rho": 0.11999525976716541, + "acc": 0.846 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 456, + "StateErr": 0.01331474657356739, + "Gamma": 0.23938010726124048, + "rho": 0.21203574631363153, + "acc": 0.759 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.06404305621981621, + "rho": 0.13004278298467398, + "acc": 0.6555 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.8145705312490463, + "rho": 0.9999416694045067, + "acc": 0.9135 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.07243995321914554, + "rho": 0.09743102500215173, + "acc": 0.8415 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 789, + "StateErr": 0.004964958165772259, + "Gamma": 0.25941951386630535, + "rho": 0.22139000706374645, + "acc": 0.7625 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.0831079410854727, + "rho": 0.15918657230213284, + "acc": 0.6245 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.8478909432888031, + "rho": 0.9999396428465843, + "acc": 0.92 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.06737133057322353, + "rho": 0.08042471471708268, + "acc": 0.8615 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 1024, + "StateErr": 0.006657648294698447, + "Gamma": 0.2543684197589755, + "rho": 0.2292898604646325, + "acc": 0.7645 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.08684566424926743, + "rho": 0.1955197062343359, + "acc": 0.6315 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.8799556866288185, + "rho": 0.9999411776661873, + "acc": 0.9135 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.049802992260083556, + "rho": 0.09564788592979312, + "acc": 0.862 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 2048, + "StateErr": 0.005097676486708224, + "Gamma": 0.26504675671458244, + "rho": 0.2259912956506014, + "acc": 0.7915 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.10936234518885612, + "rho": 0.16827034018933773, + "acc": 0.6145 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.8378414362668991, + "rho": 0.9999293759465218, + "acc": 0.9255 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.11397410440258682, + "rho": 0.1381535604596138, + "acc": 0.847 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 3000, + "StateErr": 0.014880129693076014, + "Gamma": 0.2201737705618143, + "rho": 0.19643162470310926, + "acc": 0.7825 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.08571063168346882, + "rho": 0.13454539375379682, + "acc": 0.6535 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.8487726971507072, + "rho": 0.9999396353960037, + "acc": 0.923 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.05830177618190646, + "rho": 0.09378852555528283, + "acc": 0.8485 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 4000, + "StateErr": 0.022656571633182465, + "Gamma": 0.23578881658613682, + "rho": 0.21631191670894623, + "acc": 0.7165 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.07291368849109858, + "rho": 0.1467280697543174, + "acc": 0.623 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.8527729287743568, + "rho": 0.9999343007802963, + "acc": 0.912 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.0880711309146136, + "rho": 0.07768160989508033, + "acc": 0.8435 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 5000, + "StateErr": 0.001290266184194479, + "Gamma": 0.26903260312974453, + "rho": 0.21923481859266758, + "acc": 0.706 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.08217159216292202, + "rho": 0.15023266337811947, + "acc": 0.6875 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "bp", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.8804621174931526, + "rho": 0.9999351277947426, + "acc": 0.9185 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "dfa", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.06979017774574459, + "rho": 0.08295673737302423, + "acc": 0.836 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "state_bridge", + "seed": 6000, + "StateErr": 0.003901493979152292, + "Gamma": 0.2838806174695492, + "rho": 0.23159940354526043, + "acc": 0.7935 + }, + { + "alpha": 0.0, + "depth": 8, + "method": "credit_bridge", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.1168073476292193, + "rho": 0.21190964989364147, + "acc": 0.6195 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.778934895992279, + "rho": 0.9999890029430389, + "acc": 0.8875 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.0771798980422318, + "rho": 0.07931134616956115, + "acc": 0.822 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 42, + "StateErr": 0.014639395724982024, + "Gamma": 0.3877125680446625, + "rho": 0.4016897827386856, + "acc": 0.777 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.18997271731495857, + "rho": 0.37320059537887573, + "acc": 0.653 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.7060435861349106, + "rho": 0.9999908953905106, + "acc": 0.8855 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.1366678001359105, + "rho": 0.14800320356152952, + "acc": 0.8155 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 123, + "StateErr": 0.00966854555811733, + "Gamma": 0.3970109149813652, + "rho": 0.34398024156689644, + "acc": 0.7615 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.1873281691223383, + "rho": 0.25071992352604866, + "acc": 0.656 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.712785392999649, + "rho": 0.9999827593564987, + "acc": 0.8955 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.06838363222777843, + "rho": 0.06962036329787225, + "acc": 0.839 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 456, + "StateErr": 0.008301737054251135, + "Gamma": 0.3867475911974907, + "rho": 0.36412956193089485, + "acc": 0.76 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.20988117344677448, + "rho": 0.33718378841876984, + "acc": 0.615 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.7440494745969772, + "rho": 0.99999038875103, + "acc": 0.89 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.08207487314939499, + "rho": 0.12179941032081842, + "acc": 0.8245 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 789, + "StateErr": 0.0079743139334023, + "Gamma": 0.4024505019187927, + "rho": 0.36630794778466225, + "acc": 0.747 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.1982774008065462, + "rho": 0.3794977590441704, + "acc": 0.68 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.7264262437820435, + "rho": 0.9999918192625046, + "acc": 0.905 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.0874363575130701, + "rho": 0.12305337563157082, + "acc": 0.848 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 1024, + "StateErr": 0.0007635582342045382, + "Gamma": 0.36391081660985947, + "rho": 0.2906326614320278, + "acc": 0.738 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.17063568718731403, + "rho": 0.2797263041138649, + "acc": 0.687 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.7792519479990005, + "rho": 0.9999875724315643, + "acc": 0.8935 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.08299929136410356, + "rho": 0.07884005922824144, + "acc": 0.819 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 2048, + "StateErr": 0.0069182014418765906, + "Gamma": 0.4167069047689438, + "rho": 0.38259734585881233, + "acc": 0.785 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.19401324167847633, + "rho": 0.35267358273267746, + "acc": 0.64 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.7533685863018036, + "rho": 0.9999895542860031, + "acc": 0.8935 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.06140868738293648, + "rho": 0.0712057501077652, + "acc": 0.8265 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 3000, + "StateErr": 0.005231155015993864, + "Gamma": 0.41574693098664284, + "rho": 0.35996725410223007, + "acc": 0.768 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.19257186725735664, + "rho": 0.3234494589269161, + "acc": 0.658 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.7910260111093521, + "rho": 0.9999895095825195, + "acc": 0.894 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.09473285032436252, + "rho": 0.14322808384895325, + "acc": 0.8505 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 4000, + "StateErr": 0.010673340854234993, + "Gamma": 0.38491013273596764, + "rho": 0.3446698300540447, + "acc": 0.7965 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.1998334638774395, + "rho": 0.3253607787191868, + "acc": 0.6195 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.7515025287866592, + "rho": 0.9999890476465225, + "acc": 0.885 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.08981088036671281, + "rho": 0.10898786783218384, + "acc": 0.8325 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 5000, + "StateErr": 0.008934525582939386, + "Gamma": 0.4031970761716366, + "rho": 0.35525544732809067, + "acc": 0.764 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.18700676038861275, + "rho": 0.3295666575431824, + "acc": 0.6465 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "bp", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.7618523091077805, + "rho": 0.9999861419200897, + "acc": 0.883 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "dfa", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.06073849997483194, + "rho": 0.10279720090329647, + "acc": 0.8225 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "state_bridge", + "seed": 6000, + "StateErr": 0.0113879487933591, + "Gamma": 0.4088409934192896, + "rho": 0.35849119909107685, + "acc": 0.7885 + }, + { + "alpha": 0.5, + "depth": 4, + "method": "credit_bridge", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.18101266399025917, + "rho": 0.2402007132768631, + "acc": 0.626 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.7800825014710426, + "rho": 0.9999619349837303, + "acc": 0.874 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.0680798904504627, + "rho": 0.07815029704943299, + "acc": 0.8105 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 42, + "StateErr": 0.013940593439154327, + "Gamma": 0.22329587768763304, + "rho": 0.18133462546393275, + "acc": 0.775 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.16422098223119974, + "rho": 0.2585284411907196, + "acc": 0.5765 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.741077221930027, + "rho": 0.9999566599726677, + "acc": 0.8835 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.06835221056826413, + "rho": 0.07870115921832621, + "acc": 0.826 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 123, + "StateErr": 0.01196373869618401, + "Gamma": 0.2609132928773761, + "rho": 0.22901193983852863, + "acc": 0.732 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.1594983646646142, + "rho": 0.28827742394059896, + "acc": 0.631 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.8024904653429985, + "rho": 0.9999617040157318, + "acc": 0.8605 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.09767440240830183, + "rho": 0.10227222001412883, + "acc": 0.804 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 456, + "StateErr": 0.007250209745950997, + "Gamma": 0.308225903660059, + "rho": 0.26882948353886604, + "acc": 0.746 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.17834078380838037, + "rho": 0.28439006581902504, + "acc": 0.607 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.7425128743052483, + "rho": 0.9999456480145454, + "acc": 0.862 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.07170737814158201, + "rho": 0.10886842722538859, + "acc": 0.795 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 789, + "StateErr": 0.010051667786203325, + "Gamma": 0.22407432738691568, + "rho": 0.19511110894382, + "acc": 0.755 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.15873884549364448, + "rho": 0.2204220425337553, + "acc": 0.6015 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.802269235253334, + "rho": 0.9999621286988258, + "acc": 0.889 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.07059638481587172, + "rho": 0.09517476172186434, + "acc": 0.844 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 1024, + "StateErr": 0.015034918455407024, + "Gamma": 0.26445331890136003, + "rho": 0.2382330046966672, + "acc": 0.754 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.14965758845210075, + "rho": 0.22262850403785706, + "acc": 0.5705 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.7915605679154396, + "rho": 0.9999616295099258, + "acc": 0.876 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.054607931757345796, + "rho": 0.0988450946751982, + "acc": 0.831 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 2048, + "StateErr": 0.011838351630140097, + "Gamma": 0.25473486352711916, + "rho": 0.22712249495089054, + "acc": 0.774 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.18231265805661678, + "rho": 0.23427345044910908, + "acc": 0.623 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.7524143233895302, + "rho": 0.9999579712748528, + "acc": 0.878 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.12144442298449576, + "rho": 0.1239226256730035, + "acc": 0.812 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 3000, + "StateErr": 0.01295656328741461, + "Gamma": 0.2410168293863535, + "rho": 0.21426095999777317, + "acc": 0.7855 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.13227692258078605, + "rho": 0.1761333434842527, + "acc": 0.5945 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.7426219433546066, + "rho": 0.9999610707163811, + "acc": 0.867 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.061901216860860586, + "rho": 0.09875185182318091, + "acc": 0.8075 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 4000, + "StateErr": 0.0033825204903259872, + "Gamma": 0.277063325047493, + "rho": 0.2407076209783554, + "acc": 0.7165 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.1314755715429783, + "rho": 0.2386790281161666, + "acc": 0.61 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.7783375456929207, + "rho": 0.9999541118741035, + "acc": 0.867 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.07623820775188506, + "rho": 0.07604160532355309, + "acc": 0.8255 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 5000, + "StateErr": 0.017978039579465987, + "Gamma": 0.24194164481014013, + "rho": 0.20475764898583293, + "acc": 0.736 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.1662514479830861, + "rho": 0.24278739467263222, + "acc": 0.6285 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "bp", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.8337910175323486, + "rho": 0.999962106347084, + "acc": 0.8735 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "dfa", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.07081685795856174, + "rho": 0.07124830968677998, + "acc": 0.794 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "state_bridge", + "seed": 6000, + "StateErr": 0.008247435386758298, + "Gamma": 0.2887079808861017, + "rho": 0.28017569333314896, + "acc": 0.737 + }, + { + "alpha": 0.5, + "depth": 8, + "method": "credit_bridge", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.1532246011774987, + "rho": 0.24239609204232693, + "acc": 0.546 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.837363988161087, + "rho": 0.9999942779541016, + "acc": 0.796 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.05314898584038019, + "rho": 0.04739766335114837, + "acc": 0.6945 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 42, + "StateErr": 0.05111445432715118, + "Gamma": 0.3092905580997467, + "rho": 0.2985522858798504, + "acc": 0.654 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.5798117071390152, + "rho": 0.5770738422870636, + "acc": 0.455 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.7834264636039734, + "rho": 0.9999949932098389, + "acc": 0.7945 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.03143235971219838, + "rho": 0.04966995166614652, + "acc": 0.6755 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 123, + "StateErr": 0.039300061028450725, + "Gamma": 0.32962899655103683, + "rho": 0.3271522969007492, + "acc": 0.653 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.5211702585220337, + "rho": 0.5396891608834267, + "acc": 0.4955 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.7738194167613983, + "rho": 0.9999952465295792, + "acc": 0.802 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.03672327077947557, + "rho": 0.03823344281408936, + "acc": 0.687 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 456, + "StateErr": 0.041286803239956496, + "Gamma": 0.3122059181332588, + "rho": 0.3080013431608677, + "acc": 0.6535 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.5516191199421883, + "rho": 0.5655025988817215, + "acc": 0.4575 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.8138538002967834, + "rho": 0.9999965578317642, + "acc": 0.7785 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.044787557795643806, + "rho": 0.050711866933852434, + "acc": 0.6645 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 789, + "StateErr": 0.047810701571404934, + "Gamma": 0.321536161005497, + "rho": 0.294049471616745, + "acc": 0.6505 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.498397096991539, + "rho": 0.5393850430846214, + "acc": 0.4785 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.8156489282846451, + "rho": 0.9999967068433762, + "acc": 0.796 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.05224289372563362, + "rho": 0.051481935661286116, + "acc": 0.7 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 1024, + "StateErr": 0.03910857268422842, + "Gamma": 0.32806089147925377, + "rho": 0.3095804899930954, + "acc": 0.659 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.4827726259827614, + "rho": 0.5368957445025444, + "acc": 0.5295 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.8019291013479233, + "rho": 0.999995157122612, + "acc": 0.789 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.04420588002540171, + "rho": 0.054026867961511016, + "acc": 0.693 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 2048, + "StateErr": 0.04625624809600413, + "Gamma": 0.31831347569823265, + "rho": 0.30145254731178284, + "acc": 0.6415 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.6112141609191895, + "rho": 0.6130285188555717, + "acc": 0.49 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.8269992768764496, + "rho": 0.9999961256980896, + "acc": 0.8055 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.031051788479089737, + "rho": 0.04200031189247966, + "acc": 0.68 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 3000, + "StateErr": 0.03373044667486101, + "Gamma": 0.33542975783348083, + "rho": 0.3166932016611099, + "acc": 0.6695 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.5547316148877144, + "rho": 0.574584349989891, + "acc": 0.4775 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.8469432443380356, + "rho": 0.9999967068433762, + "acc": 0.7985 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.05680405022576451, + "rho": 0.058447605930268764, + "acc": 0.6885 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 4000, + "StateErr": 0.05037181778624654, + "Gamma": 0.31409140303730965, + "rho": 0.3157280907034874, + "acc": 0.661 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.5717481672763824, + "rho": 0.5648023337125778, + "acc": 0.4815 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.7843969911336899, + "rho": 0.9999955743551254, + "acc": 0.7815 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.05237697158008814, + "rho": 0.05225806776434183, + "acc": 0.696 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 5000, + "StateErr": 0.020391553843859583, + "Gamma": 0.3294149599969387, + "rho": 0.32860465347766876, + "acc": 0.6545 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.5680579468607903, + "rho": 0.5536182671785355, + "acc": 0.4935 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "bp", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.8201684504747391, + "rho": 0.9999956488609314, + "acc": 0.777 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "dfa", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.043721775291487575, + "rho": 0.053218568209558725, + "acc": 0.6775 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "state_bridge", + "seed": 6000, + "StateErr": 0.03792587007582188, + "Gamma": 0.3274501822888851, + "rho": 0.3072219528257847, + "acc": 0.6375 + }, + { + "alpha": 1.0, + "depth": 4, + "method": "credit_bridge", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.5522571504116058, + "rho": 0.566972091794014, + "acc": 0.4445 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.7569558992981911, + "rho": 0.9999850168824196, + "acc": 0.7715 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.04026596760377288, + "rho": 0.0463115515303798, + "acc": 0.6555 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 42, + "StateErr": 0.024733127796091138, + "Gamma": 0.21983085572719574, + "rho": 0.21069114468991756, + "acc": 0.636 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.42342473939061165, + "rho": 0.46175311505794525, + "acc": 0.501 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.7589660733938217, + "rho": 0.9999836310744286, + "acc": 0.775 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.052690219948999584, + "rho": 0.04398861777735874, + "acc": 0.6655 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 123, + "StateErr": 0.00904931088979356, + "Gamma": 0.23909619636833668, + "rho": 0.21755888871848583, + "acc": 0.6665 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.4901758562773466, + "rho": 0.5194796472787857, + "acc": 0.499 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.8152446746826172, + "rho": 0.9999835714697838, + "acc": 0.7615 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.06644452596083283, + "rho": 0.07211897929664701, + "acc": 0.6595 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 456, + "StateErr": 0.027026508945971726, + "Gamma": 0.17726875049993396, + "rho": 0.14926194958388805, + "acc": 0.634 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.4374451842159033, + "rho": 0.469217861071229, + "acc": 0.471 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.7641392648220062, + "rho": 0.9999837800860405, + "acc": 0.77 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.06174341728910804, + "rho": 0.08079314092174172, + "acc": 0.663 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 789, + "StateErr": 0.005507162362337112, + "Gamma": 0.27462095208466053, + "rho": 0.25958841666579247, + "acc": 0.6305 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.4231471223756671, + "rho": 0.49050698801875114, + "acc": 0.5545 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.8173553571105003, + "rho": 0.9999844655394554, + "acc": 0.7665 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.05211032921215519, + "rho": 0.04730354994535446, + "acc": 0.657 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 1024, + "StateErr": 0.03733631442394108, + "Gamma": 0.24442856572568417, + "rho": 0.2330861296504736, + "acc": 0.651 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.43784002028405666, + "rho": 0.45684560760855675, + "acc": 0.4465 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.7980030849575996, + "rho": 0.9999835342168808, + "acc": 0.778 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.03056039405055344, + "rho": 0.019816300831735134, + "acc": 0.6515 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 2048, + "StateErr": 0.019828419615048914, + "Gamma": 0.21816869638860226, + "rho": 0.2006466817110777, + "acc": 0.639 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.44690870121121407, + "rho": 0.49068378657102585, + "acc": 0.491 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.8051622360944748, + "rho": 0.9999843910336494, + "acc": 0.774 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.039206627872772515, + "rho": 0.04525771760381758, + "acc": 0.666 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 3000, + "StateErr": 0.0073138324548490345, + "Gamma": 0.23834207840263844, + "rho": 0.22312144935131073, + "acc": 0.6255 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.48071201518177986, + "rho": 0.4987096134573221, + "acc": 0.494 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.7561301067471504, + "rho": 0.9999867826700211, + "acc": 0.7555 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.046561805153032765, + "rho": 0.04669333342462778, + "acc": 0.651 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 4000, + "StateErr": 0.021662812023889273, + "Gamma": 0.24263181537389755, + "rho": 0.23203729651868343, + "acc": 0.625 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.5189926661550999, + "rho": 0.5291757248342037, + "acc": 0.4825 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.8032103478908539, + "rho": 0.9999829307198524, + "acc": 0.769 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.04911017371341586, + "rho": 0.044786704413127154, + "acc": 0.6785 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 5000, + "StateErr": 0.0282969608604908, + "Gamma": 0.2040038648992777, + "rho": 0.2028592685237527, + "acc": 0.6355 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.4662684202194214, + "rho": 0.48669758066534996, + "acc": 0.5155 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "bp", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.8568788692355156, + "rho": 0.9999835342168808, + "acc": 0.759 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "dfa", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.047016877244459465, + "rho": 0.04356467560864985, + "acc": 0.6415 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "state_bridge", + "seed": 6000, + "StateErr": 0.011806304380530491, + "Gamma": 0.23779606446623802, + "rho": 0.22786595951765776, + "acc": 0.637 + }, + { + "alpha": 1.0, + "depth": 8, + "method": "credit_bridge", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.45259521529078484, + "rho": 0.4737100824713707, + "acc": 0.4825 + } +]
\ No newline at end of file diff --git a/results/confirmatory/A2_cifar_state_vs_credit.csv b/results/confirmatory/A2_cifar_state_vs_credit.csv new file mode 100644 index 0000000..68156b5 --- /dev/null +++ b/results/confirmatory/A2_cifar_state_vs_credit.csv @@ -0,0 +1,31 @@ +method,seed,StateErr,Gamma,rho,acc
+dfa,42,nan,0.10076353600015864,-0.00476757250726223,0.3116
+state_bridge,42,9.164297834395258e-07,0.02469697833294049,-0.012663604691624641,0.1737
+credit_bridge_eT,42,nan,0.17921950668096542,0.008699589408934116,0.283
+dfa,123,nan,0.11738517042249441,0.005885639227926731,0.3106
+state_bridge,123,4.6238185521913695e-05,0.17851687408983707,0.11297287911293097,0.2115
+credit_bridge_eT,123,nan,0.17617796920239925,0.0019302130676805973,0.2753
+dfa,456,nan,0.09892195643624291,-0.004888533148914576,0.297
+state_bridge,456,8.734992210497147e-05,0.26485365629196167,0.11413963884115219,0.2607
+credit_bridge_eT,456,nan,0.23417379707098007,-0.002338360995054245,0.2893
+dfa,789,nan,0.09602321637794375,-0.012195338495075703,0.3051
+state_bridge,789,0.0001798719183267094,0.219103729352355,0.1035128419753164,0.2542
+credit_bridge_eT,789,nan,0.1968853361904621,0.009125714423134923,0.2825
+dfa,1024,nan,0.10625142272328958,0.007073512766510248,0.3083
+state_bridge,1024,3.6692365324357524e-05,0.1903094481676817,0.08768247324042022,0.2338
+credit_bridge_eT,1024,nan,0.1989332279190421,0.006513587199151516,0.2975
+dfa,2048,nan,0.10264200350502506,0.0015445498283952475,0.3102
+state_bridge,2048,5.471739053819328e-05,0.1533527011051774,0.021173154236748815,0.2015
+credit_bridge_eT,2048,nan,0.21371350064873695,0.01922372356057167,0.2862
+dfa,3000,nan,0.09254411532310769,-0.0020813816227018833,0.3054
+state_bridge,3000,1.4380053736007537e-05,0.11326553428079933,0.03250931575894356,0.197
+credit_bridge_eT,3000,nan,0.19290144089609385,0.0142808947712183,0.2705
+dfa,4000,nan,0.09669098260928877,-0.006473553366959095,0.3016
+state_bridge,4000,2.7812045817612668e-05,0.09340349677950144,0.0637101479806006,0.2155
+credit_bridge_eT,4000,nan,0.15450781304389238,0.0030927499756217003,0.2966
+dfa,5000,nan,0.1122395604616031,0.0056443302892148495,0.3139
+state_bridge,5000,0.00019077022730000314,0.11574495000240859,0.15696023870259523,0.1957
+credit_bridge_eT,5000,nan,0.18241790123283863,0.007469391683116555,0.3002
+dfa,6000,nan,0.09259073645807803,0.0021172836422920227,0.282
+state_bridge,6000,0.00011229384421696886,0.2289915308356285,0.14905795291997492,0.2964
+credit_bridge_eT,6000,nan,0.20601818710565567,-0.0038647495675832033,0.2882
diff --git a/results/confirmatory/A2_cifar_state_vs_credit.json b/results/confirmatory/A2_cifar_state_vs_credit.json new file mode 100644 index 0000000..e994d03 --- /dev/null +++ b/results/confirmatory/A2_cifar_state_vs_credit.json @@ -0,0 +1,242 @@ +[ + { + "method": "dfa", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.10076353600015864, + "rho": -0.00476757250726223, + "acc": 0.3116 + }, + { + "method": "state_bridge", + "seed": 42, + "StateErr": 9.164297834395258e-07, + "Gamma": 0.02469697833294049, + "rho": -0.012663604691624641, + "acc": 0.1737 + }, + { + "method": "credit_bridge_eT", + "seed": 42, + "StateErr": NaN, + "Gamma": 0.17921950668096542, + "rho": 0.008699589408934116, + "acc": 0.283 + }, + { + "method": "dfa", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.11738517042249441, + "rho": 0.005885639227926731, + "acc": 0.3106 + }, + { + "method": "state_bridge", + "seed": 123, + "StateErr": 4.6238185521913695e-05, + "Gamma": 0.17851687408983707, + "rho": 0.11297287911293097, + "acc": 0.2115 + }, + { + "method": "credit_bridge_eT", + "seed": 123, + "StateErr": NaN, + "Gamma": 0.17617796920239925, + "rho": 0.0019302130676805973, + "acc": 0.2753 + }, + { + "method": "dfa", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.09892195643624291, + "rho": -0.004888533148914576, + "acc": 0.297 + }, + { + "method": "state_bridge", + "seed": 456, + "StateErr": 8.734992210497147e-05, + "Gamma": 0.26485365629196167, + "rho": 0.11413963884115219, + "acc": 0.2607 + }, + { + "method": "credit_bridge_eT", + "seed": 456, + "StateErr": NaN, + "Gamma": 0.23417379707098007, + "rho": -0.002338360995054245, + "acc": 0.2893 + }, + { + "method": "dfa", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.09602321637794375, + "rho": -0.012195338495075703, + "acc": 0.3051 + }, + { + "method": "state_bridge", + "seed": 789, + "StateErr": 0.0001798719183267094, + "Gamma": 0.219103729352355, + "rho": 0.1035128419753164, + "acc": 0.2542 + }, + { + "method": "credit_bridge_eT", + "seed": 789, + "StateErr": NaN, + "Gamma": 0.1968853361904621, + "rho": 0.009125714423134923, + "acc": 0.2825 + }, + { + "method": "dfa", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.10625142272328958, + "rho": 0.007073512766510248, + "acc": 0.3083 + }, + { + "method": "state_bridge", + "seed": 1024, + "StateErr": 3.6692365324357524e-05, + "Gamma": 0.1903094481676817, + "rho": 0.08768247324042022, + "acc": 0.2338 + }, + { + "method": "credit_bridge_eT", + "seed": 1024, + "StateErr": NaN, + "Gamma": 0.1989332279190421, + "rho": 0.006513587199151516, + "acc": 0.2975 + }, + { + "method": "dfa", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.10264200350502506, + "rho": 0.0015445498283952475, + "acc": 0.3102 + }, + { + "method": "state_bridge", + "seed": 2048, + "StateErr": 5.471739053819328e-05, + "Gamma": 0.1533527011051774, + "rho": 0.021173154236748815, + "acc": 0.2015 + }, + { + "method": "credit_bridge_eT", + "seed": 2048, + "StateErr": NaN, + "Gamma": 0.21371350064873695, + "rho": 0.01922372356057167, + "acc": 0.2862 + }, + { + "method": "dfa", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.09254411532310769, + "rho": -0.0020813816227018833, + "acc": 0.3054 + }, + { + "method": "state_bridge", + "seed": 3000, + "StateErr": 1.4380053736007537e-05, + "Gamma": 0.11326553428079933, + "rho": 0.03250931575894356, + "acc": 0.197 + }, + { + "method": "credit_bridge_eT", + "seed": 3000, + "StateErr": NaN, + "Gamma": 0.19290144089609385, + "rho": 0.0142808947712183, + "acc": 0.2705 + }, + { + "method": "dfa", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.09669098260928877, + "rho": -0.006473553366959095, + "acc": 0.3016 + }, + { + "method": "state_bridge", + "seed": 4000, + "StateErr": 2.7812045817612668e-05, + "Gamma": 0.09340349677950144, + "rho": 0.0637101479806006, + "acc": 0.2155 + }, + { + "method": "credit_bridge_eT", + "seed": 4000, + "StateErr": NaN, + "Gamma": 0.15450781304389238, + "rho": 0.0030927499756217003, + "acc": 0.2966 + }, + { + "method": "dfa", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.1122395604616031, + "rho": 0.0056443302892148495, + "acc": 0.3139 + }, + { + "method": "state_bridge", + "seed": 5000, + "StateErr": 0.00019077022730000314, + "Gamma": 0.11574495000240859, + "rho": 0.15696023870259523, + "acc": 0.1957 + }, + { + "method": "credit_bridge_eT", + "seed": 5000, + "StateErr": NaN, + "Gamma": 0.18241790123283863, + "rho": 0.007469391683116555, + "acc": 0.3002 + }, + { + "method": "dfa", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.09259073645807803, + "rho": 0.0021172836422920227, + "acc": 0.282 + }, + { + "method": "state_bridge", + "seed": 6000, + "StateErr": 0.00011229384421696886, + "Gamma": 0.2289915308356285, + "rho": 0.14905795291997492, + "acc": 0.2964 + }, + { + "method": "credit_bridge_eT", + "seed": 6000, + "StateErr": NaN, + "Gamma": 0.20601818710565567, + "rho": -0.0038647495675832033, + "acc": 0.2882 + } +]
\ No newline at end of file diff --git a/results/confirmatory/A3_frozen_vs_online.csv b/results/confirmatory/A3_frozen_vs_online.csv new file mode 100644 index 0000000..60ef6ad --- /dev/null +++ b/results/confirmatory/A3_frozen_vs_online.csv @@ -0,0 +1,61 @@ +regime,method,seed,Gamma,rho,nudge,acc
+frozen,dfa,42,0.004505785531364381,0.008119155652821064,-9.162341120827477e-05,nan
+frozen,scalar_cb,42,0.11589984688907862,0.18948250077664852,-0.001349875790765509,nan
+frozen,vec_eT_M4,42,0.3822024315595627,0.3902328796684742,-0.004724124912172556,nan
+online,dfa,42,0.10076353600015864,-0.00476757250726223,-1.0885996744036674e-06,0.3116
+online,scalar_cb,42,0.17921950668096542,0.008699589408934116,-2.653570845723152e-06,0.283
+online,vec_eT_M4,42,0.002346692723222077,-0.000655805692076683,4.1443854570388794e-08,0.3116
+frozen,dfa,123,0.006769153871573508,0.006851167418062687,-4.7400368202943355e-05,nan
+frozen,scalar_cb,123,0.12154238484799862,0.20577848330140114,-0.0014780515775782987,nan
+frozen,vec_eT_M4,123,0.3757530301809311,0.3804876245558262,-0.004601294989697635,nan
+online,dfa,123,0.11738517042249441,0.005885639227926731,-1.4995457604527473e-06,0.3106
+online,scalar_cb,123,0.17617796920239925,0.0019302130676805973,-2.4832552298903465e-06,0.2753
+online,vec_eT_M4,123,-0.003648026817245409,0.008452285081148148,2.468004822731018e-08,0.3106
+frozen,dfa,456,-0.0022321121068671346,-0.025214229593984783,9.127522389462683e-06,nan
+frozen,scalar_cb,456,0.14351800736039877,0.13401155546307564,-0.0015096227143658325,nan
+frozen,vec_eT_M4,456,0.37154657021164894,0.3922838978469372,-0.004483302065636963,nan
+online,dfa,456,0.09892195643624291,-0.004888533148914576,-9.865034371614456e-07,0.297
+online,scalar_cb,456,0.23417379707098007,-0.002338360995054245,-1.8764985725283623e-06,0.2893
+online,vec_eT_M4,456,8.133021765388548e-05,0.0004630112089216709,-3.562308847904205e-08,0.297
+frozen,dfa,789,0.0010723177110776305,-9.368173778057098e-06,-3.215731521777343e-05,nan
+frozen,scalar_cb,789,0.116337139159441,0.1693769097328186,-0.0013340917212190107,nan
+frozen,vec_eT_M4,789,0.3803345151245594,0.40245913714170456,-0.00463068817043677,nan
+online,dfa,789,0.09602321637794375,-0.012195338495075703,-1.2458767741918564e-06,0.3051
+online,scalar_cb,789,0.1968853361904621,0.009125714423134923,-2.344953827559948e-06,0.2825
+online,vec_eT_M4,789,-0.0013487973556038924,-0.0010448158718645573,1.5133991837501526e-09,0.3051
+frozen,dfa,1024,0.004047981696203351,-0.004657504265196621,-5.443185364129022e-05,nan
+frozen,scalar_cb,1024,0.13928020559251308,0.17055200971662998,-0.00163848337251693,nan
+frozen,vec_eT_M4,1024,0.3866974972188473,0.3883730284869671,-0.0044556696666404605,nan
+online,dfa,1024,0.10625142272328958,0.007073512766510248,-1.2062955647706985e-06,0.3083
+online,scalar_cb,1024,0.1989332279190421,0.006513587199151516,-2.512824721634388e-06,0.2975
+online,vec_eT_M4,1024,-0.0008005678537301719,0.0017584874294698238,-2.60770320892334e-08,0.3083
+frozen,dfa,2048,-0.01007436728104949,-0.022396742831915617,9.364452944282675e-05,nan
+frozen,scalar_cb,2048,0.11622164957225323,0.09785426966845989,-0.001329392776824534,nan
+frozen,vec_eT_M4,2048,0.37112750113010406,0.4292800724506378,-0.004542236507404596,nan
+online,dfa,2048,0.10264200350502506,0.0015445498283952475,-1.2172386050224304e-06,0.3102
+online,scalar_cb,2048,0.21371350064873695,0.01922372356057167,-1.786043867468834e-06,0.2862
+online,vec_eT_M4,2048,-0.0013046257517999038,-0.0022081234492361546,2.7939677238464355e-09,0.3102
+frozen,dfa,3000,0.012637533247470856,-0.003309569787234068,-0.0001136179780587554,nan
+frozen,scalar_cb,3000,0.12735093012452126,0.16207957826554775,-0.0014450718881562352,nan
+frozen,vec_eT_M4,3000,0.3722042217850685,0.4153371751308441,-0.005018443625885993,nan
+online,dfa,3000,0.09254411532310769,-0.0020813816227018833,-8.647330105304718e-07,0.3054
+online,scalar_cb,3000,0.19290144089609385,0.0142808947712183,-1.846812665462494e-06,0.2705
+online,vec_eT_M4,3000,0.0027015579398721457,0.0031934939324855804,-4.7497451305389404e-08,0.3054
+frozen,dfa,4000,0.012827006517909467,-0.022367039695382118,-0.00015865873137954623,nan
+frozen,scalar_cb,4000,0.14385110139846802,0.1320151798427105,-0.001822189544327557,nan
+frozen,vec_eT_M4,4000,0.37244511023163795,0.4071832448244095,-0.004847258620429784,nan
+online,dfa,4000,0.09669098260928877,-0.006473553366959095,-1.225038431584835e-06,0.3016
+online,scalar_cb,4000,0.15450781304389238,0.0030927499756217003,-2.410844899713993e-06,0.2966
+online,vec_eT_M4,4000,-0.0019520780188031495,-0.0038583979476243258,1.885928213596344e-08,0.3016
+frozen,dfa,5000,0.0013664336875081062,0.006194833666086197,-4.495742837207217e-05,nan
+frozen,scalar_cb,5000,0.11013340763747692,0.1481193359941244,-0.0015028833877295256,nan
+frozen,vec_eT_M4,5000,0.3717799335718155,0.38883912563323975,-0.004724961589090526,nan
+online,dfa,5000,0.1122395604616031,0.0056443302892148495,-1.3115350157022476e-06,0.3139
+online,scalar_cb,5000,0.18241790123283863,0.007469391683116555,-2.3095635697245598e-06,0.3002
+online,vec_eT_M4,5000,-0.0025941856001736596,-0.005957933608442545,6.891787052154541e-08,0.3139
+frozen,dfa,6000,0.0008208117797039449,-0.004433086141943932,-5.1620487283798866e-05,nan
+frozen,scalar_cb,6000,0.11223629210144281,0.1833980716764927,-0.0013175156782381237,nan
+frozen,vec_eT_M4,6000,0.37406010553240776,0.3813984990119934,-0.004561110574286431,nan
+online,dfa,6000,0.09259073645807803,0.0021172836422920227,-7.421476766467094e-07,0.282
+online,scalar_cb,6000,0.20601818710565567,-0.0038647495675832033,-1.5320256352424622e-06,0.2882
+online,vec_eT_M4,6000,-0.003383698291145265,-0.00234654126688838,-1.234002411365509e-08,0.282
diff --git a/results/confirmatory/A3_frozen_vs_online.json b/results/confirmatory/A3_frozen_vs_online.json new file mode 100644 index 0000000..f88a7c6 --- /dev/null +++ b/results/confirmatory/A3_frozen_vs_online.json @@ -0,0 +1,542 @@ +[ + { + "regime": "frozen", + "method": "dfa", + "seed": 42, + "Gamma": 0.004505785531364381, + "rho": 0.008119155652821064, + "nudge": -9.162341120827477e-05, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 42, + "Gamma": 0.11589984688907862, + "rho": 0.18948250077664852, + "nudge": -0.001349875790765509, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 42, + "Gamma": 0.3822024315595627, + "rho": 0.3902328796684742, + "nudge": -0.004724124912172556, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 42, + "Gamma": 0.10076353600015864, + "rho": -0.00476757250726223, + "nudge": -1.0885996744036674e-06, + "acc": 0.3116 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 42, + "Gamma": 0.17921950668096542, + "rho": 0.008699589408934116, + "nudge": -2.653570845723152e-06, + "acc": 0.283 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 42, + "Gamma": 0.002346692723222077, + "rho": -0.000655805692076683, + "nudge": 4.1443854570388794e-08, + "acc": 0.3116 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 123, + "Gamma": 0.006769153871573508, + "rho": 0.006851167418062687, + "nudge": -4.7400368202943355e-05, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 123, + "Gamma": 0.12154238484799862, + "rho": 0.20577848330140114, + "nudge": -0.0014780515775782987, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 123, + "Gamma": 0.3757530301809311, + "rho": 0.3804876245558262, + "nudge": -0.004601294989697635, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 123, + "Gamma": 0.11738517042249441, + "rho": 0.005885639227926731, + "nudge": -1.4995457604527473e-06, + "acc": 0.3106 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 123, + "Gamma": 0.17617796920239925, + "rho": 0.0019302130676805973, + "nudge": -2.4832552298903465e-06, + "acc": 0.2753 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 123, + "Gamma": -0.003648026817245409, + "rho": 0.008452285081148148, + "nudge": 2.468004822731018e-08, + "acc": 0.3106 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 456, + "Gamma": -0.0022321121068671346, + "rho": -0.025214229593984783, + "nudge": 9.127522389462683e-06, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 456, + "Gamma": 0.14351800736039877, + "rho": 0.13401155546307564, + "nudge": -0.0015096227143658325, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 456, + "Gamma": 0.37154657021164894, + "rho": 0.3922838978469372, + "nudge": -0.004483302065636963, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 456, + "Gamma": 0.09892195643624291, + "rho": -0.004888533148914576, + "nudge": -9.865034371614456e-07, + "acc": 0.297 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 456, + "Gamma": 0.23417379707098007, + "rho": -0.002338360995054245, + "nudge": -1.8764985725283623e-06, + "acc": 0.2893 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 456, + "Gamma": 8.133021765388548e-05, + "rho": 0.0004630112089216709, + "nudge": -3.562308847904205e-08, + "acc": 0.297 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 789, + "Gamma": 0.0010723177110776305, + "rho": -9.368173778057098e-06, + "nudge": -3.215731521777343e-05, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 789, + "Gamma": 0.116337139159441, + "rho": 0.1693769097328186, + "nudge": -0.0013340917212190107, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 789, + "Gamma": 0.3803345151245594, + "rho": 0.40245913714170456, + "nudge": -0.00463068817043677, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 789, + "Gamma": 0.09602321637794375, + "rho": -0.012195338495075703, + "nudge": -1.2458767741918564e-06, + "acc": 0.3051 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 789, + "Gamma": 0.1968853361904621, + "rho": 0.009125714423134923, + "nudge": -2.344953827559948e-06, + "acc": 0.2825 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 789, + "Gamma": -0.0013487973556038924, + "rho": -0.0010448158718645573, + "nudge": 1.5133991837501526e-09, + "acc": 0.3051 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 1024, + "Gamma": 0.004047981696203351, + "rho": -0.004657504265196621, + "nudge": -5.443185364129022e-05, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 1024, + "Gamma": 0.13928020559251308, + "rho": 0.17055200971662998, + "nudge": -0.00163848337251693, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 1024, + "Gamma": 0.3866974972188473, + "rho": 0.3883730284869671, + "nudge": -0.0044556696666404605, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 1024, + "Gamma": 0.10625142272328958, + "rho": 0.007073512766510248, + "nudge": -1.2062955647706985e-06, + "acc": 0.3083 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 1024, + "Gamma": 0.1989332279190421, + "rho": 0.006513587199151516, + "nudge": -2.512824721634388e-06, + "acc": 0.2975 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 1024, + "Gamma": -0.0008005678537301719, + "rho": 0.0017584874294698238, + "nudge": -2.60770320892334e-08, + "acc": 0.3083 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 2048, + "Gamma": -0.01007436728104949, + "rho": -0.022396742831915617, + "nudge": 9.364452944282675e-05, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 2048, + "Gamma": 0.11622164957225323, + "rho": 0.09785426966845989, + "nudge": -0.001329392776824534, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 2048, + "Gamma": 0.37112750113010406, + "rho": 0.4292800724506378, + "nudge": -0.004542236507404596, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 2048, + "Gamma": 0.10264200350502506, + "rho": 0.0015445498283952475, + "nudge": -1.2172386050224304e-06, + "acc": 0.3102 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 2048, + "Gamma": 0.21371350064873695, + "rho": 0.01922372356057167, + "nudge": -1.786043867468834e-06, + "acc": 0.2862 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 2048, + "Gamma": -0.0013046257517999038, + "rho": -0.0022081234492361546, + "nudge": 2.7939677238464355e-09, + "acc": 0.3102 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 3000, + "Gamma": 0.012637533247470856, + "rho": -0.003309569787234068, + "nudge": -0.0001136179780587554, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 3000, + "Gamma": 0.12735093012452126, + "rho": 0.16207957826554775, + "nudge": -0.0014450718881562352, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 3000, + "Gamma": 0.3722042217850685, + "rho": 0.4153371751308441, + "nudge": -0.005018443625885993, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 3000, + "Gamma": 0.09254411532310769, + "rho": -0.0020813816227018833, + "nudge": -8.647330105304718e-07, + "acc": 0.3054 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 3000, + "Gamma": 0.19290144089609385, + "rho": 0.0142808947712183, + "nudge": -1.846812665462494e-06, + "acc": 0.2705 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 3000, + "Gamma": 0.0027015579398721457, + "rho": 0.0031934939324855804, + "nudge": -4.7497451305389404e-08, + "acc": 0.3054 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 4000, + "Gamma": 0.012827006517909467, + "rho": -0.022367039695382118, + "nudge": -0.00015865873137954623, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 4000, + "Gamma": 0.14385110139846802, + "rho": 0.1320151798427105, + "nudge": -0.001822189544327557, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 4000, + "Gamma": 0.37244511023163795, + "rho": 0.4071832448244095, + "nudge": -0.004847258620429784, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 4000, + "Gamma": 0.09669098260928877, + "rho": -0.006473553366959095, + "nudge": -1.225038431584835e-06, + "acc": 0.3016 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 4000, + "Gamma": 0.15450781304389238, + "rho": 0.0030927499756217003, + "nudge": -2.410844899713993e-06, + "acc": 0.2966 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 4000, + "Gamma": -0.0019520780188031495, + "rho": -0.0038583979476243258, + "nudge": 1.885928213596344e-08, + "acc": 0.3016 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 5000, + "Gamma": 0.0013664336875081062, + "rho": 0.006194833666086197, + "nudge": -4.495742837207217e-05, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 5000, + "Gamma": 0.11013340763747692, + "rho": 0.1481193359941244, + "nudge": -0.0015028833877295256, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 5000, + "Gamma": 0.3717799335718155, + "rho": 0.38883912563323975, + "nudge": -0.004724961589090526, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 5000, + "Gamma": 0.1122395604616031, + "rho": 0.0056443302892148495, + "nudge": -1.3115350157022476e-06, + "acc": 0.3139 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 5000, + "Gamma": 0.18241790123283863, + "rho": 0.007469391683116555, + "nudge": -2.3095635697245598e-06, + "acc": 0.3002 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 5000, + "Gamma": -0.0025941856001736596, + "rho": -0.005957933608442545, + "nudge": 6.891787052154541e-08, + "acc": 0.3139 + }, + { + "regime": "frozen", + "method": "dfa", + "seed": 6000, + "Gamma": 0.0008208117797039449, + "rho": -0.004433086141943932, + "nudge": -5.1620487283798866e-05, + "acc": NaN + }, + { + "regime": "frozen", + "method": "scalar_cb", + "seed": 6000, + "Gamma": 0.11223629210144281, + "rho": 0.1833980716764927, + "nudge": -0.0013175156782381237, + "acc": NaN + }, + { + "regime": "frozen", + "method": "vec_eT_M4", + "seed": 6000, + "Gamma": 0.37406010553240776, + "rho": 0.3813984990119934, + "nudge": -0.004561110574286431, + "acc": NaN + }, + { + "regime": "online", + "method": "dfa", + "seed": 6000, + "Gamma": 0.09259073645807803, + "rho": 0.0021172836422920227, + "nudge": -7.421476766467094e-07, + "acc": 0.282 + }, + { + "regime": "online", + "method": "scalar_cb", + "seed": 6000, + "Gamma": 0.20601818710565567, + "rho": -0.0038647495675832033, + "nudge": -1.5320256352424622e-06, + "acc": 0.2882 + }, + { + "regime": "online", + "method": "vec_eT_M4", + "seed": 6000, + "Gamma": -0.003383698291145265, + "rho": -0.00234654126688838, + "nudge": -1.234002411365509e-08, + "acc": 0.282 + } +]
\ No newline at end of file diff --git a/results/confirmatory/A4_protocol_dependence.csv b/results/confirmatory/A4_protocol_dependence.csv new file mode 100644 index 0000000..509e51f --- /dev/null +++ b/results/confirmatory/A4_protocol_dependence.csv @@ -0,0 +1,83 @@ +slice,method,metric,value
+snapshot_exploit_ep100,dfa,gamma,0.008470005544950254
+snapshot_exploit_ep100,dfa,rho,-0.022525018197484314
+snapshot_exploit_ep100,dfa,nudge,-2.3932622525535407e-05
+snapshot_exploit_ep100,dfa,loss_before,1.1802499413490295
+snapshot_exploit_ep100,dfa,loss_after_1step,1.1798909544944762
+snapshot_exploit_ep100,dfa,delta_loss_1step,-0.00035898685455326707
+snapshot_exploit_ep100,dfa,delta_acc_1step,0.002343749999999978
+snapshot_exploit_ep100,scalar_cb,gamma,0.12223921716213226
+snapshot_exploit_ep100,scalar_cb,rho,0.09000759199261665
+snapshot_exploit_ep100,scalar_cb,nudge,-0.00036903251748299226
+snapshot_exploit_ep100,scalar_cb,loss_before,1.1802499413490295
+snapshot_exploit_ep100,scalar_cb,loss_after_1step,1.1835142850875855
+snapshot_exploit_ep100,scalar_cb,delta_loss_1step,0.003264343738555997
+snapshot_exploit_ep100,scalar_cb,delta_acc_1step,0.0007812499999999556
+snapshot_exploit_ep100,vec_eT_M4,gamma,0.37760399281978607
+snapshot_exploit_ep100,vec_eT_M4,rho,0.41141805797815323
+snapshot_exploit_ep100,vec_eT_M4,nudge,-0.0013705549063161016
+snapshot_exploit_ep100,vec_eT_M4,loss_before,1.1802499413490295
+snapshot_exploit_ep100,vec_eT_M4,loss_after_1step,1.1832772016525268
+snapshot_exploit_ep100,vec_eT_M4,delta_loss_1step,0.0030272603034973145
+snapshot_exploit_ep100,vec_eT_M4,delta_acc_1step,-0.0031250000000000444
+snapshot_exploit_ep100,oracle_bp,gamma,1.0
+snapshot_exploit_ep100,oracle_bp,rho,0.9979714304208755
+snapshot_exploit_ep100,oracle_bp,nudge,-0.0031722619314678013
+snapshot_exploit_ep100,oracle_bp,loss_before,1.1802499413490295
+snapshot_exploit_ep100,oracle_bp,loss_after_1step,1.1796290874481201
+snapshot_exploit_ep100,oracle_bp,delta_loss_1step,-0.0006208539009093794
+snapshot_exploit_ep100,oracle_bp,delta_acc_1step,0.005468750000000022
+snapshot_ep5,dfa,dl_held_1,6.842613220214844e-05
+snapshot_ep5,dfa,dl_same_1,-0.011249661445617676
+snapshot_ep5,dfa,dl_held_5,0.0032531023025512695
+snapshot_ep5,dfa,dl_same_5,-0.004767775535583496
+snapshot_ep5,vec_eT_M4,dl_held_1,-0.0002554655075073242
+snapshot_ep5,vec_eT_M4,dl_same_1,-0.011568784713745117
+snapshot_ep5,vec_eT_M4,dl_held_5,-0.005142092704772949
+snapshot_ep5,vec_eT_M4,dl_same_5,-0.007385373115539551
+snapshot_ep5,oracle_bp,dl_held_1,-0.0002244710922241211
+snapshot_ep5,oracle_bp,dl_same_1,-0.011359930038452148
+snapshot_ep5,oracle_bp,dl_held_5,-0.008973956108093262
+snapshot_ep5,oracle_bp,dl_same_5,-0.008551597595214844
+snapshot_ep20,dfa,dl_held_1,0.0006439685821533203
+snapshot_ep20,dfa,dl_same_1,-0.0050983428955078125
+snapshot_ep20,dfa,dl_held_5,0.0009417533874511719
+snapshot_ep20,dfa,dl_same_5,-0.001807570457458496
+snapshot_ep20,vec_eT_M4,dl_held_1,4.982948303222656e-05
+snapshot_ep20,vec_eT_M4,dl_same_1,-0.005342721939086914
+snapshot_ep20,vec_eT_M4,dl_held_5,0.0015401840209960938
+snapshot_ep20,vec_eT_M4,dl_same_5,-0.00039839744567871094
+snapshot_ep20,oracle_bp,dl_held_1,6.651878356933594e-05
+snapshot_ep20,oracle_bp,dl_same_1,-0.004968762397766113
+snapshot_ep20,oracle_bp,dl_held_5,0.0003821849822998047
+snapshot_ep20,oracle_bp,dl_same_5,-0.0005745887756347656
+scaffold_3seed,dfa,final_mean,0.30593333333333333
+scaffold_3seed,dfa,final_std,0.006104825049818285
+scaffold_3seed,dfa,acc20_mean,0.30383333333333334
+scaffold_3seed,dfa,acc20_std,0.007825741001478518
+scaffold_3seed,perlayer,final_mean,0.30396666666666666
+scaffold_3seed,perlayer,final_std,0.006049977043115317
+scaffold_3seed,perlayer,acc20_mean,0.2874666666666667
+scaffold_3seed,perlayer,acc20_std,0.011382540235914942
+scaffold_3seed,vec,final_mean,0.3133333333333333
+scaffold_3seed,vec,final_std,0.007196912918436331
+scaffold_3seed,vec,acc20_mean,0.30333333333333334
+scaffold_3seed,vec,acc20_std,0.007024401912065005
+online_3seed,dfa_s42,test_acc,0.3116
+online_3seed,dfa_s42,mean_gamma,0.10076353600015864
+online_3seed,dfa_s42,mean_rho,-0.00476757250726223
+online_3seed,cb_eT_s42,test_acc,0.283
+online_3seed,cb_eT_s42,mean_gamma,0.17921950668096542
+online_3seed,cb_eT_s42,mean_rho,0.008699589408934116
+online_3seed,dfa_s123,test_acc,0.3106
+online_3seed,dfa_s123,mean_gamma,0.11738517042249441
+online_3seed,dfa_s123,mean_rho,0.005885639227926731
+online_3seed,cb_eT_s123,test_acc,0.2753
+online_3seed,cb_eT_s123,mean_gamma,0.17617796920239925
+online_3seed,cb_eT_s123,mean_rho,0.0019302130676805973
+online_3seed,dfa_s456,test_acc,0.297
+online_3seed,dfa_s456,mean_gamma,0.09892195643624291
+online_3seed,dfa_s456,mean_rho,-0.004888533148914576
+online_3seed,cb_eT_s456,test_acc,0.2893
+online_3seed,cb_eT_s456,mean_gamma,0.23417379707098007
+online_3seed,cb_eT_s456,mean_rho,-0.002338360995054245
diff --git a/results/confirmatory/A4_protocol_dependence.json b/results/confirmatory/A4_protocol_dependence.json new file mode 100644 index 0000000..59671d4 --- /dev/null +++ b/results/confirmatory/A4_protocol_dependence.json @@ -0,0 +1,494 @@ +[ + { + "slice": "snapshot_exploit_ep100", + "method": "dfa", + "metric": "gamma", + "value": 0.008470005544950254 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "dfa", + "metric": "rho", + "value": -0.022525018197484314 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "dfa", + "metric": "nudge", + "value": -2.3932622525535407e-05 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "dfa", + "metric": "loss_before", + "value": 1.1802499413490295 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "dfa", + "metric": "loss_after_1step", + "value": 1.1798909544944762 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "dfa", + "metric": "delta_loss_1step", + "value": -0.00035898685455326707 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "dfa", + "metric": "delta_acc_1step", + "value": 0.002343749999999978 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "scalar_cb", + "metric": "gamma", + "value": 0.12223921716213226 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "scalar_cb", + "metric": "rho", + "value": 0.09000759199261665 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "scalar_cb", + "metric": "nudge", + "value": -0.00036903251748299226 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "scalar_cb", + "metric": "loss_before", + "value": 1.1802499413490295 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "scalar_cb", + "metric": "loss_after_1step", + "value": 1.1835142850875855 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "scalar_cb", + "metric": "delta_loss_1step", + "value": 0.003264343738555997 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "scalar_cb", + "metric": "delta_acc_1step", + "value": 0.0007812499999999556 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "vec_eT_M4", + "metric": "gamma", + "value": 0.37760399281978607 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "vec_eT_M4", + "metric": "rho", + "value": 0.41141805797815323 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "vec_eT_M4", + "metric": "nudge", + "value": -0.0013705549063161016 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "vec_eT_M4", + "metric": "loss_before", + "value": 1.1802499413490295 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "vec_eT_M4", + "metric": "loss_after_1step", + "value": 1.1832772016525268 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "vec_eT_M4", + "metric": "delta_loss_1step", + "value": 0.0030272603034973145 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "vec_eT_M4", + "metric": "delta_acc_1step", + "value": -0.0031250000000000444 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "oracle_bp", + "metric": "gamma", + "value": 1.0 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "oracle_bp", + "metric": "rho", + "value": 0.9979714304208755 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "oracle_bp", + "metric": "nudge", + "value": -0.0031722619314678013 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "oracle_bp", + "metric": "loss_before", + "value": 1.1802499413490295 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "oracle_bp", + "metric": "loss_after_1step", + "value": 1.1796290874481201 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "oracle_bp", + "metric": "delta_loss_1step", + "value": -0.0006208539009093794 + }, + { + "slice": "snapshot_exploit_ep100", + "method": "oracle_bp", + "metric": "delta_acc_1step", + "value": 0.005468750000000022 + }, + { + "slice": "snapshot_ep5", + "method": "dfa", + "metric": "dl_held_1", + "value": 6.842613220214844e-05 + }, + { + "slice": "snapshot_ep5", + "method": "dfa", + "metric": "dl_same_1", + "value": -0.011249661445617676 + }, + { + "slice": "snapshot_ep5", + "method": "dfa", + "metric": "dl_held_5", + "value": 0.0032531023025512695 + }, + { + "slice": "snapshot_ep5", + "method": "dfa", + "metric": "dl_same_5", + "value": -0.004767775535583496 + }, + { + "slice": "snapshot_ep5", + "method": "vec_eT_M4", + "metric": "dl_held_1", + "value": -0.0002554655075073242 + }, + { + "slice": "snapshot_ep5", + "method": "vec_eT_M4", + "metric": "dl_same_1", + "value": -0.011568784713745117 + }, + { + "slice": "snapshot_ep5", + "method": "vec_eT_M4", + "metric": "dl_held_5", + "value": -0.005142092704772949 + }, + { + "slice": "snapshot_ep5", + "method": "vec_eT_M4", + "metric": "dl_same_5", + "value": -0.007385373115539551 + }, + { + "slice": "snapshot_ep5", + "method": "oracle_bp", + "metric": "dl_held_1", + "value": -0.0002244710922241211 + }, + { + "slice": "snapshot_ep5", + "method": "oracle_bp", + "metric": "dl_same_1", + "value": -0.011359930038452148 + }, + { + "slice": "snapshot_ep5", + "method": "oracle_bp", + "metric": "dl_held_5", + "value": -0.008973956108093262 + }, + { + "slice": "snapshot_ep5", + "method": "oracle_bp", + "metric": "dl_same_5", + "value": -0.008551597595214844 + }, + { + "slice": "snapshot_ep20", + "method": "dfa", + "metric": "dl_held_1", + "value": 0.0006439685821533203 + }, + { + "slice": "snapshot_ep20", + "method": "dfa", + "metric": "dl_same_1", + "value": -0.0050983428955078125 + }, + { + "slice": "snapshot_ep20", + "method": "dfa", + "metric": "dl_held_5", + "value": 0.0009417533874511719 + }, + { + "slice": "snapshot_ep20", + "method": "dfa", + "metric": "dl_same_5", + "value": -0.001807570457458496 + }, + { + "slice": "snapshot_ep20", + "method": "vec_eT_M4", + "metric": "dl_held_1", + "value": 4.982948303222656e-05 + }, + { + "slice": "snapshot_ep20", + "method": "vec_eT_M4", + "metric": "dl_same_1", + "value": -0.005342721939086914 + }, + { + "slice": "snapshot_ep20", + "method": "vec_eT_M4", + "metric": "dl_held_5", + "value": 0.0015401840209960938 + }, + { + "slice": "snapshot_ep20", + "method": "vec_eT_M4", + "metric": "dl_same_5", + "value": -0.00039839744567871094 + }, + { + "slice": "snapshot_ep20", + "method": "oracle_bp", + "metric": "dl_held_1", + "value": 6.651878356933594e-05 + }, + { + "slice": "snapshot_ep20", + "method": "oracle_bp", + "metric": "dl_same_1", + "value": -0.004968762397766113 + }, + { + "slice": "snapshot_ep20", + "method": "oracle_bp", + "metric": "dl_held_5", + "value": 0.0003821849822998047 + }, + { + "slice": "snapshot_ep20", + "method": "oracle_bp", + "metric": "dl_same_5", + "value": -0.0005745887756347656 + }, + { + "slice": "scaffold_3seed", + "method": "dfa", + "metric": "final_mean", + "value": 0.30593333333333333 + }, + { + "slice": "scaffold_3seed", + "method": "dfa", + "metric": "final_std", + "value": 0.006104825049818285 + }, + { + "slice": "scaffold_3seed", + "method": "dfa", + "metric": "acc20_mean", + "value": 0.30383333333333334 + }, + { + "slice": "scaffold_3seed", + "method": "dfa", + "metric": "acc20_std", + "value": 0.007825741001478518 + }, + { + "slice": "scaffold_3seed", + "method": "perlayer", + "metric": "final_mean", + "value": 0.30396666666666666 + }, + { + "slice": "scaffold_3seed", + "method": "perlayer", + "metric": "final_std", + "value": 0.006049977043115317 + }, + { + "slice": "scaffold_3seed", + "method": "perlayer", + "metric": "acc20_mean", + "value": 0.2874666666666667 + }, + { + "slice": "scaffold_3seed", + "method": "perlayer", + "metric": "acc20_std", + "value": 0.011382540235914942 + }, + { + "slice": "scaffold_3seed", + "method": "vec", + "metric": "final_mean", + "value": 0.3133333333333333 + }, + { + "slice": "scaffold_3seed", + "method": "vec", + "metric": "final_std", + "value": 0.007196912918436331 + }, + { + "slice": "scaffold_3seed", + "method": "vec", + "metric": "acc20_mean", + "value": 0.30333333333333334 + }, + { + "slice": "scaffold_3seed", + "method": "vec", + "metric": "acc20_std", + "value": 0.007024401912065005 + }, + { + "slice": "online_3seed", + "method": "dfa_s42", + "metric": "test_acc", + "value": 0.3116 + }, + { + "slice": "online_3seed", + "method": "dfa_s42", + "metric": "mean_gamma", + "value": 0.10076353600015864 + }, + { + "slice": "online_3seed", + "method": "dfa_s42", + "metric": "mean_rho", + "value": -0.00476757250726223 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s42", + "metric": "test_acc", + "value": 0.283 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s42", + "metric": "mean_gamma", + "value": 0.17921950668096542 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s42", + "metric": "mean_rho", + "value": 0.008699589408934116 + }, + { + "slice": "online_3seed", + "method": "dfa_s123", + "metric": "test_acc", + "value": 0.3106 + }, + { + "slice": "online_3seed", + "method": "dfa_s123", + "metric": "mean_gamma", + "value": 0.11738517042249441 + }, + { + "slice": "online_3seed", + "method": "dfa_s123", + "metric": "mean_rho", + "value": 0.005885639227926731 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s123", + "metric": "test_acc", + "value": 0.2753 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s123", + "metric": "mean_gamma", + "value": 0.17617796920239925 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s123", + "metric": "mean_rho", + "value": 0.0019302130676805973 + }, + { + "slice": "online_3seed", + "method": "dfa_s456", + "metric": "test_acc", + "value": 0.297 + }, + { + "slice": "online_3seed", + "method": "dfa_s456", + "metric": "mean_gamma", + "value": 0.09892195643624291 + }, + { + "slice": "online_3seed", + "method": "dfa_s456", + "metric": "mean_rho", + "value": -0.004888533148914576 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s456", + "metric": "test_acc", + "value": 0.2893 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s456", + "metric": "mean_gamma", + "value": 0.23417379707098007 + }, + { + "slice": "online_3seed", + "method": "cb_eT_s456", + "metric": "mean_rho", + "value": -0.002338360995054245 + } +]
\ No newline at end of file |
