1
|
{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 20, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "train_examples": 0, "task_seed": 0, "task_train_examples": 50000, "task_test_examples": 10000, "task_levels": 8, "task_n_in": 128, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d20_s0", "n_in": 3072, "n_out": 10}, "provenance": {"git_commit": "d4aef1a571c21e6e4e4125c90aa58da7f14e62e3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.5764718055725098, "cos_r_negg": [-0.019500190392136574, -0.06851186603307724, -0.022075217217206955, -0.029041359201073647, 0.002836341504007578, -0.015061014331877232, 0.04729481786489487, -0.020570902153849602, -0.009287690743803978, 0.08202917873859406, 0.03213351219892502, -0.012884090654551983, -0.0028710030019283295, 0.015713732689619064, 0.04467347264289856, 0.052076369524002075, 0.015987787395715714, -0.008609583601355553, -0.00859350711107254, -0.07578613609075546], "cos_innovation_negg": [-0.019500190392136574, -0.06851186603307724, -0.022075217217206955, -0.029041359201073647, 0.002836341504007578, -0.015061014331877232, 0.04729481786489487, -0.020570902153849602, -0.009287690743803978, 0.08202917873859406, 0.03213351219892502, -0.012884090654551983, -0.0028710030019283295, 0.015713732689619064, 0.04467347264289856, 0.052076369524002075, 0.015987787395715714, -0.008609583601355553, -0.00859350711107254, -0.07578613609075546], "cos_apical_negg": [-0.019500190392136574, -0.06851186603307724, -0.022075217217206955, -0.029041359201073647, 0.002836341504007578, -0.015061014331877232, 0.04729481786489487, -0.020570902153849602, -0.009287690743803978, 0.08202917873859406, 0.03213351219892502, -0.012884090654551983, -0.0028710030019283295, 0.015713732689619064, 0.04467347264289856, 0.052076369524002075, 0.015987787395715714, -0.008609583601355553, -0.00859350711107254, -0.07578613609075546], "cos_Ac_negg": [-0.019500190392136574, -0.06851186603307724, -0.022075217217206955, -0.029041359201073647, 0.002836341504007578, -0.015061014331877232, 0.04729481786489487, -0.020570902153849602, -0.009287690743803978, 0.08202917873859406, 0.03213351219892502, -0.012884090654551983, -0.0028710030019283295, 0.015713732689619064, 0.04467347264289856, 0.052076369524002075, 0.015987787395715714, -0.008609583601355553, -0.00859350711107254, -0.07578613609075546], "r_norm": [2.617947578430176, 2.44466495513916, 2.5444884300231934, 2.4182395935058594, 2.561816453933716, 2.4764671325683594, 2.490081310272217, 2.5378732681274414, 2.5621564388275146, 2.5308473110198975, 2.617791175842285, 2.518054723739624, 2.5913219451904297, 2.4762251377105713, 2.5998520851135254, 2.4868688583374023, 2.535977840423584, 2.4587440490722656, 2.5869359970092773, 2.659351348876953]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3356, "test_loss": 1.8784977172851562}, {"step": 500, "epoch": 1, "train_loss": 1.769270420074463, "cos_r_negg": [0.46736034750938416, -0.039863839745521545, -0.02915871888399124, -0.0499245710670948, -0.051531825214624405, 0.012173362076282501, -0.05229507386684418, -0.012186378240585327, -0.020817182958126068, 0.02329660952091217, 0.024037808179855347, 0.03547365963459015, 0.05455750972032547, 0.020796187222003937, 0.03703618049621582, -0.056826572865247726, -0.014485000632703304, 0.07331870496273041, -0.05329829081892967, -0.05832912027835846], "cos_innovation_negg": [0.46736034750938416, -0.039863839745521545, -0.02915871888399124, -0.0499245710670948, -0.051531825214624405, 0.012173362076282501, -0.05229507386684418, -0.012186378240585327, -0.020817182958126068, 0.02329660952091217, 0.024037808179855347, 0.03547365963459015, 0.05455750972032547, 0.020796187222003937, 0.03703618049621582, -0.056826572865247726, -0.014485000632703304, 0.07331870496273041, -0.05329829081892967, -0.05832912027835846], "cos_apical_negg": [0.46736034750938416, -0.039863839745521545, -0.02915871888399124, -0.0499245710670948, -0.051531825214624405, 0.012173362076282501, -0.05229507386684418, -0.012186378240585327, -0.020817182958126068, 0.02329660952091217, 0.024037808179855347, 0.03547365963459015, 0.05455750972032547, 0.020796187222003937, 0.03703618049621582, -0.056826572865247726, -0.014485000632703304, 0.07331870496273041, -0.05329829081892967, -0.05832912027835846], "cos_Ac_negg": [0.46736034750938416, -0.039863839745521545, -0.02915871888399124, -0.0499245710670948, -0.051531825214624405, 0.012173362076282501, -0.05229507386684418, -0.012186378240585327, -0.020817182958126068, 0.02329660952091217, 0.024037808179855347, 0.03547365963459015, 0.05455750972032547, 0.020796187222003937, 0.03703618049621582, -0.056826572865247726, -0.014485000632703304, 0.07331870496273041, -0.05329829081892967, -0.05832912027835846], "r_norm": [2.220099687576294, 2.0305495262145996, 2.1996383666992188, 2.070868492126465, 2.169755220413208, 2.1038951873779297, 2.147573471069336, 2.1749067306518555, 2.1951982975006104, 2.1746554374694824, 2.2588601112365723, 2.1441798210144043, 2.2195777893066406, 2.138514518737793, 2.198246479034424, 2.1041417121887207, 2.1653456687927246, 2.0914852619171143, 2.1940460205078125, 2.229619264602661]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3614, "test_loss": 1.8460565551757813}, {"step": 1000, "epoch": 2, "train_loss": 1.8356685638427734, "cos_r_negg": [0.5715234875679016, -0.06747676432132721, -0.03454872965812683, -0.043657153844833374, -0.07381317019462585, 0.03620036691427231, 0.01425466500222683, 0.041283246129751205, 0.04687042534351349, 0.08967145532369614, 0.03432723134756088, 0.04623639956116676, 0.028272368013858795, 0.057662442326545715, 0.048781804740428925, 0.0023845972027629614, 0.02812224254012108, 0.02978171408176422, -0.017251819372177124, -0.03878463804721832], "cos_innovation_negg": [0.5715234875679016, -0.06747676432132721, -0.03454872965812683, -0.043657153844833374, -0.07381317019462585, 0.03620036691427231, 0.01425466500222683, 0.041283246129751205, 0.04687042534351349, 0.08967145532369614, 0.03432723134756088, 0.04623639956116676, 0.028272368013858795, 0.057662442326545715, 0.048781804740428925, 0.0023845972027629614, 0.02812224254012108, 0.02978171408176422, -0.017251819372177124, -0.03878463804721832], "cos_apical_negg": [0.5715234875679016, -0.06747676432132721, -0.03454872965812683, -0.043657153844833374, -0.07381317019462585, 0.03620036691427231, 0.01425466500222683, 0.041283246129751205, 0.04687042534351349, 0.08967145532369614, 0.03432723134756088, 0.04623639956116676, 0.028272368013858795, 0.057662442326545715, 0.048781804740428925, 0.0023845972027629614, 0.02812224254012108, 0.02978171408176422, -0.017251819372177124, -0.03878463804721832], "cos_Ac_negg": [0.5715234875679016, -0.06747676432132721, -0.03454872965812683, -0.043657153844833374, -0.07381317019462585, 0.03620036691427231, 0.01425466500222683, 0.041283246129751205, 0.04687042534351349, 0.08967145532369614, 0.03432723134756088, 0.04623639956116676, 0.028272368013858795, 0.057662442326545715, 0.048781804740428925, 0.0023845972027629614, 0.02812224254012108, 0.02978171408176422, -0.017251819372177124, -0.03878463804721832], "r_norm": [2.279806137084961, 2.093953847885132, 2.2574238777160645, 2.1158483028411865, 2.2333312034606934, 2.1709227561950684, 2.206829071044922, 2.234591484069824, 2.25022029876709, 2.2335329055786133, 2.3249573707580566, 2.199455738067627, 2.278775215148926, 2.1961967945098877, 2.2643239498138428, 2.1605212688446045, 2.225320816040039, 2.1496522426605225, 2.25720477104187, 2.2894253730773926]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3472, "test_loss": 1.865696337890625}, {"step": 1500, "epoch": 3, "train_loss": 1.8100554943084717, "cos_r_negg": [0.5905617475509644, -0.0020461962558329105, 0.001780700869858265, 0.03228110820055008, 0.04243522882461548, 0.044074319303035736, 0.040963418781757355, 0.051450349390506744, 0.07257775962352753, 0.07289508730173111, 0.002765776589512825, 0.05449425056576729, 0.027422791346907616, 0.05823320895433426, 0.10960214585065842, 0.006491017993539572, 0.03821496665477753, 0.016838474199175835, 0.007161032874137163, -0.028348051011562347], "cos_innovation_negg": [0.5905617475509644, -0.0020461962558329105, 0.001780700869858265, 0.03228110820055008, 0.04243522882461548, 0.044074319303035736, 0.040963418781757355, 0.051450349390506744, 0.07257775962352753, 0.07289508730173111, 0.002765776589512825, 0.05449425056576729, 0.027422791346907616, 0.05823320895433426, 0.10960214585065842, 0.006491017993539572, 0.03821496665477753, 0.016838474199175835, 0.007161032874137163, -0.028348051011562347], "cos_apical_negg": [0.5905617475509644, -0.0020461962558329105, 0.001780700869858265, 0.03228110820055008, 0.04243522882461548, 0.044074319303035736, 0.040963418781757355, 0.051450349390506744, 0.07257775962352753, 0.07289508730173111, 0.002765776589512825, 0.05449425056576729, 0.027422791346907616, 0.05823320895433426, 0.10960214585065842, 0.006491017993539572, 0.03821496665477753, 0.016838474199175835, 0.007161032874137163, -0.028348051011562347], "cos_Ac_negg": [0.5905617475509644, -0.0020461962558329105, 0.001780700869858265, 0.03228110820055008, 0.04243522882461548, 0.044074319303035736, 0.040963418781757355, 0.051450349390506744, 0.07257775962352753, 0.07289508730173111, 0.002765776589512825, 0.05449425056576729, 0.027422791346907616, 0.05823320895433426, 0.10960214585065842, 0.006491017993539572, 0.03821496665477753, 0.016838474199175835, 0.007161032874137163, -0.028348051011562347], "r_norm": [2.221247673034668, 2.0441832542419434, 2.2031712532043457, 2.072357177734375, 2.1802282333374023, 2.118795871734619, 2.1560540199279785, 2.168017864227295, 2.2014622688293457, 2.179969310760498, 2.2639594078063965, 2.148977756500244, 2.224015235900879, 2.1506896018981934, 2.2046356201171875, 2.1058201789855957, 2.1626486778259277, 2.098271369934082, 2.194507598876953, 2.2384145259857178]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3573, "test_loss": 1.8498263061523437}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3824, "test_loss": 1.7975571655273437}], "final": {"test_acc": 0.3824, "test_loss": 1.7975571655273437, "wall_s": 17.575655460357666, "cos_r_negg": [0.5324325561523438, 0.03298833221197128, 0.04252372682094574, 0.06904610991477966, 0.08148430287837982, 0.0934550017118454, 0.05379283428192139, 0.08711553364992142, 0.05030951276421547, 0.09972405433654785, 0.0680934488773346, 0.062188148498535156, 0.07344379276037216, -0.00676838168874383, 0.05246010050177574, 0.012371981516480446, 0.027268335223197937, 0.09751579165458679, 0.08608158677816391, -0.017415359616279602], "cos_innovation_negg": [0.5324325561523438, 0.03298833221197128, 0.04252372682094574, 0.06904610991477966, 0.08148430287837982, 0.0934550017118454, 0.05379283428192139, 0.08711553364992142, 0.05030951276421547, 0.09972405433654785, 0.0680934488773346, 0.062188148498535156, 0.07344379276037216, -0.00676838168874383, 0.05246010050177574, 0.012371981516480446, 0.027268335223197937, 0.09751579165458679, 0.08608158677816391, -0.017415359616279602], "cos_apical_negg": [0.5324325561523438, 0.03298833221197128, 0.04252372682094574, 0.06904610991477966, 0.08148430287837982, 0.0934550017118454, 0.05379283428192139, 0.08711553364992142, 0.05030951276421547, 0.09972405433654785, 0.0680934488773346, 0.062188148498535156, 0.07344379276037216, -0.00676838168874383, 0.05246010050177574, 0.012371981516480446, 0.027268335223197937, 0.09751579165458679, 0.08608158677816391, -0.017415359616279602], "cos_Ac_negg": [0.5324325561523438, 0.03298833221197128, 0.04252372682094574, 0.06904610991477966, 0.08148430287837982, 0.0934550017118454, 0.05379283428192139, 0.08711553364992142, 0.05030951276421547, 0.09972405433654785, 0.0680934488773346, 0.062188148498535156, 0.07344379276037216, -0.00676838168874383, 0.05246010050177574, 0.012371981516480446, 0.027268335223197937, 0.09751579165458679, 0.08608158677816391, -0.017415359616279602]}}
|
