diff options
Diffstat (limited to 'experiments/analyze_depth_tasks.py')
| -rw-r--r-- | experiments/analyze_depth_tasks.py | 32 |
1 files changed, 21 insertions, 11 deletions
diff --git a/experiments/analyze_depth_tasks.py b/experiments/analyze_depth_tasks.py index f1a8e19..b0df690 100644 --- a/experiments/analyze_depth_tasks.py +++ b/experiments/analyze_depth_tasks.py @@ -24,27 +24,37 @@ def main(): key = (args["dataset"], args["mode"], args["width"], args["depth"]) groups.setdefault(key, []).append(row) - print("| task | method | width | depth | task/model runs | validation acc (%) |") - print("|:---|:---|---:|---:|---:|---:|") + print("| task | method | width | depth | task/model runs | last val (%) | best val (%) |") + print("|:---|:---|---:|---:|---:|---:|---:|") by_curve = {} for key in sorted(groups): rows = groups[key] - values = [100 * row["final"]["val_acc"] for row in rows] - mean = statistics.mean(values) - sd = statistics.stdev(values) if len(values) > 1 else None - value = f"{mean:.3f}" if sd is None else f"{mean:.3f} ± {sd:.3f}" - print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | {value} |") - by_curve.setdefault(key[:3], {})[key[3]] = mean + last_values = [100 * row["final"]["val_acc"] for row in rows] + best_values = [100 * max(step["val_acc"] for step in row["steps"] + if "val_acc" in step) for row in rows] + last_mean = statistics.mean(last_values) + best_mean = statistics.mean(best_values) + last_sd = statistics.stdev(last_values) if len(last_values) > 1 else None + best_sd = statistics.stdev(best_values) if len(best_values) > 1 else None + last_text = (f"{last_mean:.3f}" if last_sd is None + else f"{last_mean:.3f} ± {last_sd:.3f}") + best_text = (f"{best_mean:.3f}" if best_sd is None + else f"{best_mean:.3f} ± {best_sd:.3f}") + print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | " + f"{last_text} | {best_text} |") + by_curve.setdefault(key[:3], {})[key[3]] = (last_mean, best_mean) print() for key, curve in sorted(by_curve.items()): if len(curve) < 2: continue shallow, deep = min(curve), max(curve) - gain = curve[deep] - curve[shallow] - verdict = "PASS" if gain >= 5.0 else "FAIL" + last_gain = curve[deep][0] - curve[shallow][0] + best_gain = curve[deep][1] - curve[shallow][1] + verdict = "PASS" if best_gain >= 5.0 else "FAIL" print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} " - f"gain {gain:+.3f} points [{verdict} C2 BP-screen threshold]") + f"last gain {last_gain:+.3f}, best-epoch gain {best_gain:+.3f} points " + f"[{verdict} C2 BP-screen threshold]") if __name__ == "__main__": |