1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
From c88ebbc422cb7ff8ae117d924e6fa28c41729022 Mon Sep 17 00:00:00 2001
From: YurenHao0426 <Blackhao0426@gmail.com>
Date: Thu, 30 Jul 2026 16:31:36 -0500
Subject: [PATCH 19/19] experiment: add portable A6000 P2 profile
---
analyze_p2.py | 16 +++++++++--
crossover_grid.py | 71 +++++++++++++++++++++++++++++++++++++++--------
2 files changed, 74 insertions(+), 13 deletions(-)
diff --git a/analyze_p2.py b/analyze_p2.py
index d997ec5..696fef0 100644
--- a/analyze_p2.py
+++ b/analyze_p2.py
@@ -137,8 +137,18 @@ def audit_record(job, launch):
assert manifest["selector_sha256"] == crossover_grid.SELECTOR_SHA256
assert manifest["work"] == crossover_grid.p2_work_report(job)
hardware = manifest["hardware"]
- assert hardware["physical_index"] in (5, 7)
- assert hardware["name"] == "NVIDIA GeForce GTX 1080"
+ hardware_policy = launch["hardware_policy"]
+ assert hardware["hardware_profile"] == hardware_policy["profile"]
+ assert (
+ hardware["physical_index"]
+ in hardware_policy["allowed_physical_gpu_indices"]
+ )
+ assert (
+ hardware["cuda_visible_devices"]
+ == str(hardware["physical_index"])
+ )
+ assert hardware["uuid"]
+ assert hardware["name"] in hardware_policy["allowed_device_names"]
assert hardware["peak_process_gpu_memory_mib"] >= 0
histogram = manifest["histogram"]
if histogram is None:
@@ -188,6 +198,8 @@ def main():
jobs)
assert launch["jobs"] == jobs
assert launch["selector_sha256"] == crossover_grid.SELECTOR_SHA256
+ assert launch["hardware_policy"]["profile"] in (
+ "timan107_gtx1080", "a6000")
assert launch["source"]["cifar10_tfds_tree"]["num_files"] == 5
expected = {job["experiment_name"]: job for job in jobs}
actual = {
diff --git a/crossover_grid.py b/crossover_grid.py
index bb6d9d5..1f14d21 100644
--- a/crossover_grid.py
+++ b/crossover_grid.py
@@ -12,7 +12,7 @@ import time
ROOT = os.path.dirname(os.path.abspath(__file__))
-MAIN_ROOT = "/home/yurenh2/sdil"
+MAIN_ROOT = os.environ.get("SDIL_MAIN_ROOT", "/home/yurenh2/sdil")
PROTOCOL = os.path.join(MAIN_ROOT, "PLAIN_CNN_CROSSOVER.md")
SELECTOR = os.path.join(
MAIN_ROOT, "results", "plain_cnn_p1_selector.json")
@@ -22,6 +22,16 @@ P2_METHODS = (
"bp", "fa", "dfa", "pepita", "ff", "ep", "dualprop", "clean_kp",
"sdil")
P2_ARCHITECTURES = ("minicnn", "vgglike", "vgg16")
+HARDWARE_PROFILES = {
+ "timan107_gtx1080": {
+ "allowed_physical_gpu_indices": [5, 7],
+ "allowed_device_names": ["NVIDIA GeForce GTX 1080"],
+ },
+ "a6000": {
+ "allowed_physical_gpu_indices": None,
+ "allowed_device_names": ["NVIDIA RTX A6000"],
+ },
+}
def output(*args):
@@ -185,7 +195,7 @@ def p2_registry_sha256(jobs):
return hashlib.sha256(encoded).hexdigest()
-def ensure_p2_launch(source, jobs):
+def ensure_p2_launch(source, jobs, hardware_policy):
path = os.path.join(ROOT, "runs", "plain-p2-launch.json")
immutable = {
"stage": "p2",
@@ -194,6 +204,7 @@ def ensure_p2_launch(source, jobs):
"selector_sha256": SELECTOR_SHA256,
"num_registered_cells": len(jobs),
"registry_sha256": p2_registry_sha256(jobs),
+ "hardware_policy": hardware_policy,
"jobs": jobs,
}
if os.path.exists(path):
@@ -413,11 +424,22 @@ def completed_histogram(experiment_name):
return paths[0]
-def physical_gpu_report():
+def physical_gpu_report(profile):
index = os.environ.get("CUDA_VISIBLE_DEVICES")
- if index not in ("5", "7"):
+ if index is None or not index.isdigit():
raise RuntimeError(
- "formal P2 is authorized only on physical GPUs 5 and 7")
+ "formal P2 requires CUDA_VISIBLE_DEVICES to contain one "
+ "physical numeric GPU index")
+ if profile not in HARDWARE_PROFILES:
+ raise ValueError(f"unknown hardware profile: {profile}")
+ configured = HARDWARE_PROFILES[profile]
+ allowed_indices = configured["allowed_physical_gpu_indices"]
+ if (
+ allowed_indices is not None
+ and int(index) not in allowed_indices
+ ):
+ raise RuntimeError(
+ f"physical GPU {index} is not allowed by {profile}")
query = output(
"nvidia-smi",
"--query-gpu=index,uuid,name,memory.total",
@@ -427,7 +449,13 @@ def physical_gpu_report():
if len(matches) != 1:
raise RuntimeError(f"cannot resolve physical GPU {index}")
physical_index, uuid, name, memory_mib = matches[0]
+ if name not in configured["allowed_device_names"]:
+ raise RuntimeError(
+ f"{profile} requires {configured['allowed_device_names']}, "
+ f"found {name}")
return {
+ "hardware_profile": profile,
+ "cuda_visible_devices": index,
"physical_index": int(physical_index),
"uuid": uuid,
"name": name,
@@ -435,6 +463,19 @@ def physical_gpu_report():
}
+def hardware_policy(profile, gpu):
+ configured = HARDWARE_PROFILES[profile]
+ allowed_indices = configured["allowed_physical_gpu_indices"]
+ if allowed_indices is None:
+ allowed_indices = [gpu["physical_index"]]
+ return {
+ "profile": profile,
+ "allowed_physical_gpu_indices": list(allowed_indices),
+ "allowed_device_names": list(configured["allowed_device_names"]),
+ "single_gpu_per_process": True,
+ }
+
+
def descendant_pids(root_pid):
listing = subprocess.run(
["ps", "-eo", "pid=,ppid="], check=True, capture_output=True,
@@ -597,7 +638,7 @@ def terminate_process_group(process):
process.wait()
-def run_p2_job(job, source, dry_run):
+def run_p2_job(job, source, gpu, dry_run):
existing = record_path(job["experiment_name"])
if existing is not None:
print(f"preserving {job['experiment_name']} -> {existing}", flush=True)
@@ -609,7 +650,6 @@ def run_p2_job(job, source, dry_run):
print("RUN", " ".join(job["command"]), flush=True)
if dry_run:
return
- gpu = physical_gpu_report()
started = time.time()
process = subprocess.Popen(
job["command"], cwd=ROOT, start_new_session=True)
@@ -666,9 +706,9 @@ def run_p2_job(job, source, dry_run):
flush=True)
-def run_job(job, source, dry_run):
+def run_job(job, source, gpu, dry_run):
if job["stage"] == "p2":
- return run_p2_job(job, source, dry_run)
+ return run_p2_job(job, source, gpu, dry_run)
result = completed_histogram(job["experiment_name"])
if result is not None:
print(f"preserving {job['experiment_name']} -> {result}", flush=True)
@@ -703,14 +743,23 @@ def main():
parser.add_argument("--shard-index", type=int, default=0)
parser.add_argument("--num-shards", type=int, default=1)
parser.add_argument("--method")
+ parser.add_argument(
+ "--hardware-profile",
+ choices=tuple(HARDWARE_PROFILES),
+ default=os.environ.get(
+ "SDIL_HARDWARE_PROFILE", "timan107_gtx1080"),
+ )
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
if not 0 <= args.shard_index < args.num_shards:
raise ValueError("invalid shard")
source = source_report()
jobs = p1_jobs() if args.stage == "p1" else p2_jobs()
+ gpu = None
if args.stage == "p2" and not args.dry_run:
- launch = ensure_p2_launch(source, jobs)
+ gpu = physical_gpu_report(args.hardware_profile)
+ launch = ensure_p2_launch(
+ source, jobs, hardware_policy(args.hardware_profile, gpu))
print(f"P2 launch lock: {launch}", flush=True)
if args.method:
jobs = [job for job in jobs if job["method"] == args.method]
@@ -721,7 +770,7 @@ def main():
if not jobs:
raise ValueError("no jobs selected")
for job in jobs:
- run_job(job, source, args.dry_run)
+ run_job(job, source, gpu, args.dry_run)
if __name__ == "__main__":
--
2.54.0
|
