1 files changed, 20 insertions, 4 deletions
diff --git a/scripts/scrape.js b/scripts/scrape.js
index 2bfca5e..d4b2ecb 100644
--- a/scripts/scrape.js
+++ b/scripts/scrape.js
@@ -3,25 +3,41 @@ import fs from "fs/promises";
 import path from "path";
 import dns from "node:dns";
 
+import { ProxyAgent } from "undici";
+
 dns.setDefaultResultOrder("ipv4first");
 
+const proxy = process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy;
+const dispatcher = proxy ? new ProxyAgent(proxy) : undefined;
+
 const [ , , YEAR = "2025", TERM = "fall" ] = process.argv;
-const parser = new XMLParser({ ignoreAttributes: false });
+// UIUC's API uses XML namespaces (e.g. `<ns2:term>`). In order for the
+// returned object to have plain keys like `term` and `subject`, we instruct
+// fast-xml-parser to strip the namespace prefixes.
+const parser = new XMLParser({ ignoreAttributes: false, removeNSPrefix: true });
 const BASE = `https://courses.illinois.edu/cisapp/explorer`;
 
 async function getXML(url) {
-  return parser.parse(await (await fetch(url)).text());
+  const res = await fetch(url, { dispatcher });
+  if (!res.ok) throw new Error(`Request failed: ${res.status} ${url}`);
+  return parser.parse(await res.text());
+
 }
 
 async function scrapeSchedule(year, term) {
   const catalog = {};
   const termRoot = await getXML(`${BASE}/schedule/${year}/${term}.xml`);
-  const subjects = termRoot.term.subject;
+
+  const subjects = termRoot.term?.subjects?.subject;
+  if (!subjects) throw new Error(`Unexpected XML structure for ${year} ${term}`);
+
   const subjHrefs = Array.isArray(subjects) ? subjects.map(s => s['@_href']) : [subjects['@_href']];
 
   for (const subjURL of subjHrefs) {
     const subjXML = await getXML(subjURL);
-    const courses = subjXML.subject.course || [];
+
+    const courses = subjXML.subject?.courses?.course || [];
+
     const courseList = Array.isArray(courses) ? courses : [courses];
     for (const c of courseList) {
       const courseURL = c['@_href'];