From ca98ccca0bc1bd008e95efc81d62fbfa3c9df22c Mon Sep 17 00:00:00 2001 From: mehallhm Date: Mon, 18 Nov 2024 16:49:12 -0500 Subject: [PATCH] Scrape professors in an additional request --- package.json | 4 +- scrapers/classes/parsersxe/termParser.ts | 27 +++++++- services/dumpProcessor.ts | 18 +----- yarn.lock | 78 ++++++++++++------------ 4 files changed, 68 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index 39dd6a01..ff0043c6 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "@babel/node": "^7.0.0", "@babel/register": "^7.0.0", "@elastic/elasticsearch": "7.17.0", - "@prisma/client": "^5.0.0", + "@prisma/client": "5.22.0", "@typescript-eslint/typescript-estree": "^8.10.0", "amplitude": "^6.0.0", "apollo-server": "^3.13.0", @@ -137,7 +137,7 @@ "nock": "^13.5.5", "prettier": "^3.0.0", "pretty-quick": "4", - "prisma": "^5.0.0", + "prisma": "5.22.0", "typescript": "^5.6.0" }, "about:engines": [ diff --git a/scrapers/classes/parsersxe/termParser.ts b/scrapers/classes/parsersxe/termParser.ts index 732ba054..cabca57d 100644 --- a/scrapers/classes/parsersxe/termParser.ts +++ b/scrapers/classes/parsersxe/termParser.ts @@ -203,10 +203,31 @@ class TermParser { ); const bodyObj = JSON.parse(req.body); - if (bodyObj.success) { - return { items: bodyObj.data, totalCount: bodyObj.totalCount }; + + if (!bodyObj.success) { + return false; } - return false; + + bodyObj.data = await Promise.all( + bodyObj.data.map(async (sr: SectionSR) => { + const resp = await request.get( + "https://nubanner.neu.edu/StudentRegistrationSsb/ssb/searchResults/getFacultyMeetingTimes", + { + searchParams: { + term: termId, + courseReferenceNumber: sr.courseReferenceNumber, + }, + }, + ); + + const body = await JSON.parse(resp.body); + sr.faculty = body.fmt[0]?.faculty ?? []; + + return sr; + }), + ); + + return { items: bodyObj.data, totalCount: bodyObj.totalCount }; })) as SectionSR[]; } catch (error) { macros.error(`Could not get section data for ${termId}`); diff --git a/services/dumpProcessor.ts b/services/dumpProcessor.ts index b38c5969..50b4adb0 100644 --- a/services/dumpProcessor.ts +++ b/services/dumpProcessor.ts @@ -42,7 +42,6 @@ class DumpProcessor { convertSectionToPrismaType(section), ); await this.saveSectionsToDatabase(processedSections); - await this.updateSectionsLastUpdateTime(termDump.sections); await this.saveSubjectsToDatabase(termDump.subjects); @@ -131,7 +130,10 @@ class DumpProcessor { for (const sections of groupedSections) { const upsertQueries = sections.map((prismaSection) => { + // Updates thelast update time for tracking sections that haven't been updated + // in a while for eventual removal prismaSection.lastUpdateTime = updateTime; + return prisma.section.upsert({ create: prismaSection, update: prismaSection, @@ -148,20 +150,6 @@ class DumpProcessor { macros.log("Finished with sections"); } - /** - * Update the lastUpdateTime attribute on all given sections. - * We use this to track sections that haven't been updated in a while, - * which means that they're no longer on Banner & should be removed from our database. - */ - async updateSectionsLastUpdateTime(sections: Section[]): Promise { - await prisma.course.updateMany({ - where: { id: { in: sections.map((s) => keys.getClassHash(s)) } }, - data: { lastUpdateTime: new Date() }, - }); - - macros.log("Finished updating times"); - } - /** * Saves all subject data to the database. This does NOT delete existing subjects, but will * overwrite the data if there is any new data for those subjects. diff --git a/yarn.lock b/yarn.lock index 1cc154b6..d2e6a88e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2118,61 +2118,61 @@ __metadata: languageName: node linkType: hard -"@prisma/client@npm:^5.0.0": - version: 5.21.1 - resolution: "@prisma/client@npm:5.21.1" +"@prisma/client@npm:5.22.0": + version: 5.22.0 + resolution: "@prisma/client@npm:5.22.0" peerDependencies: prisma: "*" peerDependenciesMeta: prisma: optional: true - checksum: 10c0/066bf678d6bda048893eb9080818339a99bba995a0ba19935cccda30a10eb46c6674bc445bbbbb37881e52d3010c72ce4a28565156b80bfea5438cafd9868941 + checksum: 10c0/ad111b931f184249794f811637456eb38dcfd0d7047e5a1f64804e5256f462fa3ee7c7a552fc8fa181e8daacc2b958af7426cb57bf0dc66605c3ef7c4aef4afb languageName: node linkType: hard -"@prisma/debug@npm:5.21.1": - version: 5.21.1 - resolution: "@prisma/debug@npm:5.21.1" - checksum: 10c0/5d69a13ae83efe3748efabb16ceae1dc0e9c2cc0c132e7fe46016a85a6d403eaa994094e8b30fb660b42e23abf5034b8ac1733bca30d6d66cb3fd763281e64e1 +"@prisma/debug@npm:5.22.0": + version: 5.22.0 + resolution: "@prisma/debug@npm:5.22.0" + checksum: 10c0/ff7c5e84d9f9b568a2a1992eb39fc7c3ab6d9e326c77fdefec1655ccfbf6c0ee268a5dcf087867848eb00a0328c9cc75a164880ec7cd60f7fd634e2fca2943d9 languageName: node linkType: hard -"@prisma/engines-version@npm:5.21.1-1.bf0e5e8a04cada8225617067eaa03d041e2bba36": - version: 5.21.1-1.bf0e5e8a04cada8225617067eaa03d041e2bba36 - resolution: "@prisma/engines-version@npm:5.21.1-1.bf0e5e8a04cada8225617067eaa03d041e2bba36" - checksum: 10c0/3b4ca9f8911c36da847740951f73b2ab66f2f3619530dea7a9667764e8853675d2b7fbbd24e2568ece5ad9c5a78c5498e44be94af2979372d7b8743c83ee27a1 +"@prisma/engines-version@npm:5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2": + version: 5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2 + resolution: "@prisma/engines-version@npm:5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2" + checksum: 10c0/95df49c6f35f99c0977e9690d9ae0776327df07c6f035e8c4a247a9c21108f0760c02ca6dbe63a35013e5cd1eb77a896edb340f1e28ea04373c6cefc4ce09b51 languageName: node linkType: hard -"@prisma/engines@npm:5.21.1": - version: 5.21.1 - resolution: "@prisma/engines@npm:5.21.1" +"@prisma/engines@npm:5.22.0": + version: 5.22.0 + resolution: "@prisma/engines@npm:5.22.0" dependencies: - "@prisma/debug": "npm:5.21.1" - "@prisma/engines-version": "npm:5.21.1-1.bf0e5e8a04cada8225617067eaa03d041e2bba36" - "@prisma/fetch-engine": "npm:5.21.1" - "@prisma/get-platform": "npm:5.21.1" - checksum: 10c0/5a6a76457da5cb075e39a21e566b7e03295cacd357182b0f96f849e9950676c2a9d49cdaac7b974b1b9af6a350d96791a8fd30cf7814a669fa77e14ba17a7531 + "@prisma/debug": "npm:5.22.0" + "@prisma/engines-version": "npm:5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2" + "@prisma/fetch-engine": "npm:5.22.0" + "@prisma/get-platform": "npm:5.22.0" + checksum: 10c0/81a439e5064a1036e05da8feaf7ffb19b5dba98d5ea68dc385f7e24e3cc25ef9f0b78a1aa1d7d7234a9afd916ee18b707863cee48274912315bc933429f26ce7 languageName: node linkType: hard -"@prisma/fetch-engine@npm:5.21.1": - version: 5.21.1 - resolution: "@prisma/fetch-engine@npm:5.21.1" +"@prisma/fetch-engine@npm:5.22.0": + version: 5.22.0 + resolution: "@prisma/fetch-engine@npm:5.22.0" dependencies: - "@prisma/debug": "npm:5.21.1" - "@prisma/engines-version": "npm:5.21.1-1.bf0e5e8a04cada8225617067eaa03d041e2bba36" - "@prisma/get-platform": "npm:5.21.1" - checksum: 10c0/068663fa15cb204d033b166588157915be11571a1584c914372968248faab0f7c236ec1086aaf194929ece3ae2a3d2d0b28e33dac114a1545e0dee1011dc669c + "@prisma/debug": "npm:5.22.0" + "@prisma/engines-version": "npm:5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2" + "@prisma/get-platform": "npm:5.22.0" + checksum: 10c0/0d3dcaffbadcf9185cca37bb50ce3a92fad2a205451731e5c3d02b1ee96a918dde99a9ce231e62bddde5ec8dcb3035338ff1312f8416f2cdaf31a69a19448baf languageName: node linkType: hard -"@prisma/get-platform@npm:5.21.1": - version: 5.21.1 - resolution: "@prisma/get-platform@npm:5.21.1" +"@prisma/get-platform@npm:5.22.0": + version: 5.22.0 + resolution: "@prisma/get-platform@npm:5.22.0" dependencies: - "@prisma/debug": "npm:5.21.1" - checksum: 10c0/3e6f2378e8a6086db0cc02e95e5b872b602263d0b2e55c3e18f6041bbbf9edea5edbfb9efce331527e75e13411eee68d2b34c24b9aeac9c579e75aac2a39432c + "@prisma/debug": "npm:5.22.0" + checksum: 10c0/b934f9bbba1e8dae01721a213a5b166a4dd9f35bf04ad93fb9e24afb67945c09562d4e2c5b4d33b1830cdc7adeb03775309c55ab2c470048ed4cfc947fe495f5 languageName: node linkType: hard @@ -4382,7 +4382,7 @@ __metadata: "@babel/preset-typescript": "npm:^7.8.3" "@babel/register": "npm:^7.0.0" "@elastic/elasticsearch": "npm:7.17.0" - "@prisma/client": "npm:^5.0.0" + "@prisma/client": "npm:5.22.0" "@types/async-retry": "npm:^1.4.9" "@types/cheerio": "npm:^0.22.30" "@types/fs-extra": "npm:^11.0.4" @@ -4439,7 +4439,7 @@ __metadata: pg: "npm:^8.13.0" prettier: "npm:^3.0.0" pretty-quick: "npm:4" - prisma: "npm:^5.0.0" + prisma: "npm:5.22.0" regenerator-runtime: "npm:^0.14.1" request: "npm:^2.81.0" request-promise-native: "npm:^1.0.9" @@ -9219,18 +9219,18 @@ __metadata: languageName: node linkType: hard -"prisma@npm:^5.0.0": - version: 5.21.1 - resolution: "prisma@npm:5.21.1" +"prisma@npm:5.22.0": + version: 5.22.0 + resolution: "prisma@npm:5.22.0" dependencies: - "@prisma/engines": "npm:5.21.1" + "@prisma/engines": "npm:5.22.0" fsevents: "npm:2.3.3" dependenciesMeta: fsevents: optional: true bin: prisma: build/index.js - checksum: 10c0/14a276fb1891e1fec832da1632a4c3c9ae6909a094067d1ba94012fe2063b80f44680a0a373fc86e0e73b8a81455f20374303f70d93451060e06b93f0983b82b + checksum: 10c0/63d1fe828394d1a1fabc0d29245ae35be53c1bdb26da4dbcfaf17fba3733c1c120b72f28b548dba47b2b80ff7fad670717be1316c1ea4bb12ebe1937415a1ddb languageName: node linkType: hard