Fix professors for sections not being scraped

infrastructure/dev/docker-compose-server.yml

@@ -9,8 +9,9 @@ services:
       - ./docker-postgresql-multiple-databases:/docker-entrypoint-initdb.d
       - pg:/var/lib/postgresql/data
     environment:
-      - POSTGRES_MULTIPLE_DATABASES=searchneu_dev,searchneu_test
-      - POSTGRES_USER=postgres
+      POSTGRES_MULTIPLE_DATABASES: searchneu_dev,searchneu_test
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: default_password
   es:
     image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
     ports:
@@ -22,12 +23,12 @@ services:
     depends_on:
       - es
       - postgresql
-    command: yarn prod
     ports:
       - 4000:4000
       - 8080:8080
     environment:
-      DATABASE_URL: postgresql://postgres@postgresql:5432/searchneu_dev
+      DATABASE_URL: postgresql://postgres:default_password@postgresql:5432/searchneu_dev
+      POSTGRES_PASSWORD: default_password
       elasticURL: http://es:9200
       TWILIO_PHONE_NUMBER:
       TWILIO_ACCOUNT_SID:
@@ -36,5 +37,6 @@ services:
       CLIENT_ORIGIN: http://localhost:5000
       JWT_SECRET:
       SLACK_WEBHOOK_URL:
+
 volumes:
   pg:

package.json

@@ -67,7 +67,7 @@
     "@babel/node": "^7.0.0",
     "@babel/register": "^7.0.0",
     "@elastic/elasticsearch": "7.17.0",
-    "@prisma/client": "^5.0.0",
+    "@prisma/client": "5.22.0",
     "@typescript-eslint/typescript-estree": "^8.10.0",
     "amplitude": "^6.0.0",
     "apollo-server": "^3.13.0",
@@ -149,5 +149,6 @@
   ],
   "engines": {
     "node": ">=20"
-  }
+  },
+  "packageManager": "yarn@1.22.19+sha1.4ba7fc5c6e704fce2066ecbfb0b0d8976fe62447"
 }

scrapers/classes/parsersxe/termParser.ts

@@ -203,10 +203,31 @@ class TermParser {
         );
 
         const bodyObj = JSON.parse(req.body);
-        if (bodyObj.success) {
-          return { items: bodyObj.data, totalCount: bodyObj.totalCount };
+
+        if (!bodyObj.success) {
+          return false;
         }
-        return false;
+
+        bodyObj.data = await Promise.all(
+          bodyObj.data.map(async (sr: SectionSR) => {
+            const resp = await request.get(
+              "https://nubanner.neu.edu/StudentRegistrationSsb/ssb/searchResults/getFacultyMeetingTimes",
+              {
+                searchParams: {
+                  term: termId,
+                  courseReferenceNumber: sr.courseReferenceNumber,
+                },
+              },
+            );
+
+            const body = await JSON.parse(resp.body);
+            sr.faculty = body.fmt[0]?.faculty ?? [];
+
+            return sr;
+          }),
+        );
+
+        return { items: bodyObj.data, totalCount: bodyObj.totalCount };
       })) as SectionSR[];
     } catch (error) {
       macros.error(`Could not get section data for ${termId}`);

services/dumpProcessor.ts

@@ -7,13 +7,13 @@
 import path from "path";
 import { Prisma, TermInfo } from "@prisma/client";
 import prisma from "./prisma";
 import keys from "../utils/keys";
 import macros from "../utils/macros";
 import { populateES } from "../scripts/populateES";
 import {
   Dump,
   Employee,
   Section,
   convertSectionToPrismaType,
 } from "../types/types";
 import { convertCourseToPrismaType } from "../types/scraperTypes";
@@ -42,7 +42,6 @@
       convertSectionToPrismaType(section),
     );
     await this.saveSectionsToDatabase(processedSections);
-    await this.updateSectionsLastUpdateTime(termDump.sections);
 
     await this.saveSubjectsToDatabase(termDump.subjects);
 
@@ -131,7 +130,10 @@
 
     for (const sections of groupedSections) {
       const upsertQueries = sections.map((prismaSection) => {
+        // Updates thelast update time for tracking sections that haven't been updated
+        // in a while for eventual removal
         prismaSection.lastUpdateTime = updateTime;
+
         return prisma.section.upsert({
           create: prismaSection,
           update: prismaSection,
@@ -148,20 +150,6 @@
     macros.log("Finished with sections");
   }
 
-  /**
-   * Update the lastUpdateTime attribute on all given sections.
-   * We use this to track sections that haven't been updated in a while,
-   * which means that they're no longer on Banner & should be removed from our database.
-   */
-  async updateSectionsLastUpdateTime(sections: Section[]): Promise<void> {
-    await prisma.course.updateMany({
-      where: { id: { in: sections.map((s) => keys.getClassHash(s)) } },
-      data: { lastUpdateTime: new Date() },
-    });
-
-    macros.log("Finished updating times");
-  }
-
   /**
    * Saves all subject data to the database. This does NOT delete existing subjects, but will
    * overwrite the data if there is any new data for those subjects.

tests/database/updater.test.seq.ts

@@ -48,7 +48,7 @@ const defaultClassProps = {
   prettyUrl: "pretty",
   desc: "a class",
   url: "url",
-  lastUpdateTime: 20,
+  lastUpdateTime: Date.now(),
   maxCredits: 4,
   minCredits: 0,
   coreqs: EMPTY_REQ,
@@ -717,6 +717,9 @@ describe("Updater", () => {
       const fundies1SectionsUpdated = await prisma.section.findMany({
         where: { classHash: Keys.getClassHash(FUNDIES_ONE) },
       });
+
+      console.log("AAAAAAAAAAAAAA", fundies1Sections);
+
       expect(fundies1SectionsUpdated.length).toBe(2); // new fundies 1 section
       const fundies1Section1 = fundies1SectionsUpdated.find(
         (section) => section.crn === FUNDIES_ONE_S1.crn,

yarn.lock

@@ -1499,10 +1499,10 @@
     "@nodelib/fs.scandir" "2.1.5"
     fastq "^1.6.0"
 
-"@prisma/client@^5.0.0":
-  version "5.21.1"
-  resolved "https://registry.yarnpkg.com/@prisma/client/-/client-5.21.1.tgz#ad51ef220eb80173f882e859960d81e626b73898"
-  integrity sha512-3n+GgbAZYjaS/k0M03yQsQfR1APbr411r74foknnsGpmhNKBG49VuUkxIU6jORgvJPChoD4WC4PqoHImN1FP0w==
+"@prisma/client@5.22.0":
+  version "5.22.0"
+  resolved "https://registry.yarnpkg.com/@prisma/client/-/client-5.22.0.tgz#da1ca9c133fbefe89e0da781c75e1c59da5f8802"
+  integrity sha512-M0SVXfyHnQREBKxCgyo7sffrKttwE6R8PMq330MIUF0pTwjUhLbW84pFDlf06B27XyCR++VtjugEnIHdr07SVA==
 
 "@prisma/debug@5.21.1":
   version "5.21.1"