first successful scrape of the new api

Author: ravern

scrapers/nus-v2/src/services/io/elastic.ts

@@ -148,6 +148,10 @@ export default class ElasticPersist implements Persist {
       }
     }
 
+    if (bulkBody.length === 0) {
+      return;
+    }
+
     const client = await this.client;
     const res = await client.bulk({
       index: INDEX_NAME,

scrapers/nus-v2/src/services/nus-api.ts

@@ -20,6 +20,7 @@ import type {
 import type { ModuleCode } from '../types/modules';
 
 import { AuthError, NotFoundError, UnknownApiError } from '../utils/errors';
+import { fromTermCode } from '../utils/api';
 import config from '../config';
 
 // Interface extracted for easier mocking
@@ -142,6 +143,40 @@ function mapErrorCode(code: string, msg: string) {
   return error;
 }
 
+/**
+ * Maps the 4-digit term code to the parameters expected by the CourseNUSMods API.
+ */
+function mapTermToApiParams(term: string) {
+  const [acadYear, semester] = fromTermCode(term);
+
+  // 2024/2025 -> 2024/25
+  const yearParts = acadYear.split('/');
+  const shortYear = `${yearParts[0]}/${yearParts[1].slice(2)}`;
+
+  let applicableInSem = '';
+  switch (semester) {
+    case 1:
+      applicableInSem = 'Semester 1';
+      break;
+    case 2:
+      applicableInSem = 'Semester 2';
+      break;
+    case 3:
+      applicableInSem = 'Special Semester (Part 1)';
+      break;
+    case 4:
+      applicableInSem = 'Special Semester (Part 2)';
+      break;
+    default:
+      applicableInSem = `Semester ${semester}`;
+  }
+
+  return {
+    applicableInYear: shortYear,
+    applicableInSem,
+  };
+}
+
 /* eslint-disable camelcase */
 
 /**
@@ -263,16 +298,59 @@ class NusApi implements INusApi {
    * Calls the modules endpoint
    */
   callModulesEndpoint = async (term: string, params: ApiParams): Promise<ModuleInfo[]> => {
+    const termParams = mapTermToApiParams(term);
+    const maxItems = 1000;
+    const baseParams = {
+      ...termParams,
+      ...params,
+      latestVersionOnly: 'True',
+      publishedOnly: 'True',
+      maxItems: String(maxItems),
+    };
+
     try {
-      const { data: modules } = await this.callApi<ModuleInfo[]>(
+      // 1. Fetch the first page to get the total itemCount
+      const firstResponse = await this.callApi<{ data: ModuleInfo[]; itemCount: number }>(
         'CourseNUSMods',
         {
-          term,
-          ...params,
+          ...baseParams,
+          offset: '0',
         },
         courseHeaders,
       );
-      return modules;
+
+      const allModules = [...firstResponse.data.data];
+      const { itemCount } = firstResponse.data;
+
+      // 2. If there are more items, fetch the remaining pages in parallel.
+      // Since this.callApi uses a queue, concurrency will still be limited.
+      const remainingPages = [];
+      for (let offset = allModules.length; offset < itemCount; offset += maxItems) {
+        remainingPages.push(
+          this.callApi<{ data: ModuleInfo[]; itemCount: number }>(
+            'CourseNUSMods',
+            {
+              ...baseParams,
+              offset: String(offset),
+            },
+            courseHeaders,
+          ),
+        );
+      }
+
+      if (remainingPages.length > 0) {
+        const responses = await Promise.all(remainingPages);
+        responses.forEach((response) => {
+          allModules.push(...response.data.data);
+        });
+      }
+
+      console.log(
+        `[API] CourseNUSMods fetched ${allModules.length}/${itemCount} results`,
+        baseParams,
+      );
+
+      return allModules;
     } catch (e) {
       // The modules endpoint will return NotFound even for valid inputs
       // that just happen to have no records, so we ignore this error
@@ -313,26 +391,35 @@ class NusApi implements INusApi {
     }
 
     // catalognbr = Catalog number
-    const [subject, catalognbr] = parts;
-    const { data: modules } = await this.callApi<ModuleInfo[]>(
+    const [, subject, catalognbr] = parts;
+    const termParams = mapTermToApiParams(term);
+    const { data: response } = await this.callApi<{ data: ModuleInfo[]; itemCount: number }>(
       'CourseNUSMods',
       {
-        term,
-        subject,
-        catalognbr,
+        ...termParams,
+        subjectArea: subject,
+        catalogNbr: catalognbr,
+        latestVersionOnly: 'True',
+        publishedOnly: 'True',
       },
       courseHeaders,
     );
+    const modules = response.data;
 
+    console.log(`[API] CourseNUSMods returned ${response.itemCount} result(s) for ${moduleCode}`);
     if (modules.length === 0) throw new NotFoundError(`Module ${moduleCode} cannot be found`);
     return modules[0];
   };
 
   getFacultyModules = async (term: string, facultyCode: string) =>
-    this.callModulesEndpoint(term, { acadgroup: facultyCode });
+    this.callModulesEndpoint(term, { acadGroupCode: facultyCode.slice(0, 3) });
 
-  getDepartmentModules = async (term: string, departmentCode: string): Promise<ModuleInfo[]> =>
-    this.callModulesEndpoint(term, { acadorg: departmentCode });
+  getDepartmentModules = async (term: string, departmentCode: string): Promise<ModuleInfo[]> => {
+    const modules = await this.callModulesEndpoint(term, {
+      acadGroupCode: departmentCode.slice(0, 3),
+    });
+    return modules.filter((module) => module.OrganisationCode === departmentCode);
+  };
 
   getModuleTimetable = async (term: string, module: ModuleCode): Promise<TimetableLesson[]> =>
     this.callV1Api(

scrapers/nus-v2/src/tasks/GetSemesterData.ts

@@ -144,7 +144,9 @@ export function cleanModuleInfo(module: SemesterModule) {
  * Parse the workload string into a mapping of individual components to their hours.
  * If the string is unparsable, it is returned without any modification.
  */
-export function parseWorkload(workloadString: string): Workload {
+export function parseWorkload(workloadString: string | null | undefined): Workload {
+  if (!workloadString) return '';
+
   const cleanedWorkloadString = workloadString
     .replace(/\(.*?\)/g, '') // Remove stuff in parenthesis
     .replace(/NA/gi, '0') // Replace 'NA' with 0
@@ -178,52 +180,54 @@ const mapModuleInfo = (
   departmentMap: DepartmentCodeMap,
   facultyMap: FacultyCodeMap,
   logger: Logger,
+  acadYear: string,
 ): SemesterModule => {
   const {
-    Term,
-    AcademicOrganisation,
+    OrganisationCode,
     AcademicGroup,
-    CourseTitle,
-    AdditionalInformation,
-    WorkLoadHours,
-    GradingBasisDesc,
-    Preclusion,
+    Title,
+    WorkloadHoursNUSMods,
+    PreclusionSummary,
     PreclusionRule,
-    PreRequisite,
-    PreRequisiteRule,
     PreRequisiteAdvisory,
-    CoRequisite,
-    CoRequisiteRule,
-    ModularCredit,
-    Description,
-    Subject,
+    PrerequisiteRule,
+    PrerequisiteSummary,
+    CorequisiteRule,
+    CorequisiteSummary,
+    UnitsMin,
+    CourseDesc,
+    SubjectArea,
     CatalogNumber,
-    ModuleAttributes = [],
+    CourseAttributes = [],
   } = moduleInfo;
 
-  const [AcadYear] = fromTermCode(Term);
-
   // We map department from our department list because
   // AcademicOrganisation.Description is empty for some reason
   return {
-    acadYear: AcadYear,
-    preclusion: Preclusion,
+    acadYear,
+    preclusion: PreclusionSummary,
     preclusionRule: PreclusionRule,
-    description: Description,
-    title: CourseTitle,
-    additionalInformation: AdditionalInformation,
-    department: departmentMap[AcademicOrganisation.Code],
-    faculty: facultyMap[AcademicGroup.Code],
-    workload: parseWorkload(WorkLoadHours),
-    gradingBasisDescription: GradingBasisDesc,
-    prerequisite: PreRequisite,
-    prerequisiteRule: PreRequisiteRule,
+    description: CourseDesc,
+    title: Title,
+    additionalInformation: '', // Missing in new API?
+    department: departmentMap[OrganisationCode],
+    faculty: facultyMap[AcademicGroup],
+    workload: parseWorkload(WorkloadHoursNUSMods),
+    gradingBasisDescription: '', // Missing in new API?
+    prerequisite: PrerequisiteSummary,
+    prerequisiteRule: PrerequisiteRule,
     prerequisiteAdvisory: PreRequisiteAdvisory,
-    corequisite: CoRequisite,
-    corequisiteRule: CoRequisiteRule,
-    moduleCredit: ModularCredit,
-    moduleCode: Subject + CatalogNumber,
-    attributes: mapAttributes(ModuleAttributes, logger),
+    corequisite: CorequisiteSummary,
+    corequisiteRule: CorequisiteRule,
+    moduleCredit: String(UnitsMin),
+    moduleCode: SubjectArea + CatalogNumber,
+    attributes: mapAttributes(
+      CourseAttributes.map((attr) => ({
+        CourseAttribute: attr.Code,
+        CourseAttributeValue: attr.Value,
+      })),
+      logger,
+    ),
   };
 };
 
@@ -284,10 +288,7 @@ export default class GetSemesterData extends BaseTask implements Task<Input, Out
     const facultyMap = getFacultyCodeMap(input.faculties);
 
     // Key modules by their module code for easier mapping
-    const modulesMap = keyBy(
-      modules,
-      (moduleInfo) => moduleInfo.Subject + moduleInfo.CatalogNumber,
-    );
+    const modulesMap = keyBy(modules, (moduleInfo) => moduleInfo.SubjectArea + moduleInfo.CatalogNumber);
 
     // Combine all three source of data into one set of semester module info.
     //
@@ -300,7 +301,7 @@ export default class GetSemesterData extends BaseTask implements Task<Input, Out
 
       // Map module info to the shape expected by our frontend and clean up
       // the data by removing nil fields and fixing data issues
-      const rawModule = mapModuleInfo(moduleInfo, departmentMap, facultyMap, logger);
+      const rawModule = mapModuleInfo(moduleInfo, departmentMap, facultyMap, logger, academicYear);
       const module = cleanModuleInfo(rawModule);
 
       const timetable = timetables[moduleCode];

scrapers/nus-v2/src/tasks/GetSemesterModules.ts

@@ -54,38 +54,49 @@ export default class GetSemesterModules extends BaseTask implements Task<Input,
 
     const term = getTermCode(this.semester, this.academicYear);
 
-    // We make a new request for each department because the API will timeout if
+    let downloadedCount = 0;
+    const totalFaculties = input.faculties.length;
+
+    // We make a new request for each faculty because the API will timeout if
     // we try to request for all of them in one shot
-    const requests = input.departments.map(async (department) => {
+    const requests = input.faculties.map(async (faculty, index) => {
       try {
-        const getModules = () =>
-          this.api.getDepartmentModules(term, department.AcademicOrganisation);
+        const getModules = () => this.api.getFacultyModules(term, faculty.AcademicGroup);
         const modules = await retry(getModules, 3, (error) => error instanceof UnknownApiError);
 
         // Only return modules which are visible in the system
         const [printed, hidden] = partition(
           modules,
-          (module: ModuleInfo) => module.PrintCatalog === 'Y',
+          (module: ModuleInfo) => module.PrintCatalog !== 'N',
+        );
+
+        downloadedCount += printed.length;
+        this.logger.info(
+          '[%d/%d] Downloaded %i modules from %s (Total: %d)',
+          index + 1,
+          totalFaculties,
+          printed.length,
+          faculty.Description,
+          downloadedCount,
         );
 
-        this.logger.debug('Downloaded %i modules from %s', printed.length, department.Description);
         if (hidden.length > 0) {
           this.logger.debug('Filtered out %i non-print modules', hidden.length);
         }
 
         printed.forEach(
           (module) =>
-            !!containsNbsps(module.Description) &&
+            !!containsNbsps(module.CourseDesc) &&
             this.logger.error(
-              { moduleCode: `${module.Subject}${module.CatalogNumber}` },
-              `${module.Subject}${module.CatalogNumber}: Module description contains non-breaking spaces`,
+              { moduleCode: `${module.SubjectArea}${module.CatalogNumber}` },
+              `${module.SubjectArea}${module.CatalogNumber}: Module description contains non-breaking spaces`,
             ),
         );
 
         return printed;
       } catch (e) {
-        this.logger.error(e, `Cannot get modules from ${department.Description}`);
-        throw new TaskError(`Cannot get modules from ${department.Description}`, this, e);
+        this.logger.error(e, `Cannot get modules from ${faculty.Description}`);
+        throw new TaskError(`Cannot get modules from ${faculty.Description}`, this, e);
       }
     });