diff --git a/scrapers/cpex-scraper/src/index.ts b/scrapers/cpex-scraper/src/index.ts index 065d255219..ae13fb27af 100644 --- a/scrapers/cpex-scraper/src/index.ts +++ b/scrapers/cpex-scraper/src/index.ts @@ -7,8 +7,8 @@ import env from '../env.json'; const TERM = '2520'; // Sanity check to see if there are at least this many modules before overwriting cpexModules.json -// The last time I ran this fully there were 3418 modules -const threshold = 1500; +// The last time I ran this fully there were 4038 modules +const threshold = 3000; const baseUrl = env['baseUrl'].endsWith('/') ? env['baseUrl'].slice(0, -1) : env['baseUrl']; @@ -19,23 +19,6 @@ axios.defaults.headers.common = { 'X-APP-API': env['appKey'], }; -function getTimestampForFilename(): string { - function pad2(n: number): string { - return n < 10 ? '0' + n : String(n); - } - - const date = new Date(); - - return ( - date.getFullYear().toString() + - pad2(date.getMonth() + 1) + - pad2(date.getDate()) + - pad2(date.getHours()) + - pad2(date.getMinutes()) + - pad2(date.getSeconds()) - ); -} - type ApiResponse = { msg: string; code: string; @@ -61,6 +44,33 @@ type Module = { }[]; }; +class CPExModuleExport { + public lastUpdated: string; + public modules: CPExModule[]; + + constructor(modules: CPExModule[]) { + this.modules = modules; + this.lastUpdated = CPExModuleExport.getCurrentTimestamp(); + } + + private static pad2(n: number): string { + return n < 10 ? '0' + n : String(n); + } + + private static getCurrentTimestamp(): string { + const date = new Date(); + + return ( + date.getFullYear().toString() + + CPExModuleExport.pad2(date.getMonth() + 1) + + CPExModuleExport.pad2(date.getDate()) + + CPExModuleExport.pad2(date.getHours()) + + CPExModuleExport.pad2(date.getMinutes()) + + CPExModuleExport.pad2(date.getSeconds()) + ); + } +} + export type CPExModule = { title: string; moduleCode: string; @@ -165,8 +175,8 @@ async function scraper() { } } - const collatedCPExModules = Array.from(collatedCPExModulesMap.values()); - console.log(`Collated ${collatedCPExModules.length} modules.`); + const collatedCPExModules = new CPExModuleExport(Array.from(collatedCPExModulesMap.values())); + console.log(`Collated ${collatedCPExModules.modules.length} modules.`); const DATA_DIR = path.join(__dirname, '../../data'); if (!fs.existsSync(DATA_DIR)) { @@ -177,18 +187,18 @@ async function scraper() { fs.mkdirSync(OLD_DATA_DIR); } - if (collatedCPExModules.length >= threshold) { + if (collatedCPExModules.modules.length >= threshold) { fs.writeFileSync(path.join(DATA_DIR, 'cpexModules.json'), JSON.stringify(collatedCPExModules)); - console.log(`Wrote ${collatedCPExModules.length} modules to cpexModules.json.`); + console.log(`Wrote ${collatedCPExModules.modules.length} modules to cpexModules.json.`); } else { console.log( - `Not writing to cpexModules.json because the number of modules ${collatedCPExModules.length} is less than the threshold of ${threshold}.`, + `Not writing to cpexModules.json because the number of modules ${collatedCPExModules.modules.length} is less than the threshold of ${threshold}.`, ); } - const archiveFilename = `cpexModules-${getTimestampForFilename()}.json`; + const archiveFilename = `cpexModules-${collatedCPExModules.lastUpdated}.json`; fs.writeFileSync(path.join(OLD_DATA_DIR, archiveFilename), JSON.stringify(collatedCPExModules)); - console.log(`Wrote ${collatedCPExModules.length} modules to archive ${archiveFilename}.`); + console.log(`Wrote ${collatedCPExModules.modules.length} modules to archive ${archiveFilename}.`); console.log('Done!'); } diff --git a/website/src/apis/mpe.ts b/website/src/apis/mpe.ts index 1d8c1416d6..f5cb1b2fcc 100644 --- a/website/src/apis/mpe.ts +++ b/website/src/apis/mpe.ts @@ -4,7 +4,7 @@ import { Location, History } from 'history'; import { produce } from 'immer'; import getLocalStorage from 'storage/localStorage'; import NUSModsApi from './nusmods'; -import { MpeSubmission, MpeModule } from '../types/mpe'; +import { MpeSubmission, MpeModuleExport } from '../types/mpe'; import { NUS_AUTH_TOKEN } from '../storage/keys'; export class MpeSessionExpiredError extends Error {} @@ -55,9 +55,9 @@ export const getLoginState = (location: Location, history: History): boolean => return getToken() !== null; }; -export const fetchMpeModuleList = (): Promise => +export const fetchMpeModuleList = (): Promise => // Using MPE_AY instead to fetch from the respective AY's scraper - axios.get(NUSModsApi.mpeModuleListUrl()).then((resp) => resp.data); + axios.get(NUSModsApi.mpeModuleListUrl()).then((resp) => resp.data); export const getSSOLink = (): Promise => mpe diff --git a/website/src/types/mpe.ts b/website/src/types/mpe.ts index c6c9e470c6..b9eae73b9d 100644 --- a/website/src/types/mpe.ts +++ b/website/src/types/mpe.ts @@ -14,6 +14,11 @@ export type MpeSubmission = { preferences: Array; }; +export type MpeModuleExport = { + lastUpdated: Date; + modules: MpeModule[]; +}; + export type MpeModule = { title: ModuleTitle; moduleCode: ModuleCode; diff --git a/website/src/views/mpe/form/ModuleForm.tsx b/website/src/views/mpe/form/ModuleForm.tsx index f5c2766c0d..44bc37e9d2 100644 --- a/website/src/views/mpe/form/ModuleForm.tsx +++ b/website/src/views/mpe/form/ModuleForm.tsx @@ -2,7 +2,7 @@ import { useMemo, useRef, useState } from 'react'; import { Draggable, DragDropContext, Droppable, DropResult } from 'react-beautiful-dnd'; import classnames from 'classnames'; -import type { MpeSubmission, MpePreference, MpeModule } from 'types/mpe'; +import type { MpeSubmission, MpePreference, MpeModuleExport } from 'types/mpe'; import type { ModuleCode } from 'types/modules'; import { MAX_MODULES, MPE_SEMESTER } from '../constants'; @@ -23,7 +23,7 @@ function reorder(items: T[], startIndex: number, endIndex: number) { type Props = { initialSubmission: MpeSubmission; - mpeModuleList: MpeModule[]; + mpeModuleList: MpeModuleExport; updateSubmission: (submission: MpeSubmission) => Promise; }; @@ -46,7 +46,7 @@ const ModuleForm: React.FC = ({ const moduleSelectList = useMemo(() => { const selectedModules = new Set(preferences.map((preference) => preference.moduleCode)); const semesterProperty = MPE_SEMESTER === 1 ? 'inS1CPEx' : 'inS2CPEx'; - return mpeModuleList + return mpeModuleList.modules .filter((module) => module[semesterProperty]) .map((module) => ({ ...module, @@ -84,7 +84,7 @@ const ModuleForm: React.FC = ({ return; } - const selectedModule = mpeModuleList.find((module) => module.moduleCode === moduleCode); + const selectedModule = mpeModuleList.modules.find((module) => module.moduleCode === moduleCode); if (selectedModule == null) { return; } diff --git a/website/src/views/mpe/form/MpeFormContainer.tsx b/website/src/views/mpe/form/MpeFormContainer.tsx index b000303516..9312acccb8 100644 --- a/website/src/views/mpe/form/MpeFormContainer.tsx +++ b/website/src/views/mpe/form/MpeFormContainer.tsx @@ -1,6 +1,6 @@ import { useEffect, useState } from 'react'; import { fetchMpeModuleList } from 'apis/mpe'; -import type { MpeSubmission, MpeModule } from 'types/mpe'; +import type { MpeSubmission, MpeModuleExport } from 'types/mpe'; import LoadingSpinner from 'views/components/LoadingSpinner'; import ModuleForm from './ModuleForm'; @@ -16,7 +16,10 @@ const MpeFormContainer: React.FC = ({ getSubmission, updateSubmission }) intendedMCs: 0, preferences: [], }); - const [mpeModuleList, setMpeModuleList] = useState([]); + const [mpeModuleList, setMpeModuleList] = useState({ + lastUpdated: new Date(0), + modules: [], + }); // fetch mpe modules and preferences useEffect(() => { @@ -26,7 +29,7 @@ const MpeFormContainer: React.FC = ({ getSubmission, updateSubmission }) .then(([fetchedMpeModuleList, fetchedSubmission]) => { setMpeModuleList(fetchedMpeModuleList); const moduleLookup = new Map( - fetchedMpeModuleList.map((module) => [module.moduleCode, module]), + fetchedMpeModuleList.modules.map((module) => [module.moduleCode, module]), ); setSubmission({ ...fetchedSubmission,