From 3d9debe2ea6113fc451e19e8656672fb330ac995 Mon Sep 17 00:00:00 2001 From: Jonathan Loh <36648707+jloh02@users.noreply.github.com> Date: Mon, 17 Nov 2025 14:29:37 +0000 Subject: [PATCH 1/2] feat(cpex): add last update date time to cpex scraper result --- scrapers/cpex-scraper/src/index.ts | 62 +++++++++++-------- website/src/apis/mpe.ts | 6 +- website/src/types/mpe.ts | 5 ++ website/src/views/mpe/form/ModuleForm.tsx | 8 +-- .../src/views/mpe/form/MpeFormContainer.tsx | 9 ++- 5 files changed, 54 insertions(+), 36 deletions(-) diff --git a/scrapers/cpex-scraper/src/index.ts b/scrapers/cpex-scraper/src/index.ts index 065d255219..ae13fb27af 100644 --- a/scrapers/cpex-scraper/src/index.ts +++ b/scrapers/cpex-scraper/src/index.ts @@ -7,8 +7,8 @@ import env from '../env.json'; const TERM = '2520'; // Sanity check to see if there are at least this many modules before overwriting cpexModules.json -// The last time I ran this fully there were 3418 modules -const threshold = 1500; +// The last time I ran this fully there were 4038 modules +const threshold = 3000; const baseUrl = env['baseUrl'].endsWith('/') ? env['baseUrl'].slice(0, -1) : env['baseUrl']; @@ -19,23 +19,6 @@ axios.defaults.headers.common = { 'X-APP-API': env['appKey'], }; -function getTimestampForFilename(): string { - function pad2(n: number): string { - return n < 10 ? '0' + n : String(n); - } - - const date = new Date(); - - return ( - date.getFullYear().toString() + - pad2(date.getMonth() + 1) + - pad2(date.getDate()) + - pad2(date.getHours()) + - pad2(date.getMinutes()) + - pad2(date.getSeconds()) - ); -} - type ApiResponse = { msg: string; code: string; @@ -61,6 +44,33 @@ type Module = { }[]; }; +class CPExModuleExport { + public lastUpdated: string; + public modules: CPExModule[]; + + constructor(modules: CPExModule[]) { + this.modules = modules; + this.lastUpdated = CPExModuleExport.getCurrentTimestamp(); + } + + private static pad2(n: number): string { + return n < 10 ? '0' + n : String(n); + } + + private static getCurrentTimestamp(): string { + const date = new Date(); + + return ( + date.getFullYear().toString() + + CPExModuleExport.pad2(date.getMonth() + 1) + + CPExModuleExport.pad2(date.getDate()) + + CPExModuleExport.pad2(date.getHours()) + + CPExModuleExport.pad2(date.getMinutes()) + + CPExModuleExport.pad2(date.getSeconds()) + ); + } +} + export type CPExModule = { title: string; moduleCode: string; @@ -165,8 +175,8 @@ async function scraper() { } } - const collatedCPExModules = Array.from(collatedCPExModulesMap.values()); - console.log(`Collated ${collatedCPExModules.length} modules.`); + const collatedCPExModules = new CPExModuleExport(Array.from(collatedCPExModulesMap.values())); + console.log(`Collated ${collatedCPExModules.modules.length} modules.`); const DATA_DIR = path.join(__dirname, '../../data'); if (!fs.existsSync(DATA_DIR)) { @@ -177,18 +187,18 @@ async function scraper() { fs.mkdirSync(OLD_DATA_DIR); } - if (collatedCPExModules.length >= threshold) { + if (collatedCPExModules.modules.length >= threshold) { fs.writeFileSync(path.join(DATA_DIR, 'cpexModules.json'), JSON.stringify(collatedCPExModules)); - console.log(`Wrote ${collatedCPExModules.length} modules to cpexModules.json.`); + console.log(`Wrote ${collatedCPExModules.modules.length} modules to cpexModules.json.`); } else { console.log( - `Not writing to cpexModules.json because the number of modules ${collatedCPExModules.length} is less than the threshold of ${threshold}.`, + `Not writing to cpexModules.json because the number of modules ${collatedCPExModules.modules.length} is less than the threshold of ${threshold}.`, ); } - const archiveFilename = `cpexModules-${getTimestampForFilename()}.json`; + const archiveFilename = `cpexModules-${collatedCPExModules.lastUpdated}.json`; fs.writeFileSync(path.join(OLD_DATA_DIR, archiveFilename), JSON.stringify(collatedCPExModules)); - console.log(`Wrote ${collatedCPExModules.length} modules to archive ${archiveFilename}.`); + console.log(`Wrote ${collatedCPExModules.modules.length} modules to archive ${archiveFilename}.`); console.log('Done!'); } diff --git a/website/src/apis/mpe.ts b/website/src/apis/mpe.ts index 1d8c1416d6..f5cb1b2fcc 100644 --- a/website/src/apis/mpe.ts +++ b/website/src/apis/mpe.ts @@ -4,7 +4,7 @@ import { Location, History } from 'history'; import { produce } from 'immer'; import getLocalStorage from 'storage/localStorage'; import NUSModsApi from './nusmods'; -import { MpeSubmission, MpeModule } from '../types/mpe'; +import { MpeSubmission, MpeModuleExport } from '../types/mpe'; import { NUS_AUTH_TOKEN } from '../storage/keys'; export class MpeSessionExpiredError extends Error {} @@ -55,9 +55,9 @@ export const getLoginState = (location: Location, history: History): boolean => return getToken() !== null; }; -export const fetchMpeModuleList = (): Promise => +export const fetchMpeModuleList = (): Promise => // Using MPE_AY instead to fetch from the respective AY's scraper - axios.get(NUSModsApi.mpeModuleListUrl()).then((resp) => resp.data); + axios.get(NUSModsApi.mpeModuleListUrl()).then((resp) => resp.data); export const getSSOLink = (): Promise => mpe diff --git a/website/src/types/mpe.ts b/website/src/types/mpe.ts index c6c9e470c6..b9eae73b9d 100644 --- a/website/src/types/mpe.ts +++ b/website/src/types/mpe.ts @@ -14,6 +14,11 @@ export type MpeSubmission = { preferences: Array; }; +export type MpeModuleExport = { + lastUpdated: Date; + modules: MpeModule[]; +}; + export type MpeModule = { title: ModuleTitle; moduleCode: ModuleCode; diff --git a/website/src/views/mpe/form/ModuleForm.tsx b/website/src/views/mpe/form/ModuleForm.tsx index f5c2766c0d..44bc37e9d2 100644 --- a/website/src/views/mpe/form/ModuleForm.tsx +++ b/website/src/views/mpe/form/ModuleForm.tsx @@ -2,7 +2,7 @@ import { useMemo, useRef, useState } from 'react'; import { Draggable, DragDropContext, Droppable, DropResult } from 'react-beautiful-dnd'; import classnames from 'classnames'; -import type { MpeSubmission, MpePreference, MpeModule } from 'types/mpe'; +import type { MpeSubmission, MpePreference, MpeModuleExport } from 'types/mpe'; import type { ModuleCode } from 'types/modules'; import { MAX_MODULES, MPE_SEMESTER } from '../constants'; @@ -23,7 +23,7 @@ function reorder(items: T[], startIndex: number, endIndex: number) { type Props = { initialSubmission: MpeSubmission; - mpeModuleList: MpeModule[]; + mpeModuleList: MpeModuleExport; updateSubmission: (submission: MpeSubmission) => Promise; }; @@ -46,7 +46,7 @@ const ModuleForm: React.FC = ({ const moduleSelectList = useMemo(() => { const selectedModules = new Set(preferences.map((preference) => preference.moduleCode)); const semesterProperty = MPE_SEMESTER === 1 ? 'inS1CPEx' : 'inS2CPEx'; - return mpeModuleList + return mpeModuleList.modules .filter((module) => module[semesterProperty]) .map((module) => ({ ...module, @@ -84,7 +84,7 @@ const ModuleForm: React.FC = ({ return; } - const selectedModule = mpeModuleList.find((module) => module.moduleCode === moduleCode); + const selectedModule = mpeModuleList.modules.find((module) => module.moduleCode === moduleCode); if (selectedModule == null) { return; } diff --git a/website/src/views/mpe/form/MpeFormContainer.tsx b/website/src/views/mpe/form/MpeFormContainer.tsx index b000303516..a4e4b04a2a 100644 --- a/website/src/views/mpe/form/MpeFormContainer.tsx +++ b/website/src/views/mpe/form/MpeFormContainer.tsx @@ -1,6 +1,6 @@ import { useEffect, useState } from 'react'; import { fetchMpeModuleList } from 'apis/mpe'; -import type { MpeSubmission, MpeModule } from 'types/mpe'; +import type { MpeSubmission, MpeModuleExport } from 'types/mpe'; import LoadingSpinner from 'views/components/LoadingSpinner'; import ModuleForm from './ModuleForm'; @@ -16,7 +16,10 @@ const MpeFormContainer: React.FC = ({ getSubmission, updateSubmission }) intendedMCs: 0, preferences: [], }); - const [mpeModuleList, setMpeModuleList] = useState([]); + const [mpeModuleList, setMpeModuleList] = useState({ + lastUpdated: new Date(0), + modules:[], + }); // fetch mpe modules and preferences useEffect(() => { @@ -26,7 +29,7 @@ const MpeFormContainer: React.FC = ({ getSubmission, updateSubmission }) .then(([fetchedMpeModuleList, fetchedSubmission]) => { setMpeModuleList(fetchedMpeModuleList); const moduleLookup = new Map( - fetchedMpeModuleList.map((module) => [module.moduleCode, module]), + fetchedMpeModuleList.modules.map((module) => [module.moduleCode, module]), ); setSubmission({ ...fetchedSubmission, From b798f090f7e667007488f8553ab9316ccb04ae4e Mon Sep 17 00:00:00 2001 From: Jonathan Loh <36648707+jloh02@users.noreply.github.com> Date: Tue, 18 Nov 2025 03:37:35 +0000 Subject: [PATCH 2/2] fix: format lint --- website/src/views/mpe/form/MpeFormContainer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/views/mpe/form/MpeFormContainer.tsx b/website/src/views/mpe/form/MpeFormContainer.tsx index a4e4b04a2a..9312acccb8 100644 --- a/website/src/views/mpe/form/MpeFormContainer.tsx +++ b/website/src/views/mpe/form/MpeFormContainer.tsx @@ -18,7 +18,7 @@ const MpeFormContainer: React.FC = ({ getSubmission, updateSubmission }) }); const [mpeModuleList, setMpeModuleList] = useState({ lastUpdated: new Date(0), - modules:[], + modules: [], }); // fetch mpe modules and preferences