refactor(nus-api): migrate to new vendor

scrapers/nus-v2/env.example.json

@@ -1,7 +1,11 @@
 {
   "appKey": "",
   "studentKey": "",
+  "ttApiKey": "",
+  "courseApiKey": "",
+  "acadApiKey": "",
+  "acadAppKey": "",
   "baseUrl": "",
   "apiConcurrency": 5,
   "elasticConfig": null
-}
+}

scrapers/nus-v2/src/__mocks__/config.ts

@@ -4,6 +4,10 @@ const config: Config = {
   // From env
   appKey: '',
   studentKey: '',
+  ttApiKey: '',
+  courseApiKey: '',
+  acadApiKey: '',
+  acadAppKey: '',
   baseUrl: 'https://example.com/api',
 
   // Other config

scrapers/nus-v2/src/config.ts

@@ -5,6 +5,10 @@ import { ClientOptions } from '@elastic/elasticsearch';
 export type Config = Readonly<{
   appKey: string;
   studentKey: string;
+  ttApiKey: string;
+  courseApiKey: string;
+  acadApiKey: string;
+  acadAppKey: string;
 
   // Base URL for all API requests
   baseUrl: string;
@@ -39,6 +43,10 @@ const config: Config = {
   // From env
   appKey: env.appKey,
   studentKey: env.studentKey,
+  ttApiKey: env.ttApiKey,
+  courseApiKey: env.courseApiKey,
+  acadApiKey: env.acadApiKey,
+  acadAppKey: env.acadAppKey,
   elasticConfig: env.elasticConfig,
   baseUrl: addTrailingSlash(env.baseUrl),
   apiConcurrency: env.apiConcurrency || 5,

scrapers/nus-v2/src/services/__mocks__/nus-api.ts

@@ -4,11 +4,9 @@ import type { INusApi } from '../nus-api';
 const mockApi: INusApi = {
   getFacultyModules: jest.fn(),
   getModuleExam: jest.fn(),
-  getModuleInfo: jest.fn(),
 
   getFaculty: jest.fn(),
   getDepartment: jest.fn(),
-  getDepartmentModules: jest.fn(),
 
   getModuleTimetable: jest.fn(),
   getDepartmentTimetables: jest.fn(),

scrapers/nus-v2/src/services/io/elastic.ts

@@ -148,6 +148,10 @@ export default class ElasticPersist implements Persist {
       }
     }
 
+    if (bulkBody.length === 0) {
+      return;
+    }
+
     const client = await this.client;
     const res = await client.bulk({
       index: INDEX_NAME,

scrapers/nus-v2/src/services/nus-api.test.ts

@@ -1,74 +1,76 @@
 import axios from 'axios';
-import { NusApi, callApi } from './nus-api';
+import { NusApi, callApi, callV1Api } from './nus-api';
 import { AuthError, NotFoundError, UnknownApiError } from '../utils/errors';
 import { mockResponse } from '../utils/test-utils';
 
 const mockedAxios: jest.Mocked<typeof axios> = axios as any;
 
 beforeEach(() => {
-  jest.spyOn(axios, 'post');
+  jest.spyOn(axios, 'get');
 });
 
 afterEach(() => {
-  mockedAxios.post.mockRestore();
+  mockedAxios.get.mockRestore();
 });
 
-describe(callApi, () => {
+describe(callV1Api, () => {
   test('should return data if everything is okay', async () => {
-    mockedAxios.post.mockResolvedValue(
+    mockedAxios.get.mockResolvedValue(
       mockResponse({ code: '00000', msg: '', data: 'Turn down for whaaaaat?' }),
     );
 
-    const result = callApi('test', {});
+    const result = callV1Api('test', {}, {});
     await expect(result).resolves.toEqual('Turn down for whaaaaat?');
   });
 
   test('should throw auth error', async () => {
-    mockedAxios.post.mockResolvedValue(
+    mockedAxios.get.mockResolvedValue(
       mockResponse({ code: '10000', msg: 'Incorrect user key', data: [] }),
     );
 
-    const result = callApi('test', {});
+    const result = callV1Api('test', {}, {});
     await expect(result).rejects.toBeInstanceOf(AuthError);
     await expect(result).rejects.toHaveProperty('message', 'Incorrect user key');
     await expect(result).rejects.toHaveProperty('response');
   });
 
   test('should throw not found error', async () => {
-    mockedAxios.post.mockResolvedValue(
+    mockedAxios.get.mockResolvedValue(
       mockResponse({ code: '10001', msg: 'Record not found', data: [] }),
     );
 
-    const result = callApi('test', {});
+    const result = callV1Api('test', {}, {});
     await expect(result).rejects.toBeInstanceOf(NotFoundError);
     await expect(result).rejects.toHaveProperty('message', 'Record not found');
     await expect(result).rejects.toHaveProperty('response');
   });
 
   test('should throw on unknown error', async () => {
-    mockedAxios.post.mockResolvedValue(
+    mockedAxios.get.mockResolvedValue(
       mockResponse({ code: '20000', msg: 'The server is on fire', data: [] }),
     );
 
-    const result = callApi('test', {});
+    const result = callV1Api('test', {}, {});
     await expect(result).rejects.toBeInstanceOf(UnknownApiError);
     await expect(result).rejects.toHaveProperty('message', 'The server is on fire');
     await expect(result).rejects.toHaveProperty('response');
   });
+});
 
+describe(callApi, () => {
   test('should throw if the server returns non-200 response', async () => {
     const config = {
       url: 'http://api.example.com',
-      method: 'post',
+      method: 'get',
       data: '{"hello": 200}',
     };
 
-    mockedAxios.post.mockRejectedValue({
+    mockedAxios.get.mockRejectedValue({
       config,
       response: mockResponse('The server is on fire', { status: 500 }),
     });
 
-    const result = callApi('test', {});
+    const result = callApi('test', {}, {});
     await expect(result).rejects.toBeInstanceOf(UnknownApiError);
     await expect(result).rejects.toHaveProperty(
       'message',
@@ -80,15 +82,15 @@ describe(callApi, () => {
   test('should throw if the request could not be made', async () => {
     const config = {
       url: 'http://api.example.com',
-      method: 'post',
+      method: 'get',
       data: '{"hello": 200}',
     };
 
-    mockedAxios.post.mockRejectedValue({
+    mockedAxios.get.mockRejectedValue({
       config,
     });
 
-    const result = callApi('test', {});
+    const result = callApi('test', {}, {});
     await expect(result).rejects.toBeInstanceOf(UnknownApiError);
     await expect(result).rejects.toHaveProperty('requestConfig', config);
   });
@@ -98,27 +100,27 @@ describe(NusApi, () => {
   test('should enforce maximum concurrency', async () => {
     expect.assertions(7);
 
-    mockedAxios.post.mockResolvedValue(
+    mockedAxios.get.mockResolvedValue(
       mockResponse({ code: '00000', msg: '', data: 'Turn down for whaaaaat?' }),
     );
 
     const api = new NusApi(2);
 
-    const p1 = api.callApi('test-1', {});
-    const p2 = api.callApi('test-2', {});
-    const p3 = api.callApi('test-3', {});
-    const p4 = api.callApi('test-4', {});
+    const p1 = api.callApi('test-1', {}, {});
+    const p2 = api.callApi('test-2', {}, {});
+    const p3 = api.callApi('test-3', {}, {});
+    const p4 = api.callApi('test-4', {}, {});
 
     // Expect 2 requests to have started, with 2 more waiting to be started.
-    expect(mockedAxios.post).toBeCalledTimes(2);
+    expect(mockedAxios.get).toBeCalledTimes(2);
     expect(api.queue.getPendingLength()).toEqual(2);
     expect(api.queue.getQueueLength()).toEqual(2);
 
     await p1;
     await p2;
 
     // Expect remaining 2 requests to have started.
-    expect(mockedAxios.post).toBeCalledTimes(4);
+    expect(mockedAxios.get).toBeCalledTimes(4);
     expect(api.queue.getQueueLength()).toEqual(0);
 
     await p3;