Feat/Introduce AI semantic search!

.env.template

@@ -7,6 +7,7 @@ NODE_ENV=development
 MONGODB_URI=mongodb://mongodb:27017/bt?replicaSet=rs0
 REDIS_URI=redis://redis:6379
 BACKEND_URL=http://backend:8080
+SEMANTIC_SEARCH_URL=http://semantic-search:8000
 TZ=America/Los_Angeles # for tslog
 
 SIS_CLASS_APP_ID=_

apps/backend/package.json

@@ -42,6 +42,7 @@
     "@repo/gql-typedefs": "*",
     "@repo/shared": "*",
     "@repo/sis-api": "*",
+    "undici": "^6.20.1",
     "compression": "^1.8.1",
     "connect-redis": "^9.0.0",
     "cors": "^2.8.5",

apps/backend/scripts/prepare-typedefs.js

@@ -11,7 +11,7 @@ const typedefFiles = fs.readdirSync(sourceDir)
   .sort();
 
 // Get all module directories from backend/src/modules (excluding non-module directories)
-const excludedDirs = ['cache', 'generated-types'];
+const excludedDirs = ['cache', 'generated-types', 'semantic-search'];
 const moduleDirs = fs.readdirSync(modulesDir, { withFileTypes: true })
   .filter(dirent => dirent.isDirectory() && !excludedDirs.includes(dirent.name))
   .map(dirent => dirent.name)

apps/backend/src/bootstrap/loaders/express.ts

@@ -7,6 +7,7 @@ import helmet from "helmet";
 import { RedisClientType } from "redis";
 
 import { config } from "../../config";
+import semanticSearchRoutes from "../../modules/semantic-search/routes";
 import passportLoader from "./passport";
 
 export default async (
@@ -58,6 +59,8 @@ export default async (
   // load authentication
   passportLoader(app, redis);
 
+  app.use("/semantic-search", semanticSearchRoutes);
+
   app.use(
     config.graphqlPath,
     expressMiddleware(server, {

apps/backend/src/config.ts

@@ -20,6 +20,9 @@ export interface Config {
   backendPath: string;
   graphqlPath: string;
   isDev: boolean;
+  semanticSearch: {
+    url: string;
+  };
   mongoDB: {
     uri: string;
   };
@@ -45,6 +48,9 @@ export const config: Config = {
   backendPath: env("BACKEND_PATH"),
   graphqlPath: env("GRAPHQL_PATH"),
   isDev: env("NODE_ENV") === "development",
+  semanticSearch: {
+    url: env("SEMANTIC_SEARCH_URL"),
+  },
   mongoDB: {
     uri: env("MONGODB_URI"),
   },

apps/backend/src/modules/semantic-search/client.ts

@@ -0,0 +1,67 @@
+import { config } from "../../config";
+
+interface SemanticSearchResult {
+  subject: string;
+  courseNumber: string;
+  title: string;
+  description: string;
+  score: number;
+  text: string;
+}
+
+interface SemanticSearchResponse {
+  query: string;
+  threshold: number;
+  count: number;
+  year: number;
+  semester: string;
+  allowed_subjects: string[] | null;
+  last_refreshed: string;
+  results: SemanticSearchResult[];
+}
+
+export async function searchSemantic(
+  query: string,
+  year: number,
+  semester: string,
+  allowedSubjects?: string[],
+  threshold: number = 0.3
+): Promise<SemanticSearchResponse> {
+  const params = new URLSearchParams({
+    query,
+    threshold: String(threshold),
+    year: String(year),
+    semester,
+  });
+
+  if (allowedSubjects && allowedSubjects.length > 0) {
+    allowedSubjects.forEach((subject) => {
+      params.append("allowed_subjects", subject);
+    });
+  }
+
+  const url = `${config.semanticSearch.url}/search?${params}`;
+
+  try {
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      throw new Error(`Semantic search failed: ${response.statusText}`);
+    }
+
+    return (await response.json()) as SemanticSearchResponse;
+  } catch (error) {
+    console.error("Semantic search error:", error);
+    // Return empty results on error, gracefully falling back
+    return {
+      query,
+      threshold,
+      count: 0,
+      year,
+      semester,
+      allowed_subjects: allowedSubjects || null,
+      last_refreshed: new Date().toISOString(),
+      results: [],
+    };
+  }
+}

apps/backend/src/modules/semantic-search/controller.ts

@@ -0,0 +1,50 @@
+import { Request, Response } from "express";
+
+import { searchSemantic } from "./client";
+
+/**
+ * Lightweight semantic search endpoint that only returns course identifiers
+ * Frontend will use these to filter the already-loaded catalog
+ */
+export async function searchCourses(req: Request, res: Response) {
+  const { query, year, semester, threshold } = req.query;
+
+  if (!query || typeof query !== "string") {
+    return res.status(400).json({ error: "query parameter is required" });
+  }
+
+  const yearNum = year ? parseInt(year as string, 10) : undefined;
+  const semesterStr = semester as string | undefined;
+  const thresholdNum = threshold ? parseFloat(threshold as string) : 0.3;
+
+  try {
+    const results = await searchSemantic(
+      query,
+      yearNum!,
+      semesterStr!,
+      undefined,
+      thresholdNum
+    );
+
+    // Return lightweight response: only subject + courseNumber + score
+    const courseIds = results.results.map((r) => ({
+      subject: r.subject,
+      courseNumber: r.courseNumber,
+      score: r.score,
+    }));
+
+    return res.json({
+      query,
+      threshold: thresholdNum,
+      results: courseIds,
+      count: courseIds.length,
+    });
+  } catch (error) {
+    console.error("Semantic search error:", error);
+    return res.status(500).json({
+      error: "Semantic search failed",
+      results: [],
+      count: 0,
+    });
+  }
+}

apps/backend/src/modules/semantic-search/routes.ts

@@ -0,0 +1,113 @@
+import { type Response, Router } from "express";
+import type { ParsedQs } from "qs";
+import { RequestInit, fetch } from "undici";
+
+import { config } from "../../config";
+import { searchCourses } from "./controller";
+
+const router = Router();
+const baseUrl = config.semanticSearch.url.replace(/\/$/, "");
+
+type QueryValue = string | ParsedQs | Array<string | ParsedQs> | undefined;
+
+const asString = (value: QueryValue): string | undefined => {
+  if (!value) return undefined;
+  if (typeof value === "string") return value;
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      const found = asString(entry as QueryValue);
+      if (found) return found;
+    }
+  }
+  return undefined;
+};
+
+const toStringList = (value: QueryValue): string[] => {
+  if (!value) return [];
+  if (Array.isArray(value)) {
+    const items: string[] = [];
+    for (const entry of value) {
+      items.push(...toStringList(entry as QueryValue));
+    }
+    return items;
+  }
+  return typeof value === "string" && value.length > 0 ? [value] : [];
+};
+
+async function forward(
+  target: string,
+  init: RequestInit,
+  res: Response
+): Promise<void> {
+  try {
+    const response = await fetch(target, init);
+    const contentType = response.headers.get("content-type") ?? "";
+    const raw = await response.text();
+
+    if (contentType.includes("application/json")) {
+      const payload = raw ? JSON.parse(raw) : {};
+      res.status(response.status).json(payload);
+    } else {
+      res.status(response.status).send(raw);
+    }
+  } catch (error) {
+    console.error("Semantic search proxy error:", error);
+    res.status(502).json({
+      error: "Unable to reach semantic search service",
+      details: String(error),
+    });
+  }
+}
+
+router.get("/health", async (_req, res) => {
+  await forward(`${baseUrl}/health`, { method: "GET" }, res);
+});
+
+router.post("/refresh", async (req, res) => {
+  const body = req.body ?? {};
+  await forward(
+    `${baseUrl}/refresh`,
+    {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body),
+    },
+    res
+  );
+});
+
+// Lightweight endpoint: returns only course identifiers for frontend filtering
+router.get("/courses", searchCourses);
+
+// Full proxy endpoint (kept for backwards compatibility)
+router.get("/search", async (req, res) => {
+  const query = asString(req.query.query);
+  if (!query || !query.trim()) {
+    res.status(400).json({ error: "query parameter is required" });
+    return;
+  }
+
+  const params = new URLSearchParams({ query });
+
+  const topK = asString(req.query.top_k);
+  if (topK) params.set("top_k", topK);
+
+  const year = asString(req.query.year);
+  if (year) params.set("year", year);
+
+  const semester = asString(req.query.semester);
+  if (semester) params.set("semester", semester);
+
+  const allowedSubjects = toStringList(req.query.allowed_subjects);
+  allowedSubjects.forEach((subject) =>
+    params.append("allowed_subjects", subject)
+  );
+
+  await forward(
+    `${baseUrl}/search?${params.toString()}`,
+    { method: "GET" },
+    res
+  );
+});
+
+export default router;

apps/frontend/src/components/ClassBrowser/Header/Header.module.scss

@@ -44,6 +44,29 @@
         color: var(--paragraph-color);
       }
     }
+
+    .sparksButton {
+      margin: 0 -8px 0 0;
+      padding: 0;
+
+      &.active {
+        color: var(--blue-500);
+        background-color: color-mix(in srgb, var(--blue-500) 10%, transparent);
+        border-radius: 4px;
+      }
+    }
+  }
+
+  .aiSearchButton {
+    width: 100%;
+    background-color: var(--blue-500);
+    color: white;
+    border: 1px solid var(--blue-500);
+
+    &:hover {
+      background-color: var(--blue-600);
+      border-color: var(--blue-600);
+    }
   }
 
   .filterButton {