feat(url-store): draft API spec for URL store

.cursor/rules/openapi-api-specification-implementation.mdc

@@ -0,0 +1,79 @@
+---
+title: OpenAPI Specification and Implementation
+description: Guidelines for maintaining OpenAPI specifications in sync with API implementation
+globs:
+  - "docs/openapi/**"
+  - "src/**"
+alwaysApply: false
+version: "1.0.0"
+---
+
+# OpenAPI Specification and Implementation
+
+## Context
+When changes touch API specification files under `docs/openapi` **or** implementation files under `src`, this rule applies.
+
+## Core Principles
+
+### Specification-Implementation Sync
+- Any new endpoint, modification of existing endpoint, or removal must include **both** the OpenAPI spec change **and** the matching implementation in `src`
+- If the implementation is deferred, the spec's endpoint description **must start with** `Not implemented yet`
+and the actual handler must return HTTP **501** with a clear JSON message:
+```json
+{ "error": "Not implemented yet: <feature-name>" }
+```
+- Keep examples in sync with actual API behavior
+
+### Schema Management
+- All request/response bodies and parameters **must reference** schemas in `schemas.yaml`
+- Reuse existing schemas wherever possible to avoid duplication
+- Avoid redefining data structures inline unless absolutely necessary
+- Use **composition, inheritance, polymorphism** (as supported by OpenAPI 3.1.1) to reuse common parts
+- When introducing new audit types, update:
+  - Audit type enum
+  - Audit result schema
+  - At least one complete example
+
+### Examples
+- Add examples to `examples.yaml` and reference them (don't inline)
+- Include at least one request and response example per endpoint
+- Examples must validate against the schema and represent realistic, production-like data
+
+### Precision & Consistency
+- Keep definitions as precise as possible (correct types, constraints, formats, etc.)
+- Endpoints under the same tag must follow consistent naming, path structure, response format, etc.
+
+### Validation & Documentation Build
+- After modifying OpenAPI specs, **always run** `npm run docs:lint` to validate the specification
+- Before completing implementation, **must run** `npm run docs:build` to generate documentation
+- Fix any linting errors or build failures before considering the task complete
+
+## API Design Patterns
+
+### Pagination
+- Set strict limits on collection resource items returned
+- Support optional `limit` query parameter with documented defaults, min/max values
+- Use cursor-based pagination with `cursor` query parameter
+- Response format: `{ "cursor": "next-token", "items": [...] }`
+- Omit `cursor` property when no more items exist
+- Use consistent naming: `limit` and `cursor` across all paginated endpoints
+
+### Bulk Operations
+- **Prefer bulk endpoints over separate single/multi-item endpoints**
+- Accept arrays containing one or more items
+- Document explicitly in description that single items can be passed in arrays
+- Set strict limits on bulk POST/PUT/PATCH/DELETE request sizes
+- Response format: `{ "metadata": { "total": N, "success": N, "failure": N }, "failures": [...], "items": [...] }`
+- Specify atomicity behavior and per-item error reporting
+
+### Parameters
+- For URLs passed as path parameters, use {base64Url} and encode as URL-safe base64 without padding (RFC 4648 §5) to avoid +/= issues in paths
+- Maintain consistent naming across all endpoints
+
+### POST Requests
+- Document idempotency behavior explicitly
+- Specify duplicate detection mechanism if applicable
+- Indicate whether upserts are supported
+
+### PATCH Requests
+- Implementation must update only fields provided in request. If other fields are modified or reset, document this behavior clearly in the endpoint description

.gitignore

@@ -14,4 +14,5 @@ admin-idp-p*.json
 *.code-workspace
 .prettierrc
 .vscode/settings.json
-.cursor
+.cursor/*
+!.cursor/rules/

docs/openapi/api.yaml

@@ -67,6 +67,8 @@ tags:
     description: Report generation and management operations
   - name: project
     description: Project management operations
+  - name: url-store
+    description: URL Store operations for managing audit URLs
 
 paths:
   /audits/latest/{auditType}:
@@ -200,6 +202,14 @@ paths:
     $ref: './site-enrollments-api.yaml#/site-enrollments-by-site'
   /sites/{siteId}/user-activities:
     $ref: './user-activities-api.yaml#/paths/~1sites~1{siteId}~1user-activities'
+  /sites/{siteId}/url-store:
+    $ref: './url-store-api.yaml#/list-all-urls'
+  /sites/{siteId}/url-store/by-source/{source}:
+    $ref: './url-store-api.yaml#/list-urls-by-source'
+  /sites/{siteId}/url-store/by-audit/{auditType}:
+    $ref: './url-store-api.yaml#/list-urls-by-audit'
+  /sites/{siteId}/url-store/{base64Url}:
+    $ref: './url-store-api.yaml#/get-url'
   /sites/{siteId}/key-events:
     $ref: './key-events-api.yaml#/key-events'
   /sites/{siteId}/key-events/{keyEventId}:

docs/openapi/parameters.yaml

@@ -33,6 +33,15 @@ deliveryType:
   required: true
   schema:
     $ref: './schemas.yaml#/DeliveryType'
+base64Url:
+  name: base64Url
+  description: Base64-encoded URL
+  in: path
+  required: true
+  schema:
+    type: string
+    format: base64url
+    example: "aHR0cHM6Ly9leGFtcGxlLmNvbS9wYWdl"
 base64BaseUrl:
   name: base64BaseUrl
   description: Base64-encoded base URL
@@ -49,6 +58,14 @@ base64PageUrl:
   schema:
     type: string
     format: base64url
+urlSource:
+  name: source
+  description: The source of the URLs (e.g., "manual")
+  in: path
+  required: true
+  schema:
+    type: string
+    example: "manual"
 ascending:
   name: ascending
   description: Whether to sort ascending or descending
@@ -342,4 +359,4 @@ llmoConfigVersion:
   schema:
     type: string
     description: S3 object version ID
-    example: "abc123def456"
+    example: "abc123def456"

docs/openapi/schemas.yaml

@@ -4630,3 +4630,239 @@ PreflightResultItem:
       type: string
       enum: [identify, suggest]
       description: The step of the preflight job when this result was generated
+
+AuditUrl:
+  type: object
+  required: [siteId, url, source, audits, createdAt, updatedAt, createdBy, updatedBy]
+  properties:
+    siteId:
+      $ref: '#/Id'
+    url:
+      $ref: '#/URL'
+    source:
+      type: string
+      description: Origin of the URL (e.g., "manual", "sitemap")
+      default: "manual"
+      example: "manual"
+    audits:
+      type: array
+      items:
+        type: string
+      description: Enabled audit types for this URL (can be empty)
+      example: ["accessibility", "broken-backlinks"]
+    createdAt:
+      $ref: '#/DateTime'
+    updatedAt:
+      $ref: '#/DateTime'
+    createdBy:
+      type: string
+      description: User or service who created the URL entry
+      example: "user-alice"
+    updatedBy:
+      type: string
+      description: Last user or service to modify the URL entry
+      example: "user-bob"
+  example:
+    siteId: "550e8400-e29b-41d4-a716-446655440000"
+    url: "https://example.com/foo/bar"
+    source: "manual"
+    audits: ["accessibility", "broken-backlinks"]
+    createdAt: "2025-10-10T12:00:00Z"
+    updatedAt: "2025-10-10T15:30:00Z"
+    createdBy: "user-alice"
+    updatedBy: "user-bob"
+
+AuditUrlList:
+  type: object
+  properties:
+    items:
+      type: array
+      items:
+        $ref: '#/AuditUrl'
+      description: Array of audit URLs
+    cursor:
+      type: string
+      description: Pagination cursor for retrieving the next page of results
+      example: "eyJzaXRlSWQiOiIxMjMiLCJ1cmwiOiJodHRwczovL2V4YW1wbGUuY29tIn0"
+  example:
+    items:
+      - siteId: "550e8400-e29b-41d4-a716-446655440000"
+        url: "https://example.com/documents/report.pdf"
+        source: "manual"
+        audits: ["accessibility"]
+        createdAt: "2025-10-10T12:00:00Z"
+        updatedAt: "2025-10-10T12:00:00Z"
+        createdBy: "user-alice"
+        updatedBy: "user-alice"
+      - siteId: "550e8400-e29b-41d4-a716-446655440000"
+        url: "https://example.com/page2"
+        source: "sitemap"
+        audits: ["broken-backlinks"]
+        createdAt: "2025-10-10T12:00:00Z"
+        updatedAt: "2025-10-10T12:00:00Z"
+        createdBy: "system"
+        updatedBy: "system"
+    cursor: "eyJzaXRlSWQiOiIxMjMiLCJ1cmwiOiJodHRwczovL2V4YW1wbGUuY29tIn0"
+
+AuditUrlBulkResponse:
+  type: object
+  properties:
+    metadata:
+      type: object
+      properties:
+        total:
+          type: integer
+          description: Total number of URLs in the request
+          example: 2
+        success:
+          type: integer
+          description: Number of URLs processed successfully
+          example: 1
+        failure:
+          type: integer
+          description: Number of URLs that failed to process
+          example: 1
+    failures:
+      type: array
+      items:
+        type: object
+        properties:
+          url:
+            type: string
+            description: The URL that failed to process
+            example: "https://example.com/invalid"
+          reason:
+            type: string
+            description: Reason for the failure
+            example: "URL does not belong to site domain"
+      description: Array of failed operations with reasons
+    items:
+      type: array
+      items:
+        $ref: '#/AuditUrl'
+      description: Array of successfully processed audit URLs
+  example:
+    metadata:
+      total: 2
+      success: 1
+      failure: 1
+    failures:
+      - url: "https://example.com/invalid"
+        reason: "URL does not belong to site domain"
+    items:
+      - siteId: "550e8400-e29b-41d4-a716-446655440000"
+        url: "https://example.com/page1"
+        source: "manual"
+        audits: ["accessibility", "broken-backlinks"]
+        createdAt: "2025-10-10T12:00:00Z"
+        updatedAt: "2025-10-10T15:30:00Z"
+        createdBy: "user-alice"
+        updatedBy: "user-bob"
+
+AuditUrlDeleteResponse:
+  type: object
+  properties:
+    metadata:
+      type: object
+      properties:
+        total:
+          type: integer
+          description: Total number of URLs in the delete request
+          example: 2
+        success:
+          type: integer
+          description: Number of URLs deleted successfully
+          example: 1
+        failure:
+          type: integer
+          description: Number of URLs that failed to delete
+          example: 1
+    failures:
+      type: array
+      items:
+        type: object
+        properties:
+          url:
+            type: string
+            description: The URL that failed to delete
+            example: "https://example.com/from-sitemap"
+          reason:
+            type: string
+            description: Reason for the deletion failure
+            example: "Can only delete URLs with source='manual'"
+      description: Array of failed delete operations with reasons
+  example:
+    metadata:
+      total: 2
+      success: 1
+      failure: 1
+    failures:
+      - url: "https://example.com/from-sitemap"
+        reason: "Can only delete URLs with source='manual'"
+
+AuditUrlAddRequest:
+  type: array
+  minItems: 1
+  maxItems: 100
+  items:
+    type: object
+    required: [url, audits]
+    properties:
+      url:
+        type: string
+        format: uri
+        description: The URL to add (will be canonicalized)
+      source:
+        type: string
+        description: Source of the URL (defaults to "manual")
+        default: manual
+      audits:
+        type: array
+        items:
+          type: string
+        description: Array of audit types to enable for this URL
+  example:
+    - url: "https://example.com/page1.html"
+      source: "manual"
+      audits: ["accessibility", "broken-backlinks"]
+    - url: "https://example.com/page2.html"
+      audits: ["broken-backlinks"]
+
+AuditUrlUpdateRequest:
+  type: array
+  minItems: 1
+  maxItems: 100
+  items:
+    type: object
+    required: [url, audits]
+    properties:
+      url:
+        type: string
+        format: uri
+        description: The URL to update
+      audits:
+        type: array
+        items:
+          type: string
+        description: New audits array (replaces existing)
+  example:
+    - url: "https://example.com/page1.html"
+      audits: ["accessibility"]
+    - url: "https://example.com/page2.html"
+      audits: []
+
+AuditUrlDeleteRequest:
+  type: object
+  required: [urls]
+  properties:
+    urls:
+      type: array
+      minItems: 1
+      maxItems: 100
+      items:
+        type: string
+        format: uri
+  example:
+    urls:
+      - "https://example.com/page1.html"
+      - "https://example.com/page2.html"