From 58db8d4e2d98690bbf2b052024bb51a8fd4b0f43 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Tue, 4 Nov 2025 15:09:35 +0100 Subject: [PATCH 01/18] docs: settings doc fix: annoying wrapper script docs: fix task doc fix: add ci skip settings-doc-check --- .github/workflows/generate_pixi_tasks_doc.py | 2 + .pre-commit-config.yaml | 11 +- diracx-core/src/diracx/core/settings.py | 105 ++++++++++++- docs/dev/reference/env-variables.md | 153 ++++++++++++++++++- docs/dev/reference/pixi-tasks.md | 5 + pixi.toml | 22 ++- scripts/check_settings_doc.sh | 42 +++++ 7 files changed, 328 insertions(+), 12 deletions(-) create mode 100755 scripts/check_settings_doc.sh diff --git a/.github/workflows/generate_pixi_tasks_doc.py b/.github/workflows/generate_pixi_tasks_doc.py index d9c1ef321..85c6f4cd7 100644 --- a/.github/workflows/generate_pixi_tasks_doc.py +++ b/.github/workflows/generate_pixi_tasks_doc.py @@ -18,6 +18,8 @@ def get_task_group(task_name): return "Client Generation" if task_name == "shellcheck": return "Shellcheck" + if task_name.startswith("generate-settings-"): + return "Settings" return "Default" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9aa54eb6e..9d2e49583 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: python: python3 ci: - skip: [generate-pixi-docs] + skip: [generate-pixi-docs, settings-doc-check] default_stages: [pre-commit] @@ -84,3 +84,12 @@ repos: language: system pass_filenames: false files: ^pixi\.toml$|^pixi\.lock$ # only run if pixi files change + + - repo: local + hooks: + - id: settings-doc-check + name: Generate settings documentation + entry: scripts/check_settings_doc.sh + language: system + pass_filenames: false + files: ^diracx-core/src/diracx/core/settings\.py$ diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index ef11459f8..1d90ebdb6 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -132,11 +132,16 @@ async def lifetime_function(self) -> AsyncIterator[None]: class DevelopmentSettings(ServiceSettingsBase): """Settings for the Development Configuration that can influence run time.""" - model_config = SettingsConfigDict(env_prefix="DIRACX_DEV_") + model_config = SettingsConfigDict( + env_prefix="DIRACX_DEV_", use_attribute_docstrings=True + ) - # When then to true (only for demo/CI), crash if an access policy isn't - # called crash_on_missed_access_policy: bool = False + """When set to true (only for demo/CI), crash if an access policy isn't called. + + This is useful for development and testing to ensure all endpoints have proper + access control policies defined. + """ @classmethod def create(cls) -> Self: @@ -146,39 +151,123 @@ def create(cls) -> Self: class AuthSettings(ServiceSettingsBase): """Settings for the authentication service.""" - model_config = SettingsConfigDict(env_prefix="DIRACX_SERVICE_AUTH_") + model_config = SettingsConfigDict( + env_prefix="DIRACX_SERVICE_AUTH_", use_attribute_docstrings=True + ) dirac_client_id: str = "myDIRACClientID" - # TODO: This should be taken dynamically - # ["http://pclhcb211:8000/docs/oauth2-redirect"] + """OAuth2 client identifier for DIRAC services. + + This should match the client ID registered with the identity provider. + """ + allowed_redirects: list[str] = [] + """List of allowed redirect URLs for OAuth2 authorization flow. + + These URLs must be pre-registered and should match the redirect URIs + configured in the OAuth2 client registration. + Example: ["http://localhost:8000/docs/oauth2-redirect"] + """ + device_flow_expiration_seconds: int = 600 + """Expiration time in seconds for device flow authorization requests. + + After this time, the device code becomes invalid and users must restart + the device flow process. Default: 10 minutes. + """ + authorization_flow_expiration_seconds: int = 300 + """Expiration time in seconds for authorization code flow. + + The time window during which the authorization code remains valid + before it must be exchanged for tokens. Default: 5 minutes. + """ - # State key is used to encrypt/decrypt the state dict passed to the IAM state_key: FernetKey + """Encryption key used to encrypt/decrypt the state parameter passed to the IAM. + + This key ensures the integrity and confidentiality of state information + during OAuth2 flows. Must be a valid Fernet key. + """ token_issuer: str + """The issuer identifier for JWT tokens. + + This should be a URI that uniquely identifies the token issuer and + matches the 'iss' claim in issued JWT tokens. + """ + token_keystore: TokenSigningKeyStore + """Keystore containing the cryptographic keys used for signing JWT tokens. + + This includes both public and private keys for token signature + generation and verification. + """ + token_allowed_algorithms: list[str] = ["RS256", "EdDSA"] # noqa: S105 + """List of allowed cryptographic algorithms for JWT token signing. + + Supported algorithms include RS256 (RSA with SHA-256) and EdDSA + (Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] + """ + access_token_expire_minutes: int = 20 + """Expiration time in minutes for access tokens. + + After this duration, access tokens become invalid and must be refreshed + or re-obtained. Default: 20 minutes. + """ + refresh_token_expire_minutes: int = 60 + """Expiration time in minutes for refresh tokens. + + The maximum lifetime of refresh tokens before they must be re-issued + through a new authentication flow. Default: 60 minutes. + """ available_properties: set[SecurityProperty] = Field( default_factory=SecurityProperty.available_properties ) + """Set of security properties available in this DIRAC installation. + + These properties define various authorization capabilities and are used + for access control decisions. Defaults to all available security properties. + """ class SandboxStoreSettings(ServiceSettingsBase): """Settings for the sandbox store.""" - model_config = SettingsConfigDict(env_prefix="DIRACX_SANDBOX_STORE_") + model_config = SettingsConfigDict( + env_prefix="DIRACX_SANDBOX_STORE_", use_attribute_docstrings=True + ) bucket_name: str + """Name of the S3 bucket used for storing job sandboxes. + + This bucket will contain input and output sandbox files for DIRAC jobs. + The bucket must exist or auto_create_bucket must be enabled. + """ + s3_client_kwargs: dict[str, str] + """Configuration parameters passed to the S3 client.""" + auto_create_bucket: bool = False + """Whether to automatically create the S3 bucket if it doesn't exist.""" + url_validity_seconds: int = 5 * 60 + """Validity duration in seconds for pre-signed S3 URLs. + + This determines how long generated download/upload URLs remain valid + before expiring. Default: 300 seconds (5 minutes). + """ + se_name: str = "SandboxSE" + """Logical name of the Storage Element for the sandbox store. + + This name is used within DIRAC to refer to this sandbox storage + endpoint in job descriptions and file catalogs. + """ _client: S3Client = PrivateAttr() @contextlib.asynccontextmanager diff --git a/docs/dev/reference/env-variables.md b/docs/dev/reference/env-variables.md index efbcb3330..41cc34b0b 100644 --- a/docs/dev/reference/env-variables.md +++ b/docs/dev/reference/env-variables.md @@ -1,5 +1,154 @@ # List of development environment variables -## Development: +*This page is auto-generated. Do not edit directly.* -- `DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY`: If set to true, the server will crash if an access policy is not called. + + +## `DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY` + +*Optional*, default value: `False` + +When set to true (only for demo/CI), crash if an access policy isn't called. + +This is useful for development and testing to ensure all endpoints have proper +access control policies defined. + +## `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID` + +*Optional*, default value: `myDIRACClientID` + +OAuth2 client identifier for DIRAC services. + +This should match the client ID registered with the identity provider. + +## `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS` + +*Optional*, default value: `[]` + +List of allowed redirect URLs for OAuth2 authorization flow. + +These URLs must be pre-registered and should match the redirect URIs +configured in the OAuth2 client registration. +Example: ["http://localhost:8000/docs/oauth2-redirect"] + +## `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS` + +*Optional*, default value: `600` + +Expiration time in seconds for device flow authorization requests. + +After this time, the device code becomes invalid and users must restart +the device flow process. Default: 10 minutes. + +## `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS` + +*Optional*, default value: `300` + +Expiration time in seconds for authorization code flow. + +The time window during which the authorization code remains valid +before it must be exchanged for tokens. Default: 5 minutes. + +## `DIRACX_SERVICE_AUTH_STATE_KEY` + +**Required** + +Encryption key used to encrypt/decrypt the state parameter passed to the IAM. + +This key ensures the integrity and confidentiality of state information +during OAuth2 flows. Must be a valid Fernet key. + +## `DIRACX_SERVICE_AUTH_TOKEN_ISSUER` + +**Required** + +The issuer identifier for JWT tokens. + +This should be a URI that uniquely identifies the token issuer and +matches the 'iss' claim in issued JWT tokens. + +## `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE` + +**Required** + +Keystore containing the cryptographic keys used for signing JWT tokens. + +This includes both public and private keys for token signature +generation and verification. + +## `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS` + +*Optional*, default value: `['RS256', 'EdDSA']` + +List of allowed cryptographic algorithms for JWT token signing. + +Supported algorithms include RS256 (RSA with SHA-256) and EdDSA +(Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] + +## `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES` + +*Optional*, default value: `20` + +Expiration time in minutes for access tokens. + +After this duration, access tokens become invalid and must be refreshed +or re-obtained. Default: 20 minutes. + +## `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES` + +*Optional*, default value: `60` + +Expiration time in minutes for refresh tokens. + +The maximum lifetime of refresh tokens before they must be re-issued +through a new authentication flow. Default: 60 minutes. + +## `DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES` + +*Optional* + +Set of security properties available in this DIRAC installation. + +These properties define various authorization capabilities and are used +for access control decisions. Defaults to all available security properties. + +## `DIRACX_SANDBOX_STORE_BUCKET_NAME` + +**Required** + +Name of the S3 bucket used for storing job sandboxes. + +This bucket will contain input and output sandbox files for DIRAC jobs. +The bucket must exist or auto_create_bucket must be enabled. + +## `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS` + +**Required** + +Configuration parameters passed to the S3 client. + +## `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET` + +*Optional*, default value: `False` + +Whether to automatically create the S3 bucket if it doesn't exist. + +## `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS` + +*Optional*, default value: `300` + +Validity duration in seconds for pre-signed S3 URLs. + +This determines how long generated download/upload URLs remain valid +before expiring. Default: 300 seconds (5 minutes). + +## `DIRACX_SANDBOX_STORE_SE_NAME` + +*Optional*, default value: `SandboxSE` + +Logical name of the Storage Element for the sandbox store. + +This name is used within DIRAC to refer to this sandbox storage +endpoint in job descriptions and file catalogs. + + diff --git a/docs/dev/reference/pixi-tasks.md b/docs/dev/reference/pixi-tasks.md index f2737acde..b7866fde8 100644 --- a/docs/dev/reference/pixi-tasks.md +++ b/docs/dev/reference/pixi-tasks.md @@ -46,6 +46,11 @@ This page documents the available pixi tasks. - `pre-commit`: pre-commit +## Settings Tasks + +- `generate-settings-doc`: Generate settings documentation in Markdown format +- `generate-settings-dotenv`: Generate a dotenv file from settings + ## Shellcheck Tasks - `shellcheck`: Run shellcheck on all shell scripts diff --git a/pixi.toml b/pixi.toml index e9ae48d61..ab64784bb 100644 --- a/pixi.toml +++ b/pixi.toml @@ -30,6 +30,10 @@ diracx-logic = { path = "diracx-logic", editable = true, extras = ["testing"] } [feature.diracx-routers.pypi-dependencies] diracx-routers = { path = "diracx-routers", editable = true, extras = ["testing"] } +# Settings documentation feature +[feature.settings-doc.pypi-dependencies] +settings-doc = "*" + # DiracX features for providing tasks. This is needed to make it so that running # "pixi run pytest-diracx-core -vvv --pdb" passes the arguments as expected. # See: https://github.com/prefix-dev/pixi/issues/1519#issuecomment-2651078457 @@ -57,6 +61,22 @@ description = "Run the tests for diracx-logic" cmd = "cd diracx-routers/ && pytest" description = "Run the tests for diracx-routers" +# Settings documentation task +[feature.settings-doc.tasks.generate-settings-doc] +cmd = """settings-doc generate \ + --module diracx.core.settings \ + --output-format markdown \ + --update docs/dev/reference/env-variables.md \ + --between "" "" \ + --heading-offset 1""" +description = "Generate settings documentation in Markdown format" + +[feature.settings-doc.tasks.generate-settings-dotenv] +cmd = """settings-doc generate \ + --module diracx.core.settings \ + --output-format dotenv > .env""" +description = "Generate a dotenv file from settings" + # Gubbins features for providing dependencies [feature.gubbins.pypi-dependencies] gubbins = { path = "extensions/gubbins", editable = true, extras = ["testing"] } @@ -142,7 +162,7 @@ description = "Run shellcheck on all shell scripts" [environments] # DiracX environments -default = {features = ["task-diracx", "diracx", "diracx-core", "diracx-api", "diracx-cli", "diracx-client", "diracx-db", "diracx-logic", "diracx-routers"], solve-group = "diracx"} +default = {features = ["task-diracx", "diracx", "diracx-core", "diracx-api", "diracx-cli", "diracx-client", "diracx-db", "diracx-logic", "diracx-routers", "settings-doc"], solve-group = "diracx"} diracx-core = {features = ["task-diracx-core", "diracx-core"], solve-group = "diracx"} diracx-api = {features = ["task-diracx-api", "diracx-api", "diracx-client", "diracx-core"], solve-group = "diracx"} diracx-cli = {features = ["task-diracx-cli", "diracx-cli", "diracx-api", "diracx-client", "diracx-core"], solve-group = "diracx"} diff --git a/scripts/check_settings_doc.sh b/scripts/check_settings_doc.sh new file mode 100755 index 000000000..dc0b18b2c --- /dev/null +++ b/scripts/check_settings_doc.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run the settings-doc generator via pixi and fail if the committed file would change. +# This allows a pre-commit hook to check that generated documentation is up-to-date +# without letting the hook itself modify the repository and cause the hook to fail. + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$repo_root" + +TARGET="docs/dev/reference/env-variables.md" + +# Save the current state of the target file +cp "$TARGET" "${TARGET}.backup" + +echo "Running settings-doc generator (pixi run generate-settings-doc)..." +pixi run generate-settings-doc + +# Normalize both files by removing empty lines, then compare +# This handles cases where settings-doc adds/removes blank lines inconsistently +normalize_file() { + grep -v '^[[:space:]]*$' "$1" || true +} + +if diff -q <(normalize_file "$TARGET") <(normalize_file "${TARGET}.backup") > /dev/null 2>&1; then + # Files are equivalent ignoring blank lines, restore original and pass + cp "${TARGET}.backup" "$TARGET" + rm "${TARGET}.backup" + echo "Generated documentation is up-to-date." + exit 0 +fi + +# Files have meaningful differences +echo +echo "ERROR: Generated settings documentation is out of date." +echo "Run 'pixi run generate-settings-doc' and commit the updated file." +echo +echo "Diff (generated vs current):" +diff "$TARGET" "${TARGET}.backup" || true +# Restore the original file so pre-commit doesn't see modifications +mv "${TARGET}.backup" "$TARGET" +exit 1 From 0170a87ece5bdada429061f0122a1f8d2a760d17 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 14:44:39 +0100 Subject: [PATCH 02/18] docs: env variable / settings documentation generator fix: cleanups --- .env.example | 121 +++++++ .gitignore | 1 + .pre-commit-config.yaml | 4 +- docs/admin/reference/env-variables.md | 218 ++++++++++-- docs/admin/reference/env-variables.md.j2 | 11 + docs/dev/reference/env-variables.md | 150 +------- docs/dev/reference/env-variables.md.j2 | 7 + docs/dev/reference/pixi-tasks.md | 3 +- docs/templates/_render_class.jinja | 40 +++ pixi.toml | 15 +- scripts/check_settings_doc.sh | 42 --- scripts/generate_settings_docs.py | 422 +++++++++++++++++++++++ 12 files changed, 794 insertions(+), 240 deletions(-) create mode 100644 .env.example create mode 100644 docs/admin/reference/env-variables.md.j2 create mode 100644 docs/dev/reference/env-variables.md.j2 create mode 100644 docs/templates/_render_class.jinja delete mode 100755 scripts/check_settings_doc.sh create mode 100644 scripts/generate_settings_docs.py diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..632a502c3 --- /dev/null +++ b/.env.example @@ -0,0 +1,121 @@ +# Auto-generated .env file with all DiracX settings +# This file contains all available environment variables. +# Uncomment and set the values you need. + +# AuthSettings (from diracx.core.settings) +# OAuth2 client identifier for DIRAC services. +# +# This should match the client ID registered with the identity provider. +# DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID=myDIRACClientID + +# List of allowed redirect URLs for OAuth2 authorization flow. +# +# These URLs must be pre-registered and should match the redirect URIs +# configured in the OAuth2 client registration. +# Example: ["http://localhost:8000/docs/oauth2-redirect"] +# DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS=[] + +# Expiration time in seconds for device flow authorization requests. +# +# After this time, the device code becomes invalid and users must restart +# the device flow process. Default: 10 minutes. +# DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS=600 + +# Expiration time in seconds for authorization code flow. +# +# The time window during which the authorization code remains valid +# before it must be exchanged for tokens. Default: 5 minutes. +# DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS=300 + +# Encryption key used to encrypt/decrypt the state parameter passed to the IAM. +# +# This key ensures the integrity and confidentiality of state information +# during OAuth2 flows. Must be a valid Fernet key. +DIRACX_SERVICE_AUTH_STATE_KEY= + +# The issuer identifier for JWT tokens. +# +# This should be a URI that uniquely identifies the token issuer and +# matches the 'iss' claim in issued JWT tokens. +DIRACX_SERVICE_AUTH_TOKEN_ISSUER= + +# Keystore containing the cryptographic keys used for signing JWT tokens. +# +# This includes both public and private keys for token signature +# generation and verification. +DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE= + +# List of allowed cryptographic algorithms for JWT token signing. +# +# Supported algorithms include RS256 (RSA with SHA-256) and EdDSA +# (Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] +# DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS=['RS256', 'EdDSA'] + +# Expiration time in minutes for access tokens. +# +# After this duration, access tokens become invalid and must be refreshed +# or re-obtained. Default: 20 minutes. +# DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES=20 + +# Expiration time in minutes for refresh tokens. +# +# The maximum lifetime of refresh tokens before they must be re-issued +# through a new authentication flow. Default: 60 minutes. +# DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES=60 + +# Set of security properties available in this DIRAC installation. +# +# These properties define various authorization capabilities and are used +# for access control decisions. Defaults to all available security properties. +# DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES= + + + +# DevelopmentSettings (from diracx.core.settings) +# When set to true (only for demo/CI), crash if an access policy isn't called. +# +# This is useful for development and testing to ensure all endpoints have proper +# access control policies defined. +# DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY=False + + + +# OTELSettings (from diracx.routers.otel) +# DIRACX_OTEL_ENABLED=False + +# DIRACX_OTEL_APPLICATION_NAME=diracx + +# DIRACX_OTEL_GRPC_ENDPOINT= + +# DIRACX_OTEL_GRPC_INSECURE=True + +# DIRACX_OTEL_HEADERS= + + + +# SandboxStoreSettings (from diracx.core.settings) +# Name of the S3 bucket used for storing job sandboxes. +# +# This bucket will contain input and output sandbox files for DIRAC jobs. +# The bucket must exist or auto_create_bucket must be enabled. +DIRACX_SANDBOX_STORE_BUCKET_NAME= + +# Configuration parameters passed to the S3 client. +DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS= + +# Whether to automatically create the S3 bucket if it doesn't exist. +# DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET=False + +# Validity duration in seconds for pre-signed S3 URLs. +# +# This determines how long generated download/upload URLs remain valid +# before expiring. Default: 300 seconds (5 minutes). +# DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS=300 + +# Logical name of the Storage Element for the sandbox store. +# +# This name is used within DIRAC to refer to this sandbox storage +# endpoint in job descriptions and file catalogs. +# DIRACX_SANDBOX_STORE_SE_NAME=SandboxSE + + diff --git a/.gitignore b/.gitignore index 900a0e643..b898f7da7 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,4 @@ docs/source/_build .pixi pixi.lock *.egg-info +docs/templates/_builtin_markdown.jinja diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d2e49583..31501d1d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -89,7 +89,7 @@ repos: hooks: - id: settings-doc-check name: Generate settings documentation - entry: scripts/check_settings_doc.sh + entry: pixi run -e default python scripts/generate_settings_docs.py language: system pass_filenames: false - files: ^diracx-core/src/diracx/core/settings\.py$ + files: ^(diracx-.*/src/diracx/.*/settings\.py|docs/.*\.j2|docs/templates/.*\.jinja|scripts/generate_settings_docs\.py)$ diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index c96cc7a90..9ac4ba5af 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -1,39 +1,183 @@ # List of environment variables -## Core - -- `DIRACX_CONFIG_BACKEND_URL`: The URL of the configuration backend. - -## Services: - -- `DIRACX_SERVICE_AUTH_TOKEN_ISSUER`: The issuer for the auth tokens. -- `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS`: A JSON-encoded list of allowed redirect URIs for the authorization code - flow. -- `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS`: The expiration time for the device flow in seconds. -- `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS`: The expiration time for the authorization flow in - seconds. -- `DIRACX_SERVICE_AUTH_STATE_KEY`: The key used to encrypt the state in the authorization code flow. -- `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE`: The path to the JWKS file containing the token signing keys. -- `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS`: A JSON-encoded list of allowed algorithms for token signing. -- `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES`: The expiration time for the access token in minutes. -- `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES`: The expiration time for the refresh token in minutes. -- `DIRACX_SANDBOX_STORE_BUCKET_NAME`: The name of the S3 bucket for the sandbox store. -- `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS`: A JSON-encoded dictionary of keyword arguments for the S3 client. -- `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET`: Whether to automatically create the S3 bucket if it doesn't exist. -- `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS`: The validity of the presigned URLs for the sandbox store in seconds. -- `DIRACX_SANDBOX_STORE_SE_NAME`: The name of the storage element for the sandbox store. -- `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY`: The hashed API key for the legacy exchange endpoint. -- `DIRACX_SERVICE_JOBS_ENABLED`: Whether the jobs service is enabled. - -## Databases: - -- `DIRACX_DB_URL_`: The URL for the SQL database ``. -- `DIRACX_OS_DB_`: A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database `` - -## OTEL: - -- `DIRACX_OTEL_ENABLED`: Whether OpenTelemetry is enabled. -- `DIRACX_OTEL_APPLICATION_NAME`: The name of the application for OpenTelemetry. -- `DIRACX_OTEL_GRPC_ENDPOINT`: The gRPC endpoint for the OpenTelemetry collector. -- `DIRACX_OTEL_GRPC_INSECURE`: Whether to use an insecure gRPC connection for the OpenTelemetry collector. -- `DIRACX_OTEL_HEADERS`: A JSON-encoded dictionary of headers to pass to the OpenTelemetry collector. +*This page is auto-generated from the settings classes in `diracx.core.settings`.* + + +## AuthSettings + +Settings for the authentication service. + + +### `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID` + +*Optional*, default value: `myDIRACClientID` + +OAuth2 client identifier for DIRAC services. + +This should match the client ID registered with the identity provider. + +### `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS` + +*Optional*, default value: `[]` + +List of allowed redirect URLs for OAuth2 authorization flow. + +These URLs must be pre-registered and should match the redirect URIs +configured in the OAuth2 client registration. +Example: ["http://localhost:8000/docs/oauth2-redirect"] + +### `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS` + +*Optional*, default value: `600` + +Expiration time in seconds for device flow authorization requests. + +After this time, the device code becomes invalid and users must restart +the device flow process. Default: 10 minutes. + +### `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS` + +*Optional*, default value: `300` + +Expiration time in seconds for authorization code flow. + +The time window during which the authorization code remains valid +before it must be exchanged for tokens. Default: 5 minutes. + +### `DIRACX_SERVICE_AUTH_STATE_KEY` + +**Required** + +Encryption key used to encrypt/decrypt the state parameter passed to the IAM. + +This key ensures the integrity and confidentiality of state information +during OAuth2 flows. Must be a valid Fernet key. + +### `DIRACX_SERVICE_AUTH_TOKEN_ISSUER` + +**Required** + +The issuer identifier for JWT tokens. + +This should be a URI that uniquely identifies the token issuer and +matches the 'iss' claim in issued JWT tokens. + +### `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE` + +**Required** + +Keystore containing the cryptographic keys used for signing JWT tokens. + +This includes both public and private keys for token signature +generation and verification. + +### `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS` + +*Optional*, default value: `['RS256', 'EdDSA']` + +List of allowed cryptographic algorithms for JWT token signing. + +Supported algorithms include RS256 (RSA with SHA-256) and EdDSA +(Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] + +### `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES` + +*Optional*, default value: `20` + +Expiration time in minutes for access tokens. + +After this duration, access tokens become invalid and must be refreshed +or re-obtained. Default: 20 minutes. + +### `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES` + +*Optional*, default value: `60` + +Expiration time in minutes for refresh tokens. + +The maximum lifetime of refresh tokens before they must be re-issued +through a new authentication flow. Default: 60 minutes. + +### `DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES` + +*Optional* + +Set of security properties available in this DIRAC installation. + +These properties define various authorization capabilities and are used +for access control decisions. Defaults to all available security properties. + + + + +## SandboxStoreSettings + +Settings for the sandbox store. + + +### `DIRACX_SANDBOX_STORE_BUCKET_NAME` + +**Required** + +Name of the S3 bucket used for storing job sandboxes. + +This bucket will contain input and output sandbox files for DIRAC jobs. +The bucket must exist or auto_create_bucket must be enabled. + +### `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS` + +**Required** + +Configuration parameters passed to the S3 client. + +### `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET` + +*Optional*, default value: `False` + +Whether to automatically create the S3 bucket if it doesn't exist. + +### `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS` + +*Optional*, default value: `300` + +Validity duration in seconds for pre-signed S3 URLs. + +This determines how long generated download/upload URLs remain valid +before expiring. Default: 300 seconds (5 minutes). + +### `DIRACX_SANDBOX_STORE_SE_NAME` + +*Optional*, default value: `SandboxSE` + +Logical name of the Storage Element for the sandbox store. + +This name is used within DIRAC to refer to this sandbox storage +endpoint in job descriptions and file catalogs. + + + + +## OTELSettings + +Settings for the Open Telemetry Configuration. + + +### `DIRACX_OTEL_ENABLED` + +*Optional*, default value: `False` + +### `DIRACX_OTEL_APPLICATION_NAME` + +*Optional*, default value: `diracx` + +### `DIRACX_OTEL_GRPC_ENDPOINT` + +*Optional*, default value: `` + +### `DIRACX_OTEL_GRPC_INSECURE` + +*Optional*, default value: `True` + +### `DIRACX_OTEL_HEADERS` + +*Optional*, default value: `None` diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 new file mode 100644 index 000000000..eecae741e --- /dev/null +++ b/docs/admin/reference/env-variables.md.j2 @@ -0,0 +1,11 @@ +{% from '_render_class.jinja' import render_class %} + +# List of environment variables + +*This page is auto-generated from the settings classes in `diracx.core.settings`.* + +{{ render_class('AuthSettings') }} + +{{ render_class('SandboxStoreSettings') }} + +{{ render_class('OTELSettings') }} diff --git a/docs/dev/reference/env-variables.md b/docs/dev/reference/env-variables.md index 41cc34b0b..949a06234 100644 --- a/docs/dev/reference/env-variables.md +++ b/docs/dev/reference/env-variables.md @@ -1,10 +1,12 @@ -# List of development environment variables +# List of environment variables -*This page is auto-generated. Do not edit directly.* +*This page is auto-generated from the DevelopmentSettings class in `diracx.core.settings`.* - +## DevelopmentSettings -## `DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY` +Settings for the Development Configuration that can influence run time. + +### `DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY` *Optional*, default value: `False` @@ -12,143 +14,3 @@ When set to true (only for demo/CI), crash if an access policy isn't called. This is useful for development and testing to ensure all endpoints have proper access control policies defined. - -## `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID` - -*Optional*, default value: `myDIRACClientID` - -OAuth2 client identifier for DIRAC services. - -This should match the client ID registered with the identity provider. - -## `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS` - -*Optional*, default value: `[]` - -List of allowed redirect URLs for OAuth2 authorization flow. - -These URLs must be pre-registered and should match the redirect URIs -configured in the OAuth2 client registration. -Example: ["http://localhost:8000/docs/oauth2-redirect"] - -## `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS` - -*Optional*, default value: `600` - -Expiration time in seconds for device flow authorization requests. - -After this time, the device code becomes invalid and users must restart -the device flow process. Default: 10 minutes. - -## `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS` - -*Optional*, default value: `300` - -Expiration time in seconds for authorization code flow. - -The time window during which the authorization code remains valid -before it must be exchanged for tokens. Default: 5 minutes. - -## `DIRACX_SERVICE_AUTH_STATE_KEY` - -**Required** - -Encryption key used to encrypt/decrypt the state parameter passed to the IAM. - -This key ensures the integrity and confidentiality of state information -during OAuth2 flows. Must be a valid Fernet key. - -## `DIRACX_SERVICE_AUTH_TOKEN_ISSUER` - -**Required** - -The issuer identifier for JWT tokens. - -This should be a URI that uniquely identifies the token issuer and -matches the 'iss' claim in issued JWT tokens. - -## `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE` - -**Required** - -Keystore containing the cryptographic keys used for signing JWT tokens. - -This includes both public and private keys for token signature -generation and verification. - -## `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS` - -*Optional*, default value: `['RS256', 'EdDSA']` - -List of allowed cryptographic algorithms for JWT token signing. - -Supported algorithms include RS256 (RSA with SHA-256) and EdDSA -(Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] - -## `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES` - -*Optional*, default value: `20` - -Expiration time in minutes for access tokens. - -After this duration, access tokens become invalid and must be refreshed -or re-obtained. Default: 20 minutes. - -## `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES` - -*Optional*, default value: `60` - -Expiration time in minutes for refresh tokens. - -The maximum lifetime of refresh tokens before they must be re-issued -through a new authentication flow. Default: 60 minutes. - -## `DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES` - -*Optional* - -Set of security properties available in this DIRAC installation. - -These properties define various authorization capabilities and are used -for access control decisions. Defaults to all available security properties. - -## `DIRACX_SANDBOX_STORE_BUCKET_NAME` - -**Required** - -Name of the S3 bucket used for storing job sandboxes. - -This bucket will contain input and output sandbox files for DIRAC jobs. -The bucket must exist or auto_create_bucket must be enabled. - -## `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS` - -**Required** - -Configuration parameters passed to the S3 client. - -## `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET` - -*Optional*, default value: `False` - -Whether to automatically create the S3 bucket if it doesn't exist. - -## `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS` - -*Optional*, default value: `300` - -Validity duration in seconds for pre-signed S3 URLs. - -This determines how long generated download/upload URLs remain valid -before expiring. Default: 300 seconds (5 minutes). - -## `DIRACX_SANDBOX_STORE_SE_NAME` - -*Optional*, default value: `SandboxSE` - -Logical name of the Storage Element for the sandbox store. - -This name is used within DIRAC to refer to this sandbox storage -endpoint in job descriptions and file catalogs. - - diff --git a/docs/dev/reference/env-variables.md.j2 b/docs/dev/reference/env-variables.md.j2 new file mode 100644 index 000000000..6bc891a90 --- /dev/null +++ b/docs/dev/reference/env-variables.md.j2 @@ -0,0 +1,7 @@ +{% from '_render_class.jinja' import render_class %} + +# List of environment variables + +*This page is auto-generated from the DevelopmentSettings class in `diracx.core.settings`.* + +{{ render_class('DevelopmentSettings') }} diff --git a/docs/dev/reference/pixi-tasks.md b/docs/dev/reference/pixi-tasks.md index b7866fde8..34ad9bc4f 100644 --- a/docs/dev/reference/pixi-tasks.md +++ b/docs/dev/reference/pixi-tasks.md @@ -48,8 +48,7 @@ This page documents the available pixi tasks. ## Settings Tasks -- `generate-settings-doc`: Generate settings documentation in Markdown format -- `generate-settings-dotenv`: Generate a dotenv file from settings +- `generate-settings-doc`: Auto-discover and generate settings documentation with validation ## Shellcheck Tasks diff --git a/docs/templates/_render_class.jinja b/docs/templates/_render_class.jinja new file mode 100644 index 000000000..242d6fb4b --- /dev/null +++ b/docs/templates/_render_class.jinja @@ -0,0 +1,40 @@ +{# +Reusable macro to render a settings class with all its environment variables. + +This macro encapsulates all the complexity of rendering fields, so the main +template can be kept simple and clean. +#} + +{% macro render_class(class_name) %} +{# Find the class by name from the classes dict #} +{% set cls = namespace(found=none) %} +{% for c, field_list in classes.items() %} + {% if c.__name__ == class_name %} + {% set cls.found = c %} + {% endif %} +{% endfor %} + +{% if cls.found %} +## {{ cls.found.__name__ }} + +{{ cls.found.__doc__ or "*No description available.*" }} + +{# Build the fields list in the format expected by the built-in template #} +{% set ns = namespace(fields=[]) %} +{% for field_name, field_info in cls.found.model_fields.items() %} + {% set env_prefix = cls.found.model_config.get('env_prefix', '') %} + {% set env_name = (env_prefix ~ field_name).upper() %} + {% set _ = ns.fields.append((env_name, field_info)) %} +{% endfor %} + +{# Set the fields variable and include the built-in settings_doc template #} +{% set fields = ns.fields %} +{% include '_builtin_markdown.jinja' with context %} +{% else %} +{# Class not found - provide a helpful error #} +**Error: Class '{{ class_name }}' not found in diracx.core.settings** + +Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(', ') }} +{% endif %} + +{% endmacro %} diff --git a/pixi.toml b/pixi.toml index ab64784bb..df37211bc 100644 --- a/pixi.toml +++ b/pixi.toml @@ -63,19 +63,8 @@ description = "Run the tests for diracx-routers" # Settings documentation task [feature.settings-doc.tasks.generate-settings-doc] -cmd = """settings-doc generate \ - --module diracx.core.settings \ - --output-format markdown \ - --update docs/dev/reference/env-variables.md \ - --between "" "" \ - --heading-offset 1""" -description = "Generate settings documentation in Markdown format" - -[feature.settings-doc.tasks.generate-settings-dotenv] -cmd = """settings-doc generate \ - --module diracx.core.settings \ - --output-format dotenv > .env""" -description = "Generate a dotenv file from settings" +cmd = "python scripts/generate_settings_docs.py" +description = "Auto-discover and generate settings documentation with validation" # Gubbins features for providing dependencies [feature.gubbins.pypi-dependencies] diff --git a/scripts/check_settings_doc.sh b/scripts/check_settings_doc.sh deleted file mode 100755 index dc0b18b2c..000000000 --- a/scripts/check_settings_doc.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Run the settings-doc generator via pixi and fail if the committed file would change. -# This allows a pre-commit hook to check that generated documentation is up-to-date -# without letting the hook itself modify the repository and cause the hook to fail. - -repo_root="$(cd "$(dirname "$0")/.." && pwd)" -cd "$repo_root" - -TARGET="docs/dev/reference/env-variables.md" - -# Save the current state of the target file -cp "$TARGET" "${TARGET}.backup" - -echo "Running settings-doc generator (pixi run generate-settings-doc)..." -pixi run generate-settings-doc - -# Normalize both files by removing empty lines, then compare -# This handles cases where settings-doc adds/removes blank lines inconsistently -normalize_file() { - grep -v '^[[:space:]]*$' "$1" || true -} - -if diff -q <(normalize_file "$TARGET") <(normalize_file "${TARGET}.backup") > /dev/null 2>&1; then - # Files are equivalent ignoring blank lines, restore original and pass - cp "${TARGET}.backup" "$TARGET" - rm "${TARGET}.backup" - echo "Generated documentation is up-to-date." - exit 0 -fi - -# Files have meaningful differences -echo -echo "ERROR: Generated settings documentation is out of date." -echo "Run 'pixi run generate-settings-doc' and commit the updated file." -echo -echo "Diff (generated vs current):" -diff "$TARGET" "${TARGET}.backup" || true -# Restore the original file so pre-commit doesn't see modifications -mv "${TARGET}.backup" "$TARGET" -exit 1 diff --git a/scripts/generate_settings_docs.py b/scripts/generate_settings_docs.py new file mode 100644 index 000000000..c7532e7a1 --- /dev/null +++ b/scripts/generate_settings_docs.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +"""Automatically discover and validate settings documentation. + +This script: +1. Syncs the built-in settings_doc template to avoid recursion +2. Discovers all Settings classes across the DiracX codebase +3. Checks which classes are documented in templates +4. Warns about undocumented classes +5. Generates documentation for all templates +6. Generates .env.example file with all settings +""" + +from __future__ import annotations + +import importlib +import inspect +import pkgutil +import re +import shutil +import sys +from pathlib import Path +from typing import Any + +import settings_doc +from jinja2 import Environment, FileSystemLoader, select_autoescape +from pydantic_settings import BaseSettings +from settings_doc import OutputFormat, importing, render +from settings_doc.main import _model_fields +from settings_doc.template_functions import JINJA_ENV_GLOBALS + +from diracx.core.settings import ServiceSettingsBase + + +def sync_builtin_template(docs_dir: Path) -> None: + """Sync the built-in settings_doc markdown template. + + This copies the built-in template to '_builtin_markdown.jinja' so our custom + 'markdown.jinja' can include it without causing recursion issues in Jinja2. + + Args: + docs_dir: The docs directory where templates are stored + + """ + # Get the path to the built-in settings_doc template + builtin_template_dir = Path(settings_doc.__file__).parent / "templates" + builtin_markdown = builtin_template_dir / "markdown.jinja" + + # Define our custom templates directory + custom_template_dir = docs_dir / "templates" + custom_template_dir.mkdir(parents=True, exist_ok=True) + + # Copy the built-in template with a different name to avoid recursion + target_template = custom_template_dir / "_builtin_markdown.jinja" + shutil.copy2(builtin_markdown, target_template) + + +def discover_all_settings_classes() -> dict[str, dict[str, Any]]: + """Automatically discover all Settings classes in the DiracX codebase. + + Uses pkgutil.walk_packages() to walk all diracx.* packages. + + Returns: + Dict mapping class names to their info (module, class object, etc.) + + """ + settings_classes = {} + + import diracx + + # Use pkgutil to walk all packages and subpackages + # This is the canonical way to discover modules in a package + if hasattr(diracx, "__path__"): + for _, module_name, _ in pkgutil.walk_packages( + path=diracx.__path__, + prefix="diracx.", + onerror=lambda x: None, # Silently skip modules with import errors + ): + try: + module = importlib.import_module(module_name) + + # Inspect the module for Settings classes + for name, obj in inspect.getmembers(module, inspect.isclass): + # Check if it's a subclass of ServiceSettingsBase + if ( + issubclass(obj, ServiceSettingsBase) + and obj is not ServiceSettingsBase + and obj is not BaseSettings + # Only include classes defined in this module (not imported) + and obj.__module__ == module_name + ): + settings_classes[name] = { + "module": module_name, + "class": obj, + "file": inspect.getfile(obj), + } + except (ImportError, AttributeError, TypeError): + # Skip modules that can't be imported or inspected + continue + else: + raise ImportError("Cannot find diracx package paths") + + return settings_classes + + +def discover_templates(docs_dir: Path) -> dict[str, Path]: + """Discover all Jinja2 template files in the docs directory. + + Returns: + Dict mapping template names to their paths + + """ + templates = {} + + # Look for .j2 and .jinja files + for pattern in ["**/*.j2", "**/*.jinja"]: + for template_file in docs_dir.glob(pattern): + # Skip the helper templates (those starting with _) + if template_file.name.startswith("_"): + continue + + # Use relative path from docs dir as the key + rel_path = template_file.relative_to(docs_dir) + templates[str(rel_path)] = template_file + + return templates + + +def extract_documented_classes(template_file: Path) -> set[str]: + """Extract which Settings classes are documented in a template. + + Looks for {{ render_class('ClassName') }} patterns. + """ + content = template_file.read_text() + + # Pattern to match render_class('ClassName') or render_class("ClassName") + pattern = r"render_class\(['\"](\w+)['\"]\)" + + matches = re.findall(pattern, content) + return set(matches) + + +def validate_documentation( + settings_classes: dict[str, dict], + templates: dict[str, Path], +) -> tuple[bool, list[str]]: + """Validate that all Settings classes are documented. + + Returns: + Tuple of (all_documented, warnings) + + """ + warnings = [] + all_documented = True + + # Track which classes are documented in which templates + class_to_templates: dict[str, list[str]] = {} + + # Get all documented classes across all templates + for template_name, template_file in templates.items(): + documented = extract_documented_classes(template_file) + + if documented: + print(f"šŸ“„ {template_name}: documents {len(documented)} classes") + for cls_name in sorted(documented): + # Track which template documents this class + if cls_name not in class_to_templates: + class_to_templates[cls_name] = [] + class_to_templates[cls_name].append(template_name) + + # Check if class exists + if cls_name not in settings_classes: + warnings.append( + f" āš ļø Template '{template_name}' references unknown class: {cls_name}" + ) + + # Check for classes documented in multiple templates + for cls_name, template_list in class_to_templates.items(): + if len(template_list) > 1: + warnings.append( + f"\nāš ļø Class '{cls_name}' is documented in multiple templates:" + ) + for tmpl in template_list: + warnings.append(f" - {tmpl}") + warnings.append( + " This might be intentional, but verify it's not a mistake." + ) + + # Get all documented classes (from any template) + all_documented_classes = set(class_to_templates.keys()) + + # Check for undocumented classes + undocumented = set(settings_classes.keys()) - all_documented_classes + + # Exclude ServiceSettingsBase as it's the base class + undocumented.discard("ServiceSettingsBase") + + if undocumented: + all_documented = False + warnings.append("\nāŒ Undocumented Settings classes found:") + for cls_name in sorted(undocumented): + info = settings_classes[cls_name] + warnings.append(f" - {cls_name} (in {info['module']})") + warnings.append( + f" Add to template: {{{{ render_class('{cls_name}') }}}}" + ) + + return all_documented, warnings + + +def generate_all_templates( + templates: dict[str, Path], + settings_classes: dict[str, dict], + docs_dir: Path, + repo_root: Path, +) -> None: + """Generate documentation for all discovered templates.""" + # Collect all unique modules that contain settings + modules = sorted(set(info["module"] for info in settings_classes.values())) + + # Import settings classes from all modules + settings = importing.import_module_path(tuple(modules)) + + if not settings: + raise ValueError(f"No settings classes found in {modules}") + + # Prepare data structures for templates + fields = list( + (env_name, field) for cls in settings for env_name, field in _model_fields(cls) + ) + + classes = {cls: list(cls.model_fields.values()) for cls in settings} + + # Set up Jinja2 environment + env = Environment( + loader=FileSystemLoader([str(docs_dir), str(docs_dir / "templates")]), + autoescape=select_autoescape(), + trim_blocks=True, + lstrip_blocks=True, + keep_trailing_newline=True, + ) + + env.globals.update(JINJA_ENV_GLOBALS) + env.globals.update( + { + "heading_offset": 2, + "fields": fields, + "classes": classes, + "all_classes": settings, + } + ) + + # Generate each template to .bak file + generated_files = [] + for template_name, template_file in templates.items(): + template = env.get_template(str(template_name)) + rendered = template.render().strip() + "\n" + + # Output file is the template file without the .j2/.jinja extension + if template_file.suffix == ".j2": + output_file = template_file.with_suffix("") + elif template_file.suffix == ".jinja": + output_file = template_file.with_name(template_file.stem + ".md") + else: + output_file = template_file.with_suffix(".md") + + # Write to .bak file + backup_file = output_file.with_suffix(output_file.suffix + ".bak") + backup_file.write_text(rendered) + generated_files.append((output_file, backup_file)) + + return generated_files + + +def generate_dotenv_file( + settings_classes: dict[str, dict[str, Any]], + output_path: Path, +) -> Path: + """Generate a .env file with all Settings classes. + + Args: + settings_classes: Dictionary of discovered Settings classes + output_path: Path where the .env file should be written + + Returns: + Path to the .bak file that was created + + """ + dotenv_sections = [] + + # Generate .env content for each Settings class + for class_name in sorted(settings_classes.keys()): + info = settings_classes[class_name] + + # Use settings_doc.render to generate DOTENV format + # class_path should be in format "module.ClassName" + class_path = f"{info['module']}.{class_name}" + dotenv_content = render( + output_format=OutputFormat.DOTENV, + class_path=(class_path,), # Must be a tuple + ) + + # Add a header comment for the class + section = f"# {class_name} (from {info['module']})\n{dotenv_content}\n" + dotenv_sections.append(section) + + # Combine all sections + full_content = "# Auto-generated .env file with all DiracX settings\n" + full_content += "# This file contains all available environment variables.\n" + full_content += "# Uncomment and set the values you need.\n\n" + full_content += "\n".join(dotenv_sections) + + # Write to .bak file + backup_path = output_path.with_suffix(output_path.suffix + ".bak") + backup_path.write_text(full_content) + + return backup_path + + +def compare_and_update_files( + generated_files: list[tuple[Path, Path]], repo_root: Path +) -> bool: + """Compare generated .bak files with originals and update if needed. + + Args: + generated_files: List of (original_path, backup_path) tuples + repo_root: Repository root for relative path display + + Returns: + True if all files were already up to date, False if any were updated + + """ + + # Helper to normalize content (remove empty lines for comparison) + def normalize(text: str) -> str: + return "\n".join(line for line in text.splitlines() if line.strip()) + + all_up_to_date = True + + for original_file, backup_file in generated_files: + if original_file.exists(): + original_content = normalize(original_file.read_text()) + backup_content = normalize(backup_file.read_text()) + + if original_content != backup_content: + # Replace original with backup + backup_file.replace(original_file) + print(f"āœ“ Updated {original_file.relative_to(repo_root)}") + all_up_to_date = False + else: + # Files match, remove backup + backup_file.unlink() + print(f" No changes: {original_file.relative_to(repo_root)}") + else: + # New file - move backup to original + backup_file.replace(original_file) + print(f"āœ“ Created {original_file.relative_to(repo_root)}") + all_up_to_date = False + + return all_up_to_date + + +def main(): + """Main entry point.""" + repo_root = Path(__file__).parent.parent + docs_dir = repo_root / "docs" + + print("šŸ”„ Syncing built-in template...") + sync_builtin_template(docs_dir) + + print("šŸ” Discovering Settings classes...") + settings_classes = discover_all_settings_classes() + + print(f"āœ“ Found {len(settings_classes)} Settings classes:") + for name, info in sorted(settings_classes.items()): + print(f" - {name} ({info['module']})") + + print("\nšŸ” Discovering documentation templates...") + templates = discover_templates(docs_dir) + + print(f"āœ“ Found {len(templates)} template(s):") + for name in sorted(templates.keys()): + print(f" - {name}") + + print("\nšŸ” Validating documentation coverage...") + all_documented, warnings = validate_documentation(settings_classes, templates) + + if warnings: + for warning in warnings: + print(warning) + + if not all_documented: + print("\nāŒ Documentation is incomplete!") + print(" Add missing classes to your templates.") + return 1 + + print("\nāœ“ All Settings classes are documented!") + + # Generate documentation (always to .bak files first) + print("\nšŸ“ Generating documentation from templates...") + generated_files = generate_all_templates( + templates, settings_classes, docs_dir, repo_root + ) + + print("\nšŸ“ Generating .env file...") + dotenv_path = repo_root / ".env.example" + dotenv_backup = generate_dotenv_file(settings_classes, dotenv_path) + generated_files.append((dotenv_path, dotenv_backup)) + + # Update files if they're different + print("\nšŸ“ Updating files...") + all_up_to_date = compare_and_update_files(generated_files, repo_root) + + if all_up_to_date: + print("\nāœ… All files were already up to date!") + else: + print("\nāœ… Documentation updated successfully!") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 228865b76e426fc5cea16c46974b4d77eedf6c0d Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 14:58:44 +0100 Subject: [PATCH 03/18] fix: use separate tooling env for settings-doc fix: well that didn't work fix: update tasks fix: format settings doc files --- .env.example | 121 -------------------------- .gitignore | 3 + docs/admin/reference/env-variables.md | 12 +-- pixi.toml | 26 ++++-- scripts/generate_settings_docs.py | 28 ++++-- 5 files changed, 44 insertions(+), 146 deletions(-) delete mode 100644 .env.example diff --git a/.env.example b/.env.example deleted file mode 100644 index 632a502c3..000000000 --- a/.env.example +++ /dev/null @@ -1,121 +0,0 @@ -# Auto-generated .env file with all DiracX settings -# This file contains all available environment variables. -# Uncomment and set the values you need. - -# AuthSettings (from diracx.core.settings) -# OAuth2 client identifier for DIRAC services. -# -# This should match the client ID registered with the identity provider. -# DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID=myDIRACClientID - -# List of allowed redirect URLs for OAuth2 authorization flow. -# -# These URLs must be pre-registered and should match the redirect URIs -# configured in the OAuth2 client registration. -# Example: ["http://localhost:8000/docs/oauth2-redirect"] -# DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS=[] - -# Expiration time in seconds for device flow authorization requests. -# -# After this time, the device code becomes invalid and users must restart -# the device flow process. Default: 10 minutes. -# DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS=600 - -# Expiration time in seconds for authorization code flow. -# -# The time window during which the authorization code remains valid -# before it must be exchanged for tokens. Default: 5 minutes. -# DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS=300 - -# Encryption key used to encrypt/decrypt the state parameter passed to the IAM. -# -# This key ensures the integrity and confidentiality of state information -# during OAuth2 flows. Must be a valid Fernet key. -DIRACX_SERVICE_AUTH_STATE_KEY= - -# The issuer identifier for JWT tokens. -# -# This should be a URI that uniquely identifies the token issuer and -# matches the 'iss' claim in issued JWT tokens. -DIRACX_SERVICE_AUTH_TOKEN_ISSUER= - -# Keystore containing the cryptographic keys used for signing JWT tokens. -# -# This includes both public and private keys for token signature -# generation and verification. -DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE= - -# List of allowed cryptographic algorithms for JWT token signing. -# -# Supported algorithms include RS256 (RSA with SHA-256) and EdDSA -# (Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"] -# DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS=['RS256', 'EdDSA'] - -# Expiration time in minutes for access tokens. -# -# After this duration, access tokens become invalid and must be refreshed -# or re-obtained. Default: 20 minutes. -# DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES=20 - -# Expiration time in minutes for refresh tokens. -# -# The maximum lifetime of refresh tokens before they must be re-issued -# through a new authentication flow. Default: 60 minutes. -# DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES=60 - -# Set of security properties available in this DIRAC installation. -# -# These properties define various authorization capabilities and are used -# for access control decisions. Defaults to all available security properties. -# DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES= - - - -# DevelopmentSettings (from diracx.core.settings) -# When set to true (only for demo/CI), crash if an access policy isn't called. -# -# This is useful for development and testing to ensure all endpoints have proper -# access control policies defined. -# DIRACX_DEV_CRASH_ON_MISSED_ACCESS_POLICY=False - - - -# OTELSettings (from diracx.routers.otel) -# DIRACX_OTEL_ENABLED=False - -# DIRACX_OTEL_APPLICATION_NAME=diracx - -# DIRACX_OTEL_GRPC_ENDPOINT= - -# DIRACX_OTEL_GRPC_INSECURE=True - -# DIRACX_OTEL_HEADERS= - - - -# SandboxStoreSettings (from diracx.core.settings) -# Name of the S3 bucket used for storing job sandboxes. -# -# This bucket will contain input and output sandbox files for DIRAC jobs. -# The bucket must exist or auto_create_bucket must be enabled. -DIRACX_SANDBOX_STORE_BUCKET_NAME= - -# Configuration parameters passed to the S3 client. -DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS= - -# Whether to automatically create the S3 bucket if it doesn't exist. -# DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET=False - -# Validity duration in seconds for pre-signed S3 URLs. -# -# This determines how long generated download/upload URLs remain valid -# before expiring. Default: 300 seconds (5 minutes). -# DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS=300 - -# Logical name of the Storage Element for the sandbox store. -# -# This name is used within DIRAC to refer to this sandbox storage -# endpoint in job descriptions and file catalogs. -# DIRACX_SANDBOX_STORE_SE_NAME=SandboxSE - - diff --git a/.gitignore b/.gitignore index b898f7da7..6aac865c8 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,6 @@ docs/source/_build pixi.lock *.egg-info docs/templates/_builtin_markdown.jinja + +# docs site +site diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index 9ac4ba5af..ddd1304e9 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -2,12 +2,10 @@ *This page is auto-generated from the settings classes in `diracx.core.settings`.* - ## AuthSettings Settings for the authentication service. - ### `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID` *Optional*, default value: `myDIRACClientID` @@ -107,14 +105,10 @@ Set of security properties available in this DIRAC installation. These properties define various authorization capabilities and are used for access control decisions. Defaults to all available security properties. - - - ## SandboxStoreSettings Settings for the sandbox store. - ### `DIRACX_SANDBOX_STORE_BUCKET_NAME` **Required** @@ -154,14 +148,10 @@ Logical name of the Storage Element for the sandbox store. This name is used within DIRAC to refer to this sandbox storage endpoint in job descriptions and file catalogs. - - - ## OTELSettings Settings for the Open Telemetry Configuration. - ### `DIRACX_OTEL_ENABLED` *Optional*, default value: `False` @@ -172,7 +162,7 @@ Settings for the Open Telemetry Configuration. ### `DIRACX_OTEL_GRPC_ENDPOINT` -*Optional*, default value: `` +*Optional*, default value: \`\` ### `DIRACX_OTEL_GRPC_INSECURE` diff --git a/pixi.toml b/pixi.toml index df37211bc..f3a670153 100644 --- a/pixi.toml +++ b/pixi.toml @@ -30,10 +30,6 @@ diracx-logic = { path = "diracx-logic", editable = true, extras = ["testing"] } [feature.diracx-routers.pypi-dependencies] diracx-routers = { path = "diracx-routers", editable = true, extras = ["testing"] } -# Settings documentation feature -[feature.settings-doc.pypi-dependencies] -settings-doc = "*" - # DiracX features for providing tasks. This is needed to make it so that running # "pixi run pytest-diracx-core -vvv --pdb" passes the arguments as expected. # See: https://github.com/prefix-dev/pixi/issues/1519#issuecomment-2651078457 @@ -61,11 +57,6 @@ description = "Run the tests for diracx-logic" cmd = "cd diracx-routers/ && pytest" description = "Run the tests for diracx-routers" -# Settings documentation task -[feature.settings-doc.tasks.generate-settings-doc] -cmd = "python scripts/generate_settings_docs.py" -description = "Auto-discover and generate settings documentation with validation" - # Gubbins features for providing dependencies [feature.gubbins.pypi-dependencies] gubbins = { path = "extensions/gubbins", editable = true, extras = ["testing"] } @@ -149,6 +140,22 @@ shellcheck = "*" cmd = "find . -not -wholename './.pixi/*' -name '*.sh' -print -exec shellcheck --exclude=SC1090,SC1091 --external-source '{}' ';'" description = "Run shellcheck on all shell scripts" +# Settings documentation feature +[feature.settings-doc.dependencies] +python = "*" +[feature.settings-doc.pypi-dependencies] +settings-doc = "*" +mdformat = "*" +mdformat-mkdocs = "*" +mdformat-gfm = "*" +mdformat-black = "*" +diracx-core = { path = "diracx-core", editable = true } +diracx-routers = { path = "diracx-routers", editable = true } +# Settings documentation task +[feature.settings-doc.tasks.generate-settings-doc] +cmd = "python scripts/generate_settings_docs.py" +description = "Auto-discover and generate settings documentation with validation" + [environments] # DiracX environments default = {features = ["task-diracx", "diracx", "diracx-core", "diracx-api", "diracx-cli", "diracx-client", "diracx-db", "diracx-logic", "diracx-routers", "settings-doc"], solve-group = "diracx"} @@ -177,6 +184,7 @@ gubbins-generate-client = {features = ["client-gen", "diracx-client", "gubbins-c mkdocs = {features = ["mkdocs"], no-default-feature = true} shellcheck = {features = ["shellcheck"], no-default-feature = true} pre-commit = {features = ["pre-commit"], no-default-feature = true} +settings-doc = {features = ["settings-doc"], no-default-feature = true} # Meta-tasks for running many tests at once [tasks.pytest-diracx-all-one-by-one] diff --git a/scripts/generate_settings_docs.py b/scripts/generate_settings_docs.py index c7532e7a1..0c86c9d61 100644 --- a/scripts/generate_settings_docs.py +++ b/scripts/generate_settings_docs.py @@ -17,6 +17,7 @@ import pkgutil import re import shutil +import subprocess import sys from pathlib import Path from typing import Any @@ -212,7 +213,7 @@ def generate_all_templates( settings_classes: dict[str, dict], docs_dir: Path, repo_root: Path, -) -> None: +) -> list: """Generate documentation for all discovered templates.""" # Collect all unique modules that contain settings modules = sorted(set(info["module"] for info in settings_classes.values())) @@ -334,6 +335,23 @@ def compare_and_update_files( def normalize(text: str) -> str: return "\n".join(line for line in text.splitlines() if line.strip()) + # First, run mdformat on all .bak files to ensure consistent formatting + backup_files = [backup for _, backup in generated_files] + if backup_files: + try: + subprocess.run( # noqa: S603 + ["mdformat", "--number"] + [str(f) for f in backup_files], + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as e: + print(f"āš ļø mdformat failed: {e.stderr}") + return False + except FileNotFoundError: + print("āš ļø mdformat not found in environment") + return False + all_up_to_date = True for original_file, backup_file in generated_files: @@ -401,10 +419,10 @@ def main(): templates, settings_classes, docs_dir, repo_root ) - print("\nšŸ“ Generating .env file...") - dotenv_path = repo_root / ".env.example" - dotenv_backup = generate_dotenv_file(settings_classes, dotenv_path) - generated_files.append((dotenv_path, dotenv_backup)) + # print("\nšŸ“ Generating .env file...") + # dotenv_path = repo_root / ".env.example" + # dotenv_backup = generate_dotenv_file(settings_classes, dotenv_path) + # generated_files.append((dotenv_path, dotenv_backup)) # Update files if they're different print("\nšŸ“ Updating files...") From 24d95b167cc48970dad43a8246a7f49f7f424f31 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 18:21:42 +0100 Subject: [PATCH 04/18] feat: trying griffe and griffe pydantic --- .pre-commit-config.yaml | 11 +- docs/dev/reference/api/README.md | 164 +++++++ docs/dev/reference/api/check_coverage.py | 239 ++++++++++ docs/dev/reference/api/cli/index.md | 65 +++ docs/dev/reference/api/core/config.md | 23 + docs/dev/reference/api/core/exceptions.md | 11 + docs/dev/reference/api/core/extensions.md | 10 + docs/dev/reference/api/core/index.md | 18 + docs/dev/reference/api/core/models.md | 10 + docs/dev/reference/api/core/preferences.md | 10 + docs/dev/reference/api/core/properties.md | 10 + docs/dev/reference/api/core/resources.md | 11 + docs/dev/reference/api/core/s3.md | 11 + docs/dev/reference/api/core/settings.md | 10 + docs/dev/reference/api/core/utils.md | 10 + docs/dev/reference/api/db/auth.md | 23 + docs/dev/reference/api/db/dummy.md | 23 + docs/dev/reference/api/db/exceptions.md | 10 + docs/dev/reference/api/db/index.md | 22 + docs/dev/reference/api/db/job.md | 23 + docs/dev/reference/api/db/job_logging.md | 23 + docs/dev/reference/api/db/opensearch.md | 23 + docs/dev/reference/api/db/pilot_agents.md | 23 + docs/dev/reference/api/db/sandbox_metadata.md | 23 + docs/dev/reference/api/db/task_queue.md | 23 + docs/dev/reference/api/db/utils.md | 33 ++ docs/dev/reference/api/index.md | 61 +++ docs/dev/reference/api/logic/auth.md | 63 +++ docs/dev/reference/api/logic/index.md | 12 + docs/dev/reference/api/logic/jobs.md | 53 +++ docs/dev/reference/api/logic/task_queues.md | 13 + .../reference/api/routers/access_policies.md | 11 + docs/dev/reference/api/routers/auth.md | 74 +++ .../reference/api/routers/configuration.md | 14 + .../dev/reference/api/routers/dependencies.md | 11 + docs/dev/reference/api/routers/factory.md | 11 + .../reference/api/routers/fastapi_classes.md | 11 + docs/dev/reference/api/routers/health.md | 25 + docs/dev/reference/api/routers/index.md | 18 + docs/dev/reference/api/routers/jobs.md | 79 ++++ docs/dev/reference/api/routers/otel.md | 11 + docs/dev/reference/api/routers/utils.md | 13 + docs/dev/reference/api/writing-api-docs.md | 447 ++++++++++++++++++ mkdocs.yml | 72 +++ pixi.toml | 14 +- 45 files changed, 1873 insertions(+), 2 deletions(-) create mode 100644 docs/dev/reference/api/README.md create mode 100644 docs/dev/reference/api/check_coverage.py create mode 100644 docs/dev/reference/api/cli/index.md create mode 100644 docs/dev/reference/api/core/config.md create mode 100644 docs/dev/reference/api/core/exceptions.md create mode 100644 docs/dev/reference/api/core/extensions.md create mode 100644 docs/dev/reference/api/core/index.md create mode 100644 docs/dev/reference/api/core/models.md create mode 100644 docs/dev/reference/api/core/preferences.md create mode 100644 docs/dev/reference/api/core/properties.md create mode 100644 docs/dev/reference/api/core/resources.md create mode 100644 docs/dev/reference/api/core/s3.md create mode 100644 docs/dev/reference/api/core/settings.md create mode 100644 docs/dev/reference/api/core/utils.md create mode 100644 docs/dev/reference/api/db/auth.md create mode 100644 docs/dev/reference/api/db/dummy.md create mode 100644 docs/dev/reference/api/db/exceptions.md create mode 100644 docs/dev/reference/api/db/index.md create mode 100644 docs/dev/reference/api/db/job.md create mode 100644 docs/dev/reference/api/db/job_logging.md create mode 100644 docs/dev/reference/api/db/opensearch.md create mode 100644 docs/dev/reference/api/db/pilot_agents.md create mode 100644 docs/dev/reference/api/db/sandbox_metadata.md create mode 100644 docs/dev/reference/api/db/task_queue.md create mode 100644 docs/dev/reference/api/db/utils.md create mode 100644 docs/dev/reference/api/index.md create mode 100644 docs/dev/reference/api/logic/auth.md create mode 100644 docs/dev/reference/api/logic/index.md create mode 100644 docs/dev/reference/api/logic/jobs.md create mode 100644 docs/dev/reference/api/logic/task_queues.md create mode 100644 docs/dev/reference/api/routers/access_policies.md create mode 100644 docs/dev/reference/api/routers/auth.md create mode 100644 docs/dev/reference/api/routers/configuration.md create mode 100644 docs/dev/reference/api/routers/dependencies.md create mode 100644 docs/dev/reference/api/routers/factory.md create mode 100644 docs/dev/reference/api/routers/fastapi_classes.md create mode 100644 docs/dev/reference/api/routers/health.md create mode 100644 docs/dev/reference/api/routers/index.md create mode 100644 docs/dev/reference/api/routers/jobs.md create mode 100644 docs/dev/reference/api/routers/otel.md create mode 100644 docs/dev/reference/api/routers/utils.md create mode 100644 docs/dev/reference/api/writing-api-docs.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31501d1d2..da2bd2c84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: python: python3 ci: - skip: [generate-pixi-docs, settings-doc-check] + skip: [generate-pixi-docs, settings-doc-check, api-doc-coverage-check] default_stages: [pre-commit] @@ -93,3 +93,12 @@ repos: language: system pass_filenames: false files: ^(diracx-.*/src/diracx/.*/settings\.py|docs/.*\.j2|docs/templates/.*\.jinja|scripts/generate_settings_docs\.py)$ + + - repo: local + hooks: + - id: api-doc-coverage-check + name: Check API documentation coverage + entry: pixi run -e default python docs/dev/reference/api/check_coverage.py + language: system + pass_filenames: false + files: ^(diracx-.*/src/diracx/.*\.py|docs/dev/reference/api/.*\.md|docs/dev/reference/api/check_coverage\.py)$ diff --git a/docs/dev/reference/api/README.md b/docs/dev/reference/api/README.md new file mode 100644 index 000000000..5819ac613 --- /dev/null +++ b/docs/dev/reference/api/README.md @@ -0,0 +1,164 @@ +# API Reference Documentation + +This directory contains auto-generated API reference documentation for DiracX packages. + +## Structure + +The API reference is organized by package: + +- **core/** - Core functionality (models, settings, configuration) +- **routers/** - FastAPI routers and endpoints +- **logic/** - Business logic layer +- **db/** - Database models and access layers +- **cli/** - Command-line interface + +## Maintaining Documentation + +### Checking Coverage + +To ensure all modules are documented, run the coverage check script: + +```bash +cd docs/dev/reference/api +python check_coverage.py +``` + +This will report any Python modules that are missing documentation pages. + +### Adding Documentation for New Modules + +When you add a new Python module, you need to add corresponding documentation: + +1. **Create a new .md file** in the appropriate section directory (core/, routers/, logic/, db/, or cli/) + +2. **Add module references** using the `:::` syntax: + + ```markdown + # My New Module + + Description of what this module does. + + ::: diracx.package.module + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true + ``` + +3. **Update the navigation** in `mkdocs.yml`: + + ```yaml + - Logic: + - dev/reference/api/logic/index.md + - Jobs: dev/reference/api/logic/jobs.md + - Auth: dev/reference/api/logic/auth.md + - My New Module: dev/reference/api/logic/my-new-module.md # Add here + ``` + +4. **Update the index page** (e.g., `logic/index.md`) to link to your new page + +5. **Run the coverage check** to verify your documentation is complete + +### Documentation Options + +The `:::` directive supports various options to control how documentation is rendered: + +- `show_root_heading: true` - Show the module name as a heading +- `show_source: true` - Show source code links +- `members_order: source` - Order members as they appear in source +- `group_by_category: true` - Group by functions, classes, etc. +- `show_if_no_docstring: true` - Show members even without docstrings +- `filters: ["!^_"]` - Hide private members (starting with \_) + +For more options, see the [mkdocstrings-python documentation](https://mkdocstrings.github.io/python/usage/). + +## Writing Good Docstrings + +The API reference is auto-generated from docstrings in the source code. Follow these guidelines: + +1. **Use Google-style docstrings**: + + ```python + def my_function(arg1: str, arg2: int) -> bool: + """Brief description of the function. + + More detailed description if needed. This can span + multiple lines. + + Args: + arg1: Description of arg1 + arg2: Description of arg2 + + Returns: + Description of return value + + Raises: + ValueError: When arg2 is negative + """ + ``` + +2. **Document Pydantic models** using Field descriptions: + + ```python + from pydantic import BaseModel, Field + + + class MyModel(BaseModel): + """Brief description of the model.""" + + name: str = Field(..., description="The name field") + age: int = Field(..., ge=0, description="Age must be non-negative") + ``` + +3. **Document FastAPI endpoints** with clear descriptions: + + ```python + @router.post("/jobs") + async def submit_job(job: JobDefinition) -> InsertedJob: + """Submit a new job to the system. + + This endpoint accepts a job definition and submits it + to the task queue for processing. + + Args: + job: The job definition to submit + + Returns: + Information about the inserted job including job ID + """ + ``` + +## Troubleshooting + +### Module not showing up + +If a module isn't rendering in the docs: + +1. Check that the module path is correct (must point to actual `.py` files, not empty `__init__.py`) +2. Verify the module is in the `paths` list in `mkdocs.yml` under the mkdocstrings config +3. Run `python check_coverage.py` to see if it's detected + +### Decorators not showing + +FastAPI route decorators (like `@router.post("/path")`) are visible in the "Source code" section when you expand it. They are not displayed separately by default. + +### Empty documentation + +If a module shows up but has no content: + +1. Check that the module actually has functions/classes (not just an empty `__init__.py`) +2. Ensure docstrings are present in the source code +3. Use `show_if_no_docstring: true` to show members even without docs + +## Setup + +The API reference is generated using: + +- **mkdocstrings** - Plugin for generating API documentation +- **mkdocstrings-python** - Python handler for mkdocstrings +- **griffe** - Python code parser +- **griffe-pydantic** - Extension for enhanced Pydantic model documentation + +Configuration is in `mkdocs.yml` (search for `mkdocstrings` plugin). diff --git a/docs/dev/reference/api/check_coverage.py b/docs/dev/reference/api/check_coverage.py new file mode 100644 index 000000000..e2d2e8c25 --- /dev/null +++ b/docs/dev/reference/api/check_coverage.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Check that all Python modules have corresponding API reference documentation pages. + +This script ensures that the API reference documentation stays in sync with the codebase +by checking that every Python module has a corresponding .md file in docs/dev/reference/api/. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import NamedTuple + +# Root directory of the diracx project +ROOT_DIR = Path(__file__).parent.parent.parent.parent.parent +DOCS_API_DIR = Path(__file__).parent + +# Package directories to check +PACKAGES = { + "diracx-core": ROOT_DIR / "diracx-core" / "src" / "diracx" / "core", + "diracx-routers": ROOT_DIR / "diracx-routers" / "src" / "diracx" / "routers", + "diracx-logic": ROOT_DIR / "diracx-logic" / "src" / "diracx" / "logic", + "diracx-db": ROOT_DIR / "diracx-db" / "src" / "diracx" / "db", + "diracx-cli": ROOT_DIR / "diracx-cli" / "src" / "diracx" / "cli", +} + +# Modules to ignore (typically __pycache__, __init__.py, py.typed, etc.) +IGNORED_PATTERNS = { + "__pycache__", + "__init__.py", + "__main__.py", + "py.typed", + ".pyc", +} + + +class ModuleInfo(NamedTuple): + """Information about a Python module.""" + + package: str + module_path: str # e.g., "routers.jobs.submission" + file_path: Path + + +def find_python_modules(package_name: str, package_path: Path) -> list[ModuleInfo]: + """Find all Python modules in a package. + + Args: + package_name: Name of the package (e.g., "diracx-routers") + package_path: Path to the package source directory + + Returns: + List of ModuleInfo objects for each Python module + + """ + modules = [] + + if not package_path.exists(): + print(f"Warning: Package path does not exist: {package_path}") + return modules + + # Find all .py files + for py_file in package_path.rglob("*.py"): + # Skip ignored files + if any(pattern in str(py_file) for pattern in IGNORED_PATTERNS): + continue + + # Get relative path from package root + rel_path = py_file.relative_to(package_path) + + # Convert path to module notation (e.g., jobs/submission.py -> jobs.submission) + module_parts = list(rel_path.parts[:-1]) + [rel_path.stem] + module_path = ".".join(module_parts) + + modules.append( + ModuleInfo( + package=package_name, + module_path=module_path, + file_path=py_file, + ) + ) + + return sorted(modules, key=lambda m: m.module_path) + + +def get_expected_doc_path(module: ModuleInfo) -> Path: + """Get the expected documentation file path for a module. + + Args: + module: ModuleInfo object + + Returns: + Expected path to the .md documentation file + + """ + # Map package names to doc sections + section_map = { + "diracx-core": "core", + "diracx-routers": "routers", + "diracx-logic": "logic", + "diracx-db": "db", + "diracx-cli": "cli", + } + + section = section_map.get(module.package) + if not section: + return None + + # Convert module path to doc path + # e.g., jobs.submission -> jobs.md (we document at package level, not individual files) + # Or for deeper nesting: sql.job.db -> job.md + parts = module.module_path.split(".") + + # For most cases, we document at the first or second level + # This is a heuristic - adjust based on your documentation structure + if section == "db": + # Special handling for db: sql.job.db -> job.md, os.job_parameters -> opensearch.md + if parts[0] == "sql": + doc_name = parts[1] if len(parts) > 1 else "index" + elif parts[0] == "os": + doc_name = "opensearch" + elif parts[0] == "exceptions": + doc_name = "exceptions" + else: + doc_name = parts[0] + elif section in ["routers", "logic"]: + # routers.jobs.submission -> jobs.md + # logic.jobs.submission -> jobs.md + doc_name = parts[0] if parts else "index" + elif section == "core": + # core.models -> models.md + doc_name = parts[0] if parts else "index" + elif section == "cli": + # cli.jobs -> index.md (CLI is all in one file currently) + doc_name = "index" + else: + doc_name = parts[0] if parts else "index" + + doc_path = DOCS_API_DIR / section / f"{doc_name}.md" + return doc_path + + +def check_module_documented(module: ModuleInfo) -> tuple[bool, Path]: + """Check if a module is documented. + + Args: + module: ModuleInfo object + + Returns: + Tuple of (is_documented, expected_doc_path) + + """ + expected_path = get_expected_doc_path(module) + if expected_path is None: + return False, None + + # Check if the doc file exists + exists = expected_path.exists() + + # If it exists, check if it references this module + if exists: + content = expected_path.read_text() + # Check for the module reference in the doc (loose check) + # This could be improved to parse the ::: directives more carefully + referenced = ( + f"diracx.{module.package.split('-')[1]}.{module.module_path}" in content + ) + return referenced, expected_path + + return False, expected_path + + +def main(): + """Main function to check documentation coverage.""" + print("Checking API reference documentation coverage...\n") + + all_modules = [] + undocumented = [] + documented = [] + + # Collect all modules + for package_name, package_path in PACKAGES.items(): + modules = find_python_modules(package_name, package_path) + all_modules.extend(modules) + print(f"Found {len(modules)} modules in {package_name}") + + print(f"\nTotal modules found: {len(all_modules)}\n") + + # Check documentation coverage + for module in all_modules: + is_documented, doc_path = check_module_documented(module) + + if is_documented: + documented.append((module, doc_path)) + else: + undocumented.append((module, doc_path)) + + # Print results + print(f"āœ“ Documented modules: {len(documented)}") + print(f"āœ— Undocumented modules: {len(undocumented)}\n") + + if undocumented: + print("Undocumented modules:") + print("-" * 80) + + # Group by package + by_package = {} + for module, doc_path in undocumented: + by_package.setdefault(module.package, []).append((module, doc_path)) + + for package_name, modules in sorted(by_package.items()): + print(f"\n{package_name}:") + for module, doc_path in modules: + if doc_path and doc_path.exists(): + status = f"Doc exists but missing reference: {doc_path}" + elif doc_path: + status = f"Missing doc file: {doc_path}" + else: + status = "No doc path determined" + print(f" - {module.module_path:40} ā†’ {status}") + + print("\n" + "=" * 80) + print( + f"Documentation coverage: {len(documented)}/{len(all_modules)} " + f"({len(documented) * 100 / len(all_modules):.1f}%)" + ) + print("=" * 80) + + # Return non-zero exit code if there are undocumented modules + sys.exit(1) + else: + print("āœ“ All modules are documented!") + print("=" * 80) + print(f"Documentation coverage: 100% ({len(all_modules)} modules)") + print("=" * 80) + + +if __name__ == "__main__": + main() diff --git a/docs/dev/reference/api/cli/index.md b/docs/dev/reference/api/cli/index.md new file mode 100644 index 000000000..b4a007b6a --- /dev/null +++ b/docs/dev/reference/api/cli/index.md @@ -0,0 +1,65 @@ +# Command Line Interface + +DiracX command-line interface modules and commands. + +## Jobs Commands + +::: diracx.cli.jobs +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Auth Commands + +::: diracx.cli.auth +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Config Commands + +::: diracx.cli.config +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Internal Commands + +### Legacy Commands + +::: diracx.cli.internal.legacy +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +### Internal Config + +::: diracx.cli.internal.config +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## CLI Utilities + +::: diracx.cli.utils +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/config.md b/docs/dev/reference/api/core/config.md new file mode 100644 index 000000000..0437f66d1 --- /dev/null +++ b/docs/dev/reference/api/core/config.md @@ -0,0 +1,23 @@ +# Configuration + +Configuration schema and sources for DiracX. + +## Config Schema + +::: diracx.core.config.schema +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Config Sources + +::: diracx.core.config.sources +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/exceptions.md b/docs/dev/reference/api/core/exceptions.md new file mode 100644 index 000000000..561b87018 --- /dev/null +++ b/docs/dev/reference/api/core/exceptions.md @@ -0,0 +1,11 @@ +# Exceptions + +Core exception classes used throughout DiracX. + +::: diracx.core.exceptions +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/extensions.md b/docs/dev/reference/api/core/extensions.md new file mode 100644 index 000000000..df8216ead --- /dev/null +++ b/docs/dev/reference/api/core/extensions.md @@ -0,0 +1,10 @@ +# Extensions + +Extension system for extending DiracX functionality. + +::: diracx.core.extensions +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true diff --git a/docs/dev/reference/api/core/index.md b/docs/dev/reference/api/core/index.md new file mode 100644 index 000000000..691f1f385 --- /dev/null +++ b/docs/dev/reference/api/core/index.md @@ -0,0 +1,18 @@ +# Core + +Core components of DiracX including models, settings, configuration, and utilities. + +## Modules + +- [Models](models.md) - Core Pydantic models for data validation +- [Settings](settings.md) - Configuration settings +- [Preferences](preferences.md) - User preferences +- [Configuration](config.md) - Configuration schema and sources +- [Exceptions](exceptions.md) - Core exception classes +- [Resources](resources.md) - Resource management and dependency injection +- [S3](s3.md) - S3-compatible object storage integration +- [Properties](properties.md) - Security properties +- [Extensions](extensions.md) - Extension system +- [Utilities](utils.md) - Core utilities + +The core package provides the foundational components used throughout DiracX. diff --git a/docs/dev/reference/api/core/models.md b/docs/dev/reference/api/core/models.md new file mode 100644 index 000000000..2f6e2db37 --- /dev/null +++ b/docs/dev/reference/api/core/models.md @@ -0,0 +1,10 @@ +# Core Models + +Core Pydantic models used throughout DiracX for data validation and serialization. + +::: diracx.core.models +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true diff --git a/docs/dev/reference/api/core/preferences.md b/docs/dev/reference/api/core/preferences.md new file mode 100644 index 000000000..3e8627912 --- /dev/null +++ b/docs/dev/reference/api/core/preferences.md @@ -0,0 +1,10 @@ +# Preferences + +User preferences and configuration options. + +::: diracx.core.preferences +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true diff --git a/docs/dev/reference/api/core/properties.md b/docs/dev/reference/api/core/properties.md new file mode 100644 index 000000000..3e90051e4 --- /dev/null +++ b/docs/dev/reference/api/core/properties.md @@ -0,0 +1,10 @@ +# Properties + +Security properties and property management. + +::: diracx.core.properties +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true diff --git a/docs/dev/reference/api/core/resources.md b/docs/dev/reference/api/core/resources.md new file mode 100644 index 000000000..2fa3d509f --- /dev/null +++ b/docs/dev/reference/api/core/resources.md @@ -0,0 +1,11 @@ +# Resources + +Resource management and dependency injection utilities. + +::: diracx.core.resources +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/s3.md b/docs/dev/reference/api/core/s3.md new file mode 100644 index 000000000..892d3f1af --- /dev/null +++ b/docs/dev/reference/api/core/s3.md @@ -0,0 +1,11 @@ +# S3 Integration + +S3-compatible object storage integration utilities. + +::: diracx.core.s3 +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/settings.md b/docs/dev/reference/api/core/settings.md new file mode 100644 index 000000000..893f66813 --- /dev/null +++ b/docs/dev/reference/api/core/settings.md @@ -0,0 +1,10 @@ +# Settings + +Configuration settings for DiracX services and components. + +::: diracx.core.settings +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true diff --git a/docs/dev/reference/api/core/utils.md b/docs/dev/reference/api/core/utils.md new file mode 100644 index 000000000..58d8380d0 --- /dev/null +++ b/docs/dev/reference/api/core/utils.md @@ -0,0 +1,10 @@ +# Utilities + +Core utility functions and helpers. + +::: diracx.core.utils +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true diff --git a/docs/dev/reference/api/db/auth.md b/docs/dev/reference/api/db/auth.md new file mode 100644 index 000000000..4880d6899 --- /dev/null +++ b/docs/dev/reference/api/db/auth.md @@ -0,0 +1,23 @@ +# Auth Database + +Authentication and authorization database. + +## Database Schema + +::: diracx.db.sql.auth.schema +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Database Access Layer + +::: diracx.db.sql.auth.db +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/dummy.md b/docs/dev/reference/api/db/dummy.md new file mode 100644 index 000000000..ddb54c406 --- /dev/null +++ b/docs/dev/reference/api/db/dummy.md @@ -0,0 +1,23 @@ +# Dummy Database + +Dummy database for testing and development purposes. + +## Schema + +::: diracx.db.sql.dummy.schema +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Database Access + +::: diracx.db.sql.dummy.db +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/exceptions.md b/docs/dev/reference/api/db/exceptions.md new file mode 100644 index 000000000..4f69b965d --- /dev/null +++ b/docs/dev/reference/api/db/exceptions.md @@ -0,0 +1,10 @@ +# Database Exceptions + +Database-related exception classes. + +::: diracx.db.exceptions +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true diff --git a/docs/dev/reference/api/db/index.md b/docs/dev/reference/api/db/index.md new file mode 100644 index 000000000..3270f7f5d --- /dev/null +++ b/docs/dev/reference/api/db/index.md @@ -0,0 +1,22 @@ +# Database + +Database models, schemas, and access layers. + +## SQL Databases + +- [Job DB](job.md) - Job database +- [Job Logging DB](job_logging.md) - Job logging and history +- [Auth DB](auth.md) - Authentication and authorization +- [Sandbox Metadata DB](sandbox_metadata.md) - Sandbox file metadata +- [Task Queue DB](task_queue.md) - Task queue management +- [Pilot Agents DB](pilot_agents.md) - Pilot agent tracking +- [Dummy DB](dummy.md) - Dummy database for testing + +## OpenSearch + +- [OpenSearch](opensearch.md) - OpenSearch-based databases + +## Utilities + +- [SQL Utilities](utils.md) - SQL database utilities +- [Exceptions](exceptions.md) - Database exceptions diff --git a/docs/dev/reference/api/db/job.md b/docs/dev/reference/api/db/job.md new file mode 100644 index 000000000..1c2376e2c --- /dev/null +++ b/docs/dev/reference/api/db/job.md @@ -0,0 +1,23 @@ +# Job Database + +Job database models and access layer. + +## Database Schema + +::: diracx.db.sql.job.schema +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Database Access Layer + +::: diracx.db.sql.job.db +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/job_logging.md b/docs/dev/reference/api/db/job_logging.md new file mode 100644 index 000000000..235caba98 --- /dev/null +++ b/docs/dev/reference/api/db/job_logging.md @@ -0,0 +1,23 @@ +# Job Logging Database + +Job logging and history database. + +## Database Schema + +::: diracx.db.sql.job_logging.schema +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Database Access Layer + +::: diracx.db.sql.job_logging.db +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/opensearch.md b/docs/dev/reference/api/db/opensearch.md new file mode 100644 index 000000000..4ffbc4ea0 --- /dev/null +++ b/docs/dev/reference/api/db/opensearch.md @@ -0,0 +1,23 @@ +# OpenSearch Databases + +OpenSearch-based database implementations. + +## Job Parameters + +::: diracx.db.os.job_parameters +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Utilities + +::: diracx.db.os.utils +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/pilot_agents.md b/docs/dev/reference/api/db/pilot_agents.md new file mode 100644 index 000000000..4ff448567 --- /dev/null +++ b/docs/dev/reference/api/db/pilot_agents.md @@ -0,0 +1,23 @@ +# Pilot Agents Database + +Pilot agent tracking and management database. + +## Database Schema + +::: diracx.db.sql.pilot_agents.schema +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Database Access Layer + +::: diracx.db.sql.pilot_agents.db +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/sandbox_metadata.md b/docs/dev/reference/api/db/sandbox_metadata.md new file mode 100644 index 000000000..334db230c --- /dev/null +++ b/docs/dev/reference/api/db/sandbox_metadata.md @@ -0,0 +1,23 @@ +# Sandbox Metadata Database + +Sandbox file metadata and tracking database. + +## Database Schema + +::: diracx.db.sql.sandbox_metadata.schema +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Database Access Layer + +::: diracx.db.sql.sandbox_metadata.db +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/task_queue.md b/docs/dev/reference/api/db/task_queue.md new file mode 100644 index 000000000..7f3ebccd9 --- /dev/null +++ b/docs/dev/reference/api/db/task_queue.md @@ -0,0 +1,23 @@ +# Task Queue Database + +Task queue management database. + +## Database Schema + +::: diracx.db.sql.task_queue.schema +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Database Access Layer + +::: diracx.db.sql.task_queue.db +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/utils.md b/docs/dev/reference/api/db/utils.md new file mode 100644 index 000000000..085c010b4 --- /dev/null +++ b/docs/dev/reference/api/db/utils.md @@ -0,0 +1,33 @@ +# SQL Utilities + +Utilities for SQL database operations. + +## Base Classes + +::: diracx.db.sql.utils.base +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Functions + +::: diracx.db.sql.utils.functions +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Types + +::: diracx.db.sql.utils.types +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/index.md b/docs/dev/reference/api/index.md new file mode 100644 index 000000000..fe07b2681 --- /dev/null +++ b/docs/dev/reference/api/index.md @@ -0,0 +1,61 @@ +# API Reference + +This section provides detailed API reference documentation for DiracX modules, automatically generated from the source code using [Griffe](https://mkdocstrings.github.io/griffe/) and [Griffe-Pydantic](https://mkdocstrings.github.io/griffe-pydantic/). + +## [Core](core/index.md) + +Core components including models, settings, configuration, and utilities. + +- **[Models](core/models.md)** - Core Pydantic models for data validation +- **[Settings](core/settings.md)** - Configuration settings +- **[Preferences](core/preferences.md)** - User preferences +- **[Config Schema](core/config-schema.md)** - Configuration schema definitions +- **[Properties](core/properties.md)** - Security properties +- **[Extensions](core/extensions.md)** - Extension system +- **[Utilities](core/utils.md)** - Core utilities + +## [Routers](routers/index.md) + +FastAPI routers providing the REST API endpoints. + +- **[Jobs](routers/jobs.md)** - Job management endpoints +- **[Auth](routers/auth.md)** - Authentication and authorization +- **[Configuration](routers/configuration.md)** - Configuration management +- **[Health](routers/health.md)** - Health check and monitoring + +## [Logic](logic/index.md) + +Business logic layer providing service implementations and orchestration. + +## [Database](db/index.md) + +Database models, schemas, and access layers. + +- **[Job DB](db/job.md)** - Job database +- **[Job Logging DB](db/job-logging.md)** - Job logging and history +- **[Auth DB](db/auth.md)** - Authentication and authorization +- **[Sandbox Metadata DB](db/sandbox-metadata.md)** - Sandbox file metadata +- **[Task Queue DB](db/task-queue.md)** - Task queue management +- **[Pilot Agents DB](db/pilot-agents.md)** - Pilot agent tracking +- **[OpenSearch](db/opensearch.md)** - OpenSearch-based databases +- **[SQL Utilities](db/sql-utils.md)** - SQL database utilities +- **[Exceptions](db/exceptions.md)** - Database exceptions + +## [CLI](cli/index.md) + +Command-line interface modules and commands. + +______________________________________________________________________ + +## How to Use + +Each module page contains automatically generated documentation including: + +- **Pydantic Models**: Field descriptions, types, defaults, constraints, and validation rules +- **Functions & Methods**: Parameters, return types, and docstrings +- **Type Annotations**: Full type information for all public APIs +- **Source Links**: Direct links to source code on GitHub + +## Contributing Documentation + +See [Writing API Docs](writing-api-docs.md) for guidelines on documenting your code. diff --git a/docs/dev/reference/api/logic/auth.md b/docs/dev/reference/api/logic/auth.md new file mode 100644 index 000000000..fcc8e3bba --- /dev/null +++ b/docs/dev/reference/api/logic/auth.md @@ -0,0 +1,63 @@ +# Auth Logic + +Authentication and authorization business logic. + +## Token Management + +::: diracx.logic.auth.token +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Authorization Code Flow + +::: diracx.logic.auth.authorize_code_flow +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Device Flow + +::: diracx.logic.auth.device_flow +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## User Management + +::: diracx.logic.auth.management +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Well Known Endpoints + +::: diracx.logic.auth.well_known +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Auth Utilities + +::: diracx.logic.auth.utils +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/logic/index.md b/docs/dev/reference/api/logic/index.md new file mode 100644 index 000000000..c71b6d9fd --- /dev/null +++ b/docs/dev/reference/api/logic/index.md @@ -0,0 +1,12 @@ +# Logic + +Business logic layer providing service implementations and orchestration. + +The logic layer sits between the routers and the database, implementing business rules, +validation, and complex operations. + +## Modules + +- [Jobs Logic](jobs.md) - Job submission, querying, status management, and sandboxes +- [Auth Logic](auth.md) - Authentication and authorization flows +- [Task Queues Logic](task-queues.md) - Task queue management and priority diff --git a/docs/dev/reference/api/logic/jobs.md b/docs/dev/reference/api/logic/jobs.md new file mode 100644 index 000000000..8ab72a06a --- /dev/null +++ b/docs/dev/reference/api/logic/jobs.md @@ -0,0 +1,53 @@ +# Jobs Logic + +Job-related business logic including submission, querying, status management, and sandboxes. + +## Job Submission + +::: diracx.logic.jobs.submission +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Job Query + +::: diracx.logic.jobs.query +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Job Status + +::: diracx.logic.jobs.status +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Job Sandboxes + +::: diracx.logic.jobs.sandboxes +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Job Utilities + +::: diracx.logic.jobs.utils +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/logic/task_queues.md b/docs/dev/reference/api/logic/task_queues.md new file mode 100644 index 000000000..0e731886d --- /dev/null +++ b/docs/dev/reference/api/logic/task_queues.md @@ -0,0 +1,13 @@ +# Task Queues Logic + +Task queue management and priority logic. + +## Priority Management + +::: diracx.logic.task_queues.priority +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/access_policies.md b/docs/dev/reference/api/routers/access_policies.md new file mode 100644 index 000000000..24f8a4748 --- /dev/null +++ b/docs/dev/reference/api/routers/access_policies.md @@ -0,0 +1,11 @@ +# Access Policies + +Access control policies for DiracX routers. + +::: diracx.routers.access_policies +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/auth.md b/docs/dev/reference/api/routers/auth.md new file mode 100644 index 000000000..6e341bf82 --- /dev/null +++ b/docs/dev/reference/api/routers/auth.md @@ -0,0 +1,74 @@ +# Auth Router + +Authentication and authorization endpoints. + +::: diracx.routers.auth +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true +filters: +\- "!^\_" +\- "!^logger" + +## Token Management + +::: diracx.routers.auth.token +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Authorization Code Flow + +::: diracx.routers.auth.authorize_code_flow +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Device Flow + +::: diracx.routers.auth.device_flow +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Management + +::: diracx.routers.auth.management +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Well Known + +::: diracx.routers.auth.well_known +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true + +## Utilities + +::: diracx.routers.auth.utils +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/configuration.md b/docs/dev/reference/api/routers/configuration.md new file mode 100644 index 000000000..5e3cdf273 --- /dev/null +++ b/docs/dev/reference/api/routers/configuration.md @@ -0,0 +1,14 @@ +# Configuration Router + +Configuration management endpoints. + +::: diracx.routers.configuration +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true +filters: +\- "!^\_" +\- "!^logger" diff --git a/docs/dev/reference/api/routers/dependencies.md b/docs/dev/reference/api/routers/dependencies.md new file mode 100644 index 000000000..c84baed73 --- /dev/null +++ b/docs/dev/reference/api/routers/dependencies.md @@ -0,0 +1,11 @@ +# Dependencies + +FastAPI dependency injection utilities and common dependencies. + +::: diracx.routers.dependencies +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/factory.md b/docs/dev/reference/api/routers/factory.md new file mode 100644 index 000000000..f5c5c1606 --- /dev/null +++ b/docs/dev/reference/api/routers/factory.md @@ -0,0 +1,11 @@ +# Router Factory + +Factory functions and utilities for creating DiracX routers. + +::: diracx.routers.factory +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/fastapi_classes.md b/docs/dev/reference/api/routers/fastapi_classes.md new file mode 100644 index 000000000..5bccd0f39 --- /dev/null +++ b/docs/dev/reference/api/routers/fastapi_classes.md @@ -0,0 +1,11 @@ +# FastAPI Classes + +Custom FastAPI router and application classes for DiracX. + +::: diracx.routers.fastapi_classes +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/health.md b/docs/dev/reference/api/routers/health.md new file mode 100644 index 000000000..133b45fa5 --- /dev/null +++ b/docs/dev/reference/api/routers/health.md @@ -0,0 +1,25 @@ +# Health Router + +Health check and monitoring endpoints for service status. + +::: diracx.routers.health +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true +filters: +\- "!^\_" + +## Health Probes + +::: diracx.routers.health.probes +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true +filters: +\- "!^\_" diff --git a/docs/dev/reference/api/routers/index.md b/docs/dev/reference/api/routers/index.md new file mode 100644 index 000000000..009fb690a --- /dev/null +++ b/docs/dev/reference/api/routers/index.md @@ -0,0 +1,18 @@ +# Routers + +FastAPI routers providing the REST API endpoints for DiracX. + +## Available Routers + +- [Jobs](jobs.md) - Job management endpoints +- [Auth](auth.md) - Authentication and authorization +- [Configuration](configuration.md) - Configuration management +- [Health](health.md) - Health check and monitoring endpoints +- [Access Policies](access_policies.md) - Access control policies +- [Dependencies](dependencies.md) - FastAPI dependency injection utilities +- [Factory](factory.md) - Router factory functions +- [FastAPI Classes](fastapi_classes.md) - Custom FastAPI router classes +- [OpenTelemetry](otel.md) - Tracing and instrumentation +- [Utilities](utils.md) - Router utilities + +Each router module contains endpoint definitions, request/response models, and route handlers. diff --git a/docs/dev/reference/api/routers/jobs.md b/docs/dev/reference/api/routers/jobs.md new file mode 100644 index 000000000..1794566ee --- /dev/null +++ b/docs/dev/reference/api/routers/jobs.md @@ -0,0 +1,79 @@ +# Jobs Router + +Job management API endpoints including submission, querying, and status updates. + +The Jobs router is composed of multiple sub-routers: + +- **Submission**: Job submission endpoints +- **Query**: Job search and filtering +- **Status**: Job status management +- **Sandboxes**: Sandbox upload/download + +## Router + +::: diracx.routers.jobs +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true +filters: +\- "!^\_" +\- "!^logger" + +## Sub-Routers + +### Submission + +::: diracx.routers.jobs.submission +options: +show_root_heading: false +show_source: true +members_order: source +group_by_category: true + +### Query + +::: diracx.routers.jobs.query +options: +show_root_heading: false +show_source: true +members_order: source +group_by_category: true + +### Status + +::: diracx.routers.jobs.status +options: +show_root_heading: false +show_source: true +members_order: source +group_by_category: true + +### Sandboxes + +::: diracx.routers.jobs.sandboxes +options: +show_root_heading: false +show_source: true +members_order: source +group_by_category: true + +### Access Policies + +::: diracx.routers.jobs.access_policies +options: +show_root_heading: false +show_source: true +members_order: source +group_by_category: true + +### Legacy + +::: diracx.routers.jobs.legacy +options: +show_root_heading: false +show_source: true +members_order: source +group_by_category: true diff --git a/docs/dev/reference/api/routers/otel.md b/docs/dev/reference/api/routers/otel.md new file mode 100644 index 000000000..15e325fbb --- /dev/null +++ b/docs/dev/reference/api/routers/otel.md @@ -0,0 +1,11 @@ +# OpenTelemetry + +OpenTelemetry instrumentation and tracing utilities for DiracX routers. + +::: diracx.routers.otel +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/utils.md b/docs/dev/reference/api/routers/utils.md new file mode 100644 index 000000000..35d232a26 --- /dev/null +++ b/docs/dev/reference/api/routers/utils.md @@ -0,0 +1,13 @@ +# Router Utilities + +Utility functions for DiracX routers. + +## User Utilities + +::: diracx.routers.utils.users +options: +show_root_heading: true +show_source: true +members_order: source +group_by_category: true +show_if_no_docstring: true diff --git a/docs/dev/reference/api/writing-api-docs.md b/docs/dev/reference/api/writing-api-docs.md new file mode 100644 index 000000000..eedc19471 --- /dev/null +++ b/docs/dev/reference/api/writing-api-docs.md @@ -0,0 +1,447 @@ +# Writing API Documentation + +This guide explains how to write documentation that will be automatically picked up by Griffe and rendered in the API reference. + +## Docstring Style + +DiracX uses **Google-style docstrings**. Here's the general format: + +```python +def example_function(param1: str, param2: int = 0) -> dict[str, Any]: + """Brief one-line summary. + + Longer description of the function if needed. This can span + multiple paragraphs. + + Args: + param1: Description of param1. + param2: Description of param2. Defaults to 0. + + Returns: + Description of the return value. + + Raises: + ValueError: When param2 is negative. + TypeError: When param1 is not a string. + + Examples: + >>> example_function("test", 42) + {'result': 'test_42'} + """ + if param2 < 0: + raise ValueError("param2 must be non-negative") + return {"result": f"{param1}_{param2}"} +``` + +## Documenting Pydantic Models + +With Griffe-Pydantic, your Pydantic models are automatically documented with rich information: + +```python +from pydantic import BaseModel, Field + + +class UserConfig(BaseModel): + """User configuration model. + + This model represents a user's configuration in the DiracX system. + """ + + username: str = Field( + ..., + description="The user's unique username", + min_length=3, + max_length=50, + pattern="^[a-zA-Z0-9_-]+$", + ) + + email: str | None = Field( + None, description="User's email address for notifications" + ) + + age: int = Field(default=0, description="User's age", ge=0, le=150) + + roles: list[str] = Field( + default_factory=list, description="List of roles assigned to the user" + ) +``` + +### What Gets Documented Automatically + +Griffe-Pydantic extracts and displays: + +- **Field names and types** (including Union types, Optional, etc.) +- **Field descriptions** from `Field(description=...)` +- **Default values** from `Field(default=...)` +- **Validation constraints**: `min_length`, `max_length`, `ge`, `le`, `pattern`, etc. +- **Required vs optional** fields +- **Field examples** from `Field(examples=...)` +- **Validators** (custom field validators) + +## Class Documentation + +```python +class MyService: + """Service for handling business logic. + + This service provides methods for processing data and + interacting with the database. + + Attributes: + db: The database connection. + cache: Optional cache instance. + """ + + def __init__(self, db: Database, cache: Cache | None = None): + """Initialize the service. + + Args: + db: Database connection instance. + cache: Optional cache for performance optimization. + """ + self.db = db + self.cache = cache + + async def process_data(self, data: dict[str, Any]) -> ProcessResult: + """Process the input data. + + Args: + data: Raw data dictionary to process. + + Returns: + Processed result with metadata. + + Raises: + ValidationError: If data is invalid. + DatabaseError: If database operation fails. + """ + # Implementation here + pass +``` + +## Module Documentation + +Add module-level documentation at the top of your Python files: + +```python +"""Job management utilities. + +This module provides utilities for managing jobs in DiracX, +including submission, monitoring, and status updates. + +Example: + >>> from diracx.routers.jobs import submit_job + >>> result = await submit_job(job_data) +""" + +from __future__ import annotations + +# Rest of your code... +``` + +## Auto-Discovery of Classes and Functions + +The `::: module.path` syntax automatically discovers and documents all public members. + +### What Gets Auto-Discovered + +By default, mkdocstrings will discover and document: + +1. **Functions**: All functions not starting with `_` +2. **Classes**: All classes not starting with `_` +3. **Constants**: Module-level constants +4. **Pydantic Models**: With field information from griffe-pydantic + +### Controlling What's Documented + +#### Show Everything (Including Undocumented) + +```markdown +::: diracx.core.models + options: + show_if_no_docstring: true + filters: + - "!^_" # Exclude private members +``` + +#### Show Only Specific Members + +```markdown +::: diracx.core.models + options: + members: + - JobStatus + - UserInfo + - SandboxInfo +``` + +#### Exclude Specific Members + +```markdown +::: diracx.routers.jobs + options: + filters: + - "!^_" # Exclude private members + - "!^logger" # Exclude logger + - "!router" # Exclude router instance +``` + +#### Show Inherited Members + +```markdown +::: diracx.core.settings.ServiceSettingsBase + options: + inherited_members: true + show_bases: true +``` + +### Common Patterns + +#### Document a Whole Module + +```markdown +# My Module + +::: diracx.module.name + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true +``` + +This will show: + +- Module docstring +- All public classes +- All public functions +- All public constants +- Grouped by category (attributes, classes, functions, etc.) + +#### Document a Specific Class + +```markdown +# JobStatus Model + +::: diracx.core.models.JobStatus + options: + show_root_heading: true + members_order: alphabetical +``` + +#### Document Multiple Related Classes + +```markdown +# Job Models + +## JobStatus + +::: diracx.core.models.JobStatus + +## JobInfo + +::: diracx.core.models.JobInfo + +## JobSubmission + +::: diracx.core.models.JobSubmission +``` + +### Debugging Empty Documentation + +If a module page appears empty, check: + +1. **Module exists and is importable** + + ```bash + pixi run -e mkdocs python -c "import diracx.module.name; print(dir(diracx.module.name))" + ``` + +2. **Members are public (not starting with `_`)** + + ```python + # This will be documented + def public_function(): + pass + + + # This will NOT be documented + def _private_function(): + pass + ``` + +3. **Set `show_if_no_docstring: true`** to see undocumented members + +4. **Check the build output** for import errors or warnings + +### Example: Full Router Documentation + +Here's how to document a complete router with all its routes: + +```markdown +# Jobs Router + +Job management endpoints. + +## Router Module + +::: diracx.routers.jobs + options: + show_root_heading: true + show_source: true + show_if_no_docstring: true + filters: + - "!^_" + - "!^logger" + +## Submission Routes + +::: diracx.routers.jobs.submission + options: + show_root_heading: false + members_order: source +``` + +This will show: + +- The router's docstring +- All route handler functions with their HTTP methods, paths, and parameters +- Request/response models +- Dependencies +- Source code links + +## Best Practices + +### 1. Be Descriptive but Concise + +```python +# Good +def calculate_total(amounts: list[float]) -> float: + """Calculate the sum of all amounts. + + Args: + amounts: List of numeric amounts to sum. + + Returns: + The total sum of all amounts. + """ + return sum(amounts) + + +# Avoid - too brief +def calculate_total(amounts: list[float]) -> float: + """Calculate total.""" + return sum(amounts) +``` + +### 2. Document Type Information + +Even though type hints are extracted automatically, explain complex types: + +```python +def process_config(config: dict[str, dict[str, list[str]]]) -> ProcessedConfig: + """Process the configuration dictionary. + + Args: + config: Configuration mapping where keys are section names, + values are dictionaries mapping setting names to lists + of allowed values. + + Returns: + Validated and processed configuration object. + """ + pass +``` + +### 3. Use Examples + +Examples help users understand how to use your code: + +```python +def format_job_id(vo: str, job_number: int) -> str: + """Format a job ID string. + + Args: + vo: Virtual organization name. + job_number: Job number. + + Returns: + Formatted job ID string. + + Examples: + >>> format_job_id("lhcb", 12345) + 'lhcb:12345' + >>> format_job_id("atlas", 1) + 'atlas:00001' + """ + return f"{vo}:{job_number:05d}" +``` + +### 4. Document Async Functions + +Be clear about async behavior: + +```python +async def fetch_user_data(user_id: int) -> UserData: + """Fetch user data from the database. + + This is an async function that queries the database and returns + user information. Await this function when calling. + + Args: + user_id: The unique identifier for the user. + + Returns: + User data object with all user information. + + Raises: + NotFoundError: If user doesn't exist. + DatabaseError: If database query fails. + """ + pass +``` + +## Testing Your Documentation + +To test your documentation locally: + +```bash +# Install the mkdocs environment +pixi install -e mkdocs + +# Serve the documentation locally +pixi run -e mkdocs mkdocs serve + +# Build the documentation (with strict mode) +pixi run -e mkdocs mkdocs-build +``` + +Then visit `http://127.0.0.1:8000` to see your documentation. + +## Excluding Private Members + +By default, members starting with `_` are excluded from documentation. To explicitly control what's documented: + +```python +class MyClass: + """Public class.""" + + def public_method(self): + """This will be documented.""" + pass + + def _private_method(self): + """This won't be documented (starts with _).""" + pass + + def __dunder_method__(self): + """This won't be documented (dunder method).""" + pass +``` + +## Additional Resources + +- [Google Style Docstrings Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) +- [Griffe Documentation](https://mkdocstrings.github.io/griffe/) +- [Griffe-Pydantic Documentation](https://mkdocstrings.github.io/griffe-pydantic/) +- [mkdocstrings Documentation](https://mkdocstrings.github.io/) diff --git a/mkdocs.yml b/mkdocs.yml index ab0bccb1a..fe71fc28b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -170,6 +170,52 @@ nav: - Designing functionality: dev/explanations/designing-functionality.md - Reference: - dev/reference/index.md + - API Reference: + - dev/reference/api/index.md + - Writing API Docs: dev/reference/api/writing-api-docs.md + - Core: + - dev/reference/api/core/index.md + - Models: dev/reference/api/core/models.md + - Settings: dev/reference/api/core/settings.md + - Preferences: dev/reference/api/core/preferences.md + - Configuration: dev/reference/api/core/config.md + - Exceptions: dev/reference/api/core/exceptions.md + - Resources: dev/reference/api/core/resources.md + - S3: dev/reference/api/core/s3.md + - Properties: dev/reference/api/core/properties.md + - Extensions: dev/reference/api/core/extensions.md + - Utilities: dev/reference/api/core/utils.md + - Routers: + - dev/reference/api/routers/index.md + - Jobs: dev/reference/api/routers/jobs.md + - Auth: dev/reference/api/routers/auth.md + - Configuration: dev/reference/api/routers/configuration.md + - Health: dev/reference/api/routers/health.md + - Access Policies: dev/reference/api/routers/access_policies.md + - Dependencies: dev/reference/api/routers/dependencies.md + - Factory: dev/reference/api/routers/factory.md + - FastAPI Classes: dev/reference/api/routers/fastapi_classes.md + - OpenTelemetry: dev/reference/api/routers/otel.md + - Utilities: dev/reference/api/routers/utils.md + - Logic: + - dev/reference/api/logic/index.md + - Jobs: dev/reference/api/logic/jobs.md + - Auth: dev/reference/api/logic/auth.md + - Task Queues: dev/reference/api/logic/task_queues.md + - Database: + - dev/reference/api/db/index.md + - Job DB: dev/reference/api/db/job.md + - Job Logging DB: dev/reference/api/db/job_logging.md + - Auth DB: dev/reference/api/db/auth.md + - Sandbox Metadata DB: dev/reference/api/db/sandbox_metadata.md + - Task Queue DB: dev/reference/api/db/task_queue.md + - Pilot Agents DB: dev/reference/api/db/pilot_agents.md + - Dummy DB: dev/reference/api/db/dummy.md + - OpenSearch: dev/reference/api/db/opensearch.md + - SQL Utilities: dev/reference/api/db/utils.md + - Exceptions: dev/reference/api/db/exceptions.md + - CLI: + - dev/reference/api/cli/index.md - Dev env variables: dev/reference/env-variables.md - Writing tests: dev/reference/writing-tests.md - Coding conventions: dev/reference/coding-conventions.md @@ -226,3 +272,29 @@ plugins: - search - mermaid2 - autorefs + - mkdocstrings: + enabled: true + default_handler: python + handlers: + python: + paths: [diracx-core/src, diracx-api/src, diracx-routers/src, diracx-db/src, diracx-logic/src, diracx-client/src, diracx-cli/src] + import: + - https://docs.python.org/3/objects.inv + - https://docs.pydantic.dev/latest/objects.inv + options: + docstring_style: google + docstring_section_style: table + show_source: true + show_root_heading: true + show_root_full_path: false + show_symbol_type_heading: true + show_symbol_type_toc: true + members_order: source + group_by_category: true + show_category_heading: true + show_if_no_docstring: true + inherited_members: false + filters: + - "!^_" + extensions: + - griffe_pydantic diff --git a/pixi.toml b/pixi.toml index f3a670153..abf2fc2f5 100644 --- a/pixi.toml +++ b/pixi.toml @@ -106,12 +106,24 @@ description = "Run the tests for gubbins-routers" # Features for generating the documentation [feature.mkdocs.dependencies] -python = "*" +python = ">=3.12,<3.13" [feature.mkdocs.pypi-dependencies] mkdocs-material = "*" mkdocs-mermaid2-plugin = "*" mkdocs-autorefs = "*" mkdocs-diracx-plugin = { git = "https://github.com/DIRACGrid/mkdocs-diracx-plugin.git", branch = "master"} +mkdocstrings = { version = ">=0.24", extras = ["python"] } +mkdocstrings-python = "*" +griffe = "*" +griffe-pydantic = "*" +# Install DiracX packages for documentation +diracx-core = { path = "diracx-core", editable = true } +diracx-api = { path = "diracx-api", editable = true } +diracx-routers = { path = "diracx-routers", editable = true } +diracx-db = { path = "diracx-db", editable = true } +diracx-logic = { path = "diracx-logic", editable = true } +diracx-client = { path = "diracx-client", editable = true } +diracx-cli = { path = "diracx-cli", editable = true } [feature.mkdocs.tasks] mkdocs = "mkdocs serve" mkdocs-build = "mkdocs build --strict" From 01a23866ce126dd463ac3d8dfaddae0cfc7856af Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 18:23:19 +0100 Subject: [PATCH 05/18] fix: check coverage --- docs/dev/reference/api/check_coverage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/dev/reference/api/check_coverage.py b/docs/dev/reference/api/check_coverage.py index e2d2e8c25..1dc95cf9e 100644 --- a/docs/dev/reference/api/check_coverage.py +++ b/docs/dev/reference/api/check_coverage.py @@ -53,7 +53,7 @@ def find_python_modules(package_name: str, package_path: Path) -> list[ModuleInf List of ModuleInfo objects for each Python module """ - modules = [] + modules: list[ModuleInfo] = [] if not package_path.exists(): print(f"Warning: Package path does not exist: {package_path}") @@ -83,7 +83,7 @@ def find_python_modules(package_name: str, package_path: Path) -> list[ModuleInf return sorted(modules, key=lambda m: m.module_path) -def get_expected_doc_path(module: ModuleInfo) -> Path: +def get_expected_doc_path(module: ModuleInfo) -> Path | None: """Get the expected documentation file path for a module. Args: @@ -140,7 +140,7 @@ def get_expected_doc_path(module: ModuleInfo) -> Path: return doc_path -def check_module_documented(module: ModuleInfo) -> tuple[bool, Path]: +def check_module_documented(module: ModuleInfo) -> tuple[bool, Path | None]: """Check if a module is documented. Args: From ae395721a4aa7afe40b19af9deabe539e3b8e850 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 18:30:02 +0100 Subject: [PATCH 06/18] fix: broken links --- docs/dev/reference/api/index.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/dev/reference/api/index.md b/docs/dev/reference/api/index.md index fe07b2681..c11032a4c 100644 --- a/docs/dev/reference/api/index.md +++ b/docs/dev/reference/api/index.md @@ -9,7 +9,10 @@ Core components including models, settings, configuration, and utilities. - **[Models](core/models.md)** - Core Pydantic models for data validation - **[Settings](core/settings.md)** - Configuration settings - **[Preferences](core/preferences.md)** - User preferences -- **[Config Schema](core/config-schema.md)** - Configuration schema definitions +- **[Configuration](core/config.md)** - Configuration schema and sources +- **[Exceptions](core/exceptions.md)** - Core exception classes +- **[Resources](core/resources.md)** - Resource management and dependency injection +- **[S3](core/s3.md)** - S3-compatible object storage integration - **[Properties](core/properties.md)** - Security properties - **[Extensions](core/extensions.md)** - Extension system - **[Utilities](core/utils.md)** - Core utilities @@ -22,6 +25,12 @@ FastAPI routers providing the REST API endpoints. - **[Auth](routers/auth.md)** - Authentication and authorization - **[Configuration](routers/configuration.md)** - Configuration management - **[Health](routers/health.md)** - Health check and monitoring +- **[Access Policies](routers/access_policies.md)** - Access control policies +- **[Dependencies](routers/dependencies.md)** - FastAPI dependency injection utilities +- **[Factory](routers/factory.md)** - Router factory functions +- **[FastAPI Classes](routers/fastapi_classes.md)** - Custom FastAPI router classes +- **[OpenTelemetry](routers/otel.md)** - Tracing and instrumentation +- **[Utilities](routers/utils.md)** - Router utilities ## [Logic](logic/index.md) @@ -32,13 +41,14 @@ Business logic layer providing service implementations and orchestration. Database models, schemas, and access layers. - **[Job DB](db/job.md)** - Job database -- **[Job Logging DB](db/job-logging.md)** - Job logging and history +- **[Job Logging DB](db/job_logging.md)** - Job logging and history - **[Auth DB](db/auth.md)** - Authentication and authorization -- **[Sandbox Metadata DB](db/sandbox-metadata.md)** - Sandbox file metadata -- **[Task Queue DB](db/task-queue.md)** - Task queue management -- **[Pilot Agents DB](db/pilot-agents.md)** - Pilot agent tracking +- **[Sandbox Metadata DB](db/sandbox_metadata.md)** - Sandbox file metadata +- **[Task Queue DB](db/task_queue.md)** - Task queue management +- **[Pilot Agents DB](db/pilot_agents.md)** - Pilot agent tracking +- **[Dummy DB](db/dummy.md)** - Dummy database for testing - **[OpenSearch](db/opensearch.md)** - OpenSearch-based databases -- **[SQL Utilities](db/sql-utils.md)** - SQL database utilities +- **[SQL Utilities](db/utils.md)** - SQL database utilities - **[Exceptions](db/exceptions.md)** - Database exceptions ## [CLI](cli/index.md) From 888f467afdb2b9bc7ab230473d4eb8de07f5f8a9 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 18:40:04 +0100 Subject: [PATCH 07/18] docs: add some docstrings to the JobDB to illustrate what a rendered API ref page looks like --- diracx-db/src/diracx/db/sql/job/db.py | 143 +++++++++++++++++++++----- 1 file changed, 119 insertions(+), 24 deletions(-) diff --git a/diracx-db/src/diracx/db/sql/job/db.py b/diracx-db/src/diracx/db/sql/job/db.py index ec31e7e9b..00f26c7fb 100644 --- a/diracx-db/src/diracx/db/sql/job/db.py +++ b/diracx-db/src/diracx/db/sql/job/db.py @@ -47,7 +47,16 @@ class JobDB(BaseSQLDB): async def summary( self, group_by: list[str], search: list[SearchSpec] ) -> list[dict[str, str | int]]: - """Get a summary of the jobs.""" + """Get a summary of jobs aggregated by specified fields. + + Args: + group_by: List of field names to group results by. + search: List of search specifications to filter jobs. + + Returns: + List of dictionaries containing grouped job statistics. + + """ return await self._summary(table=Jobs, group_by=group_by, search=search) async def search( @@ -60,7 +69,20 @@ async def search( per_page: int = 100, page: int | None = None, ) -> tuple[int, list[dict[Any, Any]]]: - """Search for jobs in the database.""" + """Search for jobs in the database matching specified criteria. + + Args: + parameters: List of field names to return, or None for all fields. + search: List of search specifications to filter jobs. + sorts: List of sort specifications for result ordering. + distinct: If True, return only distinct results. + per_page: Number of results per page. + page: Page number to return, or None for all results. + + Returns: + Tuple of (total_count, list of job dictionaries). + + """ return await self._search( table=Jobs, parameters=parameters, @@ -72,7 +94,15 @@ async def search( ) async def create_job(self, compressed_original_jdl: str): - """Used to insert a new job with original JDL. Returns inserted job id.""" + """Create a new job with its original JDL. + + Args: + compressed_original_jdl: The compressed original JDL string. + + Returns: + The inserted job ID. + + """ result = await self.conn.execute( JobJDLs.__table__.insert().values( JDL="", @@ -83,12 +113,22 @@ async def create_job(self, compressed_original_jdl: str): return result.lastrowid async def delete_jobs(self, job_ids: list[int]): - """Delete jobs from the database.""" + """Delete jobs and their associated JDLs from the database. + + Args: + job_ids: List of job IDs to delete. + + """ stmt = delete(JobJDLs).where(JobJDLs.job_id.in_(job_ids)) await self.conn.execute(stmt) async def insert_input_data(self, lfns: dict[int, list[str]]): - """Insert input data for jobs.""" + """Insert input data LFNs for jobs. + + Args: + lfns: Mapping of job IDs to lists of logical file names (LFNs). + + """ await self.conn.execute( InputData.__table__.insert(), [ @@ -102,7 +142,12 @@ async def insert_input_data(self, lfns: dict[int, list[str]]): ) async def insert_job_attributes(self, jobs_to_update: dict[int, dict]): - """Insert the job attributes.""" + """Insert job attributes for newly created jobs. + + Args: + jobs_to_update: Mapping of job IDs to their attribute dictionaries. + + """ await self.conn.execute( Jobs.__table__.insert(), [ @@ -115,7 +160,14 @@ async def insert_job_attributes(self, jobs_to_update: dict[int, dict]): ) async def update_job_jdls(self, jdls_to_update: dict[int, str]): - """Used to update the JDL, typically just after inserting the original JDL, or rescheduling, for example.""" + """Update the JDL for existing jobs. + + Typically used just after inserting the original JDL or when rescheduling. + + Args: + jdls_to_update: Mapping of job IDs to their compressed JDL strings. + + """ await self.conn.execute( JobJDLs.__table__.update().where( JobJDLs.__table__.c.JobID == bindparam("b_JobID") @@ -130,7 +182,17 @@ async def update_job_jdls(self, jdls_to_update: dict[int, str]): ) async def set_job_attributes(self, job_data): - """Update the parameters of the given jobs.""" + """Update the parameters of the given jobs. + + Automatically updates LastUpdateTime when Status is changed. + + Args: + job_data: Mapping of job IDs to their attribute dictionaries. + + Raises: + ValueError: If job_data is empty. + + """ # TODO: add myDate and force parameters. if not job_data: @@ -170,7 +232,16 @@ async def set_job_attributes(self, job_data): await self.conn.execute(stmt) async def get_job_jdls(self, job_ids, original: bool = False) -> dict[int, str]: - """Get the JDLs for the given jobs.""" + """Get the JDLs for the given jobs. + + Args: + job_ids: List of job IDs to retrieve JDLs for. + original: If True, return the original JDL, otherwise return the processed JDL. + + Returns: + Mapping of job IDs to their JDL strings. + + """ if original: stmt = select(JobJDLs.job_id, JobJDLs.original_jdl).where( JobJDLs.job_id.in_(job_ids) @@ -183,7 +254,12 @@ async def get_job_jdls(self, job_ids, original: bool = False) -> dict[int, str]: return {jobid: jdl for jobid, jdl in (await self.conn.execute(stmt)) if jdl} async def set_job_commands(self, commands: list[tuple[int, str, str]]) -> None: - """Store a command to be passed to the job together with the next heart beat.""" + """Store commands to be passed to jobs with the next heartbeat. + + Args: + commands: List of tuples containing (job_id, command, arguments). + + """ await self.conn.execute( JobCommands.__table__.insert(), [ @@ -200,13 +276,20 @@ async def set_job_commands(self, commands: list[tuple[int, str, str]]) -> None: async def set_properties( self, properties: dict[int, dict[str, Any]], update_timestamp: bool = False ) -> int: - """Update the job parameters - All the jobs must update the same properties. + """Update job properties in bulk. + + All jobs must update the same set of properties. + + Args: + properties: Mapping of job IDs to property dictionaries. + Example: {job_id: {prop1: val1, prop2: val2}}. + update_timestamp: If True, update the LastUpdateTime to now. - :param properties: {job_id : {prop1: val1, prop2:val2} - :param update_timestamp: if True, update the LastUpdate to now + Returns: + Number of rows updated. - :return rowcount + Raises: + NotImplementedError: If jobs attempt to update different sets of properties. """ # Check that all we always update the same set of properties @@ -237,13 +320,19 @@ async def add_heartbeat_data( ) -> None: """Add the job's heartbeat data to the database. - NOTE: This does not update the HeartBeatTime column in the Jobs table. - This is instead handled by the `diracx.logic.jobs.status.set_job_statuses` - as it involves updating multiple databases. + Note: + This does not update the HeartBeatTime column in the Jobs table. + This is instead handled by diracx.logic.jobs.status.set_job_statuses + as it involves updating multiple databases. + + Args: + job_id: The job ID. + dynamic_data: Mapping of dynamic data to store. + Example: {"AvailableDiskSpace": "123"}. + + Raises: + InvalidQueryError: If dynamic_data contains fields not in heartbeat_fields. - :param job_id: the job id - :param dynamic_data: mapping of the dynamic data to store, - e.g. {"AvailableDiskSpace": 123} """ if extra_fields := set(dynamic_data) - self.heartbeat_fields: raise InvalidQueryError( @@ -262,10 +351,16 @@ async def add_heartbeat_data( await self.conn.execute(HeartBeatLoggingInfo.__table__.insert().values(values)) async def get_job_commands(self, job_ids: Iterable[int]) -> list[JobCommand]: - """Get a command to be passed to the job together with the next heartbeat. + """Get commands to be passed to jobs with the next heartbeat. + + Commands are marked as "Sent" after retrieval. + + Args: + job_ids: The job IDs to get commands for. + + Returns: + List of JobCommand objects containing job_id, command, and arguments. - :param job_ids: the job ids - :return: mapping of job id to list of commands """ # Get the commands stmt = ( From f0bdfaf3bf7f587aeadbfd942e1c7d1952d19426 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 18:49:21 +0100 Subject: [PATCH 08/18] fix: griffe warnings --- diracx-db/src/diracx/db/sql/job/db.py | 8 +++++--- docs/dev/reference/api/logic/index.md | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/diracx-db/src/diracx/db/sql/job/db.py b/diracx-db/src/diracx/db/sql/job/db.py index 00f26c7fb..2eded2065 100644 --- a/diracx-db/src/diracx/db/sql/job/db.py +++ b/diracx-db/src/diracx/db/sql/job/db.py @@ -93,7 +93,7 @@ async def search( page=page, ) - async def create_job(self, compressed_original_jdl: str): + async def create_job(self, compressed_original_jdl: str) -> int: """Create a new job with its original JDL. Args: @@ -181,7 +181,7 @@ async def update_job_jdls(self, jdls_to_update: dict[int, str]): ], ) - async def set_job_attributes(self, job_data): + async def set_job_attributes(self, job_data: dict[int, dict[str, Any]]) -> None: """Update the parameters of the given jobs. Automatically updates LastUpdateTime when Status is changed. @@ -231,7 +231,9 @@ async def set_job_attributes(self, job_data): ) await self.conn.execute(stmt) - async def get_job_jdls(self, job_ids, original: bool = False) -> dict[int, str]: + async def get_job_jdls( + self, job_ids: Iterable[int], original: bool = False + ) -> dict[int, str]: """Get the JDLs for the given jobs. Args: diff --git a/docs/dev/reference/api/logic/index.md b/docs/dev/reference/api/logic/index.md index c71b6d9fd..8212d63d2 100644 --- a/docs/dev/reference/api/logic/index.md +++ b/docs/dev/reference/api/logic/index.md @@ -9,4 +9,4 @@ validation, and complex operations. - [Jobs Logic](jobs.md) - Job submission, querying, status management, and sandboxes - [Auth Logic](auth.md) - Authentication and authorization flows -- [Task Queues Logic](task-queues.md) - Task queue management and priority +- [Task Queues Logic](task_queues.md) - Task queue management and priority From bbfbe7de594613140662a6067c7dd51ee62e610c Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Fri, 7 Nov 2025 18:53:51 +0100 Subject: [PATCH 09/18] fix: last warning --- docs/dev/reference/api/{README.md => CONTRIBUTING.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/dev/reference/api/{README.md => CONTRIBUTING.md} (100%) diff --git a/docs/dev/reference/api/README.md b/docs/dev/reference/api/CONTRIBUTING.md similarity index 100% rename from docs/dev/reference/api/README.md rename to docs/dev/reference/api/CONTRIBUTING.md From 0114c61c4a91c516652fde98d80b14dff615615f Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 00:46:02 +0100 Subject: [PATCH 10/18] fix: use a better example ... --- docs/dev/reference/api/writing-api-docs.md | 37 ++++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/docs/dev/reference/api/writing-api-docs.md b/docs/dev/reference/api/writing-api-docs.md index eedc19471..1b5fb1df5 100644 --- a/docs/dev/reference/api/writing-api-docs.md +++ b/docs/dev/reference/api/writing-api-docs.md @@ -358,23 +358,40 @@ def process_config(config: dict[str, dict[str, list[str]]]) -> ProcessedConfig: Examples help users understand how to use your code: ```python -def format_job_id(vo: str, job_number: int) -> str: - """Format a job ID string. +def recursive_merge(base: Any, override: Any) -> Any: + """Recursively merge dictionaries; values in ``override`` take precedence. + + - If both ``base`` and ``override`` are dicts, merge keys recursively. + - Otherwise, return ``override`` if it is not ``None``; fallback to ``base``. Args: - vo: Virtual organization name. - job_number: Job number. + base: Base dictionary or value to merge. + override: Override dictionary or value to merge. Values here take precedence. Returns: - Formatted job ID string. + The merged result. Examples: - >>> format_job_id("lhcb", 12345) - 'lhcb:12345' - >>> format_job_id("atlas", 1) - 'atlas:00001' + >>> from diracx.core.utils import recursive_merge + >>> base = {"a": 1, "b": {"c": 2, "d": 3}} + >>> override = {"b": {"c": 10}, "e": 4} + >>> recursive_merge(base, override) + {'a': 1, 'b': {'c': 10, 'd': 3}, 'e': 4} + >>> recursive_merge(None, {"key": "value"}) + {'key': 'value'} """ - return f"{vo}:{job_number:05d}" + if isinstance(base, dict) and isinstance(override, dict): + merged: dict[str, Any] = {} + for key, base_val in base.items(): + if key in override: + merged[key] = recursive_merge(base_val, override[key]) + else: + merged[key] = base_val + for key, override_val in override.items(): + if key not in merged: + merged[key] = override_val + return merged + return override if override is not None else base ``` ### 4. Document Async Functions From 170f59bd608a834c57d1782aef385d85662e3bbe Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 00:52:39 +0100 Subject: [PATCH 11/18] fix: remove python ver constraint on mkdocs env --- pixi.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixi.toml b/pixi.toml index abf2fc2f5..3b562958b 100644 --- a/pixi.toml +++ b/pixi.toml @@ -106,7 +106,7 @@ description = "Run the tests for gubbins-routers" # Features for generating the documentation [feature.mkdocs.dependencies] -python = ">=3.12,<3.13" +python = "*" [feature.mkdocs.pypi-dependencies] mkdocs-material = "*" mkdocs-mermaid2-plugin = "*" From 88f39b76c2ceda83d483cfa6b6a1982f4fc8f28f Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 01:15:37 +0100 Subject: [PATCH 12/18] fix: bad options --- docs/dev/reference/api/cli/index.md | 72 ++++++++-------- docs/dev/reference/api/core/config.md | 24 +++--- docs/dev/reference/api/core/exceptions.md | 12 +-- docs/dev/reference/api/core/resources.md | 12 +-- docs/dev/reference/api/core/s3.md | 12 +-- docs/dev/reference/api/db/auth.md | 24 +++--- docs/dev/reference/api/db/dummy.md | 24 +++--- docs/dev/reference/api/db/job.md | 24 +++--- docs/dev/reference/api/db/job_logging.md | 24 +++--- docs/dev/reference/api/db/opensearch.md | 24 +++--- docs/dev/reference/api/db/pilot_agents.md | 24 +++--- docs/dev/reference/api/db/sandbox_metadata.md | 24 +++--- docs/dev/reference/api/db/task_queue.md | 24 +++--- docs/dev/reference/api/db/utils.md | 36 ++++---- docs/dev/reference/api/logic/auth.md | 72 ++++++++-------- docs/dev/reference/api/logic/jobs.md | 60 ++++++------- docs/dev/reference/api/logic/task_queues.md | 12 +-- .../reference/api/routers/access_policies.md | 12 +-- docs/dev/reference/api/routers/auth.md | 84 +++++++++---------- .../reference/api/routers/configuration.md | 12 +-- .../dev/reference/api/routers/dependencies.md | 12 +-- docs/dev/reference/api/routers/factory.md | 12 +-- .../reference/api/routers/fastapi_classes.md | 12 +-- docs/dev/reference/api/routers/health.md | 24 +++--- docs/dev/reference/api/routers/jobs.md | 12 +-- docs/dev/reference/api/routers/otel.md | 12 +-- docs/dev/reference/api/routers/utils.md | 12 +-- 27 files changed, 354 insertions(+), 354 deletions(-) diff --git a/docs/dev/reference/api/cli/index.md b/docs/dev/reference/api/cli/index.md index b4a007b6a..579d20134 100644 --- a/docs/dev/reference/api/cli/index.md +++ b/docs/dev/reference/api/cli/index.md @@ -5,61 +5,61 @@ DiracX command-line interface modules and commands. ## Jobs Commands ::: diracx.cli.jobs -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Auth Commands ::: diracx.cli.auth -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Config Commands ::: diracx.cli.config -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Internal Commands ### Legacy Commands ::: diracx.cli.internal.legacy -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ### Internal Config ::: diracx.cli.internal.config -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## CLI Utilities ::: diracx.cli.utils -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/config.md b/docs/dev/reference/api/core/config.md index 0437f66d1..f09c08c1a 100644 --- a/docs/dev/reference/api/core/config.md +++ b/docs/dev/reference/api/core/config.md @@ -5,19 +5,19 @@ Configuration schema and sources for DiracX. ## Config Schema ::: diracx.core.config.schema -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Config Sources ::: diracx.core.config.sources -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/exceptions.md b/docs/dev/reference/api/core/exceptions.md index 561b87018..adbc0c774 100644 --- a/docs/dev/reference/api/core/exceptions.md +++ b/docs/dev/reference/api/core/exceptions.md @@ -3,9 +3,9 @@ Core exception classes used throughout DiracX. ::: diracx.core.exceptions -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/resources.md b/docs/dev/reference/api/core/resources.md index 2fa3d509f..a7fb361e1 100644 --- a/docs/dev/reference/api/core/resources.md +++ b/docs/dev/reference/api/core/resources.md @@ -3,9 +3,9 @@ Resource management and dependency injection utilities. ::: diracx.core.resources -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/core/s3.md b/docs/dev/reference/api/core/s3.md index 892d3f1af..6807e363e 100644 --- a/docs/dev/reference/api/core/s3.md +++ b/docs/dev/reference/api/core/s3.md @@ -3,9 +3,9 @@ S3-compatible object storage integration utilities. ::: diracx.core.s3 -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/auth.md b/docs/dev/reference/api/db/auth.md index 4880d6899..0370d1945 100644 --- a/docs/dev/reference/api/db/auth.md +++ b/docs/dev/reference/api/db/auth.md @@ -5,19 +5,19 @@ Authentication and authorization database. ## Database Schema ::: diracx.db.sql.auth.schema -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Database Access Layer ::: diracx.db.sql.auth.db -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/dummy.md b/docs/dev/reference/api/db/dummy.md index ddb54c406..fa4f22958 100644 --- a/docs/dev/reference/api/db/dummy.md +++ b/docs/dev/reference/api/db/dummy.md @@ -5,19 +5,19 @@ Dummy database for testing and development purposes. ## Schema ::: diracx.db.sql.dummy.schema -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Database Access ::: diracx.db.sql.dummy.db -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/job.md b/docs/dev/reference/api/db/job.md index 1c2376e2c..6636ca86e 100644 --- a/docs/dev/reference/api/db/job.md +++ b/docs/dev/reference/api/db/job.md @@ -5,19 +5,19 @@ Job database models and access layer. ## Database Schema ::: diracx.db.sql.job.schema -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Database Access Layer ::: diracx.db.sql.job.db -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/job_logging.md b/docs/dev/reference/api/db/job_logging.md index 235caba98..4efe9a6e5 100644 --- a/docs/dev/reference/api/db/job_logging.md +++ b/docs/dev/reference/api/db/job_logging.md @@ -5,19 +5,19 @@ Job logging and history database. ## Database Schema ::: diracx.db.sql.job_logging.schema -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Database Access Layer ::: diracx.db.sql.job_logging.db -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/opensearch.md b/docs/dev/reference/api/db/opensearch.md index 4ffbc4ea0..ec5284354 100644 --- a/docs/dev/reference/api/db/opensearch.md +++ b/docs/dev/reference/api/db/opensearch.md @@ -5,19 +5,19 @@ OpenSearch-based database implementations. ## Job Parameters ::: diracx.db.os.job_parameters -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Utilities ::: diracx.db.os.utils -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/pilot_agents.md b/docs/dev/reference/api/db/pilot_agents.md index 4ff448567..fb059ab40 100644 --- a/docs/dev/reference/api/db/pilot_agents.md +++ b/docs/dev/reference/api/db/pilot_agents.md @@ -5,19 +5,19 @@ Pilot agent tracking and management database. ## Database Schema ::: diracx.db.sql.pilot_agents.schema -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Database Access Layer ::: diracx.db.sql.pilot_agents.db -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/sandbox_metadata.md b/docs/dev/reference/api/db/sandbox_metadata.md index 334db230c..1aa0e741c 100644 --- a/docs/dev/reference/api/db/sandbox_metadata.md +++ b/docs/dev/reference/api/db/sandbox_metadata.md @@ -5,19 +5,19 @@ Sandbox file metadata and tracking database. ## Database Schema ::: diracx.db.sql.sandbox_metadata.schema -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Database Access Layer ::: diracx.db.sql.sandbox_metadata.db -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/task_queue.md b/docs/dev/reference/api/db/task_queue.md index 7f3ebccd9..f657563bf 100644 --- a/docs/dev/reference/api/db/task_queue.md +++ b/docs/dev/reference/api/db/task_queue.md @@ -5,19 +5,19 @@ Task queue management database. ## Database Schema ::: diracx.db.sql.task_queue.schema -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Database Access Layer ::: diracx.db.sql.task_queue.db -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/db/utils.md b/docs/dev/reference/api/db/utils.md index 085c010b4..e8961d2f4 100644 --- a/docs/dev/reference/api/db/utils.md +++ b/docs/dev/reference/api/db/utils.md @@ -5,29 +5,29 @@ Utilities for SQL database operations. ## Base Classes ::: diracx.db.sql.utils.base -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Functions ::: diracx.db.sql.utils.functions -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Types ::: diracx.db.sql.utils.types -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/logic/auth.md b/docs/dev/reference/api/logic/auth.md index fcc8e3bba..f2d5c0bfb 100644 --- a/docs/dev/reference/api/logic/auth.md +++ b/docs/dev/reference/api/logic/auth.md @@ -5,59 +5,59 @@ Authentication and authorization business logic. ## Token Management ::: diracx.logic.auth.token -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Authorization Code Flow ::: diracx.logic.auth.authorize_code_flow -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Device Flow ::: diracx.logic.auth.device_flow -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## User Management ::: diracx.logic.auth.management -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Well Known Endpoints ::: diracx.logic.auth.well_known -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Auth Utilities ::: diracx.logic.auth.utils -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/logic/jobs.md b/docs/dev/reference/api/logic/jobs.md index 8ab72a06a..8ac78acfd 100644 --- a/docs/dev/reference/api/logic/jobs.md +++ b/docs/dev/reference/api/logic/jobs.md @@ -5,49 +5,49 @@ Job-related business logic including submission, querying, status management, an ## Job Submission ::: diracx.logic.jobs.submission -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Job Query ::: diracx.logic.jobs.query -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Job Status ::: diracx.logic.jobs.status -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Job Sandboxes ::: diracx.logic.jobs.sandboxes -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Job Utilities ::: diracx.logic.jobs.utils -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/logic/task_queues.md b/docs/dev/reference/api/logic/task_queues.md index 0e731886d..f938c6660 100644 --- a/docs/dev/reference/api/logic/task_queues.md +++ b/docs/dev/reference/api/logic/task_queues.md @@ -5,9 +5,9 @@ Task queue management and priority logic. ## Priority Management ::: diracx.logic.task_queues.priority -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/access_policies.md b/docs/dev/reference/api/routers/access_policies.md index 24f8a4748..e5d7f2eb5 100644 --- a/docs/dev/reference/api/routers/access_policies.md +++ b/docs/dev/reference/api/routers/access_policies.md @@ -3,9 +3,9 @@ Access control policies for DiracX routers. ::: diracx.routers.access_policies -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/auth.md b/docs/dev/reference/api/routers/auth.md index 6e341bf82..5eff90b5c 100644 --- a/docs/dev/reference/api/routers/auth.md +++ b/docs/dev/reference/api/routers/auth.md @@ -3,12 +3,12 @@ Authentication and authorization endpoints. ::: diracx.routers.auth -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true filters: \- "!^\_" \- "!^logger" @@ -16,59 +16,59 @@ filters: ## Token Management ::: diracx.routers.auth.token -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Authorization Code Flow ::: diracx.routers.auth.authorize_code_flow -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Device Flow ::: diracx.routers.auth.device_flow -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Management ::: diracx.routers.auth.management -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Well Known ::: diracx.routers.auth.well_known -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true ## Utilities ::: diracx.routers.auth.utils -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/configuration.md b/docs/dev/reference/api/routers/configuration.md index 5e3cdf273..2af11ab93 100644 --- a/docs/dev/reference/api/routers/configuration.md +++ b/docs/dev/reference/api/routers/configuration.md @@ -3,12 +3,12 @@ Configuration management endpoints. ::: diracx.routers.configuration -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true filters: \- "!^\_" \- "!^logger" diff --git a/docs/dev/reference/api/routers/dependencies.md b/docs/dev/reference/api/routers/dependencies.md index c84baed73..4c863514f 100644 --- a/docs/dev/reference/api/routers/dependencies.md +++ b/docs/dev/reference/api/routers/dependencies.md @@ -3,9 +3,9 @@ FastAPI dependency injection utilities and common dependencies. ::: diracx.routers.dependencies -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/factory.md b/docs/dev/reference/api/routers/factory.md index f5c5c1606..81b58a793 100644 --- a/docs/dev/reference/api/routers/factory.md +++ b/docs/dev/reference/api/routers/factory.md @@ -3,9 +3,9 @@ Factory functions and utilities for creating DiracX routers. ::: diracx.routers.factory -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/fastapi_classes.md b/docs/dev/reference/api/routers/fastapi_classes.md index 5bccd0f39..f9f092c61 100644 --- a/docs/dev/reference/api/routers/fastapi_classes.md +++ b/docs/dev/reference/api/routers/fastapi_classes.md @@ -3,9 +3,9 @@ Custom FastAPI router and application classes for DiracX. ::: diracx.routers.fastapi_classes -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/health.md b/docs/dev/reference/api/routers/health.md index 133b45fa5..7b4582774 100644 --- a/docs/dev/reference/api/routers/health.md +++ b/docs/dev/reference/api/routers/health.md @@ -3,23 +3,23 @@ Health check and monitoring endpoints for service status. ::: diracx.routers.health -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true filters: \- "!^\_" ## Health Probes ::: diracx.routers.health.probes -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true filters: \- "!^\_" diff --git a/docs/dev/reference/api/routers/jobs.md b/docs/dev/reference/api/routers/jobs.md index 1794566ee..8f46edfb0 100644 --- a/docs/dev/reference/api/routers/jobs.md +++ b/docs/dev/reference/api/routers/jobs.md @@ -12,12 +12,12 @@ The Jobs router is composed of multiple sub-routers: ## Router ::: diracx.routers.jobs -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true filters: \- "!^\_" \- "!^logger" diff --git a/docs/dev/reference/api/routers/otel.md b/docs/dev/reference/api/routers/otel.md index 15e325fbb..e40af0d41 100644 --- a/docs/dev/reference/api/routers/otel.md +++ b/docs/dev/reference/api/routers/otel.md @@ -3,9 +3,9 @@ OpenTelemetry instrumentation and tracing utilities for DiracX routers. ::: diracx.routers.otel -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true diff --git a/docs/dev/reference/api/routers/utils.md b/docs/dev/reference/api/routers/utils.md index 35d232a26..cd2b596b6 100644 --- a/docs/dev/reference/api/routers/utils.md +++ b/docs/dev/reference/api/routers/utils.md @@ -5,9 +5,9 @@ Utility functions for DiracX routers. ## User Utilities ::: diracx.routers.utils.users -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true -show_if_no_docstring: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true + show_if_no_docstring: true From 5fd41e89dd415cde23db89eceb3a1019e79ec4da Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 02:34:01 +0100 Subject: [PATCH 13/18] feat: entrypoints docs --- docs/dev/reference/entrypoints.md | 320 ++++++++++++++++-- mkdocs.yml | 2 + pixi.toml | 1 + scripts/generate_entrypoints_docs.py | 447 +++++++++++++++++++++++++ scripts/regenerate_entrypoints_docs.sh | 16 + 5 files changed, 764 insertions(+), 22 deletions(-) create mode 100644 scripts/generate_entrypoints_docs.py create mode 100755 scripts/regenerate_entrypoints_docs.sh diff --git a/docs/dev/reference/entrypoints.md b/docs/dev/reference/entrypoints.md index fcba6f569..f83a1909d 100644 --- a/docs/dev/reference/entrypoints.md +++ b/docs/dev/reference/entrypoints.md @@ -1,42 +1,318 @@ -# Entrypoints +# DiracX Entry Points Reference -This page documents the entrypoints used in this project. +This document catalogs all available entry points for creating DiracX extensions. +Entry points are defined in `pyproject.toml` files and discovered at runtime. -## `diracx-cli` +## Table of Contents -The `diracx-cli` package provides the `diracx` command-line interface. +- [Core Extension Registration](#diracx) +- [Access Policy Registration](#diracxaccess-policies) +- [CLI Command Registration](#diracxcli) +- [Hidden CLI Commands](#diracxclihidden) +- [OpenSearch Database Registration](#diracxdbsos) +- [SQL Database Registration](#diracxdbssql) +- [Minimum Client Version Declaration](#diracxmin-client-version) +- [Resource Management Functions](#diracxresources) +- [FastAPI Router Registration](#diracxservices) -### `[project.scripts]` +## Core Extension Registration -- `dirac = "diracx.cli:app"`: This is the main entry point for the `diracx` command-line interface. It calls the `app` function in `diracx/cli/__init__.py`. +**Entry Point Group**: `diracx` -### `[project.entry-points."diracx.cli"]` +The base entry point group for registering DiracX extensions. Extensions MUST register themselves here. -These entry points are for the subcommands of the `diracx` command-line interface. +### Entry Point Keys -- `jobs = "diracx.cli.jobs:app"`: This is an entry point for the `diracx jobs` subcommand. It calls the `app` function in `diracx/cli/jobs.py`. -- `config = "diracx.cli.config:app"`: This is an entry point for the `diracx config` subcommand. It calls the `app` function in `diracx/cli/config.py`. +- **`extension`**: Extension name (required for all extensions) +- **`properties_module`**: Module path to custom DIRAC properties +- **`config`**: Path to extended configuration schema class -### `[project.entry-points."diracx.cli.hidden"]` +### Usage Example -These entry points are for hidden subcommands of the `diracx` command-line interface. +```toml +[project.entry-points."diracx"] +extension = "myextension" +properties_module = "myextension.core.properties" +config = "myextension.core.config.schema:Config" +``` -- `internal = "diracx.cli.internal:app"`: This is a hidden entry point for the `diracx internal` subcommand. It calls the `app` function in `diracx/cli/internal.py`. +### Important Notes -## `gubbins-cli` +- The `extension` key is **required** for all extensions +- Extensions are prioritized by name (alphabetically, with 'diracx' last) +- Only one extension can be installed alongside DiracX core -The `gubbins-cli` package provides the `gubbins` command-line interface. +### Current Implementations -### `[project.scripts]` +| Package | Entry Name | Entry Point | +| ---------------- | ------------------- | -------------------------------------------- | +| `diracx-core` | `config` | `diracx.core.config.schema:Config` | +| `diracx-core` | `extension` | `diracx` | +| `diracx-core` | `properties_module` | `diracx.core.properties` | +| `gubbins-client` | `aio_client_class` | `gubbins.client.generated.aio._client:Dirac` | +| `gubbins-client` | `client_class` | `gubbins.client.generated._client:Dirac` | +| `gubbins-core` | `config` | `gubbins.core.config.schema:Config` | +| `gubbins-core` | `extension` | `gubbins` | +| `gubbins-core` | `properties_module` | `gubbins.core.properties` | -- `gubbins = "gubbins.cli:app"`: This is the main entry point for the `gubbins` command-line interface. It calls the `app` function in `gubbins/cli/__init__.py`. +## Access Policy Registration -### `[project.entry-points."diracx.cli"]` +**Entry Point Group**: `diracx.access_policies` -This entry point extends the `diracx` command with a `security` subcommand. +Register custom access policies for fine-grained authorization control. Policies can inject claims into tokens and check permissions at runtime. -- `security = "gubbins.cli.security:app"`: This is an entry point for the `diracx security` subcommand. It calls the `app` function in `gubbins/cli/security.py`. +### Entry Point Keys -## Extending the `diracx` command +- **``**: Path to BaseAccessPolicy subclass -The `diracx.cli` entrypoint group allows extending the `diracx` command with subcommands from other packages. This is used by the `gubbins-cli` package to add the `security` subcommand to the `diracx` command. +### Usage Example + +```toml +[project.entry-points."diracx.access_policies"] +WMSAccessPolicy = "myextension.routers.jobs.access_policy:WMSAccessPolicy" +CustomPolicy = "myextension.routers.custom.policy:CustomAccessPolicy" +``` + +### Important Notes + +- Policies must inherit from `BaseAccessPolicy` +- Each route must call its policy or use `@open_access` decorator +- Policies can inject data during token generation via `policy_name` claim +- CI test `test_all_routes_have_policy` enforces policy usage + +### Current Implementations + +| Package | Entry Name | Entry Point | +| ----------------- | ---------------------- | ------------------------------------------------------------- | +| `diracx-routers` | `SandboxAccessPolicy` | `diracx.routers.jobs.access_policies:SandboxAccessPolicy` | +| `diracx-routers` | `WMSAccessPolicy` | `diracx.routers.jobs.access_policies:WMSAccessPolicy` | +| `gubbins-routers` | `LollygagAccessPolicy` | `gubbins.routers.lollygag.access_policy:LollygagAccessPolicy` | + +## CLI Command Registration + +**Entry Point Group**: `diracx.cli` + +Register Typer applications as subcommands of the `dirac` CLI. Extensions can add new subcommands or extend existing ones. + +### Entry Point Keys + +- **``**: Path to Typer app (e.g., 'myext.cli.jobs:app') + +### Usage Example + +```toml +[project.entry-points."diracx.cli"] +jobs = "myextension.cli.jobs:app" # Override core 'dirac jobs' command +mycmd = "myextension.cli.custom:app" # Add 'dirac mycmd' command +``` + +### Important Notes + +- Commands are automatically integrated into the main `dirac` CLI +- Extensions can completely replace core commands by using the same name +- Use `@app.async_command()` for async operations +- Follows standard Typer patterns for argument/option parsing + +### Current Implementations + +| Package | Entry Name | Entry Point | +| ------------- | ---------- | -------------------------- | +| `diracx-cli` | `config` | `diracx.cli.config:app` | +| `diracx-cli` | `jobs` | `diracx.cli.jobs:app` | +| `gubbins-cli` | `config` | `gubbins.cli.config:app` | +| `gubbins-cli` | `lollygag` | `gubbins.cli.lollygag:app` | + +## Hidden CLI Commands + +**Entry Point Group**: `diracx.cli.hidden` + +Register CLI commands that should not appear in help text. Used for internal/debugging commands. + +### Entry Point Keys + +- **``**: Path to Typer app for hidden command + +### Usage Example + +```toml +[project.entry-points."diracx.cli.hidden"] +internal = "myextension.cli.internal:app" +debug = "myextension.cli.debug:app" +``` + +### Important Notes + +- Commands are functional but don't appear in `dirac --help` +- Useful for debugging tools and internal utilities + +### Current Implementations + +| Package | Entry Name | Entry Point | +| ------------ | ---------- | ------------------------- | +| `diracx-cli` | `internal` | `diracx.cli.internal:app` | + +## OpenSearch Database Registration + +**Entry Point Group**: `diracx.dbs.os` + +Register OpenSearch/Elasticsearch database classes for log and parameter storage. Connection parameters configured via `DIRACX_OS_DB__*` environment variables. + +### Entry Point Keys + +- **``**: Path to BaseOSDB subclass (e.g., 'myext.db.os.jobs:JobParametersDB') + +### Usage Example + +```toml +[project.entry-points."diracx.dbs.os"] +JobParametersDB = "myextension.db.os.jobs:ExtendedJobParametersDB" +``` + +### Important Notes + +- Database classes must inherit from `BaseOSDB` +- No automatic transaction management (unlike SQL databases) +- Connection pooling is handled by AsyncOpenSearch client + +### Current Implementations + +| Package | Entry Name | Entry Point | +| ----------- | ----------------- | ------------------------------ | +| `diracx-db` | `JobParametersDB` | `diracx.db.os:JobParametersDB` | + +## SQL Database Registration + +**Entry Point Group**: `diracx.dbs.sql` + +Register SQL database classes using SQLAlchemy. Database URLs are configured via `DIRACX_DB_URL_` environment variables. + +### Entry Point Keys + +- **``**: Path to BaseSQLDB subclass (e.g., 'myext.db.sql.jobs:JobDB') + +### Usage Example + +```toml +[project.entry-points."diracx.dbs.sql"] +JobDB = "myextension.db.sql.jobs:ExtendedJobDB" +MyCustomDB = "myextension.db.sql.custom:MyCustomDB" +``` + +### Important Notes + +- Database classes must inherit from `BaseSQLDB` +- Use `@declared_attr` for tables to support extension inheritance +- Transactions are auto-managed: commit on success, rollback on errors +- Connection pooling is automatic via SQLAlchemy + +### Current Implementations + +| Package | Entry Name | Entry Point | +| ------------ | ------------------- | --------------------------------- | +| `diracx-db` | `AuthDB` | `diracx.db.sql:AuthDB` | +| `diracx-db` | `JobDB` | `diracx.db.sql:JobDB` | +| `diracx-db` | `JobLoggingDB` | `diracx.db.sql:JobLoggingDB` | +| `diracx-db` | `PilotAgentsDB` | `diracx.db.sql:PilotAgentsDB` | +| `diracx-db` | `SandboxMetadataDB` | `diracx.db.sql:SandboxMetadataDB` | +| `diracx-db` | `TaskQueueDB` | `diracx.db.sql:TaskQueueDB` | +| `gubbins-db` | `JobDB` | `gubbins.db.sql:GubbinsJobDB` | +| `gubbins-db` | `LollygagDB` | `gubbins.db.sql:LollygagDB` | + +## Minimum Client Version Declaration + +**Entry Point Group**: `diracx.min_client_version` + +Declare the minimum compatible client version for the server. Used to prevent compatibility issues between client and server. + +### Entry Point Keys + +- **`diracx`**: Variable name containing version string (e.g., 'myext.routers:MIN_VERSION') + +### Usage Example + +```toml +[project.entry-points."diracx.min_client_version"] +myextension = "myextension.routers:MYEXT_MIN_CLIENT_VERSION" +``` + +### Important Notes + +- Extensions take priority over 'diracx' entry point +- Version string should follow semantic versioning +- Server rejects requests from clients below minimum version + +### Current Implementations + +| Package | Entry Name | Entry Point | +| ---------------- | ---------- | ------------------------------------------ | +| `diracx-routers` | `diracx` | `diracx.routers:DIRACX_MIN_CLIENT_VERSION` | + +## Resource Management Functions + +**Entry Point Group**: `diracx.resources` + +Register functions that can be overridden by extensions to customize resource management behavior (e.g., platform compatibility). + +### Entry Point Keys + +- **`find_compatible_platforms`**: Function to determine platform compatibility + +### Usage Example + +```toml +[project.entry-points."diracx.resources"] +find_compatible_platforms = "myext.core.resources:find_compatible_platforms" +``` + +### Important Notes + +- Uses `@supports_extending` decorator pattern +- Extension implementations automatically override core functions +- Useful for site-specific resource matching logic + +### Current Implementations + +| Package | Entry Name | Entry Point | +| ------------- | --------------------------- | ------------------------------------------------- | +| `diracx-core` | `find_compatible_platforms` | `diracx.core.resources:find_compatible_platforms` | + +## FastAPI Router Registration + +**Entry Point Group**: `diracx.services` + +Register FastAPI routers to create new API endpoints or override existing ones. Each entry creates a route under `/api//`. + +### Entry Point Keys + +- **``**: Path to DiracxRouter instance (e.g., 'myext.routers.jobs:router') + +### Usage Example + +```toml +[project.entry-points."diracx.services"] +myjobs = "myextension.routers.jobs:router" +".well-known" = "myextension.routers.well_known:router" # Special case: served at root +``` + +### Important Notes + +- Routers can be disabled with `DIRACX_SERVICE__ENABLED=false` +- Extensions can override core routers by using the same name +- All routes must have proper access policies or use `@open_access` +- The system name becomes the first tag in OpenAPI spec + +### Current Implementations + +| Package | Entry Name | Entry Point | +| ----------------- | ------------- | --------------------------------------- | +| `diracx-routers` | `.well-known` | `diracx.routers.auth.well_known:router` | +| `diracx-routers` | `auth` | `diracx.routers.auth:router` | +| `diracx-routers` | `config` | `diracx.routers.configuration:router` | +| `diracx-routers` | `health` | `diracx.routers.health:router` | +| `diracx-routers` | `jobs` | `diracx.routers.jobs:router` | +| `gubbins-routers` | `.well-known` | `gubbins.routers.well_known:router` | +| `gubbins-routers` | `lollygag` | `gubbins.routers.lollygag:router` | + +______________________________________________________________________ + +*This documentation is auto-generated. See `scripts/generate_entrypoints_docs.py` for details.* diff --git a/mkdocs.yml b/mkdocs.yml index fe71fc28b..ad719223b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -272,6 +272,8 @@ plugins: - search - mermaid2 - autorefs + - neoteroi.mkdocsoad: + use_pymdownx: true - mkdocstrings: enabled: true default_handler: python diff --git a/pixi.toml b/pixi.toml index 3b562958b..237c1d178 100644 --- a/pixi.toml +++ b/pixi.toml @@ -116,6 +116,7 @@ mkdocstrings = { version = ">=0.24", extras = ["python"] } mkdocstrings-python = "*" griffe = "*" griffe-pydantic = "*" +neoteroi-mkdocs = "*" # Install DiracX packages for documentation diracx-core = { path = "diracx-core", editable = true } diracx-api = { path = "diracx-api", editable = true } diff --git a/scripts/generate_entrypoints_docs.py b/scripts/generate_entrypoints_docs.py new file mode 100644 index 000000000..25a9f90c8 --- /dev/null +++ b/scripts/generate_entrypoints_docs.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python +"""Generate documentation for all available DiracX entry points. + +This script discovers all entry points defined across DiracX and its extensions, +providing comprehensive documentation for extension developers. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import tomllib +from collections import defaultdict +from pathlib import Path +from typing import Any + + +def get_entry_points_from_toml( + toml_file: Path, +) -> tuple[str, dict[str, dict[str, str]]]: + """Parse entry points from pyproject.toml. + + Args: + toml_file: Path to pyproject.toml file + + Returns: + Tuple of (package_name, entry_points_dict) + + """ + with open(toml_file, "rb") as f: + pyproject = tomllib.load(f) + package_name = pyproject["project"]["name"] + entry_points = pyproject.get("project", {}).get("entry-points", {}) + return package_name, entry_points + + +def discover_entry_points(repo_base: Path) -> dict[str, dict[str, dict[str, str]]]: + """Discover all entry points in the repository. + + Args: + repo_base: Base directory of the repository + + Returns: + Nested dict: {entry_point_group: {package_name: {entry_name: entry_value}}} + + """ + all_entry_points: dict[str, dict[str, dict[str, str]]] = defaultdict( + lambda: defaultdict(dict) + ) + + # Search for all pyproject.toml files in diracx-* and extensions/*/ + patterns = [ + "diracx-*/pyproject.toml", + "extensions/*/pyproject.toml", + "extensions/*/*/pyproject.toml", + ] + + for pattern in patterns: + for toml_file in repo_base.glob(pattern): + try: + package_name, entry_points = get_entry_points_from_toml(toml_file) + + # Only include diracx-related entry points + for group, entries in entry_points.items(): + if group.startswith("diracx"): + all_entry_points[group][package_name] = entries + except Exception as e: + print(f"Warning: Could not parse {toml_file}: {e}", file=sys.stderr) + + return dict(all_entry_points) + + +def get_entry_point_description(group: str) -> dict[str, Any]: + """Get description and metadata for an entry point group. + + Args: + group: Entry point group name + + Returns: + Dict with title, description, usage_example, and notes + + """ + descriptions = { + "diracx": { + "title": "Core Extension Registration", + "description": ( + "The base entry point group for registering DiracX extensions. " + "Extensions MUST register themselves here." + ), + "keys": { + "extension": "Extension name (required for all extensions)", + "properties_module": "Module path to custom DIRAC properties", + "config": "Path to extended configuration schema class", + }, + "usage_example": """ +```toml +[project.entry-points."diracx"] +extension = "myextension" +properties_module = "myextension.core.properties" +config = "myextension.core.config.schema:Config" +``` +""", + "notes": [ + "The `extension` key is **required** for all extensions", + "Extensions are prioritized by name (alphabetically, with 'diracx' last)", + "Only one extension can be installed alongside DiracX core", + ], + }, + "diracx.services": { + "title": "FastAPI Router Registration", + "description": ( + "Register FastAPI routers to create new API endpoints or override existing ones. " + "Each entry creates a route under `/api//`." + ), + "keys": { + "": "Path to DiracxRouter instance (e.g., 'myext.routers.jobs:router')", + }, + "usage_example": """ +```toml +[project.entry-points."diracx.services"] +myjobs = "myextension.routers.jobs:router" +".well-known" = "myextension.routers.well_known:router" # Special case: served at root +``` +""", + "notes": [ + "Routers can be disabled with `DIRACX_SERVICE__ENABLED=false`", + "Extensions can override core routers by using the same name", + "All routes must have proper access policies or use `@open_access`", + "The system name becomes the first tag in OpenAPI spec", + ], + }, + "diracx.dbs.sql": { + "title": "SQL Database Registration", + "description": ( + "Register SQL database classes using SQLAlchemy. " + "Database URLs are configured via `DIRACX_DB_URL_` environment variables." + ), + "keys": { + "": "Path to BaseSQLDB subclass (e.g., 'myext.db.sql.jobs:JobDB')", + }, + "usage_example": """ +```toml +[project.entry-points."diracx.dbs.sql"] +JobDB = "myextension.db.sql.jobs:ExtendedJobDB" +MyCustomDB = "myextension.db.sql.custom:MyCustomDB" +``` +""", + "notes": [ + "Database classes must inherit from `BaseSQLDB`", + "Use `@declared_attr` for tables to support extension inheritance", + "Transactions are auto-managed: commit on success, rollback on errors", + "Connection pooling is automatic via SQLAlchemy", + ], + }, + "diracx.dbs.os": { + "title": "OpenSearch Database Registration", + "description": ( + "Register OpenSearch/Elasticsearch database classes for log and parameter storage. " + "Connection parameters configured via `DIRACX_OS_DB__*` environment variables." + ), + "keys": { + "": "Path to BaseOSDB subclass (e.g., 'myext.db.os.jobs:JobParametersDB')", + }, + "usage_example": """ +```toml +[project.entry-points."diracx.dbs.os"] +JobParametersDB = "myextension.db.os.jobs:ExtendedJobParametersDB" +``` +""", + "notes": [ + "Database classes must inherit from `BaseOSDB`", + "No automatic transaction management (unlike SQL databases)", + "Connection pooling is handled by AsyncOpenSearch client", + ], + }, + "diracx.cli": { + "title": "CLI Command Registration", + "description": ( + "Register Typer applications as subcommands of the `dirac` CLI. " + "Extensions can add new subcommands or extend existing ones." + ), + "keys": { + "": "Path to Typer app (e.g., 'myext.cli.jobs:app')", + }, + "usage_example": """ +```toml +[project.entry-points."diracx.cli"] +jobs = "myextension.cli.jobs:app" # Override core 'dirac jobs' command +mycmd = "myextension.cli.custom:app" # Add 'dirac mycmd' command +``` +""", + "notes": [ + "Commands are automatically integrated into the main `dirac` CLI", + "Extensions can completely replace core commands by using the same name", + "Use `@app.async_command()` for async operations", + "Follows standard Typer patterns for argument/option parsing", + ], + }, + "diracx.cli.hidden": { + "title": "Hidden CLI Commands", + "description": ( + "Register CLI commands that should not appear in help text. " + "Used for internal/debugging commands." + ), + "keys": { + "": "Path to Typer app for hidden command", + }, + "usage_example": """ +```toml +[project.entry-points."diracx.cli.hidden"] +internal = "myextension.cli.internal:app" +debug = "myextension.cli.debug:app" +``` +""", + "notes": [ + "Commands are functional but don't appear in `dirac --help`", + "Useful for debugging tools and internal utilities", + ], + }, + "diracx.access_policies": { + "title": "Access Policy Registration", + "description": ( + "Register custom access policies for fine-grained authorization control. " + "Policies can inject claims into tokens and check permissions at runtime." + ), + "keys": { + "": "Path to BaseAccessPolicy subclass", + }, + "usage_example": """ +```toml +[project.entry-points."diracx.access_policies"] +WMSAccessPolicy = "myextension.routers.jobs.access_policy:WMSAccessPolicy" +CustomPolicy = "myextension.routers.custom.policy:CustomAccessPolicy" +``` +""", + "notes": [ + "Policies must inherit from `BaseAccessPolicy`", + "Each route must call its policy or use `@open_access` decorator", + "Policies can inject data during token generation via `policy_name` claim", + "CI test `test_all_routes_have_policy` enforces policy usage", + ], + }, + "diracx.min_client_version": { + "title": "Minimum Client Version Declaration", + "description": ( + "Declare the minimum compatible client version for the server. " + "Used to prevent compatibility issues between client and server." + ), + "keys": { + "diracx": "Variable name containing version string (e.g., 'myext.routers:MIN_VERSION')", + }, + "usage_example": """ +```toml +[project.entry-points."diracx.min_client_version"] +myextension = "myextension.routers:MYEXT_MIN_CLIENT_VERSION" +``` +""", + "notes": [ + "Extensions take priority over 'diracx' entry point", + "Version string should follow semantic versioning", + "Server rejects requests from clients below minimum version", + ], + }, + "diracx.resources": { + "title": "Resource Management Functions", + "description": ( + "Register functions that can be overridden by extensions to customize " + "resource management behavior (e.g., platform compatibility)." + ), + "keys": { + "find_compatible_platforms": "Function to determine platform compatibility", + }, + "usage_example": """ +```toml +[project.entry-points."diracx.resources"] +find_compatible_platforms = "myext.core.resources:find_compatible_platforms" +``` +""", + "notes": [ + "Uses `@supports_extending` decorator pattern", + "Extension implementations automatically override core functions", + "Useful for site-specific resource matching logic", + ], + }, + } + + return descriptions.get( + group, + { + "title": f"Entry Point Group: {group}", + "description": "Custom entry point group (not documented in core DiracX).", + "keys": {}, + "usage_example": "", + "notes": [], + }, + ) + + +def generate_markdown(entry_points: dict[str, dict[str, dict[str, str]]]) -> str: + """Generate markdown documentation for entry points. + + Args: + entry_points: Discovered entry points nested dict + + Returns: + Markdown formatted documentation + + """ + output = ["# DiracX Entry Points Reference\n"] + output.append( + "This document catalogs all available entry points for creating DiracX extensions.\n" + ) + output.append( + "Entry points are defined in `pyproject.toml` files and discovered at runtime.\n" + ) + + # Generate table of contents + output.append("## Table of Contents\n") + sorted_groups = sorted(entry_points.keys()) + for group in sorted_groups: + metadata = get_entry_point_description(group) + anchor = group.replace(".", "").replace("_", "-").lower() + output.append(f"- [{metadata['title']}](#{anchor})\n") + + # Generate detailed sections + for group in sorted_groups: + packages = entry_points[group] + metadata = get_entry_point_description(group) + + output.append(f"\n## {metadata['title']}\n") + output.append(f"**Entry Point Group**: `{group}`\n\n") + output.append(f"{metadata['description']}\n") + + # Keys/entries documentation + if metadata["keys"]: + output.append("\n### Entry Point Keys\n\n") + for key, desc in metadata["keys"].items(): + output.append(f"- **`{key}`**: {desc}\n") + + # Usage example + if metadata["usage_example"]: + output.append("\n### Usage Example\n") + output.append(metadata["usage_example"]) + + # Notes + if metadata["notes"]: + output.append("\n### Important Notes\n\n") + for note in metadata["notes"]: + output.append(f"- {note}\n") + + # Current implementations + output.append("\n### Current Implementations\n\n") + if not packages: + output.append("*No implementations found in the repository.*\n") + else: + output.append("| Package | Entry Name | Entry Point |\n") + output.append("|---------|------------|-------------|\n") + for package in sorted(packages.keys()): + entries = packages[package] + for entry_name, entry_value in sorted(entries.items()): + # Escape pipe characters in entry values + safe_value = entry_value.replace("|", "\\|") + output.append( + f"| `{package}` | `{entry_name}` | `{safe_value}` |\n" + ) + + # Footer + output.append("\n---\n\n") + output.append( + "*This documentation is auto-generated. " + "See `scripts/generate_entrypoints_docs.py` for details.*\n" + ) + + return "".join(output) + + +def main(): + """Main entry point for the script.""" + parser = argparse.ArgumentParser( + description="Generate documentation for DiracX entry points" + ) + parser.add_argument( + "--repo-base", + type=Path, + default=Path(__file__).parent.parent, + help="Base directory of the DiracX repository", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Output file path (default: docs/dev/reference/entrypoints.md)", + ) + parser.add_argument( + "--stdout", + action="store_true", + help="Print to stdout instead of writing to file", + ) + + args = parser.parse_args() + + # Discover entry points + print(f"Discovering entry points in {args.repo_base}...", file=sys.stderr) + entry_points = discover_entry_points(args.repo_base) + + print(f"Found {len(entry_points)} entry point groups:", file=sys.stderr) + for group, packages in sorted(entry_points.items()): + total_entries = sum(len(entries) for entries in packages.values()) + print(f" - {group}: {total_entries} entries", file=sys.stderr) + + # Generate markdown + markdown = generate_markdown(entry_points) + + # Output + if args.stdout: + print(markdown) + else: + output_path = ( + args.output + or args.repo_base / "docs" / "dev" / "reference" / "entrypoints.md" + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(markdown) + print(f"\nDocumentation written to: {output_path}", file=sys.stderr) + + # Format the generated markdown file with mdformat + print("Formatting with mdformat...", file=sys.stderr) + try: + subprocess.run( # noqa: S603 + ["mdformat", "--number", str(output_path.absolute())], # noqa: S607 + check=True, + capture_output=True, + text=True, + ) + print("āœ“ Markdown formatted successfully", file=sys.stderr) + except subprocess.CalledProcessError as e: + print(f"Warning: mdformat failed: {e.stderr}", file=sys.stderr) + except FileNotFoundError: + print( + "Warning: mdformat not found.", + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/regenerate_entrypoints_docs.sh b/scripts/regenerate_entrypoints_docs.sh new file mode 100755 index 000000000..f1614f420 --- /dev/null +++ b/scripts/regenerate_entrypoints_docs.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Convenience script to regenerate entry points documentation +# Usage: ./scripts/regenerate_entrypoints_docs.sh + +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "Regenerating entry points documentation..." +pixi run -e default python scripts/generate_entrypoints_docs.py + +echo "" +echo "āœ“ Documentation updated at docs/dev/reference/entrypoints.md" +echo "" +echo "To view changes:" +echo " git diff docs/dev/reference/entrypoints.md" From 264a15086cb4a18712de4bd30d507f0ecaeafd6c Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 03:22:12 +0100 Subject: [PATCH 14/18] docs: add OpenAPI route documentation with neoteroi plugin Add comprehensive REST API route documentation automatically generated from the OpenAPI specification using the neoteroi-mkdocs plugin. Changes: - Add neoteroi CSS styling (v1.1.3) for OpenAPI documentation rendering - Create generate_openapi_spec.py script to generate docs/openapi.json - Add docs/dev/reference/routes/index.md with route documentation page - Integrate neoteroi.mkdocsoad plugin in mkdocs.yml configuration - Add generate-openapi-spec pixi task for OpenAPI spec generation - Make mkdocs-build depend on generate-openapi-spec task - Add generate-entrypoints-docs pre-commit hook - Exclude docs/dev/reference/api/ from mdformat to preserve indentation - Ignore generated docs/openapi.json and Copilot instructions in git - Update routers index page to clarify REST API vs infrastructure docs - Fix mkdocstrings directive indentation across API reference docs - Enhance router documentation with better descriptions and examples The OpenAPI spec generation uses test fixtures similar to the test suite, creating a minimal DiracX app with test auth settings, config repository, and database URLs. The spec is generated before each documentation build. The neoteroi plugin renders the OpenAPI spec into human-readable route documentation with request/response examples, while mdformat exclusion preserves the required indentation for mkdocstrings YAML options. --- .gitignore | 2 + .pre-commit-config.yaml | 12 +- docs/assets/css/neoteroi-mkdocs.css | 1814 +++++++++++++++++ docs/dev/reference/api/core/extensions.md | 10 +- docs/dev/reference/api/core/models.md | 10 +- docs/dev/reference/api/core/preferences.md | 10 +- docs/dev/reference/api/core/properties.md | 10 +- docs/dev/reference/api/core/settings.md | 12 +- docs/dev/reference/api/core/utils.md | 10 +- docs/dev/reference/api/db/exceptions.md | 10 +- docs/dev/reference/api/index.md | 68 +- .../reference/api/routers/access_policies.md | 7 +- docs/dev/reference/api/routers/auth.md | 6 +- .../reference/api/routers/configuration.md | 6 +- .../dev/reference/api/routers/dependencies.md | 7 +- docs/dev/reference/api/routers/factory.md | 9 +- .../reference/api/routers/fastapi_classes.md | 7 +- docs/dev/reference/api/routers/health.md | 8 +- docs/dev/reference/api/routers/index.md | 55 +- docs/dev/reference/api/routers/jobs.md | 98 +- docs/dev/reference/api/routers/otel.md | 5 + docs/dev/reference/api/routers/utils.md | 7 +- docs/dev/reference/api/writing-api-docs.md | 42 +- docs/dev/reference/pixi-tasks.md | 1 + docs/dev/reference/routes/index.md | 17 + mkdocs.yml | 3 + pixi.toml | 7 +- scripts/generate_openapi_spec.py | 271 +++ 28 files changed, 2362 insertions(+), 162 deletions(-) create mode 100644 docs/assets/css/neoteroi-mkdocs.css create mode 100644 docs/dev/reference/routes/index.md create mode 100644 scripts/generate_openapi_spec.py diff --git a/.gitignore b/.gitignore index 6aac865c8..50e050d35 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,5 @@ docs/templates/_builtin_markdown.jinja # docs site site +.github/copilot-instructions.md +docs/openapi.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da2bd2c84..328f7136a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: python: python3 ci: - skip: [generate-pixi-docs, settings-doc-check, api-doc-coverage-check] + skip: [generate-pixi-docs, settings-doc-check, api-doc-coverage-check, generate-entrypoints-docs] default_stages: [pre-commit] @@ -60,6 +60,7 @@ repos: - mdformat-mkdocs - mdformat-gfm - mdformat-black + exclude: ^docs/dev/reference/api/ - repo: https://github.com/codespell-project/codespell rev: v2.4.1 @@ -102,3 +103,12 @@ repos: language: system pass_filenames: false files: ^(diracx-.*/src/diracx/.*\.py|docs/dev/reference/api/.*\.md|docs/dev/reference/api/check_coverage\.py)$ + + - repo: local + hooks: + - id: generate-entrypoints-docs + name: Generate entry points documentation + entry: pixi run -e default python scripts/generate_entrypoints_docs.py + language: system + pass_filenames: false + files: ^(diracx-.*/pyproject\.toml|extensions/.*/pyproject\.toml|scripts/generate_entrypoints_docs\.py)$ diff --git a/docs/assets/css/neoteroi-mkdocs.css b/docs/assets/css/neoteroi-mkdocs.css new file mode 100644 index 000000000..e9daefe84 --- /dev/null +++ b/docs/assets/css/neoteroi-mkdocs.css @@ -0,0 +1,1814 @@ +/** + * All CSS for the neoteroi-mkdocs extensions. + * + * https://github.com/Neoteroi/mkdocs-plugins +**/ +:root { + --nt-color-0: #CD853F; + --nt-color-1: #B22222; + --nt-color-2: #000080; + --nt-color-3: #4B0082; + --nt-color-4: #3CB371; + --nt-color-5: #D2B48C; + --nt-color-6: #FF00FF; + --nt-color-7: #98FB98; + --nt-color-8: #FFEBCD; + --nt-color-9: #2E8B57; + --nt-color-10: #6A5ACD; + --nt-color-11: #48D1CC; + --nt-color-12: #FFA500; + --nt-color-13: #F4A460; + --nt-color-14: #A52A2A; + --nt-color-15: #FFE4C4; + --nt-color-16: #FF4500; + --nt-color-17: #AFEEEE; + --nt-color-18: #FA8072; + --nt-color-19: #2F4F4F; + --nt-color-20: #FFDAB9; + --nt-color-21: #BC8F8F; + --nt-color-22: #FFC0CB; + --nt-color-23: #00FA9A; + --nt-color-24: #F0FFF0; + --nt-color-25: #FFFACD; + --nt-color-26: #F5F5F5; + --nt-color-27: #FF6347; + --nt-color-28: #FFFFF0; + --nt-color-29: #7FFFD4; + --nt-color-30: #E9967A; + --nt-color-31: #7B68EE; + --nt-color-32: #FFF8DC; + --nt-color-33: #0000CD; + --nt-color-34: #D2691E; + --nt-color-35: #708090; + --nt-color-36: #5F9EA0; + --nt-color-37: #008080; + --nt-color-38: #008000; + --nt-color-39: #FFE4E1; + --nt-color-40: #FFFF00; + --nt-color-41: #FFFAF0; + --nt-color-42: #DCDCDC; + --nt-color-43: #ADFF2F; + --nt-color-44: #ADD8E6; + --nt-color-45: #8B008B; + --nt-color-46: #7FFF00; + --nt-color-47: #800000; + --nt-color-48: #20B2AA; + --nt-color-49: #556B2F; + --nt-color-50: #778899; + --nt-color-51: #E6E6FA; + --nt-color-52: #FFFAFA; + --nt-color-53: #FF7F50; + --nt-color-54: #FF0000; + --nt-color-55: #F5DEB3; + --nt-color-56: #008B8B; + --nt-color-57: #66CDAA; + --nt-color-58: #808000; + --nt-color-59: #FAF0E6; + --nt-color-60: #00BFFF; + --nt-color-61: #C71585; + --nt-color-62: #00FFFF; + --nt-color-63: #8B4513; + --nt-color-64: #F0F8FF; + --nt-color-65: #FAEBD7; + --nt-color-66: #8B0000; + --nt-color-67: #4682B4; + --nt-color-68: #F0E68C; + --nt-color-69: #BDB76B; + --nt-color-70: #A0522D; + --nt-color-71: #FAFAD2; + --nt-color-72: #FFD700; + --nt-color-73: #DEB887; + --nt-color-74: #E0FFFF; + --nt-color-75: #8A2BE2; + --nt-color-76: #32CD32; + --nt-color-77: #87CEFA; + --nt-color-78: #00CED1; + --nt-color-79: #696969; + --nt-color-80: #DDA0DD; + --nt-color-81: #EE82EE; + --nt-color-82: #FFB6C1; + --nt-color-83: #8FBC8F; + --nt-color-84: #D8BFD8; + --nt-color-85: #9400D3; + --nt-color-86: #A9A9A9; + --nt-color-87: #FFFFE0; + --nt-color-88: #FFF5EE; + --nt-color-89: #FFF0F5; + --nt-color-90: #FFDEAD; + --nt-color-91: #800080; + --nt-color-92: #B0E0E6; + --nt-color-93: #9932CC; + --nt-color-94: #DAA520; + --nt-color-95: #F0FFFF; + --nt-color-96: #40E0D0; + --nt-color-97: #00FF7F; + --nt-color-98: #006400; + --nt-color-99: #808080; + --nt-color-100: #87CEEB; + --nt-color-101: #0000FF; + --nt-color-102: #6495ED; + --nt-color-103: #FDF5E6; + --nt-color-104: #B8860B; + --nt-color-105: #BA55D3; + --nt-color-106: #C0C0C0; + --nt-color-107: #000000; + --nt-color-108: #F08080; + --nt-color-109: #B0C4DE; + --nt-color-110: #00008B; + --nt-color-111: #6B8E23; + --nt-color-112: #FFE4B5; + --nt-color-113: #FFA07A; + --nt-color-114: #9ACD32; + --nt-color-115: #FFFFFF; + --nt-color-116: #F5F5DC; + --nt-color-117: #90EE90; + --nt-color-118: #1E90FF; + --nt-color-119: #7CFC00; + --nt-color-120: #FF69B4; + --nt-color-121: #F8F8FF; + --nt-color-122: #F5FFFA; + --nt-color-123: #00FF00; + --nt-color-124: #D3D3D3; + --nt-color-125: #DB7093; + --nt-color-126: #DA70D6; + --nt-color-127: #FF1493; + --nt-color-128: #228B22; + --nt-color-129: #FFEFD5; + --nt-color-130: #4169E1; + --nt-color-131: #191970; + --nt-color-132: #9370DB; + --nt-color-133: #483D8B; + --nt-color-134: #FF8C00; + --nt-color-135: #EEE8AA; + --nt-color-136: #CD5C5C; + --nt-color-137: #DC143C; +} + +:root { + --nt-group-0-main: #000000; + --nt-group-0-dark: #FFFFFF; + --nt-group-0-light: #000000; + --nt-group-0-main-bg: #F44336; + --nt-group-0-dark-bg: #BA000D; + --nt-group-0-light-bg: #FF7961; + --nt-group-1-main: #000000; + --nt-group-1-dark: #FFFFFF; + --nt-group-1-light: #000000; + --nt-group-1-main-bg: #E91E63; + --nt-group-1-dark-bg: #B0003A; + --nt-group-1-light-bg: #FF6090; + --nt-group-2-main: #FFFFFF; + --nt-group-2-dark: #FFFFFF; + --nt-group-2-light: #000000; + --nt-group-2-main-bg: #9C27B0; + --nt-group-2-dark-bg: #6A0080; + --nt-group-2-light-bg: #D05CE3; + --nt-group-3-main: #FFFFFF; + --nt-group-3-dark: #FFFFFF; + --nt-group-3-light: #000000; + --nt-group-3-main-bg: #673AB7; + --nt-group-3-dark-bg: #320B86; + --nt-group-3-light-bg: #9A67EA; + --nt-group-4-main: #FFFFFF; + --nt-group-4-dark: #FFFFFF; + --nt-group-4-light: #000000; + --nt-group-4-main-bg: #3F51B5; + --nt-group-4-dark-bg: #002984; + --nt-group-4-light-bg: #757DE8; + --nt-group-5-main: #000000; + --nt-group-5-dark: #FFFFFF; + --nt-group-5-light: #000000; + --nt-group-5-main-bg: #2196F3; + --nt-group-5-dark-bg: #0069C0; + --nt-group-5-light-bg: #6EC6FF; + --nt-group-6-main: #000000; + --nt-group-6-dark: #FFFFFF; + --nt-group-6-light: #000000; + --nt-group-6-main-bg: #03A9F4; + --nt-group-6-dark-bg: #007AC1; + --nt-group-6-light-bg: #67DAFF; + --nt-group-7-main: #000000; + --nt-group-7-dark: #000000; + --nt-group-7-light: #000000; + --nt-group-7-main-bg: #00BCD4; + --nt-group-7-dark-bg: #008BA3; + --nt-group-7-light-bg: #62EFFF; + --nt-group-8-main: #000000; + --nt-group-8-dark: #FFFFFF; + --nt-group-8-light: #000000; + --nt-group-8-main-bg: #009688; + --nt-group-8-dark-bg: #00675B; + --nt-group-8-light-bg: #52C7B8; + --nt-group-9-main: #000000; + --nt-group-9-dark: #FFFFFF; + --nt-group-9-light: #000000; + --nt-group-9-main-bg: #4CAF50; + --nt-group-9-dark-bg: #087F23; + --nt-group-9-light-bg: #80E27E; + --nt-group-10-main: #000000; + --nt-group-10-dark: #000000; + --nt-group-10-light: #000000; + --nt-group-10-main-bg: #8BC34A; + --nt-group-10-dark-bg: #5A9216; + --nt-group-10-light-bg: #BEF67A; + --nt-group-11-main: #000000; + --nt-group-11-dark: #000000; + --nt-group-11-light: #000000; + --nt-group-11-main-bg: #CDDC39; + --nt-group-11-dark-bg: #99AA00; + --nt-group-11-light-bg: #FFFF6E; + --nt-group-12-main: #000000; + --nt-group-12-dark: #000000; + --nt-group-12-light: #000000; + --nt-group-12-main-bg: #FFEB3B; + --nt-group-12-dark-bg: #C8B900; + --nt-group-12-light-bg: #FFFF72; + --nt-group-13-main: #000000; + --nt-group-13-dark: #000000; + --nt-group-13-light: #000000; + --nt-group-13-main-bg: #FFC107; + --nt-group-13-dark-bg: #C79100; + --nt-group-13-light-bg: #FFF350; + --nt-group-14-main: #000000; + --nt-group-14-dark: #000000; + --nt-group-14-light: #000000; + --nt-group-14-main-bg: #FF9800; + --nt-group-14-dark-bg: #C66900; + --nt-group-14-light-bg: #FFC947; + --nt-group-15-main: #000000; + --nt-group-15-dark: #FFFFFF; + --nt-group-15-light: #000000; + --nt-group-15-main-bg: #FF5722; + --nt-group-15-dark-bg: #C41C00; + --nt-group-15-light-bg: #FF8A50; + --nt-group-16-main: #FFFFFF; + --nt-group-16-dark: #FFFFFF; + --nt-group-16-light: #000000; + --nt-group-16-main-bg: #795548; + --nt-group-16-dark-bg: #4B2C20; + --nt-group-16-light-bg: #A98274; + --nt-group-17-main: #000000; + --nt-group-17-dark: #FFFFFF; + --nt-group-17-light: #000000; + --nt-group-17-main-bg: #9E9E9E; + --nt-group-17-dark-bg: #707070; + --nt-group-17-light-bg: #CFCFCF; + --nt-group-18-main: #000000; + --nt-group-18-dark: #FFFFFF; + --nt-group-18-light: #000000; + --nt-group-18-main-bg: #607D8B; + --nt-group-18-dark-bg: #34515E; + --nt-group-18-light-bg: #8EACBB; +} + +.nt-pastello { + --nt-group-0-main: #000000; + --nt-group-0-dark: #000000; + --nt-group-0-light: #000000; + --nt-group-0-main-bg: #EF9A9A; + --nt-group-0-dark-bg: #BA6B6C; + --nt-group-0-light-bg: #FFCCCB; + --nt-group-1-main: #000000; + --nt-group-1-dark: #000000; + --nt-group-1-light: #000000; + --nt-group-1-main-bg: #F48FB1; + --nt-group-1-dark-bg: #BF5F82; + --nt-group-1-light-bg: #FFC1E3; + --nt-group-2-main: #000000; + --nt-group-2-dark: #000000; + --nt-group-2-light: #000000; + --nt-group-2-main-bg: #CE93D8; + --nt-group-2-dark-bg: #9C64A6; + --nt-group-2-light-bg: #FFC4FF; + --nt-group-3-main: #000000; + --nt-group-3-dark: #000000; + --nt-group-3-light: #000000; + --nt-group-3-main-bg: #B39DDB; + --nt-group-3-dark-bg: #836FA9; + --nt-group-3-light-bg: #E6CEFF; + --nt-group-4-main: #000000; + --nt-group-4-dark: #000000; + --nt-group-4-light: #000000; + --nt-group-4-main-bg: #9FA8DA; + --nt-group-4-dark-bg: #6F79A8; + --nt-group-4-light-bg: #D1D9FF; + --nt-group-5-main: #000000; + --nt-group-5-dark: #000000; + --nt-group-5-light: #000000; + --nt-group-5-main-bg: #90CAF9; + --nt-group-5-dark-bg: #5D99C6; + --nt-group-5-light-bg: #C3FDFF; + --nt-group-6-main: #000000; + --nt-group-6-dark: #000000; + --nt-group-6-light: #000000; + --nt-group-6-main-bg: #81D4FA; + --nt-group-6-dark-bg: #4BA3C7; + --nt-group-6-light-bg: #B6FFFF; + --nt-group-7-main: #000000; + --nt-group-7-dark: #000000; + --nt-group-7-light: #000000; + --nt-group-7-main-bg: #80DEEA; + --nt-group-7-dark-bg: #4BACB8; + --nt-group-7-light-bg: #B4FFFF; + --nt-group-8-main: #000000; + --nt-group-8-dark: #000000; + --nt-group-8-light: #000000; + --nt-group-8-main-bg: #80CBC4; + --nt-group-8-dark-bg: #4F9A94; + --nt-group-8-light-bg: #B2FEF7; + --nt-group-9-main: #000000; + --nt-group-9-dark: #000000; + --nt-group-9-light: #000000; + --nt-group-9-main-bg: #A5D6A7; + --nt-group-9-dark-bg: #75A478; + --nt-group-9-light-bg: #D7FFD9; + --nt-group-10-main: #000000; + --nt-group-10-dark: #000000; + --nt-group-10-light: #000000; + --nt-group-10-main-bg: #C5E1A5; + --nt-group-10-dark-bg: #94AF76; + --nt-group-10-light-bg: #F8FFD7; + --nt-group-11-main: #000000; + --nt-group-11-dark: #000000; + --nt-group-11-light: #000000; + --nt-group-11-main-bg: #E6EE9C; + --nt-group-11-dark-bg: #B3BC6D; + --nt-group-11-light-bg: #FFFFCE; + --nt-group-12-main: #000000; + --nt-group-12-dark: #000000; + --nt-group-12-light: #000000; + --nt-group-12-main-bg: #FFF59D; + --nt-group-12-dark-bg: #CBC26D; + --nt-group-12-light-bg: #FFFFCF; + --nt-group-13-main: #000000; + --nt-group-13-dark: #000000; + --nt-group-13-light: #000000; + --nt-group-13-main-bg: #FFE082; + --nt-group-13-dark-bg: #CAAE53; + --nt-group-13-light-bg: #FFFFB3; + --nt-group-14-main: #000000; + --nt-group-14-dark: #000000; + --nt-group-14-light: #000000; + --nt-group-14-main-bg: #FFCC80; + --nt-group-14-dark-bg: #CA9B52; + --nt-group-14-light-bg: #FFFFB0; + --nt-group-15-main: #000000; + --nt-group-15-dark: #000000; + --nt-group-15-light: #000000; + --nt-group-15-main-bg: #FFAB91; + --nt-group-15-dark-bg: #C97B63; + --nt-group-15-light-bg: #FFDDC1; + --nt-group-16-main: #000000; + --nt-group-16-dark: #000000; + --nt-group-16-light: #000000; + --nt-group-16-main-bg: #BCAAA4; + --nt-group-16-dark-bg: #8C7B75; + --nt-group-16-light-bg: #EFDCD5; + --nt-group-17-main: #000000; + --nt-group-17-dark: #000000; + --nt-group-17-light: #000000; + --nt-group-17-main-bg: #EEEEEE; + --nt-group-17-dark-bg: #BCBCBC; + --nt-group-17-light-bg: #FFFFFF; + --nt-group-18-main: #000000; + --nt-group-18-dark: #000000; + --nt-group-18-light: #000000; + --nt-group-18-main-bg: #B0BEC5; + --nt-group-18-dark-bg: #808E95; + --nt-group-18-light-bg: #E2F1F8; +} + +.nt-group-0 .nt-plan-group-summary, +.nt-group-0 .nt-timeline-dot { + color: var(--nt-group-0-dark); + background-color: var(--nt-group-0-dark-bg); +} +.nt-group-0 .period { + color: var(--nt-group-0-main); + background-color: var(--nt-group-0-main-bg); +} + +.nt-group-1 .nt-plan-group-summary, +.nt-group-1 .nt-timeline-dot { + color: var(--nt-group-1-dark); + background-color: var(--nt-group-1-dark-bg); +} +.nt-group-1 .period { + color: var(--nt-group-1-main); + background-color: var(--nt-group-1-main-bg); +} + +.nt-group-2 .nt-plan-group-summary, +.nt-group-2 .nt-timeline-dot { + color: var(--nt-group-2-dark); + background-color: var(--nt-group-2-dark-bg); +} +.nt-group-2 .period { + color: var(--nt-group-2-main); + background-color: var(--nt-group-2-main-bg); +} + +.nt-group-3 .nt-plan-group-summary, +.nt-group-3 .nt-timeline-dot { + color: var(--nt-group-3-dark); + background-color: var(--nt-group-3-dark-bg); +} +.nt-group-3 .period { + color: var(--nt-group-3-main); + background-color: var(--nt-group-3-main-bg); +} + +.nt-group-4 .nt-plan-group-summary, +.nt-group-4 .nt-timeline-dot { + color: var(--nt-group-4-dark); + background-color: var(--nt-group-4-dark-bg); +} +.nt-group-4 .period { + color: var(--nt-group-4-main); + background-color: var(--nt-group-4-main-bg); +} + +.nt-group-5 .nt-plan-group-summary, +.nt-group-5 .nt-timeline-dot { + color: var(--nt-group-5-dark); + background-color: var(--nt-group-5-dark-bg); +} +.nt-group-5 .period { + color: var(--nt-group-5-main); + background-color: var(--nt-group-5-main-bg); +} + +.nt-group-6 .nt-plan-group-summary, +.nt-group-6 .nt-timeline-dot { + color: var(--nt-group-6-dark); + background-color: var(--nt-group-6-dark-bg); +} +.nt-group-6 .period { + color: var(--nt-group-6-main); + background-color: var(--nt-group-6-main-bg); +} + +.nt-group-7 .nt-plan-group-summary, +.nt-group-7 .nt-timeline-dot { + color: var(--nt-group-7-dark); + background-color: var(--nt-group-7-dark-bg); +} +.nt-group-7 .period { + color: var(--nt-group-7-main); + background-color: var(--nt-group-7-main-bg); +} + +.nt-group-8 .nt-plan-group-summary, +.nt-group-8 .nt-timeline-dot { + color: var(--nt-group-8-dark); + background-color: var(--nt-group-8-dark-bg); +} +.nt-group-8 .period { + color: var(--nt-group-8-main); + background-color: var(--nt-group-8-main-bg); +} + +.nt-group-9 .nt-plan-group-summary, +.nt-group-9 .nt-timeline-dot { + color: var(--nt-group-9-dark); + background-color: var(--nt-group-9-dark-bg); +} +.nt-group-9 .period { + color: var(--nt-group-9-main); + background-color: var(--nt-group-9-main-bg); +} + +.nt-group-10 .nt-plan-group-summary, +.nt-group-10 .nt-timeline-dot { + color: var(--nt-group-10-dark); + background-color: var(--nt-group-10-dark-bg); +} +.nt-group-10 .period { + color: var(--nt-group-10-main); + background-color: var(--nt-group-10-main-bg); +} + +.nt-group-11 .nt-plan-group-summary, +.nt-group-11 .nt-timeline-dot { + color: var(--nt-group-11-dark); + background-color: var(--nt-group-11-dark-bg); +} +.nt-group-11 .period { + color: var(--nt-group-11-main); + background-color: var(--nt-group-11-main-bg); +} + +.nt-group-12 .nt-plan-group-summary, +.nt-group-12 .nt-timeline-dot { + color: var(--nt-group-12-dark); + background-color: var(--nt-group-12-dark-bg); +} +.nt-group-12 .period { + color: var(--nt-group-12-main); + background-color: var(--nt-group-12-main-bg); +} + +.nt-group-13 .nt-plan-group-summary, +.nt-group-13 .nt-timeline-dot { + color: var(--nt-group-13-dark); + background-color: var(--nt-group-13-dark-bg); +} +.nt-group-13 .period { + color: var(--nt-group-13-main); + background-color: var(--nt-group-13-main-bg); +} + +.nt-group-14 .nt-plan-group-summary, +.nt-group-14 .nt-timeline-dot { + color: var(--nt-group-14-dark); + background-color: var(--nt-group-14-dark-bg); +} +.nt-group-14 .period { + color: var(--nt-group-14-main); + background-color: var(--nt-group-14-main-bg); +} + +.nt-group-15 .nt-plan-group-summary, +.nt-group-15 .nt-timeline-dot { + color: var(--nt-group-15-dark); + background-color: var(--nt-group-15-dark-bg); +} +.nt-group-15 .period { + color: var(--nt-group-15-main); + background-color: var(--nt-group-15-main-bg); +} + +.nt-group-16 .nt-plan-group-summary, +.nt-group-16 .nt-timeline-dot { + color: var(--nt-group-16-dark); + background-color: var(--nt-group-16-dark-bg); +} +.nt-group-16 .period { + color: var(--nt-group-16-main); + background-color: var(--nt-group-16-main-bg); +} + +.nt-group-17 .nt-plan-group-summary, +.nt-group-17 .nt-timeline-dot { + color: var(--nt-group-17-dark); + background-color: var(--nt-group-17-dark-bg); +} +.nt-group-17 .period { + color: var(--nt-group-17-main); + background-color: var(--nt-group-17-main-bg); +} + +.nt-group-18 .nt-plan-group-summary, +.nt-group-18 .nt-timeline-dot { + color: var(--nt-group-18-dark); + background-color: var(--nt-group-18-dark-bg); +} +.nt-group-18 .period { + color: var(--nt-group-18-main); + background-color: var(--nt-group-18-main-bg); +} + +/** + * Extra CSS file for MkDocs and the neoteroi.timeline extension. + * + * https://github.com/Neoteroi/mkdocs-plugins +**/ +.nt-error { + border: 2px dashed darkred; + padding: 0 1rem; + background: #faf9ba; + color: darkred; +} + +.nt-timeline { + margin-top: 30px; +} +.nt-timeline .nt-timeline-title { + font-size: 1.1rem; + margin-top: 0; +} +.nt-timeline .nt-timeline-sub-title { + margin-top: 0; +} +.nt-timeline .nt-timeline-content { + font-size: 0.8rem; + border-bottom: 2px dashed #ccc; + padding-bottom: 1.2rem; +} +.nt-timeline.horizontal .nt-timeline-items { + flex-direction: row; + overflow-x: scroll; +} +.nt-timeline.horizontal .nt-timeline-items > div { + min-width: 400px; + margin-right: 50px; +} +.nt-timeline.horizontal.reverse .nt-timeline-items { + flex-direction: row-reverse; +} +.nt-timeline.horizontal.center .nt-timeline-before { + background-image: linear-gradient(rgba(252, 70, 107, 0) 0%, rgb(252, 70, 107) 100%); + background-repeat: no-repeat; + background-size: 100% 2px; + background-position: 0 center; +} +.nt-timeline.horizontal.center .nt-timeline-after { + background-image: linear-gradient(180deg, rgb(252, 70, 107) 0%, rgba(252, 70, 107, 0) 100%); + background-repeat: no-repeat; + background-size: 100% 2px; + background-position: 0 center; +} +.nt-timeline.horizontal.center .nt-timeline-items { + background-image: radial-gradient(circle, rgb(63, 94, 251) 0%, rgb(252, 70, 107) 100%); + background-repeat: no-repeat; + background-size: 100% 2px; + background-position: 0 center; +} +.nt-timeline.horizontal .nt-timeline-dot { + left: 50%; +} +.nt-timeline.horizontal .nt-timeline-dot:not(.bigger) { + top: calc(50% - 4px); +} +.nt-timeline.horizontal .nt-timeline-dot.bigger { + top: calc(50% - 15px); +} +.nt-timeline.vertical .nt-timeline-items { + flex-direction: column; +} +.nt-timeline.vertical.reverse .nt-timeline-items { + flex-direction: column-reverse; +} +.nt-timeline.vertical.center .nt-timeline-before { + background: linear-gradient(rgba(252, 70, 107, 0) 0%, rgb(252, 70, 107) 100%) no-repeat center/2px 100%; +} +.nt-timeline.vertical.center .nt-timeline-after { + background: linear-gradient(rgb(252, 70, 107) 0%, rgba(252, 70, 107, 0) 100%) no-repeat center/2px 100%; +} +.nt-timeline.vertical.center .nt-timeline-items { + background: radial-gradient(circle, rgb(63, 94, 251) 0%, rgb(252, 70, 107) 100%) no-repeat center/2px 100%; +} +.nt-timeline.vertical.center .nt-timeline-dot { + left: calc(50% - 10px); +} +.nt-timeline.vertical.center .nt-timeline-dot:not(.bigger) { + top: 10px; +} +.nt-timeline.vertical.center .nt-timeline-dot.bigger { + left: calc(50% - 20px); +} +.nt-timeline.vertical.left { + padding-left: 100px; +} +.nt-timeline.vertical.left .nt-timeline-item { + padding-left: 70px; +} +.nt-timeline.vertical.left .nt-timeline-sub-title { + left: -100px; + width: 100px; +} +.nt-timeline.vertical.left .nt-timeline-before { + background: linear-gradient(rgba(252, 70, 107, 0) 0%, rgb(252, 70, 107) 100%) no-repeat 30px/2px 100%; +} +.nt-timeline.vertical.left .nt-timeline-after { + background: linear-gradient(rgb(252, 70, 107) 0%, rgba(252, 70, 107, 0) 100%) no-repeat 30px/2px 100%; +} +.nt-timeline.vertical.left .nt-timeline-items { + background: radial-gradient(circle, rgb(63, 94, 251) 0%, rgb(252, 70, 107) 100%) no-repeat 30px/2px 100%; +} +.nt-timeline.vertical.left .nt-timeline-dot { + left: 21px; + top: 8px; +} +.nt-timeline.vertical.left .nt-timeline-dot.bigger { + top: 0px; + left: 10px; +} +.nt-timeline.vertical.right { + padding-right: 100px; +} +.nt-timeline.vertical.right .nt-timeline-sub-title { + right: -100px; + text-align: left; + width: 100px; +} +.nt-timeline.vertical.right .nt-timeline-item { + padding-right: 70px; +} +.nt-timeline.vertical.right .nt-timeline-before { + background: linear-gradient(rgba(252, 70, 107, 0) 0%, rgb(252, 70, 107) 100%) no-repeat calc(100% - 30px)/2px 100%; +} +.nt-timeline.vertical.right .nt-timeline-after { + background: linear-gradient(rgb(252, 70, 107) 0%, rgba(252, 70, 107, 0) 100%) no-repeat calc(100% - 30px)/2px 100%; +} +.nt-timeline.vertical.right .nt-timeline-items { + background: radial-gradient(circle, rgb(63, 94, 251) 0%, rgb(252, 70, 107) 100%) no-repeat calc(100% - 30px)/2px 100%; +} +.nt-timeline.vertical.right .nt-timeline-dot { + right: 21px; + top: 8px; +} +.nt-timeline.vertical.right .nt-timeline-dot.bigger { + top: 10px; + right: 10px; +} + +.nt-timeline-items { + display: flex; + position: relative; +} +.nt-timeline-items > div { + min-height: 100px; + padding-top: 2px; + padding-bottom: 20px; +} + +.nt-timeline-before { + content: ""; + height: 15px; +} + +.nt-timeline-after { + content: ""; + height: 60px; + margin-bottom: 20px; +} + +.nt-timeline-sub-title { + position: absolute; + width: 50%; + top: 4px; + font-size: 18px; + color: var(--nt-color-50); +} + +[data-md-color-scheme=slate] .nt-timeline-sub-title { + color: var(--nt-color-51); +} + +.nt-timeline-item { + position: relative; +} + +.nt-timeline.vertical.center:not(.alternate) .nt-timeline-item { + padding-left: calc(50% + 40px); +} +.nt-timeline.vertical.center:not(.alternate) .nt-timeline-item .nt-timeline-sub-title { + left: 0; + padding-right: 40px; + text-align: right; +} +.nt-timeline.vertical.center.alternate .nt-timeline-item:nth-child(odd) { + padding-left: calc(50% + 40px); +} +.nt-timeline.vertical.center.alternate .nt-timeline-item:nth-child(odd) .nt-timeline-sub-title { + left: 0; + padding-right: 40px; + text-align: right; +} +.nt-timeline.vertical.center.alternate .nt-timeline-item:nth-child(even) { + text-align: right; + padding-right: calc(50% + 40px); +} +.nt-timeline.vertical.center.alternate .nt-timeline-item:nth-child(even) .nt-timeline-sub-title { + right: 0; + padding-left: 40px; + text-align: left; +} + +.nt-timeline-dot { + position: relative; + width: 20px; + height: 20px; + border-radius: 100%; + background-color: #fc5b5b; + position: absolute; + top: 0px; + z-index: 2; + display: flex; + justify-content: center; + align-items: center; + box-shadow: 0 2px 1px -1px rgba(0, 0, 0, 0.2), 0 1px 1px 0 rgba(0, 0, 0, 0.14), 0 1px 3px 0 rgba(0, 0, 0, 0.12); + border: 3px solid white; +} +.nt-timeline-dot:not(.bigger) .icon { + font-size: 10px; +} +.nt-timeline-dot.bigger { + width: 40px; + height: 40px; + padding: 3px; +} +.nt-timeline-dot .icon { + color: white; + position: relative; + top: 1px; +} + +/* Fix for webkit (Chrome, Safari) */ +@supports not (-moz-appearance: none) { + /* + This fix is necessary, for some reason, to render the timeline properly + inside `details` elements used by pymdownx. Firefox doesn't need this fix, + it renders elements properly. + */ + details .nt-timeline.vertical.center.alternate .nt-timeline-item:nth-child(odd) .nt-timeline-sub-title, +details .nt-timeline.vertical.center:not(.alternate) .nt-timeline-item .nt-timeline-sub-title { + left: -40px; + } + details .nt-timeline.vertical.center.alternate .nt-timeline-item:nth-child(even) .nt-timeline-sub-title { + right: -40px; + } + details .nt-timeline.vertical.center .nt-timeline-dot { + left: calc(50% - 12px); + } + details .nt-timeline-dot.bigger { + font-size: 1rem !important; + } +} +/* default colors */ +.nt-timeline-item:nth-child(0) .nt-timeline-dot { + background-color: var(--nt-color-0); +} + +.nt-timeline-item:nth-child(1) .nt-timeline-dot { + background-color: var(--nt-color-1); +} + +.nt-timeline-item:nth-child(2) .nt-timeline-dot { + background-color: var(--nt-color-2); +} + +.nt-timeline-item:nth-child(3) .nt-timeline-dot { + background-color: var(--nt-color-3); +} + +.nt-timeline-item:nth-child(4) .nt-timeline-dot { + background-color: var(--nt-color-4); +} + +.nt-timeline-item:nth-child(5) .nt-timeline-dot { + background-color: var(--nt-color-5); +} + +.nt-timeline-item:nth-child(6) .nt-timeline-dot { + background-color: var(--nt-color-6); +} + +.nt-timeline-item:nth-child(7) .nt-timeline-dot { + background-color: var(--nt-color-7); +} + +.nt-timeline-item:nth-child(8) .nt-timeline-dot { + background-color: var(--nt-color-8); +} + +.nt-timeline-item:nth-child(9) .nt-timeline-dot { + background-color: var(--nt-color-9); +} + +.nt-timeline-item:nth-child(10) .nt-timeline-dot { + background-color: var(--nt-color-10); +} + +.nt-timeline-item:nth-child(11) .nt-timeline-dot { + background-color: var(--nt-color-11); +} + +.nt-timeline-item:nth-child(12) .nt-timeline-dot { + background-color: var(--nt-color-12); +} + +.nt-timeline-item:nth-child(13) .nt-timeline-dot { + background-color: var(--nt-color-13); +} + +.nt-timeline-item:nth-child(14) .nt-timeline-dot { + background-color: var(--nt-color-14); +} + +.nt-timeline-item:nth-child(15) .nt-timeline-dot { + background-color: var(--nt-color-15); +} + +.nt-timeline-item:nth-child(16) .nt-timeline-dot { + background-color: var(--nt-color-16); +} + +.nt-timeline-item:nth-child(17) .nt-timeline-dot { + background-color: var(--nt-color-17); +} + +.nt-timeline-item:nth-child(18) .nt-timeline-dot { + background-color: var(--nt-color-18); +} + +.nt-timeline-item:nth-child(19) .nt-timeline-dot { + background-color: var(--nt-color-19); +} + +.nt-timeline-item:nth-child(20) .nt-timeline-dot { + background-color: var(--nt-color-20); +} + +/** + * Extra CSS for the neoteroi.projects.gantt extension. + * + * https://github.com/Neoteroi/mkdocs-plugins +**/ +:root { + --nt-scrollbar-color: #2751b0; + --nt-plan-actions-height: 24px; + --nt-units-background: #ff9800; + --nt-months-background: #2751b0; + --nt-plan-vertical-line-color: #a3a3a3ad; +} + +.nt-pastello { + --nt-scrollbar-color: #9fb8f4; + --nt-units-background: #f5dc82; + --nt-months-background: #5b7fd1; +} + +[data-md-color-scheme=slate] { + --nt-units-background: #003773; +} +[data-md-color-scheme=slate] .nt-pastello { + --nt-units-background: #3f4997; +} + +.nt-plan-root { + min-height: 200px; + scrollbar-width: 20px; + scrollbar-color: var(--nt-scrollbar-color); + display: flex; +} +.nt-plan-root ::-webkit-scrollbar { + width: 20px; +} +.nt-plan-root ::-webkit-scrollbar-track { + box-shadow: inset 0 0 5px grey; + border-radius: 10px; +} +.nt-plan-root ::-webkit-scrollbar-thumb { + background: var(--nt-scrollbar-color); + border-radius: 10px; +} +.nt-plan-root .nt-plan { + flex: 80%; +} +.nt-plan-root.no-groups .nt-plan-periods { + padding-left: 0; +} +.nt-plan-root.no-groups .nt-plan-group-summary { + display: none; +} +.nt-plan-root .nt-timeline-dot.bigger { + top: -10px; +} +.nt-plan-root .nt-timeline-dot.bigger[title] { + cursor: help; +} + +.nt-plan { + white-space: nowrap; + overflow-x: auto; + display: flex; +} +.nt-plan .ug-timeline-dot { + left: 368px; + top: -8px; + cursor: help; +} + +.months { + display: flex; +} + +.month { + flex: auto; + display: inline-block; + box-shadow: rgba(0, 0, 0, 0.2) 0px 3px 1px -2px, rgba(0, 0, 0, 0.14) 0px 2px 2px 0px, rgba(0, 0, 0, 0.12) 0px 1px 5px 0px inset; + background-color: var(--nt-months-background); + color: white; + text-transform: uppercase; + font-family: Roboto, Helvetica, Arial, sans-serif; + padding: 2px 5px; + font-size: 12px; + border: 1px solid #000; + width: 150px; + border-radius: 8px; +} + +.nt-plan-group-activities { + flex: auto; + position: relative; +} + +.nt-vline { + border-left: 1px dashed var(--nt-plan-vertical-line-color); + height: 100%; + left: 0; + position: absolute; + margin-left: -0.5px; + top: 0; + -webkit-transition: all 0.5s linear !important; + -moz-transition: all 0.5s linear !important; + -ms-transition: all 0.5s linear !important; + -o-transition: all 0.5s linear !important; + transition: all 0.5s linear !important; + z-index: -2; +} + +.nt-plan-activity { + display: flex; + margin: 2px 0; + background-color: rgba(187, 187, 187, 0.2509803922); +} + +.actions { + height: var(--nt-plan-actions-height); +} + +.actions { + position: relative; +} + +.period { + display: inline-block; + height: var(--nt-plan-actions-height); + width: 120px; + position: absolute; + left: 0px; + background: #1da1f2; + border-radius: 5px; + transition: all 0.5s; + cursor: help; + -webkit-transition: width 1s ease-in-out; + -moz-transition: width 1s ease-in-out; + -o-transition: width 1s ease-in-out; + transition: width 1s ease-in-out; +} +.period .nt-tooltip { + display: none; + top: 30px; + position: relative; + padding: 1rem; + text-align: center; + font-size: 12px; +} +.period:hover .nt-tooltip { + display: inline-block; +} + +.period-0 { + left: 340px; + visibility: visible; + background-color: rgb(69, 97, 101); +} + +.period-1 { + left: 40px; + visibility: visible; + background-color: green; +} + +.period-2 { + left: 120px; + visibility: visible; + background-color: pink; + width: 80px; +} + +.period-3 { + left: 190px; + visibility: visible; + background-color: darkred; + width: 150px; +} + +.weeks > span, +.days > span { + height: 25px; +} + +.weeks > span { + display: inline-block; + margin: 0; + padding: 0; + font-weight: bold; +} +.weeks > span .week-text { + font-size: 10px; + position: absolute; + display: inline-block; + padding: 3px 4px; +} + +.days { + z-index: -2; + position: relative; +} + +.day-text { + font-size: 10px; + position: absolute; + display: inline-block; + padding: 3px 4px; +} + +.period span { + font-size: 12px; + vertical-align: top; + margin-left: 4px; + color: black; + background: rgba(255, 255, 255, 0.6588235294); + border-radius: 6px; + padding: 0 4px; +} + +.weeks, +.days { + height: 20px; + display: flex; + box-sizing: content-box; +} + +.months { + display: flex; +} + +.week, +.day { + height: 20px; + position: relative; + border: 1; + flex: auto; + border: 2px solid white; + border-radius: 4px; + background-color: var(--nt-units-background); + cursor: help; +} + +.years { + display: flex; +} + +.year { + text-align: center; + border-right: 1px solid var(--nt-plan-vertical-line-color); + font-weight: bold; +} +.year:first-child { + border-left: 1px solid var(--nt-plan-vertical-line-color); +} +.year:first-child:last-child { + width: 100%; +} + +.quarters { + display: flex; +} + +.quarter { + width: 12.5%; + text-align: center; + border-right: 1px solid var(--nt-plan-vertical-line-color); + font-weight: bold; +} +.quarter:first-child { + border-left: 1px solid var(--nt-plan-vertical-line-color); +} + +.nt-plan-group { + margin: 20px 0; + position: relative; +} + +.nt-plan-group { + display: flex; +} + +.nt-plan-group-summary { + background: #2751b0; + width: 150px; + white-space: normal; + padding: 0.1rem 0.5rem; + border-radius: 5px; + color: #fff; + z-index: 3; +} +.nt-plan-group-summary p { + margin: 0; + padding: 0; + font-size: 0.6rem; + color: #fff; +} + +.nt-plan-group-summary, +.month, +.period, +.week, +.day, +.nt-tooltip { + border: 3px solid white; + box-shadow: 0 2px 3px -1px rgba(0, 0, 0, 0.2), 0 3px 3px 0 rgba(0, 0, 0, 0.14), 0 1px 5px 0 rgba(0, 0, 0, 0.12); +} + +.nt-plan-periods { + padding-left: 150px; +} + +.months { + z-index: 2; + position: relative; +} + +.weeks { + position: relative; + top: -2px; + z-index: 0; +} + +.month, +.quarter, +.year, +.week, +.day, +.nt-tooltip { + font-family: Roboto, Helvetica, Arial, sans-serif; + box-sizing: border-box; +} + +.nt-cards.nt-grid { + display: grid; + grid-auto-columns: 1fr; + gap: 0.5rem; + max-width: 100vw; + overflow-x: auto; + padding: 1px; +} +.nt-cards.nt-grid.cols-1 { + grid-template-columns: repeat(1, 1fr); +} +.nt-cards.nt-grid.cols-2 { + grid-template-columns: repeat(2, 1fr); +} +.nt-cards.nt-grid.cols-3 { + grid-template-columns: repeat(3, 1fr); +} +.nt-cards.nt-grid.cols-4 { + grid-template-columns: repeat(4, 1fr); +} +.nt-cards.nt-grid.cols-5 { + grid-template-columns: repeat(5, 1fr); +} +.nt-cards.nt-grid.cols-6 { + grid-template-columns: repeat(6, 1fr); +} + +@media only screen and (max-width: 400px) { + .nt-cards.nt-grid { + grid-template-columns: repeat(1, 1fr) !important; + } +} +.nt-card { + box-shadow: 0 2px 2px 0 rgba(0, 0, 0, 0.14), 0 3px 1px -2px rgba(0, 0, 0, 0.2), 0 1px 5px 0 rgba(0, 0, 0, 0.12); +} +.nt-card:hover { + box-shadow: 0 2px 2px 0 rgba(0, 0, 0, 0.24), 0 3px 1px -2px rgba(0, 0, 0, 0.3), 0 1px 5px 0 rgba(0, 0, 0, 0.22); +} + +[data-md-color-scheme=slate] .nt-card { + box-shadow: 0 2px 2px 0 rgba(4, 40, 33, 0.14), 0 3px 1px -2px rgba(40, 86, 94, 0.47), 0 1px 5px 0 rgba(139, 252, 255, 0.64); +} +[data-md-color-scheme=slate] .nt-card:hover { + box-shadow: 0 2px 2px 0 rgba(0, 255, 206, 0.14), 0 3px 1px -2px rgba(33, 156, 177, 0.47), 0 1px 5px 0 rgba(96, 251, 255, 0.64); +} + +.nt-card > a { + color: var(--md-default-fg-color); +} + +.nt-card > a > div { + cursor: pointer; +} + +.nt-card { + padding: 5px; + margin-bottom: 0.5rem; +} + +.nt-card-title { + font-size: 1rem; + font-weight: bold; + margin: 4px 0 8px 0; + line-height: 22px; +} + +.nt-card-content { + padding: 0.4rem 0.8rem 0.8rem 0.8rem; +} + +.nt-card-text { + font-size: 14px; + padding: 0; + margin: 0; +} + +.nt-card .nt-card-image { + text-align: center; + border-radius: 2px; + background-position: center center; + background-size: cover; + background-repeat: no-repeat; + min-height: 120px; +} + +.nt-card .nt-card-image.tags img { + margin-top: 12px; +} + +.nt-card .nt-card-image img { + height: 105px; + margin-top: 5px; +} + +.nt-card .nt-card-icon { + text-align: center; + padding-top: 12px; + min-height: 120px; +} + +.nt-card .nt-card-icon .icon { + font-size: 95px; + line-height: 1; +} + +.nt-card a:hover, +.nt-card a:focus { + color: var(--md-accent-fg-color); +} + +.nt-card h2 { + margin: 0; +} + +/** + * Extra CSS file recommended for MkDocs and neoteroi.spantable extension. + * + * https://github.com/Neoteroi/mkdocs-plugins +**/ +.span-table-wrapper table { + border-collapse: collapse; + margin-bottom: 2rem; + border-radius: 0.1rem; +} + +.span-table td, +.span-table th { + padding: 0.2rem; + background-color: var(--md-default-bg-color); + font-size: 0.64rem; + max-width: 100%; + overflow: auto; + touch-action: auto; + border-top: 0.05rem solid var(--md-typeset-table-color); + padding: 0.9375em 1.25em; + vertical-align: top; +} + +.span-table tr:first-child td { + font-weight: 700; + min-width: 5rem; + padding: 0.9375em 1.25em; + vertical-align: top; +} + +.span-table td:first-child { + border-left: 0.05rem solid var(--md-typeset-table-color); +} + +.span-table td:last-child { + border-right: 0.05rem solid var(--md-typeset-table-color); +} + +.span-table tr:last-child { + border-bottom: 0.05rem solid var(--md-typeset-table-color); +} + +.span-table [colspan], +.span-table [rowspan] { + font-weight: bold; + border: 0.05rem solid var(--md-typeset-table-color); +} + +.span-table tr:not(:first-child):hover td:not([colspan]):not([rowspan]), +.span-table td[colspan]:hover, +.span-table td[rowspan]:hover { + background-color: rgba(0, 0, 0, 0.035); + box-shadow: 0 0.05rem 0 var(--md-default-bg-color) inset; + transition: background-color 125ms; +} + +.nt-contribs { + margin-top: 2rem; + font-size: small; + border-top: 1px dotted lightgray; + padding-top: 0.5rem; +} +.nt-contribs .nt-contributors { + padding-top: 0.5rem; + display: flex; + flex-wrap: wrap; +} +.nt-contribs .nt-contributor { + background: lightgrey; + background-size: cover; + width: 40px; + height: 40px; + border-radius: 100%; + margin: 0 6px 6px 0; + cursor: help; + opacity: 0.7; +} +.nt-contribs .nt-contributor:hover { + opacity: 1; +} +.nt-contribs .nt-contributors-title { + font-style: italic; + margin-bottom: 0; +} +.nt-contribs .nt-initials { + text-transform: uppercase; + font-size: 20px; + text-align: center; + width: 40px; + height: 40px; + display: inline-block; + vertical-align: middle; + position: relative; + top: 4px; + color: inherit; + font-weight: bold; +} +.nt-contribs .nt-group-0 { + background-color: var(--nt-color-0); +} +.nt-contribs .nt-group-1 { + background-color: var(--nt-color-1); +} +.nt-contribs .nt-group-2 { + background-color: var(--nt-color-2); +} +.nt-contribs .nt-group-3 { + background-color: var(--nt-color-3); +} +.nt-contribs .nt-group-4 { + background-color: var(--nt-color-4); +} +.nt-contribs .nt-group-5 { + background-color: var(--nt-color-5); +} +.nt-contribs .nt-group-6 { + background-color: var(--nt-color-6); +} +.nt-contribs .nt-group-7 { + color: #000; + background-color: var(--nt-color-7); +} +.nt-contribs .nt-group-8 { + color: #000; + background-color: var(--nt-color-8); +} +.nt-contribs .nt-group-9 { + background-color: var(--nt-color-9); +} +.nt-contribs .nt-group-10 { + background-color: var(--nt-color-10); +} +.nt-contribs .nt-group-11 { + background-color: var(--nt-color-11); +} +.nt-contribs .nt-group-12 { + background-color: var(--nt-color-12); +} +.nt-contribs .nt-group-13 { + background-color: var(--nt-color-13); +} +.nt-contribs .nt-group-14 { + background-color: var(--nt-color-14); +} +.nt-contribs .nt-group-15 { + color: #000; + background-color: var(--nt-color-15); +} +.nt-contribs .nt-group-16 { + background-color: var(--nt-color-16); +} +.nt-contribs .nt-group-17 { + color: #000; + background-color: var(--nt-color-17); +} +.nt-contribs .nt-group-18 { + background-color: var(--nt-color-18); +} +.nt-contribs .nt-group-19 { + background-color: var(--nt-color-19); +} +.nt-contribs .nt-group-20 { + color: #000; + background-color: var(--nt-color-20); +} +.nt-contribs .nt-group-21 { + color: #000; + background-color: var(--nt-color-21); +} +.nt-contribs .nt-group-22 { + color: #000; + background-color: var(--nt-color-22); +} +.nt-contribs .nt-group-23 { + color: #000; + background-color: var(--nt-color-23); +} +.nt-contribs .nt-group-24 { + color: #000; + background-color: var(--nt-color-24); +} +.nt-contribs .nt-group-25 { + color: #000; + background-color: var(--nt-color-25); +} +.nt-contribs .nt-group-26 { + color: #000; + background-color: var(--nt-color-26); +} +.nt-contribs .nt-group-27 { + background-color: var(--nt-color-27); +} +.nt-contribs .nt-group-28 { + color: #000; + background-color: var(--nt-color-28); +} +.nt-contribs .nt-group-29 { + color: #000; + background-color: var(--nt-color-29); +} +.nt-contribs .nt-group-30 { + background-color: var(--nt-color-30); +} +.nt-contribs .nt-group-31 { + background-color: var(--nt-color-31); +} +.nt-contribs .nt-group-32 { + color: #000; + background-color: var(--nt-color-32); +} +.nt-contribs .nt-group-33 { + background-color: var(--nt-color-33); +} +.nt-contribs .nt-group-34 { + background-color: var(--nt-color-34); +} +.nt-contribs .nt-group-35 { + background-color: var(--nt-color-35); +} +.nt-contribs .nt-group-36 { + background-color: var(--nt-color-36); +} +.nt-contribs .nt-group-37 { + background-color: var(--nt-color-37); +} +.nt-contribs .nt-group-38 { + background-color: var(--nt-color-38); +} +.nt-contribs .nt-group-39 { + color: #000; + background-color: var(--nt-color-39); +} +.nt-contribs .nt-group-40 { + color: #000; + background-color: var(--nt-color-40); +} +.nt-contribs .nt-group-41 { + color: #000; + background-color: var(--nt-color-41); +} +.nt-contribs .nt-group-42 { + color: #000; + background-color: var(--nt-color-42); +} +.nt-contribs .nt-group-43 { + color: #000; + background-color: var(--nt-color-43); +} +.nt-contribs .nt-group-44 { + color: #000; + background-color: var(--nt-color-44); +} +.nt-contribs .nt-group-45 { + background-color: var(--nt-color-45); +} +.nt-contribs .nt-group-46 { + color: #000; + background-color: var(--nt-color-46); +} +.nt-contribs .nt-group-47 { + background-color: var(--nt-color-47); +} +.nt-contribs .nt-group-48 { + background-color: var(--nt-color-48); +} +.nt-contribs .nt-group-49 { + background-color: var(--nt-color-49); +} + +/** + * CSS for OpenAPI HTML generated with PyMdown Extensions option. + * + * This CSS file works when using the OAD plugin with pymdownx. + * See here how to use it: + * https://www.neoteroi.dev/mkdocs-plugins/web/oad/ + * + * https://github.com/Neoteroi/mkdocs-plugins +**/ +:root { + --http-get-color: green; + --http-delete-color: #dc0101; + --http-head-color: slateblue; + --http-options-color: steelblue; + --http-patch-color: darkorange; + --http-post-color: darkblue; + --http-put-color: darkmagenta; + --http-trace-color: darkcyan; + --http-route-param-color: rgb(51, 128, 210); + --oad-operation-separator-border-color: gray; + --oad-block-border-color: #00bfa5; + --oad-small-note-color: #666; + --oad-indent-border-color: #c5c5c5; +} + +@media screen { + /* Slate theme, i.e. dark mode */ + [data-md-color-scheme=slate] { + --http-get-color: #2ea82e; + --http-post-color: #0093c0; + --http-put-color: #c333c3; + --oad-small-note-color: #afafaf; + } +} +.api-tag { + font-weight: bold; +} + +span[class^=http-] { + font-weight: bold; + color: #fff; + padding: 4px 1rem; + border-radius: 2px; + margin-right: 0.5rem; +} + +.http-get { + background-color: var(--http-get-color); +} + +.http-delete { + background-color: var(--http-delete-color); +} + +.http-post { + background-color: var(--http-post-color); +} + +.http-patch { + background-color: var(--http-patch-color); +} + +.http-trace { + background-color: var(--http-trace-color); +} + +.http-put { + background-color: var(--http-put-color); +} + +.http-head { + background-color: var(--http-head-color); +} + +.http-options { + background-color: var(--http-options-color); +} + +.route-param { + color: var(--http-route-param-color); +} + +.operation-separator + h3[id^=get] .route-param { + color: var(--http-get-color); +} + +.operation-separator + h3[id^=delete] .route-param { + color: var(--http-delete-color); +} + +.operation-separator + h3[id^=post] .route-param { + color: var(--http-post-color); +} + +.operation-separator + h3[id^=patch] .route-param { + color: var(--http-patch-color); +} + +.operation-separator + h3[id^=trace] .route-param { + color: var(--http-trace-color); +} + +.operation-separator + h3[id^=put] .route-param { + color: var(--http-put-color); +} + +.operation-separator + h3[id^=head] .route-param { + color: var(--http-head-color); +} + +.operation-separator + h3[id^=options] .route-param { + color: var(--http-options-color); +} + +.api-version { + font-size: 1.2rem; +} + +.operation-separator { + margin: 0 !important; + border-bottom: 2px dotted var(--oad-operation-separator-border-color) !important; + padding-top: 0.5rem; +} + +.operation-separator + h3 { + margin-top: 1rem; +} + +.string-type { + color: var(--md-code-hl-string-color); +} + +.integer-type, .number-type { + color: var(--md-code-hl-number-color); +} + +.boolean-type { + color: var(--md-code-hl-keyword-color); +} + +.format { + color: var(--md-code-hl-name-color); +} + +.null-type { + color: var(--md-code-hl-keyword-color); +} + +a.ref-link { + color: var(--md-code-hl-special-color); +} + +.request-block + div { + padding-left: 1rem; + border-left: 2px dashed var(--oad-block-border-color); +} + +.small-note { + font-size: 14px; + color: var(--oad-small-note-color); +} + +.request-body-title { + margin-bottom: 4px; +} + +.request-body-title + .tabbed-set, +.response-title + .tabbed-set, +.message-separator + .tabbed-set, +.common-response, +.response-section { + margin-top: 2px; + padding-left: 1rem; + border-left: 2px dotted var(--oad-indent-border-color); +} + +.info-data { + font-size: 0.6rem; +} + +.message-separator { + visibility: hidden; +} + +.sub-section-title { + font-style: italic; + font-size: 14px; +} diff --git a/docs/dev/reference/api/core/extensions.md b/docs/dev/reference/api/core/extensions.md index df8216ead..3436a86d8 100644 --- a/docs/dev/reference/api/core/extensions.md +++ b/docs/dev/reference/api/core/extensions.md @@ -3,8 +3,8 @@ Extension system for extending DiracX functionality. ::: diracx.core.extensions -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true diff --git a/docs/dev/reference/api/core/models.md b/docs/dev/reference/api/core/models.md index 2f6e2db37..6167a17c1 100644 --- a/docs/dev/reference/api/core/models.md +++ b/docs/dev/reference/api/core/models.md @@ -3,8 +3,8 @@ Core Pydantic models used throughout DiracX for data validation and serialization. ::: diracx.core.models -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true diff --git a/docs/dev/reference/api/core/preferences.md b/docs/dev/reference/api/core/preferences.md index 3e8627912..f090ba2d4 100644 --- a/docs/dev/reference/api/core/preferences.md +++ b/docs/dev/reference/api/core/preferences.md @@ -3,8 +3,8 @@ User preferences and configuration options. ::: diracx.core.preferences -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true diff --git a/docs/dev/reference/api/core/properties.md b/docs/dev/reference/api/core/properties.md index 3e90051e4..08ae159dd 100644 --- a/docs/dev/reference/api/core/properties.md +++ b/docs/dev/reference/api/core/properties.md @@ -3,8 +3,8 @@ Security properties and property management. ::: diracx.core.properties -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true diff --git a/docs/dev/reference/api/core/settings.md b/docs/dev/reference/api/core/settings.md index 893f66813..4f47825fa 100644 --- a/docs/dev/reference/api/core/settings.md +++ b/docs/dev/reference/api/core/settings.md @@ -1,10 +1,10 @@ # Settings -Configuration settings for DiracX services and components. +Configuration settings for DiracX services and components. See also [environment variables](/admin/reference/env-variables) and [dev environment variables](/dev/reference/env-variables). ::: diracx.core.settings -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true diff --git a/docs/dev/reference/api/core/utils.md b/docs/dev/reference/api/core/utils.md index 58d8380d0..81f2e09d8 100644 --- a/docs/dev/reference/api/core/utils.md +++ b/docs/dev/reference/api/core/utils.md @@ -3,8 +3,8 @@ Core utility functions and helpers. ::: diracx.core.utils -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true diff --git a/docs/dev/reference/api/db/exceptions.md b/docs/dev/reference/api/db/exceptions.md index 4f69b965d..9e94f0415 100644 --- a/docs/dev/reference/api/db/exceptions.md +++ b/docs/dev/reference/api/db/exceptions.md @@ -3,8 +3,8 @@ Database-related exception classes. ::: diracx.db.exceptions -options: -show_root_heading: true -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: true + show_source: true + members_order: source + group_by_category: true diff --git a/docs/dev/reference/api/index.md b/docs/dev/reference/api/index.md index c11032a4c..7c7d8bf71 100644 --- a/docs/dev/reference/api/index.md +++ b/docs/dev/reference/api/index.md @@ -6,31 +6,31 @@ This section provides detailed API reference documentation for DiracX modules, a Core components including models, settings, configuration, and utilities. -- **[Models](core/models.md)** - Core Pydantic models for data validation -- **[Settings](core/settings.md)** - Configuration settings -- **[Preferences](core/preferences.md)** - User preferences -- **[Configuration](core/config.md)** - Configuration schema and sources -- **[Exceptions](core/exceptions.md)** - Core exception classes -- **[Resources](core/resources.md)** - Resource management and dependency injection -- **[S3](core/s3.md)** - S3-compatible object storage integration -- **[Properties](core/properties.md)** - Security properties -- **[Extensions](core/extensions.md)** - Extension system -- **[Utilities](core/utils.md)** - Core utilities + - **[Models](core/models.md)** - Core Pydantic models for data validation + - **[Settings](core/settings.md)** - Configuration settings + - **[Preferences](core/preferences.md)** - User preferences + - **[Configuration](core/config.md)** - Configuration schema and sources + - **[Exceptions](core/exceptions.md)** - Core exception classes + - **[Resources](core/resources.md)** - Resource management and dependency injection + - **[S3](core/s3.md)** - S3-compatible object storage integration + - **[Properties](core/properties.md)** - Security properties + - **[Extensions](core/extensions.md)** - Extension system + - **[Utilities](core/utils.md)** - Core utilities ## [Routers](routers/index.md) FastAPI routers providing the REST API endpoints. -- **[Jobs](routers/jobs.md)** - Job management endpoints -- **[Auth](routers/auth.md)** - Authentication and authorization -- **[Configuration](routers/configuration.md)** - Configuration management -- **[Health](routers/health.md)** - Health check and monitoring -- **[Access Policies](routers/access_policies.md)** - Access control policies -- **[Dependencies](routers/dependencies.md)** - FastAPI dependency injection utilities -- **[Factory](routers/factory.md)** - Router factory functions -- **[FastAPI Classes](routers/fastapi_classes.md)** - Custom FastAPI router classes -- **[OpenTelemetry](routers/otel.md)** - Tracing and instrumentation -- **[Utilities](routers/utils.md)** - Router utilities + - **[Jobs](routers/jobs.md)** - Job management endpoints + - **[Auth](routers/auth.md)** - Authentication and authorization + - **[Configuration](routers/configuration.md)** - Configuration management + - **[Health](routers/health.md)** - Health check and monitoring + - **[Access Policies](routers/access_policies.md)** - Access control policies + - **[Dependencies](routers/dependencies.md)** - FastAPI dependency injection utilities + - **[Factory](routers/factory.md)** - Router factory functions + - **[FastAPI Classes](routers/fastapi_classes.md)** - Custom FastAPI router classes + - **[OpenTelemetry](routers/otel.md)** - Tracing and instrumentation + - **[Utilities](routers/utils.md)** - Router utilities ## [Logic](logic/index.md) @@ -40,16 +40,16 @@ Business logic layer providing service implementations and orchestration. Database models, schemas, and access layers. -- **[Job DB](db/job.md)** - Job database -- **[Job Logging DB](db/job_logging.md)** - Job logging and history -- **[Auth DB](db/auth.md)** - Authentication and authorization -- **[Sandbox Metadata DB](db/sandbox_metadata.md)** - Sandbox file metadata -- **[Task Queue DB](db/task_queue.md)** - Task queue management -- **[Pilot Agents DB](db/pilot_agents.md)** - Pilot agent tracking -- **[Dummy DB](db/dummy.md)** - Dummy database for testing -- **[OpenSearch](db/opensearch.md)** - OpenSearch-based databases -- **[SQL Utilities](db/utils.md)** - SQL database utilities -- **[Exceptions](db/exceptions.md)** - Database exceptions + - **[Job DB](db/job.md)** - Job database + - **[Job Logging DB](db/job_logging.md)** - Job logging and history + - **[Auth DB](db/auth.md)** - Authentication and authorization + - **[Sandbox Metadata DB](db/sandbox_metadata.md)** - Sandbox file metadata + - **[Task Queue DB](db/task_queue.md)** - Task queue management + - **[Pilot Agents DB](db/pilot_agents.md)** - Pilot agent tracking + - **[Dummy DB](db/dummy.md)** - Dummy database for testing + - **[OpenSearch](db/opensearch.md)** - OpenSearch-based databases + - **[SQL Utilities](db/utils.md)** - SQL database utilities + - **[Exceptions](db/exceptions.md)** - Database exceptions ## [CLI](cli/index.md) @@ -61,10 +61,10 @@ ______________________________________________________________________ Each module page contains automatically generated documentation including: -- **Pydantic Models**: Field descriptions, types, defaults, constraints, and validation rules -- **Functions & Methods**: Parameters, return types, and docstrings -- **Type Annotations**: Full type information for all public APIs -- **Source Links**: Direct links to source code on GitHub + - **Pydantic Models**: Field descriptions, types, defaults, constraints, and validation rules + - **Functions & Methods**: Parameters, return types, and docstrings + - **Type Annotations**: Full type information for all public APIs + - **Source Links**: Direct links to source code on GitHub ## Contributing Documentation diff --git a/docs/dev/reference/api/routers/access_policies.md b/docs/dev/reference/api/routers/access_policies.md index e5d7f2eb5..4ec3c64cc 100644 --- a/docs/dev/reference/api/routers/access_policies.md +++ b/docs/dev/reference/api/routers/access_policies.md @@ -1,6 +1,8 @@ # Access Policies -Access control policies for DiracX routers. +Access control and authorization policies for DiracX routers. + +Access policies define who can access specific API endpoints based on user properties, roles, and other security context. Use these decorators and utilities to enforce authorization rules on your routes. ::: diracx.routers.access_policies options: @@ -9,3 +11,6 @@ Access control policies for DiracX routers. members_order: source group_by_category: true show_if_no_docstring: true + filters: + - "!^\_" + - "!^logger" diff --git a/docs/dev/reference/api/routers/auth.md b/docs/dev/reference/api/routers/auth.md index 5eff90b5c..007fb2627 100644 --- a/docs/dev/reference/api/routers/auth.md +++ b/docs/dev/reference/api/routers/auth.md @@ -9,9 +9,9 @@ Authentication and authorization endpoints. members_order: source group_by_category: true show_if_no_docstring: true -filters: -\- "!^\_" -\- "!^logger" + filters: + - "!^\_" + - "!^logger" ## Token Management diff --git a/docs/dev/reference/api/routers/configuration.md b/docs/dev/reference/api/routers/configuration.md index 2af11ab93..2b67d0343 100644 --- a/docs/dev/reference/api/routers/configuration.md +++ b/docs/dev/reference/api/routers/configuration.md @@ -9,6 +9,6 @@ Configuration management endpoints. members_order: source group_by_category: true show_if_no_docstring: true -filters: -\- "!^\_" -\- "!^logger" + filters: + - "!^\_" + - "!^logger" diff --git a/docs/dev/reference/api/routers/dependencies.md b/docs/dev/reference/api/routers/dependencies.md index 4c863514f..9f4405ce7 100644 --- a/docs/dev/reference/api/routers/dependencies.md +++ b/docs/dev/reference/api/routers/dependencies.md @@ -1,6 +1,8 @@ # Dependencies -FastAPI dependency injection utilities and common dependencies. +FastAPI dependency injection utilities for database access, authentication, and configuration in routers. + +Dependencies are used in route handlers to inject common resources like database connections, authenticated users, and configuration objects. ::: diracx.routers.dependencies options: @@ -9,3 +11,6 @@ FastAPI dependency injection utilities and common dependencies. members_order: source group_by_category: true show_if_no_docstring: true + filters: + - "!^\_" + - "!^logger" diff --git a/docs/dev/reference/api/routers/factory.md b/docs/dev/reference/api/routers/factory.md index 81b58a793..ce1387835 100644 --- a/docs/dev/reference/api/routers/factory.md +++ b/docs/dev/reference/api/routers/factory.md @@ -1,6 +1,8 @@ -# Router Factory +# Factory -Factory functions and utilities for creating DiracX routers. +Router factory functions for creating and registering DiracX routers with proper configuration and dependency injection. + +The factory provides the infrastructure for dynamically loading and configuring routers at application startup. ::: diracx.routers.factory options: @@ -9,3 +11,6 @@ Factory functions and utilities for creating DiracX routers. members_order: source group_by_category: true show_if_no_docstring: true + filters: + - "!^\_" + - "!^logger" diff --git a/docs/dev/reference/api/routers/fastapi_classes.md b/docs/dev/reference/api/routers/fastapi_classes.md index f9f092c61..e95f9bf10 100644 --- a/docs/dev/reference/api/routers/fastapi_classes.md +++ b/docs/dev/reference/api/routers/fastapi_classes.md @@ -1,6 +1,8 @@ # FastAPI Classes -Custom FastAPI router and application classes for DiracX. +Custom FastAPI router classes that extend the standard FastAPI functionality for DiracX-specific features. + +These classes provide enhanced router behavior such as automatic access policy enforcement, extension support, and DiracX-specific metadata. ::: diracx.routers.fastapi_classes options: @@ -9,3 +11,6 @@ Custom FastAPI router and application classes for DiracX. members_order: source group_by_category: true show_if_no_docstring: true + filters: + - "!^\_" + - "!^logger" diff --git a/docs/dev/reference/api/routers/health.md b/docs/dev/reference/api/routers/health.md index 7b4582774..b52246c80 100644 --- a/docs/dev/reference/api/routers/health.md +++ b/docs/dev/reference/api/routers/health.md @@ -9,8 +9,8 @@ Health check and monitoring endpoints for service status. members_order: source group_by_category: true show_if_no_docstring: true -filters: -\- "!^\_" + filters: + - "!^\_" ## Health Probes @@ -21,5 +21,5 @@ filters: members_order: source group_by_category: true show_if_no_docstring: true -filters: -\- "!^\_" + filters: + - "!^\_" diff --git a/docs/dev/reference/api/routers/index.md b/docs/dev/reference/api/routers/index.md index 009fb690a..fc0f64b4e 100644 --- a/docs/dev/reference/api/routers/index.md +++ b/docs/dev/reference/api/routers/index.md @@ -1,18 +1,41 @@ # Routers -FastAPI routers providing the REST API endpoints for DiracX. - -## Available Routers - -- [Jobs](jobs.md) - Job management endpoints -- [Auth](auth.md) - Authentication and authorization -- [Configuration](configuration.md) - Configuration management -- [Health](health.md) - Health check and monitoring endpoints -- [Access Policies](access_policies.md) - Access control policies -- [Dependencies](dependencies.md) - FastAPI dependency injection utilities -- [Factory](factory.md) - Router factory functions -- [FastAPI Classes](fastapi_classes.md) - Custom FastAPI router classes -- [OpenTelemetry](otel.md) - Tracing and instrumentation -- [Utilities](utils.md) - Router utilities - -Each router module contains endpoint definitions, request/response models, and route handlers. +The routers module provides the infrastructure for building FastAPI REST API endpoints in DiracX. This section documents the tools and utilities developers need to add new routes, implement access control, and structure their API endpoints. + +## For REST API Documentation + +If you're looking for documentation on the **available REST API endpoints** (HTTP methods, request/response schemas, etc.), please see: + +- **[REST API Routes](../../routes/index.md)** - Complete REST API documentation with request/response examples + +When running a DiracX instance, you can also access: + +- **Swagger UI** at `/api/docs` - Interactive API documentation where you can try endpoints +- **OpenAPI Specification** at `/api/openapi.json` - Machine-readable API schema + +## Developer Infrastructure + +This section documents the Python infrastructure for **building** routers: + +### Core Infrastructure + +- **[Dependencies](dependencies.md)** - FastAPI dependency injection utilities for database access, authentication, configuration +- **[Factory](factory.md)** - Router factory functions for creating and registering routers +- **[FastAPI Classes](fastapi_classes.md)** - Custom DiracX router classes that extend FastAPI functionality +- **[Access Policies](access_policies.md)** - Access control and authorization policy implementations +- **[Utilities](utils.md)** - Helper functions and utilities for building routes + +### OpenTelemetry Integration + +- **[OpenTelemetry](otel.md)** - Instrumentation and tracing setup for monitoring API requests + +### Example Routers + +For reference on how routers are structured, you can examine the source code of existing routers: + +- `diracx.routers.jobs` - Job management endpoints +- `diracx.routers.auth` - Authentication flows +- `diracx.routers.configuration` - Configuration management +- `diracx.routers.job_manager` - Advanced job operations + +**Note**: For complete REST API endpoint documentation with request/response examples, see the **[REST API Routes](../../routes/index.md)** section. diff --git a/docs/dev/reference/api/routers/jobs.md b/docs/dev/reference/api/routers/jobs.md index 8f46edfb0..ba6162045 100644 --- a/docs/dev/reference/api/routers/jobs.md +++ b/docs/dev/reference/api/routers/jobs.md @@ -4,10 +4,10 @@ Job management API endpoints including submission, querying, and status updates. The Jobs router is composed of multiple sub-routers: -- **Submission**: Job submission endpoints -- **Query**: Job search and filtering -- **Status**: Job status management -- **Sandboxes**: Sandbox upload/download + - **Submission**: Job submission endpoints + - **Query**: Job search and filtering + - **Status**: Job status management + - **Sandboxes**: Sandbox upload/download ## Router @@ -18,62 +18,86 @@ The Jobs router is composed of multiple sub-routers: members_order: source group_by_category: true show_if_no_docstring: true -filters: -\- "!^\_" -\- "!^logger" + filters: + - "!^\_" + - "!^logger" + - "!^router" ## Sub-Routers ### Submission ::: diracx.routers.jobs.submission -options: -show_root_heading: false -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: false + show_source: true + members_order: source + group_by_category: true + filters: + - "!^_" + - "!^router" + - "!^EXAMPLE_" + - "!^MAX\_" ### Query ::: diracx.routers.jobs.query -options: -show_root_heading: false -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: false + show_source: true + members_order: source + group_by_category: true + filters: + - "!^_" + - "!^router" + - "!^EXAMPLE_" + - "!^MAX\_" ### Status ::: diracx.routers.jobs.status -options: -show_root_heading: false -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: false + show_source: true + members_order: source + group_by_category: true + filters: + - "!^\_" + - "!^router" ### Sandboxes ::: diracx.routers.jobs.sandboxes -options: -show_root_heading: false -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: false + show_source: true + members_order: source + group_by_category: true + filters: + - "!^_" + - "!^router" + - "!^EXAMPLE_" + - "!^MAX\_" ### Access Policies ::: diracx.routers.jobs.access_policies -options: -show_root_heading: false -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: false + show_source: true + members_order: source + group_by_category: true + filters: + - "!^\_" ### Legacy ::: diracx.routers.jobs.legacy -options: -show_root_heading: false -show_source: true -members_order: source -group_by_category: true + options: + show_root_heading: false + show_source: true + members_order: source + group_by_category: true + filters: + - "!^\_" + - "!^router" diff --git a/docs/dev/reference/api/routers/otel.md b/docs/dev/reference/api/routers/otel.md index e40af0d41..5bbc8fb8a 100644 --- a/docs/dev/reference/api/routers/otel.md +++ b/docs/dev/reference/api/routers/otel.md @@ -2,6 +2,8 @@ OpenTelemetry instrumentation and tracing utilities for DiracX routers. +This module provides integration with OpenTelemetry for distributed tracing, metrics collection, and observability of API requests. + ::: diracx.routers.otel options: show_root_heading: true @@ -9,3 +11,6 @@ OpenTelemetry instrumentation and tracing utilities for DiracX routers. members_order: source group_by_category: true show_if_no_docstring: true + filters: + - "!^\_" + - "!^logger" diff --git a/docs/dev/reference/api/routers/utils.md b/docs/dev/reference/api/routers/utils.md index cd2b596b6..ee55a2897 100644 --- a/docs/dev/reference/api/routers/utils.md +++ b/docs/dev/reference/api/routers/utils.md @@ -1,6 +1,8 @@ # Router Utilities -Utility functions for DiracX routers. +Helper functions and utilities for building DiracX routers. + +These utilities provide common functionality needed across multiple routers, such as user management, response formatting, and request validation. ## User Utilities @@ -11,3 +13,6 @@ Utility functions for DiracX routers. members_order: source group_by_category: true show_if_no_docstring: true + filters: + - "!^\_" + - "!^logger" diff --git a/docs/dev/reference/api/writing-api-docs.md b/docs/dev/reference/api/writing-api-docs.md index 1b5fb1df5..d3cb7b7a7 100644 --- a/docs/dev/reference/api/writing-api-docs.md +++ b/docs/dev/reference/api/writing-api-docs.md @@ -70,13 +70,13 @@ class UserConfig(BaseModel): Griffe-Pydantic extracts and displays: -- **Field names and types** (including Union types, Optional, etc.) -- **Field descriptions** from `Field(description=...)` -- **Default values** from `Field(default=...)` -- **Validation constraints**: `min_length`, `max_length`, `ge`, `le`, `pattern`, etc. -- **Required vs optional** fields -- **Field examples** from `Field(examples=...)` -- **Validators** (custom field validators) + - **Field names and types** (including Union types, Optional, etc.) + - **Field descriptions** from `Field(description=...)` + - **Default values** from `Field(default=...)` + - **Validation constraints**: `min_length`, `max_length`, `ge`, `le`, `pattern`, etc. + - **Required vs optional** fields + - **Field examples** from `Field(examples=...)` + - **Validators** (custom field validators) ## Class Documentation @@ -212,11 +212,11 @@ By default, mkdocstrings will discover and document: This will show: -- Module docstring -- All public classes -- All public functions -- All public constants -- Grouped by category (attributes, classes, functions, etc.) + - Module docstring + - All public classes + - All public functions + - All public constants + - Grouped by category (attributes, classes, functions, etc.) #### Document a Specific Class @@ -304,11 +304,11 @@ Job management endpoints. This will show: -- The router's docstring -- All route handler functions with their HTTP methods, paths, and parameters -- Request/response models -- Dependencies -- Source code links + - The router's docstring + - All route handler functions with their HTTP methods, paths, and parameters + - Request/response models + - Dependencies + - Source code links ## Best Practices @@ -458,7 +458,7 @@ class MyClass: ## Additional Resources -- [Google Style Docstrings Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) -- [Griffe Documentation](https://mkdocstrings.github.io/griffe/) -- [Griffe-Pydantic Documentation](https://mkdocstrings.github.io/griffe-pydantic/) -- [mkdocstrings Documentation](https://mkdocstrings.github.io/) + - [Google Style Docstrings Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) + - [Griffe Documentation](https://mkdocstrings.github.io/griffe/) + - [Griffe-Pydantic Documentation](https://mkdocstrings.github.io/griffe-pydantic/) + - [mkdocstrings Documentation](https://mkdocstrings.github.io/) diff --git a/docs/dev/reference/pixi-tasks.md b/docs/dev/reference/pixi-tasks.md index 34ad9bc4f..13983d9b1 100644 --- a/docs/dev/reference/pixi-tasks.md +++ b/docs/dev/reference/pixi-tasks.md @@ -12,6 +12,7 @@ This page documents the available pixi tasks. ## Default Tasks - `description`: Run pre-commit hooks +- `generate-openapi-spec`: Generate OpenAPI specification for documentation ## DiracX Tasks diff --git a/docs/dev/reference/routes/index.md b/docs/dev/reference/routes/index.md new file mode 100644 index 000000000..96ab30ac7 --- /dev/null +++ b/docs/dev/reference/routes/index.md @@ -0,0 +1,17 @@ +This section documents the REST API endpoints available in DiracX. The route documentation is automatically generated from the OpenAPI specification. + +DiracX provides comprehensive REST API documentation through multiple formats: + +- **Route Documentation (this section)** - Human-readable documentation with request/response examples + +When running a DiracX instance, you can also access: + +- **Interactive Documentation (Swagger UI)** at `/api/docs` - Try the API endpoints directly in your browser +- **OpenAPI Specification** at `/api/openapi.json` - Machine-readable API schema for tools and clients + +If you're looking for documentation on **building** routes (the Python infrastructure), see: + +- **[Router Infrastructure](../api/routers/index.md)** - Dependencies, factory functions, access policies +- **[Adding a New Route](../../how-to/add-a-route.md)** - Tutorial for creating new API endpoints + +[OAD(../../../openapi.json)] diff --git a/mkdocs.yml b/mkdocs.yml index ad719223b..900451375 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,6 @@ extra_css: - assets/css/extra.css + - assets/css/neoteroi-mkdocs.css site_name: DiracX repo_url: https://github.com/DIRACGrid/diracx @@ -216,6 +217,7 @@ nav: - Exceptions: dev/reference/api/db/exceptions.md - CLI: - dev/reference/api/cli/index.md + - Routes: dev/reference/routes/index.md - Dev env variables: dev/reference/env-variables.md - Writing tests: dev/reference/writing-tests.md - Coding conventions: dev/reference/coding-conventions.md @@ -255,6 +257,7 @@ markdown_extensions: - pymdownx.tabbed: alternate_style: true - tables + - pymdownx.details plugins: - privacy diff --git a/pixi.toml b/pixi.toml index 237c1d178..a7e487571 100644 --- a/pixi.toml +++ b/pixi.toml @@ -127,7 +127,7 @@ diracx-client = { path = "diracx-client", editable = true } diracx-cli = { path = "diracx-cli", editable = true } [feature.mkdocs.tasks] mkdocs = "mkdocs serve" -mkdocs-build = "mkdocs build --strict" +mkdocs-build = { cmd = "mkdocs build --strict", depends-on = ["generate-openapi-spec"] } description = "Generate the documentation with mkdocs" # Features for running pre-commit hooks @@ -199,6 +199,11 @@ shellcheck = {features = ["shellcheck"], no-default-feature = true} pre-commit = {features = ["pre-commit"], no-default-feature = true} settings-doc = {features = ["settings-doc"], no-default-feature = true} +# Task for generating OpenAPI specification +[tasks.generate-openapi-spec] +cmd = "python scripts/generate_openapi_spec.py" +description = "Generate OpenAPI specification for documentation" + # Meta-tasks for running many tests at once [tasks.pytest-diracx-all-one-by-one] depends-on = [ diff --git a/scripts/generate_openapi_spec.py b/scripts/generate_openapi_spec.py new file mode 100644 index 000000000..7a2bfd718 --- /dev/null +++ b/scripts/generate_openapi_spec.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Generate OpenAPI specification from DiracX FastAPI app. + +This script creates docs/openapi.json from the DiracX FastAPI application. +It uses the same approach as the test suite to create a minimal app configuration +with test settings and a dummy git config repository. +""" + +from __future__ import annotations + +import json +import sys +import tempfile +from pathlib import Path + +from git import Repo +from joserfc.jwk import KeySet, OKPKey + +from diracx.core.config import Config +from diracx.core.extensions import select_from_extension +from diracx.core.settings import ( + AuthSettings, + DevelopmentSettings, + SandboxStoreSettings, +) +from diracx.routers import create_app_inner + + +def create_test_config_repo() -> Path: + """Create a minimal git repository with test configuration. + + Returns: + Path to the temporary git repository + + """ + tmp_path = Path(tempfile.mkdtemp(prefix="diracx-config-")) + + repo = Repo.init(tmp_path, initial_branch="master") + cs_file = tmp_path / "default.yml" + example_cs = Config.model_validate( + { + "DIRAC": {}, + "Registry": { + "lhcb": { + "DefaultGroup": "lhcb_user", + "DefaultProxyLifeTime": 432000, + "DefaultStorageQuota": 2000, + "IdP": { + "URL": "https://idp-server.invalid", + "ClientID": "test-idp", + }, + "Users": { + "b824d4dc-1f9d-4ee8-8df5-c0ae55d46041": { + "PreferedUsername": "chaen", + "Email": None, + }, + "c935e5ed-2g0e-5ff9-9eg6-d1bf66e57152": { + "PreferedUsername": "albdr", + "Email": None, + }, + }, + "Groups": { + "lhcb_user": { + "Properties": ["NormalUser", "PrivateLimitedDelegation"], + "Users": [ + "b824d4dc-1f9d-4ee8-8df5-c0ae55d46041", + "c935e5ed-2g0e-5ff9-9eg6-d1bf66e57152", + ], + }, + "lhcb_prmgr": { + "Properties": ["NormalUser", "ProductionManagement"], + "Users": ["b824d4dc-1f9d-4ee8-8df5-c0ae55d46041"], + }, + "lhcb_tokenmgr": { + "Properties": ["NormalUser", "ProxyManagement"], + "Users": ["c935e5ed-2g0e-5ff9-9eg6-d1bf66e57152"], + }, + }, + } + }, + "Operations": {}, + "Systems": { + "WorkloadManagement": { + "Databases": { + "JobDB": { + "DBName": "xyz", + "Host": "xyz", + "Port": 9999, + "MaxRescheduling": 3, + }, + "JobLoggingDB": { + "DBName": "xyz", + "Host": "xyz", + "Port": 9999, + }, + "PilotAgentsDB": { + "DBName": "xyz", + "Host": "xyz", + "Port": 9999, + }, + "SandboxMetadataDB": { + "DBName": "xyz", + "Host": "xyz", + "Port": 9999, + }, + "TaskQueueDB": { + "DBName": "xyz", + "Host": "xyz", + "Port": 9999, + }, + "ElasticJobParametersDB": { + "DBName": "xyz", + "Host": "xyz", + "Port": 9999, + }, + "VirtualMachineDB": { + "DBName": "xyz", + "Host": "xyz", + "Port": 9999, + }, + }, + }, + }, + } + ) + cs_file.write_text(example_cs.model_dump_json()) + repo.index.add(["default.yml"]) # Use relative path + repo.index.commit("Initial configuration") + return tmp_path + + +def create_test_auth_settings() -> AuthSettings: + """Create test authentication settings with a generated key. + + Returns: + AuthSettings instance with test configuration + + """ + from cryptography.fernet import Fernet + from uuid_utils import uuid7 + + private_key = OKPKey.generate_key( + parameters={ + "key_ops": ["sign", "verify"], + "alg": "EdDSA", + "kid": uuid7().hex, + } + ) + fernet_key = Fernet.generate_key().decode() + + return AuthSettings( + token_issuer="http://lalalalalalalalalalalala.invalid/", # noqa: S106 + token_keystore=json.dumps(KeySet([private_key]).as_dict(private_keys=True)), + state_key=fernet_key, + allowed_redirects=[ + "http://diracx.test.invalid:8000/api/docs/oauth2-redirect", + ], + ) + + +def create_test_sandbox_settings() -> SandboxStoreSettings: + """Create test sandbox store settings for S3. + + Returns: + SandboxStoreSettings instance with test configuration + + """ + return SandboxStoreSettings( + bucket_name="sandboxes", + s3_client_kwargs={ + "endpoint_url": "http://localhost:27132", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing", + }, + auto_create_bucket=True, + ) + + +def generate_openapi_spec(output_path: Path) -> None: + """Generate OpenAPI specification and save to file. + + Args: + output_path: Path where the openapi.json file should be saved + + """ + print("šŸ”§ Creating test configuration repository...") + config_repo = create_test_config_repo() + + try: + print("šŸ”§ Setting up test settings...") + auth_settings = create_test_auth_settings() + sandbox_settings = create_test_sandbox_settings() + dev_settings = DevelopmentSettings() + + print("šŸ”§ Discovering DiracX services...") + from diracx.core.config import ConfigSource + from diracx.routers.access_policies import BaseAccessPolicy + + enabled_systems = { + e.name for e in select_from_extension(group="diracx.services") + } + database_urls = { + e.name: "sqlite+aiosqlite:///:memory:" + for e in select_from_extension(group="diracx.dbs.sql") + } + os_database_conn_kwargs = { + e.name: {"sqlalchemy_dsn": "sqlite+aiosqlite:///:memory:"} + for e in select_from_extension(group="diracx.dbs.os") + } + + # Setup access policies + all_access_policies = { + e.name: BaseAccessPolicy.available_implementations(e.name) + for e in select_from_extension(group="diracx.access_policies") + } + + config_source = ConfigSource.create_from_url( + backend_url=f"git+file://{config_repo}" + ) + + print("šŸ”§ Creating FastAPI application...") + app = create_app_inner( + enabled_systems=enabled_systems, + all_service_settings=[auth_settings, sandbox_settings, dev_settings], + database_urls=database_urls, + os_database_conn_kwargs=os_database_conn_kwargs, + config_source=config_source, + all_access_policies=all_access_policies, + ) + + print("šŸ“ Generating OpenAPI specification...") + openapi_schema = app.openapi() + + print(f"šŸ’¾ Writing to {output_path.relative_to(output_path.parent.parent)}...") + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(openapi_schema, indent=2)) + + # Print some statistics + paths_count = len(openapi_schema.get("paths", {})) + schemas_count = len(openapi_schema.get("components", {}).get("schemas", {})) + print( + f"āœ… Generated OpenAPI spec with {paths_count} paths and {schemas_count} schemas" + ) + + finally: + # Clean up the temporary config repository + import shutil + + shutil.rmtree(config_repo, ignore_errors=True) + + +def main(): + """Main entry point.""" + repo_root = Path(__file__).parent.parent + output_path = repo_root / "docs" / "openapi.json" + + print("šŸš€ Generating OpenAPI specification for DiracX...\n") + try: + generate_openapi_spec(output_path) + print("\nāœ… OpenAPI specification generated successfully!") + return 0 + except Exception as e: + print(f"\nāŒ Failed to generate OpenAPI specification: {e}") + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) From 131e86f22feea7dfc966dc629d06c370142957be Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 03:42:06 +0100 Subject: [PATCH 15/18] fix: doc build --- pixi.toml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pixi.toml b/pixi.toml index a7e487571..30d20657a 100644 --- a/pixi.toml +++ b/pixi.toml @@ -125,10 +125,12 @@ diracx-db = { path = "diracx-db", editable = true } diracx-logic = { path = "diracx-logic", editable = true } diracx-client = { path = "diracx-client", editable = true } diracx-cli = { path = "diracx-cli", editable = true } +pyparsing = "*" [feature.mkdocs.tasks] mkdocs = "mkdocs serve" mkdocs-build = { cmd = "mkdocs build --strict", depends-on = ["generate-openapi-spec"] } description = "Generate the documentation with mkdocs" +generate-openapi-spec = "python scripts/generate_openapi_spec.py" # Features for running pre-commit hooks [feature.pre-commit.dependencies] @@ -199,11 +201,6 @@ shellcheck = {features = ["shellcheck"], no-default-feature = true} pre-commit = {features = ["pre-commit"], no-default-feature = true} settings-doc = {features = ["settings-doc"], no-default-feature = true} -# Task for generating OpenAPI specification -[tasks.generate-openapi-spec] -cmd = "python scripts/generate_openapi_spec.py" -description = "Generate OpenAPI specification for documentation" - # Meta-tasks for running many tests at once [tasks.pytest-diracx-all-one-by-one] depends-on = [ From 1ee8d6f83c052e3d4a51651c7d2579400cbb9fab Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 03:50:19 +0100 Subject: [PATCH 16/18] fix api ref index --- docs/dev/reference/api/index.md | 68 ++++++++++++++++----------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/dev/reference/api/index.md b/docs/dev/reference/api/index.md index 7c7d8bf71..c11032a4c 100644 --- a/docs/dev/reference/api/index.md +++ b/docs/dev/reference/api/index.md @@ -6,31 +6,31 @@ This section provides detailed API reference documentation for DiracX modules, a Core components including models, settings, configuration, and utilities. - - **[Models](core/models.md)** - Core Pydantic models for data validation - - **[Settings](core/settings.md)** - Configuration settings - - **[Preferences](core/preferences.md)** - User preferences - - **[Configuration](core/config.md)** - Configuration schema and sources - - **[Exceptions](core/exceptions.md)** - Core exception classes - - **[Resources](core/resources.md)** - Resource management and dependency injection - - **[S3](core/s3.md)** - S3-compatible object storage integration - - **[Properties](core/properties.md)** - Security properties - - **[Extensions](core/extensions.md)** - Extension system - - **[Utilities](core/utils.md)** - Core utilities +- **[Models](core/models.md)** - Core Pydantic models for data validation +- **[Settings](core/settings.md)** - Configuration settings +- **[Preferences](core/preferences.md)** - User preferences +- **[Configuration](core/config.md)** - Configuration schema and sources +- **[Exceptions](core/exceptions.md)** - Core exception classes +- **[Resources](core/resources.md)** - Resource management and dependency injection +- **[S3](core/s3.md)** - S3-compatible object storage integration +- **[Properties](core/properties.md)** - Security properties +- **[Extensions](core/extensions.md)** - Extension system +- **[Utilities](core/utils.md)** - Core utilities ## [Routers](routers/index.md) FastAPI routers providing the REST API endpoints. - - **[Jobs](routers/jobs.md)** - Job management endpoints - - **[Auth](routers/auth.md)** - Authentication and authorization - - **[Configuration](routers/configuration.md)** - Configuration management - - **[Health](routers/health.md)** - Health check and monitoring - - **[Access Policies](routers/access_policies.md)** - Access control policies - - **[Dependencies](routers/dependencies.md)** - FastAPI dependency injection utilities - - **[Factory](routers/factory.md)** - Router factory functions - - **[FastAPI Classes](routers/fastapi_classes.md)** - Custom FastAPI router classes - - **[OpenTelemetry](routers/otel.md)** - Tracing and instrumentation - - **[Utilities](routers/utils.md)** - Router utilities +- **[Jobs](routers/jobs.md)** - Job management endpoints +- **[Auth](routers/auth.md)** - Authentication and authorization +- **[Configuration](routers/configuration.md)** - Configuration management +- **[Health](routers/health.md)** - Health check and monitoring +- **[Access Policies](routers/access_policies.md)** - Access control policies +- **[Dependencies](routers/dependencies.md)** - FastAPI dependency injection utilities +- **[Factory](routers/factory.md)** - Router factory functions +- **[FastAPI Classes](routers/fastapi_classes.md)** - Custom FastAPI router classes +- **[OpenTelemetry](routers/otel.md)** - Tracing and instrumentation +- **[Utilities](routers/utils.md)** - Router utilities ## [Logic](logic/index.md) @@ -40,16 +40,16 @@ Business logic layer providing service implementations and orchestration. Database models, schemas, and access layers. - - **[Job DB](db/job.md)** - Job database - - **[Job Logging DB](db/job_logging.md)** - Job logging and history - - **[Auth DB](db/auth.md)** - Authentication and authorization - - **[Sandbox Metadata DB](db/sandbox_metadata.md)** - Sandbox file metadata - - **[Task Queue DB](db/task_queue.md)** - Task queue management - - **[Pilot Agents DB](db/pilot_agents.md)** - Pilot agent tracking - - **[Dummy DB](db/dummy.md)** - Dummy database for testing - - **[OpenSearch](db/opensearch.md)** - OpenSearch-based databases - - **[SQL Utilities](db/utils.md)** - SQL database utilities - - **[Exceptions](db/exceptions.md)** - Database exceptions +- **[Job DB](db/job.md)** - Job database +- **[Job Logging DB](db/job_logging.md)** - Job logging and history +- **[Auth DB](db/auth.md)** - Authentication and authorization +- **[Sandbox Metadata DB](db/sandbox_metadata.md)** - Sandbox file metadata +- **[Task Queue DB](db/task_queue.md)** - Task queue management +- **[Pilot Agents DB](db/pilot_agents.md)** - Pilot agent tracking +- **[Dummy DB](db/dummy.md)** - Dummy database for testing +- **[OpenSearch](db/opensearch.md)** - OpenSearch-based databases +- **[SQL Utilities](db/utils.md)** - SQL database utilities +- **[Exceptions](db/exceptions.md)** - Database exceptions ## [CLI](cli/index.md) @@ -61,10 +61,10 @@ ______________________________________________________________________ Each module page contains automatically generated documentation including: - - **Pydantic Models**: Field descriptions, types, defaults, constraints, and validation rules - - **Functions & Methods**: Parameters, return types, and docstrings - - **Type Annotations**: Full type information for all public APIs - - **Source Links**: Direct links to source code on GitHub +- **Pydantic Models**: Field descriptions, types, defaults, constraints, and validation rules +- **Functions & Methods**: Parameters, return types, and docstrings +- **Type Annotations**: Full type information for all public APIs +- **Source Links**: Direct links to source code on GitHub ## Contributing Documentation From badc80fe220cada6343106d353bdba545666144b Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 03:57:15 +0100 Subject: [PATCH 17/18] fix: pixi task docs --- .github/workflows/generate_pixi_tasks_doc.py | 2 +- docs/dev/reference/pixi-tasks.md | 10 +++------- pixi.toml | 7 +++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/generate_pixi_tasks_doc.py b/.github/workflows/generate_pixi_tasks_doc.py index 85c6f4cd7..ac4933765 100644 --- a/.github/workflows/generate_pixi_tasks_doc.py +++ b/.github/workflows/generate_pixi_tasks_doc.py @@ -10,7 +10,7 @@ def get_task_group(task_name): return "DiracX" if task_name.startswith("pytest-gubbins"): return "Gubbins" - if task_name.startswith("mkdocs"): + if task_name.startswith("mkdocs") or task_name == "generate-openapi-spec": return "Documentation" if task_name == "pre-commit": return "Pre-commit" diff --git a/docs/dev/reference/pixi-tasks.md b/docs/dev/reference/pixi-tasks.md index 13983d9b1..ab87c97ca 100644 --- a/docs/dev/reference/pixi-tasks.md +++ b/docs/dev/reference/pixi-tasks.md @@ -9,11 +9,6 @@ This page documents the available pixi tasks. - `generate-client`: Generate the API clients for diracx-client -## Default Tasks - -- `description`: Run pre-commit hooks -- `generate-openapi-spec`: Generate OpenAPI specification for documentation - ## DiracX Tasks - `pytest-diracx`: pytest @@ -28,8 +23,9 @@ This page documents the available pixi tasks. ## Documentation Tasks +- `generate-openapi-spec`: python scripts/generate_openapi_spec.py - `mkdocs`: mkdocs serve -- `mkdocs-build`: mkdocs build --strict +- `mkdocs-build`: Generate the documentation with mkdocs ## Gubbins Tasks @@ -45,7 +41,7 @@ This page documents the available pixi tasks. ## Pre-commit Tasks -- `pre-commit`: pre-commit +- `pre-commit`: Run pre-commit hooks ## Settings Tasks diff --git a/pixi.toml b/pixi.toml index 30d20657a..ccbc2a3b8 100644 --- a/pixi.toml +++ b/pixi.toml @@ -128,15 +128,14 @@ diracx-cli = { path = "diracx-cli", editable = true } pyparsing = "*" [feature.mkdocs.tasks] mkdocs = "mkdocs serve" -mkdocs-build = { cmd = "mkdocs build --strict", depends-on = ["generate-openapi-spec"] } -description = "Generate the documentation with mkdocs" +mkdocs-build = { cmd = "mkdocs build --strict", depends-on = ["generate-openapi-spec"], description = "Generate the documentation with mkdocs" } generate-openapi-spec = "python scripts/generate_openapi_spec.py" # Features for running pre-commit hooks [feature.pre-commit.dependencies] pre-commit = "*" -[feature.pre-commit.tasks] -pre-commit = "pre-commit" +[feature.pre-commit.tasks.pre-commit] +cmd = "pre-commit" description = "Run pre-commit hooks" # Features for generating the clients From fbc12ef2ea068ed4d942e35e52e1a546047e3404 Mon Sep 17 00:00:00 2001 From: Ryunosuke O'Neil Date: Sun, 9 Nov 2025 04:00:09 +0100 Subject: [PATCH 18/18] fix: docs indentation --- docs/dev/reference/api/routers/jobs.md | 8 ++--- docs/dev/reference/api/writing-api-docs.md | 42 +++++++++++----------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/dev/reference/api/routers/jobs.md b/docs/dev/reference/api/routers/jobs.md index ba6162045..48b1d0563 100644 --- a/docs/dev/reference/api/routers/jobs.md +++ b/docs/dev/reference/api/routers/jobs.md @@ -4,10 +4,10 @@ Job management API endpoints including submission, querying, and status updates. The Jobs router is composed of multiple sub-routers: - - **Submission**: Job submission endpoints - - **Query**: Job search and filtering - - **Status**: Job status management - - **Sandboxes**: Sandbox upload/download +- **Submission**: Job submission endpoints +- **Query**: Job search and filtering +- **Status**: Job status management +- **Sandboxes**: Sandbox upload/download ## Router diff --git a/docs/dev/reference/api/writing-api-docs.md b/docs/dev/reference/api/writing-api-docs.md index d3cb7b7a7..1b5fb1df5 100644 --- a/docs/dev/reference/api/writing-api-docs.md +++ b/docs/dev/reference/api/writing-api-docs.md @@ -70,13 +70,13 @@ class UserConfig(BaseModel): Griffe-Pydantic extracts and displays: - - **Field names and types** (including Union types, Optional, etc.) - - **Field descriptions** from `Field(description=...)` - - **Default values** from `Field(default=...)` - - **Validation constraints**: `min_length`, `max_length`, `ge`, `le`, `pattern`, etc. - - **Required vs optional** fields - - **Field examples** from `Field(examples=...)` - - **Validators** (custom field validators) +- **Field names and types** (including Union types, Optional, etc.) +- **Field descriptions** from `Field(description=...)` +- **Default values** from `Field(default=...)` +- **Validation constraints**: `min_length`, `max_length`, `ge`, `le`, `pattern`, etc. +- **Required vs optional** fields +- **Field examples** from `Field(examples=...)` +- **Validators** (custom field validators) ## Class Documentation @@ -212,11 +212,11 @@ By default, mkdocstrings will discover and document: This will show: - - Module docstring - - All public classes - - All public functions - - All public constants - - Grouped by category (attributes, classes, functions, etc.) +- Module docstring +- All public classes +- All public functions +- All public constants +- Grouped by category (attributes, classes, functions, etc.) #### Document a Specific Class @@ -304,11 +304,11 @@ Job management endpoints. This will show: - - The router's docstring - - All route handler functions with their HTTP methods, paths, and parameters - - Request/response models - - Dependencies - - Source code links +- The router's docstring +- All route handler functions with their HTTP methods, paths, and parameters +- Request/response models +- Dependencies +- Source code links ## Best Practices @@ -458,7 +458,7 @@ class MyClass: ## Additional Resources - - [Google Style Docstrings Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) - - [Griffe Documentation](https://mkdocstrings.github.io/griffe/) - - [Griffe-Pydantic Documentation](https://mkdocstrings.github.io/griffe-pydantic/) - - [mkdocstrings Documentation](https://mkdocstrings.github.io/) +- [Google Style Docstrings Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) +- [Griffe Documentation](https://mkdocstrings.github.io/griffe/) +- [Griffe-Pydantic Documentation](https://mkdocstrings.github.io/griffe-pydantic/) +- [mkdocstrings Documentation](https://mkdocstrings.github.io/)