From f96086e070044f5fa64d7a97d933a5b6b552cc3c Mon Sep 17 00:00:00 2001 From: Sagar Ghimire Date: Wed, 19 Aug 2026 10:56:17 +0545 Subject: [PATCH 1/5] feat: themed API docs and versioned route namespace Restructure the docs surface and the route namespace. Docs: serve Swagger UI from vendored assets instead of a CDN, themed to match ckanext-openapidocs so both sets of docs read as one family. The page is a Jinja template; branding comes from DOCS_* env vars, validated as CSS colours since they land in a + {%- endif %} + + +
+
+ {%- if logo_url %} + + {%- endif %} +
+

{{ site_title }}

+ {%- if api_version %} + {{ api_version }} + {%- endif %} +
+ OpenAPI spec +
+
+ +
+ + + + + diff --git a/datastore/core/config.py b/datastore/core/config.py index c201e5d..c5248f0 100644 --- a/datastore/core/config.py +++ b/datastore/core/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from functools import lru_cache from pathlib import Path from typing import Literal @@ -39,15 +40,19 @@ def _available_auth_types() -> set[str]: return _subdirs(_AUTH_DIR) +# Hex / rgb() / hsl() / named CSS colours. Deliberately narrow: these values +# land inside a `" - ) - return HTMLResponse(html) - - -# Swagger "Authorize" description per built-in AUTH_TYPE. A provider not -# listed here (a third-party drop-in under `datastore/auth//`) keeps -# the generic description declared on the scheme in `api/context.py`. -_AUTH_SCHEME_DESCRIPTIONS = { - "ckan": "A CKAN API key.", - "jwt": "A signed JWT. Accepts the raw token or `Bearer `.", -} - - -def _tailor_auth_scheme(app: FastAPI, auth_type: str) -> None: - """Shape the OpenAPI security scheme to the active AUTH_TYPE. - - The `Authorization` scheme is declared once in `api/context.py`, at - import time — before config is read — so its description can't know - which provider runs. The app factory does, and rewrites the schema - here: provider-specific wording for `ckan` / `jwt`; under `anonymous` - the scheme and every operation's `security` entry are removed, so the - docs render no Authorize button at all. - """ - default_openapi = app.openapi - - def openapi() -> dict[str, Any]: - schema = default_openapi() - schemes = schema.get("components", {}).get("securitySchemes", {}) - if auth_type == "anonymous": - schemes.pop("Authorization", None) - if not schemes: - schema.get("components", {}).pop("securitySchemes", None) - for path_item in schema.get("paths", {}).values(): - for operation in path_item.values(): - if not isinstance(operation, dict): - continue - security = [ - requirement - for requirement in operation.get("security", []) - if "Authorization" not in requirement - ] - if security: - operation["security"] = security - else: - operation.pop("security", None) - elif auth_type in _AUTH_SCHEME_DESCRIPTIONS and "Authorization" in schemes: - description = _AUTH_SCHEME_DESCRIPTIONS[auth_type] - schemes["Authorization"]["description"] = description - return schema - - app.openapi = openapi # type: ignore[method-assign] - - -def _strip_default_422(app: FastAPI) -> None: - """Drop FastAPI's auto-generated 422 from the schema. - - `RequestValidationError` is remapped to a 400 CKAN error envelope (see - `error_handlers`), so a documented 422 never actually occurs — the real - 4xx shapes are declared via `ERROR_RESPONSES`. - """ - default_openapi = app.openapi - - def openapi() -> dict[str, Any]: - schema = default_openapi() - for path_item in schema.get("paths", {}).values(): - for operation in path_item.values(): - if isinstance(operation, dict): - operation.get("responses", {}).pop("422", None) - components = schema.get("components", {}).get("schemas", {}) - components.pop("HTTPValidationError", None) - components.pop("ValidationError", None) - return schema - - app.openapi = openapi # type: ignore[method-assign] - - @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Per-process startup/shutdown. @@ -211,25 +79,11 @@ def create_app() -> FastAPI: config = get_config() app = FastAPI( title="Datastore API", - version=_api_version(), - summary="A datastore API for managing tabular data resources.", - description=( - "📮 **Postman collection** — import the " - "[Datastore API collection]" - "(https://raw.githubusercontent.com/datopian/datastore/main/" - "postman/collection.json) via Postman's **Import → Link** to " - "exercise every endpoint with worked examples." - ), + description=api_description(config.AUTH_TYPE), openapi_tags=OPENAPI_TAGS, - contact={"name": "Datopian", "url": "https://www.datopian.com/"}, - # Mount the interactive docs (and the spec they fetch) under the - # service's path prefix so this API doesn't compete with an upstream - # proxy or sibling service for the bare `/docs`. Swagger UI is - # served by `_register_swagger_docs` (CSS overrides), not FastAPI's - # stock route. docs_url=None, - redoc_url="/datastore/api/redoc", - openapi_url="/datastore/api/openapi.json", + redoc_url=f"{API_PREFIX}/redoc", + openapi_url=f"{API_PREFIX}/openapi.json", lifespan=lifespan, default_response_class=ORJSONResponse, ) @@ -252,9 +106,9 @@ def create_app() -> FastAPI: register_exception_handlers(app) app.include_router(api_router) - _register_swagger_docs(app, docs_url="/datastore/api/docs") - _strip_default_422(app) - _tailor_auth_scheme(app, config.AUTH_TYPE) + register_swagger_docs(app, docs_url=f"{API_PREFIX}/docs", config=config) + strip_default_422(app) + tailor_auth_scheme(app, config.AUTH_TYPE) return app diff --git a/datastore/schemas/request.py b/datastore/schemas/request.py index 37617f8..abe7baf 100644 --- a/datastore/schemas/request.py +++ b/datastore/schemas/request.py @@ -431,7 +431,7 @@ def _extract_sql_references(self) -> DatastoreSearchSQLRequest: class DatastoreDumpSQLRequest(DatastoreSearchSQLRequest): - """Query parameters for `GET /datastore/dump/query`. + """Query parameters for `GET /dump/query`. Same vetted-SELECT contract as `DatastoreSearchSQLRequest` (single SELECT / WITH, table + function extraction, `extra="forbid"`), but diff --git a/datastore/schemas/responses.py b/datastore/schemas/responses.py index 36d4d81..8e61b81 100644 --- a/datastore/schemas/responses.py +++ b/datastore/schemas/responses.py @@ -26,13 +26,13 @@ class ResponseModel(BaseModel): class ErrorEnvelope(BaseModel): - """CKAN-shaped error body returned for every 4xx / 5xx response.""" + """Error body returned for every 4xx / 5xx response.""" model_config = ConfigDict( populate_by_name=True, json_schema_extra={ "example": { - "help": "https://example.com/api/3/action/datastore_search", + "help": "https://example.com/datastore/api/v2/datastore_search", "success": False, "error": { "__type": "Validation Error", @@ -61,15 +61,6 @@ class Error(BaseModel): # --- health ----------------------------------------------------------------- -class WelcomeResponse(ResponseModel): - """Response for `GET /`.""" - - class Result(BaseModel): - message: str - - result: Result - - class StatusResponse(ResponseModel): """Response for `GET /health` and `GET /ready`.""" diff --git a/docker-compose.yml b/docker-compose.yml index 8240dad..1278f6a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: - .env restart: unless-stopped healthcheck: - test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=2).status == 200 else 1)"] + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/datastore/api/health', timeout=2).status == 200 else 1)"] interval: 30s timeout: 3s retries: 3 diff --git a/postman/collection.json b/postman/collection.json index d2588a4..85cd639 100644 --- a/postman/collection.json +++ b/postman/collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "2d4510dd-5f60-4f1d-860c-2da4b7b394fa", + "_postman_id": "4a423ff3-570d-4f47-bab9-dbfbfacec483", "name": "Datastore API", "description": "CKAN-compatible datastore API \u2014 auto-generated from `example_payload/`. Set `baseUrl` to your server, `apiKey` to a CKAN API key (anonymous reads are allowed; writes require a key), and `resourceId` to the table you want to hit.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" @@ -45,26 +45,8 @@ "item": [ { "name": "health", - "description": "Liveness / readiness probes live under `/datastore/api` (`/datastore/api/health`, `/datastore/api/ready`); the welcome banner is at the root `/`. Hit any of these to check the server.", + "description": "Liveness / readiness probes: `/datastore/api/health` and `/datastore/api/ready`. Hit either to check the server.", "item": [ - { - "name": "Welcome", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/", - "host": [ - "{{baseUrl}}" - ], - "path": [ - "" - ] - }, - "description": "Banner / root endpoint. Echoes `APP_MESSAGE`." - }, - "response": [] - }, { "name": "Health", "request": { @@ -131,14 +113,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_create", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_create", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_create" ] }, @@ -166,14 +148,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_create", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_create", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_create" ] }, @@ -201,14 +183,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_create", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_create", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_create" ] }, @@ -242,14 +224,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_upsert", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_upsert", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_upsert" ] }, @@ -277,14 +259,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_upsert", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_upsert", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_upsert" ] }, @@ -312,14 +294,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_upsert", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_upsert", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_upsert" ] }, @@ -339,14 +321,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_info?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_info?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_info" ], "query": [ @@ -366,14 +348,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_info?id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_info?id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_info" ], "query": [ @@ -399,14 +381,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -426,14 +408,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&filters={\"product_code\": \"DCL\", \"accepted\": true}&limit=100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&filters={\"product_code\": \"DCL\", \"accepted\": true}&limit=100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -461,14 +443,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&q=DRAX&plain=true&language=english&limit=50", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&q=DRAX&plain=true&language=english&limit=50", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -504,14 +486,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,delivery_start,clearing_price_gbp_per_mwh,volume_mwh&sort=delivery_start desc, clearing_price_gbp_per_mwh asc&limit=100&offset=0&include_total=true", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,delivery_start,clearing_price_gbp_per_mwh,volume_mwh&sort=delivery_start desc, clearing_price_gbp_per_mwh asc&limit=100&offset=0&include_total=true", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -551,14 +533,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=lists&limit=100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=lists&limit=100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -590,14 +572,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=csv&limit=100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=csv&limit=100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -635,14 +617,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true LIMIT 100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true LIMIT 100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -662,14 +644,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT product_code, AVG(clearing_price_gbp_per_mwh) AS avg_price, SUM(volume_mwh) AS total_volume FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true GROUP BY product_code ORDER BY product_code LIMIT 50", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT product_code, AVG(clearing_price_gbp_per_mwh) AS avg_price, SUM(volume_mwh) AS total_volume FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true GROUP BY product_code ORDER BY product_code LIMIT 50", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -689,14 +671,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=WITH daily AS (SELECT DATE(delivery_start) AS d, product_code, AVG(clearing_price_gbp_per_mwh) AS price FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true GROUP BY d, product_code) SELECT * FROM daily ORDER BY d DESC, product_code LIMIT 50", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=WITH daily AS (SELECT DATE(delivery_start) AS d, product_code, AVG(clearing_price_gbp_per_mwh) AS price FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true GROUP BY d, product_code) SELECT * FROM daily ORDER BY d DESC, product_code LIMIT 50", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -716,14 +698,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT _id, auction_id, product_code, delivery_start FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" ORDER BY _id ASC LIMIT 50 OFFSET 100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT _id, auction_id, product_code, delivery_start FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" ORDER BY _id ASC LIMIT 50 OFFSET 100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -743,14 +725,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT a._id AS a_id, b._id AS b_id, a.auction_id, a.product_code FROM \"balancing_auction_results_2025\" a JOIN \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" b ON a.auction_id = b.auction_id WHERE a.product_code = 'DCL' ORDER BY a.auction_id ASC LIMIT 50", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT a._id AS a_id, b._id AS b_id, a.auction_id, a.product_code FROM \"balancing_auction_results_2025\" a JOIN \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" b ON a.auction_id = b.auction_id WHERE a.product_code = 'DCL' ORDER BY a.auction_id ASC LIMIT 50", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -770,14 +752,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT '2025' AS source, auction_id, product_code, delivery_start FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" UNION ALL SELECT '2024' AS source, auction_id, product_code, delivery_start FROM \"balancing_auction_results_2024\" ORDER BY delivery_start DESC LIMIT 100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT '2025' AS source, auction_id, product_code, delivery_start FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" UNION ALL SELECT '2024' AS source, auction_id, product_code, delivery_start FROM \"balancing_auction_results_2024\" ORDER BY delivery_start DESC LIMIT 100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -817,14 +799,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_delete", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_delete", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_delete" ] }, @@ -852,14 +834,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_delete", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_delete", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_delete" ] }, @@ -887,14 +869,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_delete", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_delete", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_delete" ] }, @@ -922,14 +904,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_delete", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_delete", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_delete" ] }, diff --git a/postman/generate_postman.py b/postman/generate_postman.py index 7858d00..36d02c5 100644 --- a/postman/generate_postman.py +++ b/postman/generate_postman.py @@ -66,7 +66,6 @@ ] HEALTH_REQUESTS: list[tuple[str, str, str]] = [ - ("Welcome", "", "Banner / root endpoint. Echoes `APP_MESSAGE`."), ("Health", "datastore/api/health", "Liveness probe — always 200 while the process is up."), ("Ready", "datastore/api/ready", @@ -103,7 +102,7 @@ def _post_request(action: str, body: dict[str, Any], description: str) -> dict[s "raw": json.dumps(body, indent=2), "options": {"raw": {"language": "json"}}, }, - "url": _request_url(f"api/3/action/{action}"), + "url": _request_url(f"datastore/api/v2/{action}"), "description": description, } @@ -124,7 +123,7 @@ def _get_request(action: str, body: dict[str, Any], description: str) -> dict[st return { "method": "GET", "header": [], - "url": _request_url(f"api/3/action/{action}", query=query), + "url": _request_url(f"datastore/api/v2/{action}", query=query), "description": description, } @@ -349,9 +348,8 @@ def _build_health_folder() -> dict[str, Any]: return { "name": "health", "description": ( - "Liveness / readiness probes live under `/datastore/api` " - "(`/datastore/api/health`, `/datastore/api/ready`); the welcome " - "banner is at the root `/`. Hit any of these to check the server." + "Liveness / readiness probes: `/datastore/api/health` and " + "`/datastore/api/ready`. Hit either to check the server." ), "item": items, } diff --git a/pyproject.toml b/pyproject.toml index cb08870..86dcbce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "datastore" -version = "0.1.0" +version = "0.2.0" description = "CKAN-compatible datastore API with pluggable storage backends" readme = "README.md" requires-python = ">=3.12,<3.15" @@ -21,6 +21,10 @@ dependencies = [ "httptools>=0.6", "sqlglot>=25.0", "pyjwt>=2.8,<3", + # Renders the Swagger UI docs page (api/templates/docs.html). Arrives + # transitively via fastapi[standard] too, but declared here because + # this package imports it directly. + "jinja2>=3.1", "google-cloud-storage>=2.14", ] diff --git a/tests/conftest.py b/tests/conftest.py index 6170304..f66f8ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,6 +67,11 @@ def _isolate_bigquery_env(monkeypatch: pytest.MonkeyPatch) -> None: # Pydantic-Settings can't parse "" as int — give the dump-URL TTL a # valid placeholder so a stray .env doesn't break startup in tests. monkeypatch.setenv("BIGQUERY_EXPORT_URL_EXPIRY_HOURS", "1") + for name in ( + "DOCS_PRIMARY_COLOR", "DOCS_HEADER_COLOR", + "DOCS_SITE_TITLE", "DOCS_LOGO_URL", + ): + monkeypatch.setenv(name, "") # `Config` and engine instances are lru-cached / module-level # singletons; invalidate so the cleared env actually takes effect. get_config.cache_clear() diff --git a/tests/test_cors.py b/tests/test_cors.py index 2815cb1..f95f80b 100644 --- a/tests/test_cors.py +++ b/tests/test_cors.py @@ -53,7 +53,7 @@ def test_specific_domain_allowed(monkeypatch: pytest.MonkeyPatch) -> None: def test_preflight_options(monkeypatch: pytest.MonkeyPatch) -> None: with _client_with_origins(monkeypatch, "https://data.example.org") as client: r = client.options( - "/api/3/action/datastore_create", + "/datastore/api/v2/datastore_create", headers={ "Origin": "https://data.example.org", "Access-Control-Request-Method": "POST", diff --git a/tests/test_datastore_create.py b/tests/test_datastore_create.py index aec4a63..d06f9ec 100644 --- a/tests/test_datastore_create.py +++ b/tests/test_datastore_create.py @@ -16,7 +16,7 @@ from tests.conftest import FakeCKAN -CREATE_URL = "/api/3/action/datastore_create" +CREATE_URL = "/datastore/api/v2/datastore_create" def _valid_payload_with_resource_id() -> dict[str, Any]: diff --git a/tests/test_datastore_delete.py b/tests/test_datastore_delete.py index 16cdfa2..57df8a8 100644 --- a/tests/test_datastore_delete.py +++ b/tests/test_datastore_delete.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `POST /api/3/action/datastore_delete`. +"""End-to-end tests for `POST /datastore/api/v2/datastore_delete`. Body accepts: resource_id / id (one required) — table to delete from @@ -24,7 +24,7 @@ from tests.conftest import FakeCKAN -DELETE_URL = "/api/3/action/datastore_delete" +DELETE_URL = "/datastore/api/v2/datastore_delete" _RESOURCE_ID = "balancing_auction_results_2025" diff --git a/tests/test_datastore_dump.py b/tests/test_datastore_dump.py index 46b5096..a7ba2d2 100644 --- a/tests/test_datastore_dump.py +++ b/tests/test_datastore_dump.py @@ -1,4 +1,4 @@ -"""Tests for `GET /datastore/dump/{resource_id}`. +"""Tests for `GET /datastore/api/dump/{resource_id}`. The engine returns one signed URL (csv / gzip / ndjson shards are composed into a single object), so every format 302s. Only a sharded @@ -27,7 +27,7 @@ from tests.conftest import FakeCKAN -DUMP_URL = "/datastore/dump/balancing_auction_results_2025" +DUMP_URL = "/datastore/api/dump/balancing_auction_results_2025" def _patch_dump(urls_or_exc: list[str] | Exception): @@ -132,7 +132,7 @@ def test_unknown_format_returns_validation_error(client: TestClient) -> None: def test_dump_for_unknown_resource_returns_404(client: TestClient) -> None: - response = client.get("/datastore/dump/missing-resource") + response = client.get("/datastore/api/dump/missing-resource") assert response.status_code == 404 diff --git a/tests/test_datastore_dump_sql.py b/tests/test_datastore_dump_sql.py index 1a231a7..a9b7c72 100644 --- a/tests/test_datastore_dump_sql.py +++ b/tests/test_datastore_dump_sql.py @@ -45,7 +45,7 @@ stub_signed_urls, ) -DUMP_SQL_URL = "/datastore/dump/query" +DUMP_SQL_URL = "/datastore/api/dump/query" _NOW = dt.datetime.now(dt.timezone.utc) @@ -887,7 +887,7 @@ def test_zero_table_sql_exports_without_get_table() -> None: # ============================================================================= -# Endpoint: GET /datastore/dump/query +# Endpoint: GET /datastore/api/dump/query # ============================================================================= @@ -999,7 +999,7 @@ def test_bogus_format_rejected(client: TestClient) -> None: def test_missing_sql_names_the_field(client: TestClient) -> None: - """`/datastore/dump/query` resolves to the SQL route (declared before + """`/datastore/api/dump/query` resolves to the SQL route (declared before `/{resource_id}`, so `query` is a reserved resource name) — a missing `sql` param is a validation error on this endpoint, not a 404 dump of a table called 'query'.""" diff --git a/tests/test_datastore_info.py b/tests/test_datastore_info.py index 42a3be3..961c219 100644 --- a/tests/test_datastore_info.py +++ b/tests/test_datastore_info.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `GET /api/3/action/datastore_info`. +"""End-to-end tests for `GET /datastore/api/v2/datastore_info`. Single `resource_id` query parameter; the response envelope's `result` holds `meta` (free-form dict) + `fields` (column schema list). @@ -19,7 +19,7 @@ from tests.conftest import FakeCKAN -INFO_URL = "/api/3/action/datastore_info" +INFO_URL = "/datastore/api/v2/datastore_info" _RESOURCE_ID = "balancing_auction_results_2025" diff --git a/tests/test_datastore_search.py b/tests/test_datastore_search.py index b6567a4..45c6fc2 100644 --- a/tests/test_datastore_search.py +++ b/tests/test_datastore_search.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `GET /api/3/action/datastore_search`. +"""End-to-end tests for `GET /datastore/api/v2/datastore_search`. `datastore_search` is GET with query parameters. Complex types are encoded: @@ -37,7 +37,7 @@ from tests.conftest import FakeCKAN -SEARCH_URL = "/api/3/action/datastore_search" +SEARCH_URL = "/datastore/api/v2/datastore_search" _RESOURCE_ID = "balancing_auction_results_2025" @@ -505,7 +505,7 @@ def test_search_objects_response_includes_links(client: TestClient) -> None: assert response.status_code == 200 links = response.json()["result"]["_links"] assert set(links) == {"start", "page_size", "page", "total_pages"} - assert links["start"].startswith("http://testserver/api/3/action/datastore_search") + assert links["start"].startswith("http://testserver/datastore/api/v2/datastore_search") assert "offset" not in links["start"] assert f"resource_id={_RESOURCE_ID}" in links["start"] assert links["page_size"] == 100 # default limit diff --git a/tests/test_datastore_search_sql.py b/tests/test_datastore_search_sql.py index 867dd20..ebfa40e 100644 --- a/tests/test_datastore_search_sql.py +++ b/tests/test_datastore_search_sql.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `GET /api/3/action/datastore_search_sql`. +"""End-to-end tests for `GET /datastore/api/v2/datastore_search_sql`. Only `sql` is accepted as a query parameter; the response reuses the `datastore_search` envelope shape (same writer, same `_links` / `limit` / @@ -29,7 +29,7 @@ from tests.conftest import FakeCKAN -SQL_URL = "/api/3/action/datastore_search_sql" +SQL_URL = "/datastore/api/v2/datastore_search_sql" # 1. Happy path ------------------------------------------------------------- @@ -333,10 +333,10 @@ def test_each_table_authorized_once_for_joins( assert fake_ckan.authorize_calls - before == 2 -# 8. Download param moved to /datastore/dump/query -------------------------- +# 8. Download param moved to /datastore/api/dump/query -------------------------- def test_download_param_no_longer_accepted(client: TestClient) -> None: - """SQL downloads live at `GET /datastore/dump/query?sql=&format=` now; + """SQL downloads live at `GET /datastore/api/dump/query?sql=&format=` now; `extra="forbid"` rejects the retired `download` param here.""" response = client.get(SQL_URL, params={ "sql": "SELECT 1 LIMIT 10", "download": "csv", diff --git a/tests/test_datastore_upsert.py b/tests/test_datastore_upsert.py index 2c9139a..c2f591d 100644 --- a/tests/test_datastore_upsert.py +++ b/tests/test_datastore_upsert.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `POST /api/3/action/datastore_upsert`. +"""End-to-end tests for `POST /datastore/api/v2/datastore_upsert`. Covers: 1. all three methods (upsert, insert, update) and the default method @@ -17,7 +17,7 @@ from tests.conftest import FakeCKAN -UPSERT_URL = "/api/3/action/datastore_upsert" +UPSERT_URL = "/datastore/api/v2/datastore_upsert" _RESOURCE_ID = "balancing_auction_results_2025" diff --git a/tests/test_health.py b/tests/test_health.py index 7ebb45b..db238bd 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -1,7 +1,7 @@ -"""End-to-end tests for `GET /`, `GET /health`, `GET /ready`. +"""End-to-end tests for the health probes. Covers: - 1. / — welcome envelope + 1. / — no route; the service has no landing endpoint 2. /health — always 200 while the process is up 3. /ready — 200 when both engines pass healthcheck; 503 with a Service Unavailable envelope when either fails @@ -25,22 +25,17 @@ def _clean_engine_cache() -> Iterator[None]: reset_engine_cache() -# 1. Welcome ---------------------------------------------------------------- +# 1. No landing endpoint ---------------------------------------------------- -def test_welcome_returns_envelope(client: TestClient) -> None: - response = client.get("/") - - assert response.status_code == 200 - body = response.json() - assert body["success"] is True - assert isinstance(body["result"]["message"], str) +def test_root_is_not_routed(client: TestClient) -> None: + """There is no welcome/landing endpoint — every route lives under the + versioned API prefix.""" + assert client.get("/").status_code == 404 -def test_welcome_not_mounted_under_action_prefix(client: TestClient) -> None: - """Welcome is root-only — `/api/3/action/` is the CKAN action - namespace and shouldn't echo a generic landing message.""" - response = client.get("/api/3/action/") - assert response.status_code == 404 +def test_action_prefix_root_is_not_routed(client: TestClient) -> None: + """The action namespace itself isn't a route either.""" + assert client.get("/datastore/api/v2/").status_code == 404 # 2. /health ---------------------------------------------------------------- diff --git a/tests/test_openapi.py b/tests/test_openapi.py index 7ae98c6..f21c8a3 100644 --- a/tests/test_openapi.py +++ b/tests/test_openapi.py @@ -10,12 +10,17 @@ from __future__ import annotations +from pathlib import Path from typing import Any import pytest -from datastore.core.config import get_config +from datastore.api import docs as docs_module +from datastore.api.docs import api_description +from datastore.core.config import Config, get_config +from datastore.core.constants import API_PREFIX, API_VERSION from datastore.main import create_app from fastapi.testclient import TestClient +from pydantic import ValidationError def _build_schema( @@ -45,11 +50,13 @@ def _operations(schema: dict[str, Any]) -> list[dict[str, Any]]: def test_ckan_scheme_describes_api_key( monkeypatch: pytest.MonkeyPatch, ) -> None: + """Under CKAN auth the scheme describes an API token, and says nothing + about JWTs — the two providers' wording must not leak into each other.""" schema = _build_schema(monkeypatch, "ckan") scheme = _scheme(schema) assert scheme is not None - assert "CKAN API key" in scheme["description"] + assert "API token" in scheme["description"] assert "JWT" not in scheme["description"] @@ -101,19 +108,231 @@ def test_anonymous_operations_carry_no_security( assert "security" not in operation +# 3b. description tracks AUTH_TYPE ------------------------------------------ + +@pytest.mark.parametrize( + ("auth_type", "expected", "forbidden"), + [ + ("ckan", "CKAN API token", "JWT"), + ("jwt", "signed JWT", "CKAN API token"), + ("anonymous", "no credentials are required", "CKAN API token"), + # matched case-insensitively below, so rewording the sentence's + # opening doesn't break the assertion + ], +) +def test_description_describes_the_active_provider( + monkeypatch: pytest.MonkeyPatch, + auth_type: str, + expected: str, + forbidden: str, +) -> None: + """The description is the first thing a reader sees, so telling them to + paste a CKAN token when the service runs on JWT would mislead.""" + schema = _build_schema(monkeypatch, auth_type) + + description = schema["info"]["description"] + assert expected.lower() in description.lower() + assert forbidden.lower() not in description.lower() + + +def test_description_falls_back_for_unknown_provider() -> None: + """A third-party provider under `datastore/auth//` must not be + handed another provider's instructions.""" + description = api_description("some-third-party-provider") + + assert "Send your credentials" in description + assert "CKAN API token" not in description + assert "signed JWT" not in description + + +def test_description_placeholder_is_always_substituted() -> None: + """The auth slot is spliced with a plain replace (the text contains + literal braces, so `str.format` would raise) — make sure no raw + placeholder can reach the page.""" + for auth_type in ("ckan", "jwt", "anonymous", "unknown"): + assert "%(auth)s" not in api_description(auth_type) + + +def test_description_survives_literal_braces( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A path like `/dump/{resource_id}` in the description must survive. + + `str.format`/%-formatting would read those braces as a field and raise at + startup, so the auth slot is spliced with a plain replace. Asserted against + a description that actually contains braces, so the guarantee holds even if + the shipped prose currently has none. + """ + monkeypatch.setattr( + docs_module, + "API_DESCRIPTION", + "See `/dump/{resource_id}`.\n\n%(auth)s", + ) + + rendered = api_description("ckan") + + assert "{resource_id}" in rendered + assert "CKAN API token" in rendered + + +# 3c. contract version, not build version ----------------------------------- + +def test_info_version_is_the_api_contract_version(client: TestClient) -> None: + """`info.version` describes the API contract, not the installed build. + + The two are independent: the package can ship any number of releases + without the contract changing. A build number here would tell a client + nothing about which request/response shapes it is looking at. + """ + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + assert schema["info"]["version"] == API_VERSION + + +def test_info_version_matches_the_url_prefix(client: TestClient) -> None: + """The documented contract version and the one in the URL are the same + value, so a schema can never advertise a version its routes don't serve.""" + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + documented = schema["info"]["version"] + assert any( + path.startswith(f"/datastore/api/{documented}/") + for path in schema["paths"] + ), f"no route served under the documented version {documented!r}" + + # 4. Swagger UI page ------------------------------------------------------------ def test_docs_page_renders_swagger_ui(client: TestClient) -> None: - response = client.get("/datastore/api/docs") + response = client.get("/datastore/api/v2/docs") assert response.status_code == 200 assert "SwaggerUIBundle" in response.text - assert "/datastore/api/openapi.json" in response.text + assert "/datastore/api/v2/openapi.json" in response.text + + +def test_docs_page_serves_vendored_assets(client: TestClient) -> None: + """Swagger UI is vendored, not pulled from a CDN, so `/docs` renders + in air-gapped deployments.""" + response = client.get("/datastore/api/v2/docs") + + assert "cdn.jsdelivr.net" not in response.text + assert "/datastore/api/v2/static/swagger-ui/swagger-ui-bundle.js" in response.text + assert "/datastore/api/v2/static/theme/theme.css" in response.text + + for asset in ( + "/datastore/api/v2/static/swagger-ui/swagger-ui.css", + "/datastore/api/v2/static/swagger-ui/swagger-ui-bundle.js", + "/datastore/api/v2/static/theme/theme.css", + ): + assert client.get(asset).status_code == 200, asset -def test_docs_page_widens_authorize_input(client: TestClient) -> None: - """The Authorize modal's token input is ~230px stock — too short to - see a pasted JWT/API key. The docs page injects CSS to widen it.""" - response = client.get("/datastore/api/docs") +def test_docs_page_widens_authorize_input() -> None: + """The Authorize modal's token input is ~230px stock — too short to see + a pasted JWT/API key. The theme widens the modal and the input.""" + css = ( + Path(__file__).resolve().parent.parent + / "datastore/api/static/theme/theme.css" + ).read_text() + + assert "max-width: 900px" in css + assert ".swagger-ui .auth-container input" in css + + +def test_docs_page_applies_theme_config(monkeypatch: pytest.MonkeyPatch) -> None: + """`DOCS_*` env vars reach the page's CSS custom properties and header, + so a deployment rebrands without shipping CSS.""" + monkeypatch.setenv("DOCS_PRIMARY_COLOR", "#7A3864") + monkeypatch.setenv("DOCS_HEADER_COLOR", "#123456") + monkeypatch.setenv("DOCS_SITE_TITLE", "NESO Datastore API") + monkeypatch.setenv("DOCS_LOGO_URL", "/static/logo.png") + get_config.cache_clear() + + try: + with TestClient(create_app()) as themed_client: + body = themed_client.get("/datastore/api/v2/docs").text + finally: + get_config.cache_clear() + + assert "--docs-primary: #7A3864;" in body + assert "--docs-header-bg: #123456;" in body + assert "NESO Datastore API" in body + assert ' None: + """`DOCS_PRIMARY_COLOR` alone must brand the whole page. + + With `DOCS_HEADER_COLOR` unset no `--docs-header-bg` is emitted, so the + stylesheet's `--docs-header-bg: var(--docs-primary)` fallback applies and + the header takes the brand colour. A non-empty default here would pin the + bar to a fixed grey and make branding look broken. + """ + monkeypatch.setenv("DOCS_PRIMARY_COLOR", "#7A3864") + get_config.cache_clear() + + try: + assert Config().DOCS_HEADER_COLOR == "" + with TestClient(create_app()) as themed_client: + body = themed_client.get("/datastore/api/v2/docs").text + finally: + get_config.cache_clear() + + assert "--docs-primary: #7A3864;" in body + assert "--docs-header-bg" not in body + + +def test_docs_page_rejects_non_css_color(monkeypatch: pytest.MonkeyPatch) -> None: + """The colour lands inside a `