diff --git a/.env.example b/.env.example index d2f38e8..5dff45d 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,6 @@ # Every field below maps 1:1 to a field on app.core.config.Config. # --- Application --- -APP_MESSAGE="Datastore API is running!" MAX_REQUEST_BODY_MB=50 LOG_LEVEL=INFO # One JSON analytics event per datastore action / dump request. @@ -11,8 +10,19 @@ ANALYTICS_ENABLED=false # Cross-origin requests: `*` allows every origin, a comma-separated list # allows only those domains (e.g. https://data.example.org,https://app.example.org), # empty disables CORS entirely. +# Public base URL of this service, used to render absolute URLs in the +# OpenAPI examples. Live responses derive their URLs from the incoming +# request, so this never affects runtime behaviour. Trailing slashes trimmed. +API_URL=https://example.com + CORS_ORIGINS=* +# --- Swagger UI branding --- +DOCS_PRIMARY_COLOR= +DOCS_HEADER_COLOR= +DOCS_SITE_TITLE= +DOCS_LOGO_URL= + # --- Datastore engine --- # Selects the storage backend (must match a folder under # `datastore/infrastructure/engines/`): diff --git a/API.md b/API.md index 4368c50..2b2e1cf 100644 --- a/API.md +++ b/API.md @@ -1,11 +1,11 @@ # Datastore API Reference A standalone, CKAN-compatible datastore service: tabular CRUD + search over a -pluggable storage backend. Every action lives under `/api/3/action/` and returns +pluggable storage backend. Every action lives under `/datastore/api/v2/` and returns the CKAN envelope, so existing CKAN datastore clients work unchanged — whether this runs alongside CKAN or independently. -- **Interactive docs:** `GET /datastore/api/docs` (Swagger UI) · `GET /datastore/api/redoc` · `GET /datastore/api/openapi.json` +- **Interactive docs:** `GET /datastore/api/v2/docs` (Swagger UI) · `GET /datastore/api/v2/redoc` · `GET /datastore/api/v2/openapi.json` - **Postman:** import [postman/collection.json](postman/collection.json) — one worked request per endpoint. --- @@ -17,14 +17,18 @@ this runs alongside CKAN or independently. Every response is a CKAN envelope. On success: ```json -{ "help": "", "success": true, "result": { ... } } +{ "help": "", "success": true, "result": { ... } } ``` +`help` is a deep link into this service's Swagger UI, anchored on the operation +that served the request — so any response, including an error, points at the +documentation for the endpoint you called. + On failure: ```json { - "help": "", + "help": "", "success": false, "error": { "__type": "Validation Error", @@ -66,19 +70,19 @@ token (except under `anonymous`). | Method | Path | Summary | |---|---|---| -| POST | `/api/3/action/datastore_create` | Declare a resource (and optionally seed rows) | -| POST | `/api/3/action/datastore_upsert` | Insert / update / upsert rows | -| POST | `/api/3/action/datastore_delete` | Delete rows, drop columns, or drop the table | -| GET | `/api/3/action/datastore_search` | Search a resource (streaming) | -| GET | `/api/3/action/datastore_search_sql` | Run a read-only SQL `SELECT` (streaming) | -| GET | `/datastore/dump/query` | Download the result of a SQL `SELECT` as a file | -| GET | `/api/3/action/datastore_info` | Schema + row stats for a resource | -| GET | `/datastore/dump/{resource_id}` | Download a whole resource (CSV/NDJSON/Parquet) | -| GET | `/` · `/health` · `/ready` | Welcome / liveness / readiness | +| POST | `/datastore/api/v2/datastore_create` | Declare a resource (and optionally seed rows) | +| POST | `/datastore/api/v2/datastore_upsert` | Insert / update / upsert rows | +| POST | `/datastore/api/v2/datastore_delete` | Delete rows, drop columns, or drop the table | +| GET | `/datastore/api/v2/datastore_search` | Search a resource (streaming) | +| GET | `/datastore/api/v2/datastore_search_sql` | Run a read-only SQL `SELECT` (streaming) | +| GET | `/datastore/api/dump/query` | Download the result of a SQL `SELECT` as a file | +| GET | `/datastore/api/v2/datastore_info` | Schema + row stats for a resource | +| GET | `/datastore/api/dump/{resource_id}` | Download a whole resource (CSV/NDJSON/Parquet) | +| GET | `/datastore/api/health` · `/datastore/api/ready` | Liveness / readiness | --- -## `POST /api/3/action/datastore_create` +## `POST /datastore/api/v2/datastore_create` Declare a resource (table) and optionally seed it with rows. Re-declaring an existing resource adds columns and widens types (see below). @@ -154,7 +158,7 @@ round-tripped by `datastore_info`. --- -## `POST /api/3/action/datastore_upsert` +## `POST /datastore/api/v2/datastore_upsert` Write rows into an existing resource (declare it with `datastore_create` first). @@ -197,7 +201,7 @@ never carries it. --- -## `POST /api/3/action/datastore_delete` +## `POST /datastore/api/v2/datastore_delete` Three modes (`filters` and `fields` are mutually exclusive): @@ -244,7 +248,7 @@ follow-up `datastore_info`: --- -## `GET /api/3/action/datastore_search` +## `GET /datastore/api/v2/datastore_search` Parameterised search; the response is **streamed** (peak memory ≈ one row). @@ -268,7 +272,7 @@ Parameterised search; the response is **streamed** (peak memory ≈ one row). ### Example ```http -GET /api/3/action/datastore_search +GET /datastore/api/v2/datastore_search ?resource_id=c6153a74-43cb-4edf-8bdf-bb664feca937 &filters={"product_code":"DCL","accepted":true} &sort=delivery_start desc @@ -305,12 +309,12 @@ GET /api/3/action/datastore_search --- -## `GET /api/3/action/datastore_search_sql` +## `GET /datastore/api/v2/datastore_search_sql` Run a single read-only `SELECT` / `WITH` statement and stream the result. Tables are referenced by `resource_id`; each is authorized individually, and functions are checked against the engine's allow-list. Include a `LIMIT` (required). -To export the result as a file instead, use [`GET /datastore/dump/query`](#get-datastoredumpquery). +To export the result as a file instead, use [`GET /datastore/api/dump/query`](#get-datastoredumpquery). ### Query parameters @@ -321,7 +325,7 @@ To export the result as a file instead, use [`GET /datastore/dump/query`](#get-d ### Example ```http -GET /api/3/action/datastore_search_sql?sql= +GET /datastore/api/v2/datastore_search_sql?sql= SELECT product_code, AVG(clearing_price_gbp_per_mwh) AS avg_price FROM "c6153a74-43cb-4edf-8bdf-bb664feca937" WHERE accepted = true @@ -348,7 +352,7 @@ refuses DML/DDL. --- -## `GET /api/3/action/datastore_info` +## `GET /datastore/api/v2/datastore_info` Returns the column schema (including the `info` data dictionary, verbatim) plus row stats — a column-level metadata catalog without a side store. @@ -382,7 +386,7 @@ row stats — a column-level metadata catalog without a side store. --- -## `GET /datastore/dump/{resource_id}` +## `GET /datastore/api/dump/{resource_id}` Download an entire resource. Pick the format with `?format=csv` (default), `gzip`, `ndjson`, or `parquet`. @@ -401,13 +405,13 @@ Download an entire resource. Pick the format with `?format=csv` (default), Requires `read` permission on the resource and a configured export bucket (`BIGQUERY_EXPORT_BUCKET`). -`query` is a **reserved name** on this route — `/datastore/dump/query` is the SQL +`query` is a **reserved name** on this route — `/datastore/api/dump/query` is the SQL download endpoint below, so a resource literally named `query` can't be dumped by this URL. --- -## `GET /datastore/dump/query` +## `GET /datastore/api/dump/query` Download the result of a **SQL `SELECT`** as a single file — filtered downloads at any size. Same validation as `datastore_search_sql` (single @@ -424,14 +428,14 @@ file itself, not the CKAN envelope. ### Example ```http -GET /datastore/dump/query +GET /datastore/api/dump/query ?sql=SELECT * FROM "c6153a74-43cb-4edf-8bdf-bb664feca937" WHERE accepted = true &format=csv ``` ### Response -Identical to `/datastore/dump/{resource_id}` above: +Identical to `/datastore/api/dump/{resource_id}` above: - **csv / gzip / ndjson** — `302` to a signed GCS URL at any size (shards are composed into one object). The URL expires after @@ -461,9 +465,8 @@ All return the CKAN envelope. | Method | Path | Result | |---|---|---| -| GET | `/` | `{"message": ""}` | -| GET | `/health` | `{"status": "ok"}` — liveness; always 200 while the process runs | -| GET | `/ready` | `{"status": "ready"}` — 200 when both engines pass `healthcheck()`; `503` (`{"status": "not_ready"}`) otherwise | +| GET | `/datastore/api/health` | `{"status": "ok"}` — liveness; always 200 while the process runs | +| GET | `/datastore/api/ready` | `{"status": "ready"}` — 200 when both engines pass `healthcheck()`; `503` (`{"status": "not_ready"}`) otherwise | --- diff --git a/CLAUDE.md b/CLAUDE.md index ba41cae..008e915 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ storage backend (BigQuery Datastore or Ducklake as future support). ## 1. Goals -- CKAN-compatible request/response shapes for `/api/3/action/datastore_*`. +- CKAN-compatible request/response shapes for `/datastore/api/v2/datastore_*`. - **Pluggable storage backend** selected by `DATASTORE_ENGINE` (`bigquery` today; `ducklake` planned). - **Pluggable auth** selected by `AUTH_TYPE` (`ckan` / `jwt` / `anonymous`). Provider lives in `datastore/auth//`; only the CKAN provider touches the network, and its TTL cache is local to that provider. - **Standalone-capable** — runs without an upstream CKAN under `AUTH_TYPE=anonymous` or `AUTH_TYPE=jwt`. CKAN is only required when `AUTH_TYPE=ckan`. @@ -127,10 +127,14 @@ datastore-api/ │ │ ├── error_handlers.py # APIError / HTTPException / RequestValidationError │ │ │ # → CKAN error envelope mapping │ │ ├── middleware.py # ASGI middleware (BodySizeLimitMiddleware today) +│ │ ├── static/ # Vendored Swagger UI + docs theme (no CDN) +│ │ │ ├── swagger-ui/ # swagger-ui-dist 5.17.14 (css + bundle) +│ │ │ └── theme/theme.css # theme ported from ckanext-openapidocs +│ │ ├── templates/docs.html # Jinja template for the Swagger UI page │ │ └── endpoints/ # One module per resource group │ │ ├── __init__.py │ │ ├── health.py # /, /health, /ready (CKAN-shaped envelopes) -│ │ └── datastore.py # /api/3/action/datastore_* +│ │ └── datastore.py # /datastore/api/v2/datastore_* │ │ │ │ ── 2. AUTH PROVIDERS ─────────────────────── (one subpackage per AUTH_TYPE) │ ├── auth/ @@ -170,8 +174,7 @@ datastore-api/ │ │ │ # DatastoreSearchRequest │ │ ├── responses.py # responses.py – ResponseModel base + │ │ │ # per-endpoint envelopes -│ │ │ # (WelcomeResponse, -│ │ │ # StatusResponse, +│ │ │ # (StatusResponse, │ │ │ # DatastoreCreateResponse) │ │ └── validators.py # validators.py – FieldSpec, StringOrList, │ │ # PostgresType, helper fns @@ -259,8 +262,10 @@ adapter will live at `infrastructure/engines/ducklake/` when it lands. | `datastore/api/endpoints/` | Route declarations, request parsing, response building | SQL, engine calls, validation rules — delegate to services | | `datastore/api/context.py` | `RequestContext`, `ContextDep`, `get_context`, `get_auth_provider`, `get_ckan_client` (per-request DI bundle) | The logic those handles invoke — that lives in `services/` / `auth/` / `infrastructure/` | | `datastore/api/auth.py` | Provider-agnostic boundary policy (permission whitelist, anonymous-read rule, resource_id XOR package_id) | Concrete provider behaviour — CKAN/JWT/anonymous logic lives in `datastore/auth//` | -| `datastore/api/responses.py` | CKAN envelope helpers, `ORJSONResponse` | Anything that needs DB access | +| `datastore/api/responses.py` | Envelope helpers, `ORJSONResponse`. `_help` deep-links into Swagger via `api/docs.py`'s `help_url` | Anything that needs DB access | | `datastore/api/error_handlers.py` | Exception → CKAN error envelope mapping | Business rules — raise `APIError` from wherever the rule lives | +| `datastore/api/static/` | Vendored front-end assets served at `/datastore/api/static` — Swagger UI dist + `theme/theme.css` | Anything generated at runtime; anything Python imports | +| `datastore/api/templates/` | Jinja templates for HTML pages (`docs.html`) | Anything returning JSON — those go through `api/responses.py` | | `datastore/auth//` | Concrete `AuthProvider` implementation: `__init__.py` exports `Provider = `; `provider.py` implements `authorize` + `key_id`. CKAN provider holds its own TTL cache. | Cross-provider policy (that's `api/auth.py`); FastAPI imports | | `datastore/auth/base.py` | `AuthProvider` Protocol, `Decision` dataclass, `default_key_id` helper | Provider implementations | | `datastore/auth/registry.py` | importlib factory keyed on `AUTH_TYPE` | Instance caching — the lifespan builds once and stashes on `app.state` | @@ -385,8 +390,8 @@ flowchart LR **Pod-level shape** - One container per pod: the FastAPI app. Sidecars only for observability (e.g., OpenTelemetry collector). -- `livenessProbe` → `GET /health` (always 200 while the process is up). -- `readinessProbe` → `GET /ready` (200 only when both backends pass `healthcheck()`; pod pulled from Service when 503). +- `livenessProbe` → `GET /datastore/api/health` (always 200 while the process is up). +- `readinessProbe` → `GET /datastore/api/ready` (200 only when both backends pass `healthcheck()`; pod pulled from Service when 503). - `terminationGracePeriodSeconds: 30` so in-flight streaming responses drain before SIGKILL. - Memory bounded by `MAX_REQUEST_BODY_MB` × concurrency for writes; search responses are O(1) peak memory. @@ -402,7 +407,7 @@ flowchart LR ## 5. API Surface -All datastore endpoints sit under `/api/3/action/` to match the CKAN action API. +All datastore endpoints sit under `/datastore/api/v2/` to match the CKAN action API. Health endpoints at the root. ### 5.1 Health @@ -411,9 +416,8 @@ All three return the CKAN envelope shape `{help, success, result: {...}}`. | Method | Path | Status | Result | |---|---|---|---| -| GET | `/` | implemented | `{"message": APP_MESSAGE}` | -| GET | `/health` | implemented | `{"status": "ok"}` — liveness; always 200 if process is up | -| GET | `/ready` | implemented | `{"status": "ready"}` — calls `engine.healthcheck()` for rw + ro; 503 with a `Service Unavailable` envelope if either fails | +| GET | `/datastore/api/health` | implemented | `{"status": "ok"}` — liveness; always 200 if process is up | +| GET | `/datastore/api/ready` | implemented | `{"status": "ready"}` — calls `engine.healthcheck()` for rw + ro; 503 with a `Service Unavailable` envelope if either fails | ### 5.2 Datastore endpoints @@ -421,14 +425,14 @@ Each endpoint takes a single `ContextDep`. The handler calls `context.authorize( | Method | Path | Status | Body / Params | Response model | |---|---|---|---|---| -| POST | `/api/3/action/datastore_create` | **implemented** | `DatastoreCreateRequest` | `DatastoreCreateResponse` | -| POST | `/api/3/action/datastore_upsert` | **implemented** | `DatastoreUpsertRequest` | `DatastoreUpsertResponse` | -| POST | `/api/3/action/datastore_delete` | **implemented** | `DatastoreDeleteRequest` | `DatastoreDeleteResponse` | -| GET | `/api/3/action/datastore_search` | **implemented** (streaming) | `DatastoreSearchRequest` | `DatastoreSearchResponse` | -| GET | `/api/3/action/datastore_search_sql` | **implemented** (streaming) | `DatastoreSearchSQLRequest` | `DatastoreSearchResponse` | -| GET | `/datastore/dump/query` | **implemented** | `sql=`, `format=csv\|gzip\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) | -| GET | `/api/3/action/datastore_info` | **implemented** | `DatastoreInfoRequest` | `DatastoreInfoResponse` | -| GET | `/datastore/dump/{resource_id}` | **implemented** | `format=csv\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) | +| POST | `/datastore/api/v2/datastore_create` | **implemented** | `DatastoreCreateRequest` | `DatastoreCreateResponse` | +| POST | `/datastore/api/v2/datastore_upsert` | **implemented** | `DatastoreUpsertRequest` | `DatastoreUpsertResponse` | +| POST | `/datastore/api/v2/datastore_delete` | **implemented** | `DatastoreDeleteRequest` | `DatastoreDeleteResponse` | +| GET | `/datastore/api/v2/datastore_search` | **implemented** (streaming) | `DatastoreSearchRequest` | `DatastoreSearchResponse` | +| GET | `/datastore/api/v2/datastore_search_sql` | **implemented** (streaming) | `DatastoreSearchSQLRequest` | `DatastoreSearchResponse` | +| GET | `/datastore/api/dump/query` | **implemented** | `sql=`, `format=csv\|gzip\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) | +| GET | `/datastore/api/v2/datastore_info` | **implemented** | `DatastoreInfoRequest` | `DatastoreInfoResponse` | +| GET | `/datastore/api/dump/{resource_id}` | **implemented** | `format=csv\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) | The BigQuery engine is wired end-to-end: DDL, MERGE-based upsert, DML delete, parameterised search, native table-level metadata (the Frictionless schema + unique_key are JSON-encoded into the table's own `description` OPTION) for the schema round-trip, a row-count fast path via `INFORMATION_SCHEMA.TABLE_STORAGE`, and `EXPORT DATA`-backed dump with `table.modified`-keyed GCS caching. The DuckLake engine is the next concrete adapter — see §7. @@ -439,7 +443,7 @@ The BigQuery engine is wired end-to-end: DDL, MERGE-based upsert, DML delete, pa **Read-only guard (`AUTH_TYPE=ckan` only).** `datastore_create`, `datastore_upsert`, and `datastore_delete` refuse to write a resource whose CKAN record carries `url_type="datastore"` unless the request sets `force: true` — a `Validation Error` ("Cannot update a read-only resource. Use \"force\" to force update.") otherwise. This mirrors CKAN's protection against clobbering datastore-managed data by accident. The guard is gated on `AUTH_TYPE=ckan` and skipped entirely under any other provider (only the CKAN provider attaches a resource record). -### 5.3 `GET /datastore/dump/{resource_id}` +### 5.3 `GET /datastore/api/dump/{resource_id}` Full-table download, **one URL → one file** from the caller's point of view. Bytes never pass through API memory — the one exception is a @@ -493,9 +497,9 @@ A single SA works if both perm sets land on the same identity — `BIGQUERY_CRED A 24h object-lifecycle rule on the bucket is **required** in practice: the engine GCs older revs already, but lifecycle is the only thing that cleans abandoned `dumps//` prefixes (SQL downloads whose query is never re-issued — see below) and anything stranded by a crashed dump. -### SQL download (`GET /datastore/dump/query`) +### SQL download (`GET /datastore/api/dump/query`) -`GET /datastore/dump/query?sql=&format=csv|gzip|ndjson|parquet` exports the result of an arbitrary vetted SELECT through the same pipeline as `/datastore/dump/{resource_id}` — engine method `dump_sql` in [bigquery/export.py](datastore/infrastructure/engines/bigquery/export.py), response shaping shared via `download_response` in [api/endpoints/dump.py](datastore/api/endpoints/dump.py) (302 for the composed file · gzip streamed · JSON URL list for multi-file parquet). Same SQL validation + per-table auth as `datastore_search_sql` (`DatastoreDumpSQLRequest` subclasses its request schema); the action API itself stays pure JSON envelope. The route is declared before `/datastore/dump/{resource_id}`, making `query` a reserved resource name on the dump family. +`GET /datastore/api/dump/query?sql=&format=csv|gzip|ndjson|parquet` exports the result of an arbitrary vetted SELECT through the same pipeline as `/datastore/api/dump/{resource_id}` — engine method `dump_sql` in [bigquery/export.py](datastore/infrastructure/engines/bigquery/export.py), response shaping shared via `download_response` in [api/endpoints/dump.py](datastore/api/endpoints/dump.py) (302 for the composed file · gzip streamed · JSON URL list for multi-file parquet). Same SQL validation + per-table auth as `datastore_search_sql` (`DatastoreDumpSQLRequest` subclasses its request schema); the action API itself stays pure JSON envelope. The route is declared before `/datastore/api/dump/{resource_id}`, making `query` a reserved resource name on the dump family. Deltas vs the whole-table dump: @@ -516,7 +520,7 @@ The GCS client is built with the same credentials as the BigQuery client for the Every response is the CKAN envelope — `help`, `success`, and either `result` or `error`. The full per-endpoint reference (request bodies, query params, worked examples, and error shapes) lives in **[API.md](API.md)**. CKAN-style envelope: every response has `help`, `success`, and either `result` or `error`. -### 6.1 `POST /api/3/datastore_create` +### 6.1 `POST /datastore/api/v2/datastore_create` Running example: an electricity balancing-market auction-results table. Used consistently across the rest of §6 so the request → search → info round-trip @@ -666,7 +670,7 @@ Optional response fields (omitted from the body when not requested): - `records` — echoes the input rows back when the request sets `include_records: true`. - `total` — total row count after the write, populated when `include_total: true`. -### 6.2 `GET /api/3/datastore_search` +### 6.2 `GET /datastore/api/v2/datastore_search` **Query params** | Name | Type | Default | Notes | @@ -687,7 +691,7 @@ Optional response fields (omitted from the body when not requested): **Example request** ``` -GET /api/3/datastore_search +GET /datastore/api/v2/datastore_search ?resource_id=balancing_auction_results_2025 &filters={"product_code": "DCL", "accepted": true} &sort=delivery_start desc, clearing_price_gbp_per_mwh asc @@ -716,8 +720,8 @@ GET /api/3/datastore_search ], "total": 2, "_links": { - "start": "https://example.com/api/3/action/datastore_search?resource_id=balancing_auction_results_2025&limit=100", - "next": "https://example.com/api/3/action/datastore_search?resource_id=balancing_auction_results_2025&limit=100&offset=100" + "start": "https://example.com/datastore/api/v2/datastore_search?resource_id=balancing_auction_results_2025&limit=100", + "next": "https://example.com/datastore/api/v2/datastore_search?resource_id=balancing_auction_results_2025&limit=100&offset=100" } } } @@ -733,7 +737,7 @@ empty `records` array on the next page — there's no `prev` field today. `result.records_format` echoes back the format that was applied (always `objects` for `datastore_search_sql`), so a client can tell which `records` shape it got. -### 6.3 `POST /api/3/datastore_upsert` +### 6.3 `POST /datastore/api/v2/datastore_upsert` **Request — late-arriving correction to an auction result** ```json @@ -793,13 +797,13 @@ Optional fields appear in `result` only when requested: `null` is never serialised — fields that aren't populated are simply omitted (see `_orjson_default` in `api/responses.py`). -### 6.4 `GET /api/3/datastore_search_sql` +### 6.4 `GET /datastore/api/v2/datastore_search_sql` -**Query params**: `sql` (required; must carry a `LIMIT` literal). To export the result as a file instead of the JSON envelope, use `GET /datastore/dump/query?sql=…&format=…` (LIMIT optional + uncapped there — see §5.3 "SQL download"). +**Query params**: `sql` (required; must carry a `LIMIT` literal). To export the result as a file instead of the JSON envelope, use `GET /datastore/api/dump/query?sql=…&format=…` (LIMIT optional + uncapped there — see §5.3 "SQL download"). **Example request — daily clearing-price summary** ``` -GET /api/3/datastore_search_sql?sql= +GET /datastore/api/v2/datastore_search_sql?sql= SELECT DATE(delivery_start) AS delivery_date, product_code, @@ -836,7 +840,7 @@ GET /api/3/datastore_search_sql?sql= } ``` -### 6.5 `POST /api/3/datastore_delete` +### 6.5 `POST /datastore/api/v2/datastore_delete` **Request — purge rejected bids for a single auction window** ```json @@ -877,7 +881,7 @@ can confirm the table's new shape without a follow-up `datastore_info`: } ``` -### 6.6 `GET /api/3/datastore_info` +### 6.6 `GET /datastore/api/v2/datastore_info` Returns the same field shape that was supplied to `datastore_create`, including the `info` data dictionary verbatim — clients can use this as a column-level @@ -966,9 +970,10 @@ The original phase plan that used to live here has mostly shipped. This section - [x] **Error envelope** — handlers in [datastore/api/error_handlers.py](datastore/api/error_handlers.py); taxonomy in [datastore/core/exceptions.py](datastore/core/exceptions.py). - [x] **Pluggable auth providers** — `AUTH_TYPE` selects a folder under [datastore/auth/](datastore/auth/). Built-in: `ckan` (delegates to `datastore_authorize` with a provider-local TTL cache), `jwt` (PyJWT verify HS*/RS*/ES* + `aud`/`iss`/`exp`), `anonymous` (allow-all). Boundary policy in [datastore/api/auth.py](datastore/api/auth.py) is provider-agnostic. Adding a new provider = drop a folder; no registry / config edit. - [x] **Standalone capability** — `CKANClient` is only constructed when `AUTH_TYPE=ckan`; `RequestContext.ckan` is `CKANClient | None`. `Config` validator rejects `AUTH_TYPE=ckan` + empty `CKAN_URL` at startup. `datastore_create` `resource` dict path is gated on CKAN auth; everything else runs without an upstream CKAN. -- [x] **`/ready` healthcheck** — lifespan builds rw + ro engine instances and stashes on `app.state`; `/ready` calls `engine.healthcheck()` on both and returns 503 + `Service Unavailable` envelope if either fails. +- [x] **`/datastore/api/ready` healthcheck** — lifespan builds rw + ro engine instances and stashes on `app.state`; `/datastore/api/ready` calls `engine.healthcheck()` on both and returns 503 + `Service Unavailable` envelope if either fails. - [x] **Request context** — `RequestContext` + `ContextDep` in [datastore/api/context.py](datastore/api/context.py); CKAN client bound to the caller's `api_key` per request (or `None` under non-CKAN auth). `.authorize()` method delegates to `api/auth.py` policy + active provider. - [x] **Engine + auth registries** — `DatastoreBackend` ABC + result dataclasses in [engines/base.py](datastore/infrastructure/engines/base.py); `AuthProvider` Protocol + `Decision` in [auth/base.py](datastore/auth/base.py). Each subpackage exports `Backend` / `Provider`; `DATASTORE_ENGINE` / `AUTH_TYPE` are validated against directories on disk at startup; registries dispatch via `importlib`. +- [x] **Themed Swagger UI** — `/datastore/api/v2/docs` is served by `_register_swagger_docs` in [datastore/main.py](datastore/main.py), not FastAPI's stock route. The page is a Jinja template at [datastore/api/templates/docs.html](datastore/api/templates/docs.html), so autoescaping is structural — though it covers the HTML contexts only, which is why the colours are validated at config load and the spec URL is emitted through `tojson`. Swagger UI is **vendored** under [datastore/api/static/](datastore/api/static/) (swagger-ui-dist 5.17.14) and mounted at `/datastore/api/static`, so the page renders with no CDN and no outbound network. The stylesheet is ported from `ckanext-openapidocs` so this service's docs and the CKAN portal's read as one family: Swagger's own topbar, servers dropdown and duplicate title block are hidden in favour of a branded header, and the Authorize dialog is restyled (900px wide, so a pasted JWT / API key is readable in full). Branding comes from `DOCS_PRIMARY_COLOR` / `DOCS_HEADER_COLOR` / `DOCS_SITE_TITLE` / `DOCS_LOGO_URL`, written into CSS custom properties — the colours are validated as CSS colours at config load, since they land inside a ` + {%- endif %} + + +
+
+ {%- if logo_url %} + + {%- endif %} +
+

{{ site_title }}

+ {%- if api_version %} + {{ api_version }} + {%- endif %} +
+ OpenAPI spec +
+
+ +
+ + + + + diff --git a/datastore/auth/ckan/provider.py b/datastore/auth/ckan/provider.py index 1c49547..91d924c 100644 --- a/datastore/auth/ckan/provider.py +++ b/datastore/auth/ckan/provider.py @@ -55,7 +55,9 @@ async def authorize( decision = _decision_from_bytes(cached) log.debug( "ckan auth cache HIT scope=%s target=%s perm=%s", - scope, target, permission, + scope, + target, + permission, ) return decision except (AuthorizationError, ValueError, TypeError) as e: @@ -65,12 +67,16 @@ async def authorize( log.warning( "ckan auth cache entry malformed for scope=%s target=%s: " "%s — falling back to CKAN", - scope, target, e, + scope, + target, + e, ) log.debug( "ckan auth cache MISS scope=%s target=%s perm=%s -> CKAN", - scope, target, permission, + scope, + target, + permission, ) ckan = self._ckan.bind(credential) result = await ckan.datastore_authorize( @@ -85,7 +91,10 @@ async def authorize( package=result.get("package"), ) await _safe_set( - self._cache, cache_key, _decision_to_bytes(decision), self._cache_ttl, + self._cache, + cache_key, + _decision_to_bytes(decision), + self._cache_ttl, ) return decision @@ -125,8 +134,7 @@ async def _safe_set(cache: CachePort, key: str, value: bytes, ttl: int) -> None: def _decision_to_bytes(d: Decision) -> bytes: return orjson.dumps( - {"subject": d.subject, "claims": d.claims, - "resource": d.resource, "package": d.package}, + {"subject": d.subject, "claims": d.claims, "resource": d.resource, "package": d.package}, ) diff --git a/datastore/auth/jwt/provider.py b/datastore/auth/jwt/provider.py index d8f518e..e42732c 100644 --- a/datastore/auth/jwt/provider.py +++ b/datastore/auth/jwt/provider.py @@ -28,15 +28,11 @@ def __init__(self, *, config: Config, **_: object) -> None: self._issuer = config.JWT_ISSUER or None if algo.startswith("HS"): if not config.JWT_SECRET: - raise ValueError( - f"JWT_SECRET required when JWT_ALGORITHM={algo}" - ) + raise ValueError(f"JWT_SECRET required when JWT_ALGORITHM={algo}") self._key: str = config.JWT_SECRET else: if not config.JWT_PUBLIC_KEY: - raise ValueError( - f"JWT_PUBLIC_KEY required when JWT_ALGORITHM={algo}" - ) + raise ValueError(f"JWT_PUBLIC_KEY required when JWT_ALGORITHM={algo}") self._key = config.JWT_PUBLIC_KEY async def authorize( diff --git a/datastore/core/config.py b/datastore/core/config.py index 5f62e34..50e5d7b 100644 --- a/datastore/core/config.py +++ b/datastore/core/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from functools import lru_cache from pathlib import Path from typing import Literal @@ -7,9 +8,9 @@ from pydantic import Field, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict -_ENGINES_DIR = ( - Path(__file__).resolve().parent.parent / "infrastructure" / "engines" -) +from datastore.core.constants import DEFAULT_API_URL + +_ENGINES_DIR = Path(__file__).resolve().parent.parent / "infrastructure" / "engines" _AUTH_DIR = Path(__file__).resolve().parent.parent / "auth" @@ -17,10 +18,9 @@ def _subdirs(root: Path) -> set[str]: if not root.is_dir(): return set() return { - p.name for p in root.iterdir() - if p.is_dir() - and not p.name.startswith(("_", ".")) - and p.name != "__pycache__" + p.name + for p in root.iterdir() + if p.is_dir() and not p.name.startswith(("_", ".")) and p.name != "__pycache__" } @@ -39,15 +39,19 @@ def _available_auth_types() -> set[str]: return _subdirs(_AUTH_DIR) +# Hex / rgb() / hsl() / named CSS colours. Deliberately narrow: these values +# land inside a `" - ) - return HTMLResponse(html) - - -# Swagger "Authorize" description per built-in AUTH_TYPE. A provider not -# listed here (a third-party drop-in under `datastore/auth//`) keeps -# the generic description declared on the scheme in `api/context.py`. -_AUTH_SCHEME_DESCRIPTIONS = { - "ckan": "A CKAN API key.", - "jwt": "A signed JWT. Accepts the raw token or `Bearer `.", -} - - -def _tailor_auth_scheme(app: FastAPI, auth_type: str) -> None: - """Shape the OpenAPI security scheme to the active AUTH_TYPE. - - The `Authorization` scheme is declared once in `api/context.py`, at - import time — before config is read — so its description can't know - which provider runs. The app factory does, and rewrites the schema - here: provider-specific wording for `ckan` / `jwt`; under `anonymous` - the scheme and every operation's `security` entry are removed, so the - docs render no Authorize button at all. - """ - default_openapi = app.openapi - - def openapi() -> dict[str, Any]: - schema = default_openapi() - schemes = schema.get("components", {}).get("securitySchemes", {}) - if auth_type == "anonymous": - schemes.pop("Authorization", None) - if not schemes: - schema.get("components", {}).pop("securitySchemes", None) - for path_item in schema.get("paths", {}).values(): - for operation in path_item.values(): - if not isinstance(operation, dict): - continue - security = [ - requirement - for requirement in operation.get("security", []) - if "Authorization" not in requirement - ] - if security: - operation["security"] = security - else: - operation.pop("security", None) - elif auth_type in _AUTH_SCHEME_DESCRIPTIONS and "Authorization" in schemes: - description = _AUTH_SCHEME_DESCRIPTIONS[auth_type] - schemes["Authorization"]["description"] = description - return schema - - app.openapi = openapi # type: ignore[method-assign] - - -def _strip_default_422(app: FastAPI) -> None: - """Drop FastAPI's auto-generated 422 from the schema. - - `RequestValidationError` is remapped to a 400 CKAN error envelope (see - `error_handlers`), so a documented 422 never actually occurs — the real - 4xx shapes are declared via `ERROR_RESPONSES`. - """ - default_openapi = app.openapi - - def openapi() -> dict[str, Any]: - schema = default_openapi() - for path_item in schema.get("paths", {}).values(): - for operation in path_item.values(): - if isinstance(operation, dict): - operation.get("responses", {}).pop("422", None) - components = schema.get("components", {}).get("schemas", {}) - components.pop("HTTPValidationError", None) - components.pop("ValidationError", None) - return schema - - app.openapi = openapi # type: ignore[method-assign] - - @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Per-process startup/shutdown. @@ -182,9 +52,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: ) app.state.http = http ckan: CKANClient | None = ( - CKANClient(base_url=config.CKAN_URL, http=http) - if config.AUTH_TYPE == "ckan" - else None + CKANClient(base_url=config.CKAN_URL, http=http) if config.AUTH_TYPE == "ckan" else None ) app.state.ckan = ckan @@ -192,9 +60,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: if hasattr(cache, "close"): stack.push_async_callback(cache.close) app.state.cache = cache - + app.state.auth_provider = get_auth_provider( - config, ckan=ckan, cache=cache, cache_ttl=config.AUTH_CACHE_TTL, + config, + ckan=ckan, + cache=cache, + cache_ttl=config.AUTH_CACHE_TTL, ) # Build + initialise rw/ro engines once; surface credential @@ -217,27 +88,14 @@ def create_app() -> FastAPI: config = get_config() app = FastAPI( title="Datastore API", - version=_api_version(), - summary="A datastore API for managing tabular data resources.", - description=( - "📮 **Postman collection** — import the " - "[Datastore API collection]" - "(https://raw.githubusercontent.com/datopian/datastore/main/" - "postman/collection.json) via Postman's **Import → Link** to " - "exercise every endpoint with worked examples." - ), + description=api_description(config.AUTH_TYPE), openapi_tags=OPENAPI_TAGS, - contact={"name": "Datopian", "url": "https://www.datopian.com/"}, - # Mount the interactive docs (and the spec they fetch) under the - # service's path prefix so this API doesn't compete with an upstream - # proxy or sibling service for the bare `/docs`. Swagger UI is - # served by `_register_swagger_docs` (CSS overrides), not FastAPI's - # stock route. docs_url=None, - redoc_url="/datastore/api/redoc", - openapi_url="/datastore/api/openapi.json", + redoc_url=f"{API_PREFIX}/redoc", + openapi_url=f"{API_PREFIX}/openapi.json", lifespan=lifespan, default_response_class=ORJSONResponse, + generate_unique_id_function=operation_id, ) app.add_middleware(GZipMiddleware, minimum_size=1024) @@ -262,9 +120,10 @@ def create_app() -> FastAPI: register_exception_handlers(app) app.include_router(api_router) - _register_swagger_docs(app, docs_url="/datastore/api/docs") - _strip_default_422(app) - _tailor_auth_scheme(app, config.AUTH_TYPE) + register_swagger_docs(app, docs_url=f"{API_PREFIX}/docs", config=config) + strip_default_422(app) + tailor_auth_scheme(app, config.AUTH_TYPE) + absolutize_example_urls(app, config.API_URL) return app diff --git a/datastore/schemas/request.py b/datastore/schemas/request.py index 37617f8..18ed0e4 100644 --- a/datastore/schemas/request.py +++ b/datastore/schemas/request.py @@ -191,9 +191,7 @@ class DatastoreUpsertRequest(BaseModel): resource_id: str = Field( description="Target table — must already exist (call `datastore_create` first)." ) - records: list[dict[str, Any]] | None = Field( - default=None, description="Rows to write." - ) + records: list[dict[str, Any]] | None = Field(default=None, description="Rows to write.") method: UpsertMethod = Field( default="upsert", description=( @@ -209,9 +207,7 @@ class DatastoreUpsertRequest(BaseModel): default=False, description="Run `COUNT(*)` after the write and return `result.total`.", ) - force: bool = Field( - default=False, description="Bypass optional client-side guards (reserved)." - ) + force: bool = Field(default=False, description="Bypass optional client-side guards (reserved).") class DatastoreSearchRequest(BaseModel): @@ -267,9 +263,7 @@ class DatastoreSearchRequest(BaseModel): ge=0, description="Max rows to return (capped by `SEARCH_RESULT_ROWS_MAX`).", ) - offset: int = Field( - default=0, ge=0, description="Rows to skip — pagination offset." - ) + offset: int = Field(default=0, ge=0, description="Rows to skip — pagination offset.") fields: str | None = Field( default=None, description="Comma-separated columns to project. Default: all columns.", @@ -333,13 +327,8 @@ class DatastoreSearchSQLRequest(BaseModel): _REQUIRE_LIMIT: ClassVar[bool] = True sql: str = Field( - description=( - "A Datastore read API with `SELECT` / `WITH` statement." - ), - examples=[ - 'SELECT * FROM "balancing_auction_results_2025" ' - "WHERE accepted = true LIMIT 100" - ], + description=("A Datastore read API with `SELECT` / `WITH` statement."), + examples=['SELECT * FROM "balancing_auction_results_2025" WHERE accepted = true LIMIT 100'], ) # Set by `_extract_sql_references` after sql validates. Private so @@ -425,13 +414,14 @@ def _extract_sql_references(self) -> DatastoreSearchSQLRequest: """ self._resource_ids, self._function_names = parse_sql_references(self.sql) self._limit, self._offset = parse_sql_pagination( - self.sql, require_limit=self._REQUIRE_LIMIT, + self.sql, + require_limit=self._REQUIRE_LIMIT, ) return self class DatastoreDumpSQLRequest(DatastoreSearchSQLRequest): - """Query parameters for `GET /datastore/dump/query`. + """Query parameters for `GET /dump/query`. Same vetted-SELECT contract as `DatastoreSearchSQLRequest` (single SELECT / WITH, table + function extraction, `extra="forbid"`), but @@ -444,21 +434,13 @@ class DatastoreDumpSQLRequest(DatastoreSearchSQLRequest): _REQUIRE_LIMIT: ClassVar[bool] = False sql: str = Field( - description=( - "A Datastore read API with`SELECT` / `WITH` statement." - ), - examples=[ - 'SELECT * FROM "balancing_auction_results_2025" ' - "WHERE accepted = true" - ], + description=("A Datastore read API with`SELECT` / `WITH` statement."), + examples=['SELECT * FROM "balancing_auction_results_2025" WHERE accepted = true'], ) format: DumpFormat = Field( default="csv", - description=( - "Export format: `csv` | `gzip` (gzipped CSV) | `ndjson` | " - "`parquet`." - ), + description=("Export format: `csv` | `gzip` (gzipped CSV) | `ndjson` | `parquet`."), ) @@ -474,9 +456,7 @@ class DatastoreInfoRequest(BaseModel): model_config = ConfigDict(extra="forbid") - resource_id: str | None = Field( - default=None, description="Resource (table) to describe." - ) + resource_id: str | None = Field(default=None, description="Resource (table) to describe.") id: str | None = Field( default=None, description="CKAN alias for `resource_id`. Send exactly one." ) @@ -485,14 +465,9 @@ class DatastoreInfoRequest(BaseModel): def _require_resource_id_or_id(self) -> DatastoreInfoRequest: if self.resource_id is None and self.id is None: raise ValueError("either 'resource_id' or 'id' is required") - if ( - self.resource_id is not None - and self.id is not None - and self.resource_id != self.id - ): + if self.resource_id is not None and self.id is not None and self.resource_id != self.id: raise ValueError( - "'resource_id' and 'id' both provided with different " - "values; send exactly one" + "'resource_id' and 'id' both provided with different values; send exactly one" ) if self.resource_id is None: self.resource_id = self.id @@ -532,9 +507,7 @@ class DatastoreDeleteRequest(BaseModel): ) fields: list[str] | None = Field( default=None, - description=( - "Drop these columns instead of rows. Mutually exclusive with `filters`." - ), + description=("Drop these columns instead of rows. Mutually exclusive with `filters`."), ) force: bool = Field( default=False, description="Required to delete from a CKAN read-only resource." @@ -544,14 +517,9 @@ class DatastoreDeleteRequest(BaseModel): def _require_resource_id_or_id(self) -> DatastoreDeleteRequest: if self.resource_id is None and self.id is None: raise ValueError("either 'resource_id' or 'id' is required") - if ( - self.resource_id is not None - and self.id is not None - and self.resource_id != self.id - ): + if self.resource_id is not None and self.id is not None and self.resource_id != self.id: raise ValueError( - "'resource_id' and 'id' both provided with different " - "values; send exactly one" + "'resource_id' and 'id' both provided with different values; send exactly one" ) if self.resource_id is None: self.resource_id = self.id diff --git a/datastore/schemas/responses.py b/datastore/schemas/responses.py index 36d4d81..fafbdf6 100644 --- a/datastore/schemas/responses.py +++ b/datastore/schemas/responses.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field +from datastore.core.constants import API_PREFIX from datastore.schemas.validators import FieldSpec @@ -26,13 +27,16 @@ class ResponseModel(BaseModel): class ErrorEnvelope(BaseModel): - """CKAN-shaped error body returned for every 4xx / 5xx response.""" + """Error body returned for every 4xx / 5xx response.""" model_config = ConfigDict( populate_by_name=True, json_schema_extra={ "example": { - "help": "https://example.com/api/3/action/datastore_search", + # Relative: the public host comes from `API_URL`, which only + # `api/docs.py` can read (`schemas/` must not import config). + # That module rewrites this into an absolute URL at app build. + "help": f"{API_PREFIX}/docs#/Datastore/datastore_search", "success": False, "error": { "__type": "Validation Error", @@ -61,15 +65,6 @@ class Error(BaseModel): # --- health ----------------------------------------------------------------- -class WelcomeResponse(ResponseModel): - """Response for `GET /`.""" - - class Result(BaseModel): - message: str - - result: Result - - class StatusResponse(ResponseModel): """Response for `GET /health` and `GET /ready`.""" @@ -142,8 +137,7 @@ class Result(BaseModel): class DatastoreSearchResponse(ResponseModel): - """Response for `GET /api/3/datastore_search` - """ + """Response for `GET /api/3/datastore_search`""" class Result(BaseModel): # `_links` starts with an underscore, which pydantic treats as a diff --git a/datastore/schemas/validators.py b/datastore/schemas/validators.py index 0799589..e626c3a 100644 --- a/datastore/schemas/validators.py +++ b/datastore/schemas/validators.py @@ -326,7 +326,10 @@ def parse_sql_references(sql: str, *, dialect: str = "postgres") -> tuple[list[s def parse_sql_pagination( - sql: str, *, dialect: str = "postgres", require_limit: bool = True, + sql: str, + *, + dialect: str = "postgres", + require_limit: bool = True, ) -> tuple[int | None, int]: """Extract `(limit, offset)` from a SELECT. @@ -368,13 +371,9 @@ def parse_sql_pagination( if tree.args.get("offset") is not None: raise ValueError("OFFSET without LIMIT is not supported") else: - limit_expr = ( - limit_node.expression if isinstance(limit_node, exp.Limit) else None - ) + limit_expr = limit_node.expression if isinstance(limit_node, exp.Limit) else None if not isinstance(limit_expr, exp.Literal) or not limit_expr.is_int: - raise ValueError( - "LIMIT must be a constant integer literal" - ) + raise ValueError("LIMIT must be a constant integer literal") limit = int(limit_expr.this) if limit < 0: raise ValueError("LIMIT must be >= 0") @@ -382,14 +381,9 @@ def parse_sql_pagination( offset = 0 offset_node = tree.args.get("offset") if offset_node is not None: - offset_expr = ( - offset_node.expression - if isinstance(offset_node, exp.Offset) else None - ) + offset_expr = offset_node.expression if isinstance(offset_node, exp.Offset) else None if not isinstance(offset_expr, exp.Literal) or not offset_expr.is_int: - raise ValueError( - "OFFSET must be a constant integer literal" - ) + raise ValueError("OFFSET must be a constant integer literal") offset = int(offset_expr.this) if offset < 0: raise ValueError("OFFSET must be >= 0") @@ -398,7 +392,10 @@ def parse_sql_pagination( def rewrite_sql_offset( - sql: str, new_offset: int, *, dialect: str = "postgres", + sql: str, + new_offset: int, + *, + dialect: str = "postgres", ) -> str: """Return `sql` with its OFFSET replaced (or inserted) at `new_offset`. @@ -442,9 +439,9 @@ class FieldSpec(BaseModel): @classmethod def _check_not_reserved(cls, v: str) -> str: from datastore.core.constants import RESERVED_SYSTEM_COLUMN_NAMES + if v in RESERVED_SYSTEM_COLUMN_NAMES: raise ValueError( - f"field id {v!r} is reserved for engine-managed system " - "columns; rename the field" + f"field id {v!r} is reserved for engine-managed system columns; rename the field" ) return check_field_name(v) diff --git a/datastore/services/read.py b/datastore/services/read.py index 6b12b32..8e8ca1a 100644 --- a/datastore/services/read.py +++ b/datastore/services/read.py @@ -28,9 +28,9 @@ _WRITERS = { - "csv": stream_csv, - "tsv": stream_tsv, - "lists": stream_lists, + "csv": stream_csv, + "tsv": stream_tsv, + "lists": stream_lists, "objects": stream_objects, } @@ -87,7 +87,7 @@ async def search_datastore( ) fields, _ = frictionless_schema_to_fields(result.schema) - + envelope_kwargs = dict( help_url=request_url, resource_id=data_dict["resource_id"], @@ -147,7 +147,9 @@ async def search_sql_datastore( # Off the event loop — submitting the query + fetching the first # page blocks; streaming writer below picks up the rest in threadpool. result = await asyncio.to_thread( - engine.search_sql, sql=data_dict["sql"], limit=limit, + engine.search_sql, + sql=data_dict["sql"], + limit=limit, ) fields, _ = frictionless_schema_to_fields(result.schema) return stream_objects( @@ -175,9 +177,7 @@ async def search_sql_datastore( ) -async def dump_sql_datastore( - context: RequestContext, data_dict: dict[str, Any] -) -> list[str]: +async def dump_sql_datastore(context: RequestContext, data_dict: dict[str, Any]) -> list[str]: """Export a vetted SELECT's result via the engine; return signed URLs. The download-mode sibling of `search_sql_datastore`: same function @@ -202,7 +202,8 @@ async def dump_sql_datastore( def _ensure_allowed_sql_functions( - context: RequestContext, function_names: list[str], + context: RequestContext, + function_names: list[str], ) -> None: """Reject SQL that calls functions outside the engine's allow-list. @@ -234,7 +235,8 @@ async def info_datastore( """ engine = get_datastore_engine(context, mode="ro") result = await asyncio.to_thread( - engine.info, resource_id=data_dict["resource_id"], + engine.info, + resource_id=data_dict["resource_id"], ) schema = result.schema @@ -293,18 +295,22 @@ def _build_pagination_links( base_pairs = [(k, v) for k, v in pairs if k != "offset"] def _qs(pairs: list[tuple[str, str]]) -> str: - return urlunparse(( - parsed.scheme, parsed.netloc, parsed.path, - "", urlencode(pairs), "", - )) + return urlunparse( + ( + parsed.scheme, + parsed.netloc, + parsed.path, + "", + urlencode(pairs), + "", + ) + ) out: dict[str, Any] = {"start": _qs(base_pairs)} if offset > 0: prev_offset = max(0, offset - limit) out["prev"] = _qs(base_pairs + [("offset", str(prev_offset))]) - has_next = ( - limit > 0 and total is not None and offset + limit < total - ) + has_next = limit > 0 and total is not None and offset + limit < total if has_next: out["next"] = _qs(base_pairs + [("offset", str(offset + limit))]) if limit > 0: @@ -340,26 +346,26 @@ def _build_sql_pagination_links( of `sql` with a new OFFSET literal (LIMIT is preserved exactly). """ parsed = urlparse(url) - base_pairs = [ - (k, v) - for k, v in parse_qsl(parsed.query, keep_blank_values=True) - if k != "sql" - ] + base_pairs = [(k, v) for k, v in parse_qsl(parsed.query, keep_blank_values=True) if k != "sql"] def _link_for(target_offset: int) -> str: new_sql = rewrite_sql_offset(sql, target_offset) pairs = base_pairs + [("sql", new_sql)] - return urlunparse(( - parsed.scheme, parsed.netloc, parsed.path, - "", urlencode(pairs), "", - )) + return urlunparse( + ( + parsed.scheme, + parsed.netloc, + parsed.path, + "", + urlencode(pairs), + "", + ) + ) out: dict[str, Any] = {"start": _link_for(0)} if offset > 0: out["prev"] = _link_for(max(0, offset - limit)) - has_next = ( - limit > 0 and total is not None and offset + limit < total - ) + has_next = limit > 0 and total is not None and offset + limit < total if has_next: out["next"] = _link_for(offset + limit) if limit > 0: diff --git a/datastore/services/streaming.py b/datastore/services/streaming.py index 5917eab..b19c37e 100644 --- a/datastore/services/streaming.py +++ b/datastore/services/streaming.py @@ -264,9 +264,7 @@ def _stream_envelope( yield b"}" # close envelope -def _records_object_array( - columns: list[str], records: Iterator[tuple] -) -> Iterator[bytes]: +def _records_object_array(columns: list[str], records: Iterator[tuple]) -> Iterator[bytes]: """`[{col: value, ...}, ...]`.""" yield b"[" first = True @@ -318,9 +316,9 @@ def _delimited_row(row: Any, *, delimiter: str) -> str: `StringIO` is constant-size so memory stays bounded. """ buf = io.StringIO() - csv.writer( - buf, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL, lineterminator="\n" - ).writerow(row) + csv.writer(buf, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL, lineterminator="\n").writerow( + row + ) return buf.getvalue() @@ -403,9 +401,7 @@ async def zip_archive_writer( ) as archive: for filename, url in members: with archive.open(filename, mode="w", force_zip64=True) as entry: - async with http.stream( - "GET", url, timeout=_ZIP_FETCH_TIMEOUT - ) as response: + async with http.stream("GET", url, timeout=_ZIP_FETCH_TIMEOUT) as response: response.raise_for_status() async for chunk in response.aiter_bytes(_ZIP_CHUNK_BYTES): entry.write(chunk) diff --git a/datastore/services/write.py b/datastore/services/write.py index 41168d4..af2693d 100644 --- a/datastore/services/write.py +++ b/datastore/services/write.py @@ -126,7 +126,7 @@ async def create_datastore( # its data. Caller-supplied url_type is overridden on purpose. resource = await context.ckan.resource_create( resource={ - **resource, + **resource, "url_type": "datastore", "datastore_active": True, } @@ -181,7 +181,6 @@ async def upsert_datastore( include_total=include_total, ) - await _sync_resource_to_ckan(context, resource_id) return DatastoreUpsertResponse.Result( @@ -204,7 +203,10 @@ async def delete_datastore( engine = get_datastore_engine(context, mode="rw") result = await asyncio.to_thread( - engine.delete, resource_id=resource_id, filters=filters, fields=fields, + engine.delete, + resource_id=resource_id, + filters=filters, + fields=fields, ) # Sync CKAN per the delete variant (mirrors the engine's branching): diff --git a/docker-compose.yml b/docker-compose.yml index 8240dad..1278f6a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: - .env restart: unless-stopped healthcheck: - test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=2).status == 200 else 1)"] + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/datastore/api/health', timeout=2).status == 200 else 1)"] interval: 30s timeout: 3s retries: 3 diff --git a/postman/collection.json b/postman/collection.json index d2588a4..bea69b2 100644 --- a/postman/collection.json +++ b/postman/collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "2d4510dd-5f60-4f1d-860c-2da4b7b394fa", + "_postman_id": "fa56adc4-1610-477b-b095-713e228cad51", "name": "Datastore API", "description": "CKAN-compatible datastore API \u2014 auto-generated from `example_payload/`. Set `baseUrl` to your server, `apiKey` to a CKAN API key (anonymous reads are allowed; writes require a key), and `resourceId` to the table you want to hit.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" @@ -45,26 +45,8 @@ "item": [ { "name": "health", - "description": "Liveness / readiness probes live under `/datastore/api` (`/datastore/api/health`, `/datastore/api/ready`); the welcome banner is at the root `/`. Hit any of these to check the server.", + "description": "Liveness / readiness probes: `/datastore/api/health` and `/datastore/api/ready`. Hit either to check the server.", "item": [ - { - "name": "Welcome", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/", - "host": [ - "{{baseUrl}}" - ], - "path": [ - "" - ] - }, - "description": "Banner / root endpoint. Echoes `APP_MESSAGE`." - }, - "response": [] - }, { "name": "Health", "request": { @@ -131,14 +113,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_create", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_create", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_create" ] }, @@ -166,14 +148,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_create", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_create", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_create" ] }, @@ -201,14 +183,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_create", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_create", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_create" ] }, @@ -242,14 +224,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_upsert", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_upsert", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_upsert" ] }, @@ -277,14 +259,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_upsert", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_upsert", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_upsert" ] }, @@ -312,14 +294,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_upsert", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_upsert", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_upsert" ] }, @@ -339,14 +321,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_info?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_info?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_info" ], "query": [ @@ -366,14 +348,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_info?id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_info?id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_info" ], "query": [ @@ -399,14 +381,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -426,14 +408,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&filters={\"product_code\": \"DCL\", \"accepted\": true}&limit=100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&filters={\"product_code\": \"DCL\", \"accepted\": true}&limit=100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -461,14 +443,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&q=DRAX&plain=true&language=english&limit=50", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&q=DRAX&plain=true&language=english&limit=50", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -504,14 +486,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,delivery_start,clearing_price_gbp_per_mwh,volume_mwh&sort=delivery_start desc, clearing_price_gbp_per_mwh asc&limit=100&offset=0&include_total=true", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,delivery_start,clearing_price_gbp_per_mwh,volume_mwh&sort=delivery_start desc, clearing_price_gbp_per_mwh asc&limit=100&offset=0&include_total=true", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -551,14 +533,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=lists&limit=100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=lists&limit=100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -590,14 +572,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=csv&limit=100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=csv&limit=100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search" ], "query": [ @@ -635,14 +617,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true LIMIT 100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true LIMIT 100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -662,14 +644,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT product_code, AVG(clearing_price_gbp_per_mwh) AS avg_price, SUM(volume_mwh) AS total_volume FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true GROUP BY product_code ORDER BY product_code LIMIT 50", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT product_code, AVG(clearing_price_gbp_per_mwh) AS avg_price, SUM(volume_mwh) AS total_volume FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true GROUP BY product_code ORDER BY product_code LIMIT 50", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -689,14 +671,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=WITH daily AS (SELECT DATE(delivery_start) AS d, product_code, AVG(clearing_price_gbp_per_mwh) AS price FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true GROUP BY d, product_code) SELECT * FROM daily ORDER BY d DESC, product_code LIMIT 50", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=WITH daily AS (SELECT DATE(delivery_start) AS d, product_code, AVG(clearing_price_gbp_per_mwh) AS price FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" WHERE accepted = true GROUP BY d, product_code) SELECT * FROM daily ORDER BY d DESC, product_code LIMIT 50", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -716,14 +698,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT _id, auction_id, product_code, delivery_start FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" ORDER BY _id ASC LIMIT 50 OFFSET 100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT _id, auction_id, product_code, delivery_start FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" ORDER BY _id ASC LIMIT 50 OFFSET 100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -743,14 +725,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT a._id AS a_id, b._id AS b_id, a.auction_id, a.product_code FROM \"balancing_auction_results_2025\" a JOIN \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" b ON a.auction_id = b.auction_id WHERE a.product_code = 'DCL' ORDER BY a.auction_id ASC LIMIT 50", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT a._id AS a_id, b._id AS b_id, a.auction_id, a.product_code FROM \"balancing_auction_results_2025\" a JOIN \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" b ON a.auction_id = b.auction_id WHERE a.product_code = 'DCL' ORDER BY a.auction_id ASC LIMIT 50", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -770,14 +752,14 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_search_sql?sql=SELECT '2025' AS source, auction_id, product_code, delivery_start FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" UNION ALL SELECT '2024' AS source, auction_id, product_code, delivery_start FROM \"balancing_auction_results_2024\" ORDER BY delivery_start DESC LIMIT 100", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_search_sql?sql=SELECT '2025' AS source, auction_id, product_code, delivery_start FROM \"7a10def4-8e95-46f9-96c7-9f61bdfd1a09\" UNION ALL SELECT '2024' AS source, auction_id, product_code, delivery_start FROM \"balancing_auction_results_2024\" ORDER BY delivery_start DESC LIMIT 100", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_search_sql" ], "query": [ @@ -817,14 +799,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_delete", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_delete", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_delete" ] }, @@ -852,14 +834,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_delete", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_delete", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_delete" ] }, @@ -887,14 +869,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_delete", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_delete", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_delete" ] }, @@ -922,14 +904,14 @@ } }, "url": { - "raw": "{{baseUrl}}/api/3/action/datastore_delete", + "raw": "{{baseUrl}}/datastore/api/v2/datastore_delete", "host": [ "{{baseUrl}}" ], "path": [ + "datastore", "api", - "3", - "action", + "v2", "datastore_delete" ] }, diff --git a/postman/generate_postman.py b/postman/generate_postman.py index 7858d00..36d02c5 100644 --- a/postman/generate_postman.py +++ b/postman/generate_postman.py @@ -66,7 +66,6 @@ ] HEALTH_REQUESTS: list[tuple[str, str, str]] = [ - ("Welcome", "", "Banner / root endpoint. Echoes `APP_MESSAGE`."), ("Health", "datastore/api/health", "Liveness probe — always 200 while the process is up."), ("Ready", "datastore/api/ready", @@ -103,7 +102,7 @@ def _post_request(action: str, body: dict[str, Any], description: str) -> dict[s "raw": json.dumps(body, indent=2), "options": {"raw": {"language": "json"}}, }, - "url": _request_url(f"api/3/action/{action}"), + "url": _request_url(f"datastore/api/v2/{action}"), "description": description, } @@ -124,7 +123,7 @@ def _get_request(action: str, body: dict[str, Any], description: str) -> dict[st return { "method": "GET", "header": [], - "url": _request_url(f"api/3/action/{action}", query=query), + "url": _request_url(f"datastore/api/v2/{action}", query=query), "description": description, } @@ -349,9 +348,8 @@ def _build_health_folder() -> dict[str, Any]: return { "name": "health", "description": ( - "Liveness / readiness probes live under `/datastore/api` " - "(`/datastore/api/health`, `/datastore/api/ready`); the welcome " - "banner is at the root `/`. Hit any of these to check the server." + "Liveness / readiness probes: `/datastore/api/health` and " + "`/datastore/api/ready`. Hit either to check the server." ), "item": items, } diff --git a/pyproject.toml b/pyproject.toml index cb08870..86dcbce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "datastore" -version = "0.1.0" +version = "0.2.0" description = "CKAN-compatible datastore API with pluggable storage backends" readme = "README.md" requires-python = ">=3.12,<3.15" @@ -21,6 +21,10 @@ dependencies = [ "httptools>=0.6", "sqlglot>=25.0", "pyjwt>=2.8,<3", + # Renders the Swagger UI docs page (api/templates/docs.html). Arrives + # transitively via fastapi[standard] too, but declared here because + # this package imports it directly. + "jinja2>=3.1", "google-cloud-storage>=2.14", ] diff --git a/tests/auth/anonymous/test_provider.py b/tests/auth/anonymous/test_provider.py index 42eaf89..0491feb 100644 --- a/tests/auth/anonymous/test_provider.py +++ b/tests/auth/anonymous/test_provider.py @@ -10,24 +10,36 @@ def test_authorize_returns_empty_decision_regardless_of_inputs() -> None: provider = AnonymousProvider() - decision = asyncio.run(provider.authorize( - credential=None, - resource_id="any", - package_id=None, - permission="read", - )) + decision = asyncio.run( + provider.authorize( + credential=None, + resource_id="any", + package_id=None, + permission="read", + ) + ) assert decision == Decision() def test_authorize_does_not_care_about_credential() -> None: provider = AnonymousProvider() # Same result whether or not a token is presented. - a = asyncio.run(provider.authorize( - credential="token-1", resource_id="r", package_id=None, permission="read", - )) - b = asyncio.run(provider.authorize( - credential=None, resource_id="r", package_id=None, permission="read", - )) + a = asyncio.run( + provider.authorize( + credential="token-1", + resource_id="r", + package_id=None, + permission="read", + ) + ) + b = asyncio.run( + provider.authorize( + credential=None, + resource_id="r", + package_id=None, + permission="read", + ) + ) assert a == b == Decision() diff --git a/tests/auth/ckan/test_provider.py b/tests/auth/ckan/test_provider.py index ac64664..735bb2d 100644 --- a/tests/auth/ckan/test_provider.py +++ b/tests/auth/ckan/test_provider.py @@ -88,12 +88,14 @@ def test_authorize_binds_credential_and_maps_response_to_decision() -> None: ckan = FakeCKAN() provider = _provider(ckan=ckan) - decision = asyncio.run(provider.authorize( - credential="token-xyz", - resource_id="res-1", - package_id=None, - permission="read", - )) + decision = asyncio.run( + provider.authorize( + credential="token-xyz", + resource_id="res-1", + package_id=None, + permission="read", + ) + ) assert ckan.calls == [ { @@ -116,9 +118,14 @@ def test_authorize_propagates_ckan_authorization_error() -> None: provider = _provider(ckan=ckan) with pytest.raises(AuthorizationError, match="denied"): - asyncio.run(provider.authorize( - credential="t", resource_id="r", package_id=None, permission="read", - )) + asyncio.run( + provider.authorize( + credential="t", + resource_id="r", + package_id=None, + permission="read", + ) + ) def test_authorize_handles_missing_metadata_fields() -> None: @@ -127,9 +134,14 @@ def test_authorize_handles_missing_metadata_fields() -> None: ckan = FakeCKAN(result={"package": {"id": "pkg-1"}}) provider = _provider(ckan=ckan) - decision = asyncio.run(provider.authorize( - credential="t", resource_id=None, package_id="pkg-1", permission="create", - )) + decision = asyncio.run( + provider.authorize( + credential="t", + resource_id=None, + package_id="pkg-1", + permission="create", + ) + ) assert decision.package == {"id": "pkg-1"} assert decision.resource is None @@ -142,12 +154,22 @@ def test_cache_hit_skips_ckan_on_second_call() -> None: cache = InMemoryCache() provider = _provider(ckan=ckan, cache=cache) - asyncio.run(provider.authorize( - credential="tok", resource_id="res-1", package_id=None, permission="read", - )) - asyncio.run(provider.authorize( - credential="tok", resource_id="res-1", package_id=None, permission="read", - )) + asyncio.run( + provider.authorize( + credential="tok", + resource_id="res-1", + package_id=None, + permission="read", + ) + ) + asyncio.run( + provider.authorize( + credential="tok", + resource_id="res-1", + package_id=None, + permission="read", + ) + ) # CKAN called exactly once across both authorizations. assert len(ckan.calls) == 1 @@ -158,14 +180,24 @@ def test_cache_key_uses_anon_marker_when_no_credential() -> None: cache = InMemoryCache() provider = _provider(ckan=ckan, cache=cache) - asyncio.run(provider.authorize( - credential=None, resource_id="res-1", package_id=None, permission="read", - )) + asyncio.run( + provider.authorize( + credential=None, + resource_id="res-1", + package_id=None, + permission="read", + ) + ) # Verify by hitting again with the same shape — second call must be cached. - asyncio.run(provider.authorize( - credential=None, resource_id="res-1", package_id=None, permission="read", - )) + asyncio.run( + provider.authorize( + credential=None, + resource_id="res-1", + package_id=None, + permission="read", + ) + ) assert len(ckan.calls) == 1 @@ -174,12 +206,22 @@ def test_separate_credentials_get_separate_cache_entries() -> None: cache = InMemoryCache() provider = _provider(ckan=ckan, cache=cache) - asyncio.run(provider.authorize( - credential="user-a", resource_id="r", package_id=None, permission="read", - )) - asyncio.run(provider.authorize( - credential="user-b", resource_id="r", package_id=None, permission="read", - )) + asyncio.run( + provider.authorize( + credential="user-a", + resource_id="r", + package_id=None, + permission="read", + ) + ) + asyncio.run( + provider.authorize( + credential="user-b", + resource_id="r", + package_id=None, + permission="read", + ) + ) # Two distinct cache entries → two CKAN calls. assert len(ckan.calls) == 2 @@ -191,12 +233,22 @@ def test_package_scoped_call_uses_pkg_cache_namespace() -> None: provider = _provider(ckan=ckan, cache=cache) # res-scoped and pkg-scoped calls share neither key nor cache entry. - asyncio.run(provider.authorize( - credential="tok", resource_id="x", package_id=None, permission="read", - )) - asyncio.run(provider.authorize( - credential="tok", resource_id=None, package_id="x", permission="create", - )) + asyncio.run( + provider.authorize( + credential="tok", + resource_id="x", + package_id=None, + permission="read", + ) + ) + asyncio.run( + provider.authorize( + credential="tok", + resource_id=None, + package_id="x", + permission="create", + ) + ) assert len(ckan.calls) == 2 @@ -205,9 +257,14 @@ def test_cache_failure_falls_through_to_ckan() -> None: provider = _provider(ckan=ckan, cache=ExplodingCache()) # Fail-open: a broken cache must not break the request. - decision = asyncio.run(provider.authorize( - credential="tok", resource_id="res-1", package_id=None, permission="read", - )) + decision = asyncio.run( + provider.authorize( + credential="tok", + resource_id="res-1", + package_id=None, + permission="read", + ) + ) assert decision.resource == {"id": "res-1", "package_id": "pkg-1"} assert len(ckan.calls) == 1 @@ -220,14 +277,17 @@ def test_malformed_cache_entry_falls_through_to_ckan() -> None: ckan = FakeCKAN() cache = InMemoryCache() provider = _provider(ckan=ckan, cache=cache) - cache_key = ( - f"auth:ckan:{provider.key_id('tok')}:res:res-1:read" - ) + cache_key = f"auth:ckan:{provider.key_id('tok')}:res:res-1:read" asyncio.run(cache.set(cache_key, orjson.dumps("not-a-dict"), 60)) - decision = asyncio.run(provider.authorize( - credential="tok", resource_id="res-1", package_id=None, permission="read", - )) + decision = asyncio.run( + provider.authorize( + credential="tok", + resource_id="res-1", + package_id=None, + permission="read", + ) + ) # Fell back to CKAN and got the canned decision. assert decision.resource == {"id": "res-1", "package_id": "pkg-1"} @@ -240,10 +300,14 @@ def test_subject_never_carries_the_raw_credential() -> None: ckan = FakeCKAN() provider = _provider(ckan=ckan, cache=InMemoryCache()) - decision = asyncio.run(provider.authorize( - credential="raw-api-key-do-not-leak", - resource_id="res-1", package_id=None, permission="read", - )) + decision = asyncio.run( + provider.authorize( + credential="raw-api-key-do-not-leak", + resource_id="res-1", + package_id=None, + permission="read", + ) + ) assert decision.subject == "jhon" assert "raw-api-key-do-not-leak" not in decision.subject diff --git a/tests/auth/jwt/test_provider.py b/tests/auth/jwt/test_provider.py index b415556..bd4082a 100644 --- a/tests/auth/jwt/test_provider.py +++ b/tests/auth/jwt/test_provider.py @@ -37,12 +37,14 @@ def _provider(**overrides: Any) -> JWTAuthProvider: def _authorize(provider: JWTAuthProvider, token: str | None): - return asyncio.run(provider.authorize( - credential=token, - resource_id="r", - package_id=None, - permission="read", - )) + return asyncio.run( + provider.authorize( + credential=token, + resource_id="r", + package_id=None, + permission="read", + ) + ) def test_valid_token_returns_decision_with_subject_and_claims() -> None: @@ -79,9 +81,7 @@ def test_audience_mismatch_raises_authorization_error() -> None: def test_audience_match_passes() -> None: provider = _provider(JWT_AUDIENCE="expected-aud") - token = jwt.encode( - {"sub": "u", "aud": "expected-aud"}, SECRET, algorithm="HS256" - ) + token = jwt.encode({"sub": "u", "aud": "expected-aud"}, SECRET, algorithm="HS256") decision = _authorize(provider, token) assert decision.subject == "u" diff --git a/tests/auth/test_orchestration.py b/tests/auth/test_orchestration.py index ec6e533..bd80ea7 100644 --- a/tests/auth/test_orchestration.py +++ b/tests/auth/test_orchestration.py @@ -31,9 +31,7 @@ def __init__( decision: Decision | None = None, raises: Exception | None = None, ) -> None: - self._decision = decision or Decision( - resource={"id": "res-1"}, package={"id": "pkg-1"} - ) + self._decision = decision or Decision(resource={"id": "res-1"}, package={"id": "pkg-1"}) self._raises = raises self.calls: list[dict[str, Any]] = [] @@ -52,13 +50,15 @@ def key_id(self, credential: str) -> str: def test_provider_verdict_is_returned_as_endpoint_data_dict() -> None: provider = FakeProvider() - result = asyncio.run(authorize( - api_key="tok", - provider=provider, - resource_id="res-1", - package_id=None, - permission="read", - )) + result = asyncio.run( + authorize( + api_key="tok", + provider=provider, + resource_id="res-1", + package_id=None, + permission="read", + ) + ) assert result == { "user": None, @@ -78,13 +78,15 @@ def test_provider_verdict_is_returned_as_endpoint_data_dict() -> None: def test_verdict_without_metadata_yields_empty_dicts() -> None: # Anonymous / JWT providers answer with no user/resource/package; # endpoint code reads from the dict so we must substitute empty dicts. - result = asyncio.run(authorize( - api_key="tok", - provider=FakeProvider(decision=Decision()), - resource_id="res-1", - package_id=None, - permission="read", - )) + result = asyncio.run( + authorize( + api_key="tok", + provider=FakeProvider(decision=Decision()), + resource_id="res-1", + package_id=None, + permission="read", + ) + ) assert result == {"user": None, "resource": {}, "package": {}} @@ -93,10 +95,15 @@ def test_verdict_without_metadata_yields_empty_dicts() -> None: def test_anonymous_caller_for_read_passes_through_to_provider() -> None: provider = FakeProvider(decision=Decision()) - asyncio.run(authorize( - api_key=None, provider=provider, - resource_id="res-1", package_id=None, permission="read", - )) + asyncio.run( + authorize( + api_key=None, + provider=provider, + resource_id="res-1", + package_id=None, + permission="read", + ) + ) assert provider.calls[0]["credential"] is None @@ -104,10 +111,15 @@ def test_anonymous_caller_for_read_passes_through_to_provider() -> None: def test_anonymous_caller_rejected_for_non_read_permissions(permission: str) -> None: provider = FakeProvider() with pytest.raises(AuthorizationError, match="authenticated user"): - asyncio.run(authorize( - api_key=None, provider=provider, - resource_id="res-1", package_id=None, permission=permission, # type: ignore[arg-type] - )) + asyncio.run( + authorize( + api_key=None, + provider=provider, + resource_id="res-1", + package_id=None, + permission=permission, # type: ignore[arg-type] + ) + ) # Provider never reached — policy short-circuits first. assert provider.calls == [] @@ -118,24 +130,39 @@ def test_anonymous_caller_rejected_for_non_read_permissions(permission: str) -> def test_must_supply_exactly_one_of_resource_or_package_id() -> None: provider = FakeProvider() with pytest.raises(ValidationError, match="resource_id or package_id"): - asyncio.run(authorize( - api_key="tok", provider=provider, - resource_id="res-1", package_id="pkg-1", permission="read", - )) + asyncio.run( + authorize( + api_key="tok", + provider=provider, + resource_id="res-1", + package_id="pkg-1", + permission="read", + ) + ) with pytest.raises(ValidationError, match="resource_id or package_id"): - asyncio.run(authorize( - api_key="tok", provider=provider, - resource_id=None, package_id=None, permission="read", - )) + asyncio.run( + authorize( + api_key="tok", + provider=provider, + resource_id=None, + package_id=None, + permission="read", + ) + ) def test_invalid_permission_rejected_at_boundary() -> None: provider = FakeProvider() with pytest.raises(ValidationError, match="permission must be one of"): - asyncio.run(authorize( - api_key="tok", provider=provider, - resource_id="res-1", package_id=None, permission="execute", # type: ignore[arg-type] - )) + asyncio.run( + authorize( + api_key="tok", + provider=provider, + resource_id="res-1", + package_id=None, + permission="execute", # type: ignore[arg-type] + ) + ) assert provider.calls == [] @@ -145,10 +172,15 @@ def test_invalid_permission_rejected_at_boundary() -> None: def test_provider_authorization_error_propagates() -> None: provider = FakeProvider(raises=AuthorizationError("nope")) with pytest.raises(AuthorizationError, match="nope"): - asyncio.run(authorize( - api_key="tok", provider=provider, - resource_id="res-1", package_id=None, permission="read", - )) + asyncio.run( + authorize( + api_key="tok", + provider=provider, + resource_id="res-1", + package_id=None, + permission="read", + ) + ) # --- ensure_resource_writable (read-only force guard) ----------------------- @@ -164,20 +196,26 @@ def test_provider_authorization_error_propagates() -> None: def test_readonly_guard_blocks_non_datastore_resource_under_ckan() -> None: with pytest.raises(ValidationError, match="read-only"): ensure_resource_writable( - {"url_type": "upload"}, force=False, auth_type="ckan", + {"url_type": "upload"}, + force=False, + auth_type="ckan", ) def test_readonly_guard_allows_with_force() -> None: ensure_resource_writable( - {"url_type": "upload"}, force=True, auth_type="ckan", + {"url_type": "upload"}, + force=True, + auth_type="ckan", ) def test_readonly_guard_allows_datastore_managed_resources() -> None: """`url_type="datastore"` means the datastore owns it — writes are fine.""" ensure_resource_writable( - {"url_type": "datastore"}, force=False, auth_type="ckan", + {"url_type": "datastore"}, + force=False, + auth_type="ckan", ) @@ -186,7 +224,9 @@ def test_readonly_guard_skips_when_no_resource_record() -> None: (e.g. the dict-form of datastore_create) — nothing to guard.""" ensure_resource_writable({}, force=False, auth_type="ckan") ensure_resource_writable( - {"package_id": "pkg-1"}, force=False, auth_type="ckan", + {"package_id": "pkg-1"}, + force=False, + auth_type="ckan", ) @@ -194,5 +234,7 @@ def test_readonly_guard_is_ckan_only() -> None: """Non-CKAN auth never trips the guard, even on a non-datastore resource.""" for auth_type in ("anonymous", "jwt"): ensure_resource_writable( - {"url_type": "upload"}, force=False, auth_type=auth_type, + {"url_type": "upload"}, + force=False, + auth_type=auth_type, ) diff --git a/tests/auth/test_registry.py b/tests/auth/test_registry.py index c73445e..ab71856 100644 --- a/tests/auth/test_registry.py +++ b/tests/auth/test_registry.py @@ -29,7 +29,10 @@ def test_ckan_type_returns_ckan_provider_and_forwards_kwargs() -> None: cfg = Config(AUTH_TYPE="ckan", CKAN_URL="http://ckan.test") ckan = MagicMock() provider = get_auth_provider( - cfg, ckan=ckan, cache=InMemoryCache(), cache_ttl=60, + cfg, + ckan=ckan, + cache=InMemoryCache(), + cache_ttl=60, ) assert isinstance(provider, CKANProvider) assert provider.name == "ckan" @@ -65,8 +68,6 @@ def test_jwt_provider_raises_when_hs_secret_missing() -> None: def test_jwt_provider_raises_when_rs_public_key_missing() -> None: - cfg = Config( - AUTH_TYPE="jwt", JWT_ALGORITHM="RS256", JWT_PUBLIC_KEY="", CKAN_URL="" - ) + cfg = Config(AUTH_TYPE="jwt", JWT_ALGORITHM="RS256", JWT_PUBLIC_KEY="", CKAN_URL="") with pytest.raises(ValueError, match="JWT_PUBLIC_KEY"): get_auth_provider(cfg) diff --git a/tests/conftest.py b/tests/conftest.py index a0afdae..e72a525 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,8 +7,10 @@ # fixtures can't intercept BigQuery vars in time. Clearing them here # keeps the suite hermetic against whatever happens to be in .env. for _name in ( - "BIGQUERY_PROJECT", "BIGQUERY_DATASET", - "BIGQUERY_CREDENTIALS", "BIGQUERY_CREDENTIALS_RO", + "BIGQUERY_PROJECT", + "BIGQUERY_DATASET", + "BIGQUERY_CREDENTIALS", + "BIGQUERY_CREDENTIALS_RO", "BIGQUERY_EXPORT_BUCKET", ): os.environ[_name] = "" @@ -63,14 +65,27 @@ def _isolate_bigquery_env(monkeypatch: pytest.MonkeyPatch) -> None: from datastore.infrastructure.engines.registry import reset_engine_cache for name in ( - "BIGQUERY_PROJECT", "BIGQUERY_DATASET", - "BIGQUERY_CREDENTIALS", "BIGQUERY_CREDENTIALS_RO", + "BIGQUERY_PROJECT", + "BIGQUERY_DATASET", + "BIGQUERY_CREDENTIALS", + "BIGQUERY_CREDENTIALS_RO", "BIGQUERY_EXPORT_BUCKET", ): monkeypatch.setenv(name, "") # Pydantic-Settings can't parse "" as int — give the dump-URL TTL a # valid placeholder so a stray .env doesn't break startup in tests. monkeypatch.setenv("BIGQUERY_EXPORT_URL_EXPIRY_HOURS", "1") + for name in ( + "DOCS_PRIMARY_COLOR", + "DOCS_HEADER_COLOR", + "DOCS_SITE_TITLE", + "DOCS_LOGO_URL", + ): + monkeypatch.setenv(name, "") + # `API_URL` sets the host in the published examples, so a developer .env + # pointing at a real deployment would break the tests that assert the + # placeholder default. Tests wanting a value set it themselves. + monkeypatch.setenv("API_URL", "https://example.com") # `Config` and engine instances are lru-cached / module-level # singletons; invalidate so the cleared env actually takes effect. get_config.cache_clear() @@ -147,9 +162,7 @@ async def resource_create(self, *, resource: dict[str, Any]) -> dict[str, Any]: self.resources[str(created["id"])] = created return created - async def resource_patch( - self, *, resource_id: str, patch: dict[str, Any] - ) -> dict[str, Any]: + async def resource_patch(self, *, resource_id: str, patch: dict[str, Any]) -> dict[str, Any]: self._guard() existing = self.resources.get(resource_id) if existing is None: @@ -195,7 +208,9 @@ def client(fake_ckan: FakeCKAN, cache: InMemoryCache) -> Iterator[TestClient]: # Auth provider talks to the same FakeCKAN — tests don't go through # the real HTTP CKAN client. Mirrors what the lifespan would build. app.dependency_overrides[get_auth_provider] = lambda: CKANAuthProvider( - ckan=fake_ckan, cache=cache, cache_ttl=60, + ckan=fake_ckan, + cache=cache, + cache_ttl=60, ) with TestClient(app) as c: c.headers["Authorization"] = "test-token" diff --git a/tests/engines/bigquery/test_metadata.py b/tests/engines/bigquery/test_metadata.py index ff9d536..fa1ccc2 100644 --- a/tests/engines/bigquery/test_metadata.py +++ b/tests/engines/bigquery/test_metadata.py @@ -24,10 +24,7 @@ def _table(description: str | None = None, columns: list[tuple[str, str]] | None = None) -> Any: """A `bigquery.Table` stand-in carrying `description` + `schema`.""" - schema = [ - SimpleNamespace(name=name, field_type=ftype) - for name, ftype in (columns or []) - ] + schema = [SimpleNamespace(name=name, field_type=ftype) for name, ftype in (columns or [])] return SimpleNamespace(description=description, schema=schema) diff --git a/tests/engines/bigquery/test_tables.py b/tests/engines/bigquery/test_tables.py index d3114ec..9505f83 100644 --- a/tests/engines/bigquery/test_tables.py +++ b/tests/engines/bigquery/test_tables.py @@ -80,9 +80,7 @@ def test_can_widen_allows_supported_and_rejects_others() -> None: def test_data_table_ref_uses_backticks(mock_client: MagicMock) -> None: """Backticks let CKAN UUID-like ids parse without further escaping.""" - assert _backend(mock_client)._data_table_ref("res-abc-123") == ( - "`proj-1.ds-1.res-abc-123`" - ) + assert _backend(mock_client)._data_table_ref("res-abc-123") == ("`proj-1.ds-1.res-abc-123`") def test_create_table_sql_emits_ddl_with_options( @@ -98,7 +96,7 @@ def test_create_table_sql_emits_ddl_with_options( "res-1", { "fields": [ - {"name": "id", "type": "integer", "title": "ID"}, + {"name": "id", "type": "integer", "title": "ID"}, {"name": "label", "type": "string"}, ], "primaryKey": ["id"], @@ -116,9 +114,7 @@ def test_create_table_sql_emits_ddl_with_options( # Table-level OPTIONS contains the full user schema, verbatim. assert '"title":"ID"' in sql assert '"primaryKey":["id"]' in sql - assert sql.endswith( - ', labels = [("datastore_managed", "true")])' - ) + assert sql.endswith(', labels = [("datastore_managed", "true")])') def test_alter_adds_new_columns_widens_types_then_refreshes_options( @@ -130,10 +126,12 @@ def test_alter_adds_new_columns_widens_types_then_refreshes_options( statement).""" backend = _backend(mock_client) old = {"fields": [{"name": "a", "type": "integer"}]} - new = {"fields": [ - {"name": "a", "type": "number"}, # widen INT64 → FLOAT64 - {"name": "b", "type": "string"}, # add - ]} + new = { + "fields": [ + {"name": "a", "type": "number"}, # widen INT64 → FLOAT64 + {"name": "b", "type": "string"}, # add + ] + } backend._alter_data_table("res-1", old, new) @@ -143,9 +141,7 @@ def test_alter_adds_new_columns_widens_types_then_refreshes_options( assert "ALTER TABLE `proj-1.ds-1.res-1`" in col_sql assert "ADD COLUMN IF NOT EXISTS `b` STRING" in col_sql assert "ALTER COLUMN `a` SET DATA TYPE FLOAT64" in col_sql - assert opts_sql.startswith( - "ALTER TABLE `proj-1.ds-1.res-1` SET OPTIONS(" - ) + assert opts_sql.startswith("ALTER TABLE `proj-1.ds-1.res-1` SET OPTIONS(") # Refreshed table OPTIONS carries the new schema verbatim. assert '"name":"a","type":"number"' in opts_sql assert '"name":"b","type":"string"' in opts_sql @@ -196,10 +192,12 @@ def test_insert_records_issues_dml_insert_with_rows_param( rows go straight to storage and subsequent MERGE/UPDATE can touch them immediately.""" backend = _backend(mock_client) - schema = {"fields": [ - {"name": "auction_id", "type": "integer"}, - {"name": "bidder_metadata", "type": "object"}, - ]} + schema = { + "fields": [ + {"name": "auction_id", "type": "integer"}, + {"name": "bidder_metadata", "type": "object"}, + ] + } records = [ {"auction_id": 144, "bidder_metadata": {"unit_id": "X"}}, {"auction_id": 145, "bidder_metadata": {"unit_id": "Y"}}, @@ -221,10 +219,7 @@ def test_insert_records_issues_dml_insert_with_rows_param( # System columns auto-injected — `_id` from the inlined MAX subquery # + ROW_NUMBER(), `_updated_at` from CURRENT_TIMESTAMP(). assert "`_id`, `_updated_at`" in sql - assert ( - "(SELECT IFNULL(MAX(`_id`), 0) FROM `proj-1.ds-1.res-1`) " - "+ ROW_NUMBER() OVER ()" - ) in sql + assert ("(SELECT IFNULL(MAX(`_id`), 0) FROM `proj-1.ds-1.res-1`) + ROW_NUMBER() OVER ()") in sql assert "CURRENT_TIMESTAMP()" in sql # Only `@rows` is passed as a parameter now — no separate probe. params = {p.name: p.value for p in kwargs["job_config"].query_parameters} @@ -306,14 +301,13 @@ def test_client_query_errors_surface_as_server_error_with_context( ) -> None: """Raw BQ exceptions on `client.query` are wrapped as ServerError carrying op + resource_id — never leak as `RuntimeError`.""" - mock_client.query.return_value.result.side_effect = RuntimeError( - "Insufficient permissions" - ) + mock_client.query.return_value.result.side_effect = RuntimeError("Insufficient permissions") backend = _backend(mock_client) with pytest.raises(ServerError) as exc: backend._run_query( "CREATE TABLE foo (x INT64)", - op="CREATE TABLE", resource_id="res-1", + op="CREATE TABLE", + resource_id="res-1", ) assert "CREATE TABLE" in str(exc.value) assert "'res-1'" in str(exc.value) @@ -438,16 +432,20 @@ def test_create_with_no_records_on_existing_resource_alters_columns( records still adds the columns — `_read_schema` runs unconditionally so the diff fires whether or not the caller had rows to insert.""" backend = _backend(mock_client) - backend._read_schema = MagicMock(return_value={ - "fields": [{"name": "a", "type": "integer"}], - }) + backend._read_schema = MagicMock( + return_value={ + "fields": [{"name": "a", "type": "integer"}], + } + ) backend.create( "res-1", - schema={"fields": [ - {"name": "a", "type": "integer"}, - {"name": "b", "type": "string"}, - ]}, + schema={ + "fields": [ + {"name": "a", "type": "integer"}, + {"name": "b", "type": "string"}, + ] + }, records=None, include_total=False, ) @@ -457,9 +455,7 @@ def test_create_with_no_records_on_existing_resource_alters_columns( col_sql = mock_client.query.call_args_list[0].args[0] opts_sql = mock_client.query.call_args_list[1].args[0] assert "ADD COLUMN IF NOT EXISTS `b` STRING" in col_sql - assert opts_sql.startswith( - "ALTER TABLE `proj-1.ds-1.res-1` SET OPTIONS(" - ) + assert opts_sql.startswith("ALTER TABLE `proj-1.ds-1.res-1` SET OPTIONS(") def test_create_propagates_insert_failure_as_server_error( @@ -539,13 +535,8 @@ def test_merge_sql_renders_typed_extractors_on_match_update_no_match_insert() -> # NOT MATCHED inserts system columns + user columns. `_id` is # `(SELECT MAX(_id) FROM tbl) + S._rn` — inlined to avoid a # separate probe round-trip. - assert ( - "WHEN NOT MATCHED THEN INSERT (`_id`, `_updated_at`, " - "`id`, `label`, `meta`)" - ) in sql - assert ( - "(SELECT IFNULL(MAX(`_id`), 0) FROM `p.d.r`) + S._rn" - ) in sql + assert ("WHEN NOT MATCHED THEN INSERT (`_id`, `_updated_at`, `id`, `label`, `meta`)") in sql + assert ("(SELECT IFNULL(MAX(`_id`), 0) FROM `p.d.r`) + S._rn") in sql def test_update_sql_renders_dml_update_keyed_on_primary_key() -> None: @@ -599,17 +590,14 @@ def test_insert_conflict_count_sql_counts_batch_and_existing_dups() -> None: assert "IFNULL(SUM(_n - 1), 0)" in sql # Existing-row collisions via a composite-key JOIN against the table. assert ( - "JOIN `p.d.r` T ON d.`auction_id` = T.`auction_id` " - "AND d.`product_code` = T.`product_code`" + "JOIN `p.d.r` T ON d.`auction_id` = T.`auction_id` AND d.`product_code` = T.`product_code`" ) in sql assert sql.rstrip().endswith("AS n") def test_insert_conflict_count_sql_rejects_missing_primary_key() -> None: with pytest.raises(ValueError, match="primaryKey"): - insert_conflict_count_sql( - "`p.d.r`", {"fields": [{"name": "id", "type": "integer"}]} - ) + insert_conflict_count_sql("`p.d.r`", {"fields": [{"name": "id", "type": "integer"}]}) def test_insert_guarded_sql_wraps_check_and_insert_in_one_script() -> None: @@ -646,17 +634,13 @@ def test_insert_guarded_sql_wraps_check_and_insert_in_one_script() -> None: def test_insert_guarded_sql_rejects_missing_primary_key() -> None: with pytest.raises(ValueError, match="primaryKey"): - insert_guarded_sql( - "`p.d.r`", {"fields": [{"name": "id", "type": "integer"}]} - ) + insert_guarded_sql("`p.d.r`", {"fields": [{"name": "id", "type": "integer"}]}) # --- upsert() dispatch ---------------------------------------------------- -def _backend_with_schema( - mock_client: MagicMock, schema: dict[str, Any] -) -> BigQueryBackend: +def _backend_with_schema(mock_client: MagicMock, schema: dict[str, Any]) -> BigQueryBackend: backend = _backend(mock_client) backend._read_schema = MagicMock(return_value=schema) return backend @@ -721,8 +705,7 @@ def test_upsert_method_insert_rejects_pk_conflict( ) with pytest.raises(ValidationError, match="method='upsert'"): - backend.upsert("res-1", [{"id": 1}], method="insert", - include_total=False) + backend.upsert("res-1", [{"id": 1}], method="insert", include_total=False) assert mock_client.query.call_count == 1 # single guarded job @@ -793,9 +776,7 @@ def test_upsert_undeclared_resource_raises_not_found( backend._read_schema = MagicMock(return_value=None) with pytest.raises(NotFoundError, match="not found"): - backend.upsert( - "ghost", [{"a": 1}], method="upsert", include_total=False - ) + backend.upsert("ghost", [{"a": 1}], method="upsert", include_total=False) def test_upsert_missing_primary_key_raises_validation( @@ -808,9 +789,7 @@ def test_upsert_missing_primary_key_raises_validation( {"fields": [{"name": "id", "type": "integer"}]}, # no primaryKey ) with pytest.raises(ValidationError, match="primaryKey"): - backend.upsert( - "res-1", [{"id": 1}], method="upsert", include_total=False - ) + backend.upsert("res-1", [{"id": 1}], method="upsert", include_total=False) mock_client.query.assert_not_called() @@ -823,7 +802,10 @@ def test_upsert_unknown_method_raises_validation( ) with pytest.raises(ValidationError, match="unknown upsert method"): backend.upsert( - "res-1", [], method="merge", include_total=False # bogus + "res-1", + [], + method="merge", + include_total=False, # bogus ) @@ -835,8 +817,7 @@ def test_upsert_translates_bigquery_scalar_subquery_error_to_duplicate_pk( backend translates that into a clear ValidationError naming the actual cause.""" mock_client.query.return_value.result.side_effect = RuntimeError( - "400 Scalar subquery produced more than one element; reason: " - "invalidQuery, location: query" + "400 Scalar subquery produced more than one element; reason: invalidQuery, location: query" ) backend = _backend_with_schema( mock_client, @@ -936,7 +917,9 @@ def test_upsert_translates_bigquery_bad_int64_value_to_type_mismatch( with pytest.raises(ValidationError) as exc: backend.upsert( - "res-1", [{"id": "not-a-number"}], method="upsert", + "res-1", + [{"id": "not-a-number"}], + method="upsert", include_total=False, ) assert "'not-a-number'" in str(exc.value) @@ -961,7 +944,9 @@ def test_translate_invalid_timestamp_value(mock_client: MagicMock) -> None: with pytest.raises(ValidationError) as exc: backend.upsert( - "res-1", [{"id": 1, "ts": "2025-99-99"}], method="upsert", + "res-1", + [{"id": 1, "ts": "2025-99-99"}], + method="upsert", include_total=False, ) assert "'2025-99-99'" in str(exc.value) @@ -981,7 +966,10 @@ def test_translate_could_not_cast_literal_error(mock_client: MagicMock) -> None: ) with pytest.raises(ValidationError) as exc: backend.upsert( - "res-1", [{"id": "jk"}], method="upsert", include_total=False, + "res-1", + [{"id": "jk"}], + method="upsert", + include_total=False, ) msg = str(exc.value) assert "'jk'" in msg @@ -1006,8 +994,10 @@ def test_translate_could_not_parse_as_type_error( ) with pytest.raises(ValidationError) as exc: backend.upsert( - "res-1", [{"id": 1, "price": "abc"}], - method="upsert", include_total=False, + "res-1", + [{"id": 1, "price": "abc"}], + method="upsert", + include_total=False, ) msg = str(exc.value) assert "'abc'" in msg @@ -1018,8 +1008,7 @@ def test_translate_value_out_of_range(mock_client: MagicMock) -> None: """Numeric value that parses but exceeds the column type's range → ValidationError mentioning out-of-range.""" mock_client.query.return_value.result.side_effect = RuntimeError( - "400 Value out of range for INT64: 99999999999999999999; " - "reason: invalidQuery" + "400 Value out of range for INT64: 99999999999999999999; reason: invalidQuery" ) backend = _backend_with_schema( mock_client, @@ -1029,8 +1018,10 @@ def test_translate_value_out_of_range(mock_client: MagicMock) -> None: backend.upsert( # Use string to avoid orjson's 64-bit int limit — the test # checks the BigQuery-side error, not orjson encoding. - "res-1", [{"id": "99999999999999999999"}], - method="upsert", include_total=False, + "res-1", + [{"id": "99999999999999999999"}], + method="upsert", + include_total=False, ) msg = str(exc.value) assert "out of range" in msg @@ -1055,8 +1046,10 @@ def test_translate_bad_numeric_value(mock_client: MagicMock) -> None: ) with pytest.raises(ValidationError) as exc: backend.upsert( - "res-1", [{"id": 1, "amount": "not-a-num"}], - method="upsert", include_total=False, + "res-1", + [{"id": 1, "amount": "not-a-num"}], + method="upsert", + include_total=False, ) assert "'not-a-num'" in str(exc.value) assert "number" in str(exc.value) @@ -1165,10 +1158,7 @@ def test_info_returns_stored_schema_total_and_primary_key( result = backend.info("balancing_auction_results_2025") sql = mock_client.query.call_args[0][0] - assert sql == ( - "SELECT COUNT(*) AS n FROM " - "`proj-1.ds-1.balancing_auction_results_2025`" - ) + assert sql == ("SELECT COUNT(*) AS n FROM `proj-1.ds-1.balancing_auction_results_2025`") assert result.schema == schema assert result.meta["resource_id"] == "balancing_auction_results_2025" assert result.meta["total"] == 18420 @@ -1286,7 +1276,8 @@ def test_build_search_renders_full_param_set() -> None: assert by_name["f2"].value == "apple" # Result schema reflects the projection, in user-specified order. assert [f["name"] for f in projected["fields"]] == [ - "auction_id", "product_code", + "auction_id", + "product_code", ] @@ -1344,8 +1335,12 @@ def test_build_search_rejects_unknown_columns() -> None: table_ref="`p.d.r`", schema=schema, include_updated_at=False, - filters=None, q=None, distinct=False, sort=None, - limit=10, offset=0, + filters=None, + q=None, + distinct=False, + sort=None, + limit=10, + offset=0, ) with pytest.raises(ValueError, match="fields references unknown"): build_search(fields=["ghost"], **kwargs) @@ -1370,8 +1365,11 @@ def test_build_search_rejects_filters_on_json_columns() -> None: include_updated_at=False, fields=None, filters={"blob": {"k": "v"}}, - q=None, distinct=False, sort=None, - limit=10, offset=0, + q=None, + distinct=False, + sort=None, + limit=10, + offset=0, ) @@ -1424,8 +1422,11 @@ def test_search_returns_projection_schema_and_lazy_rows( resource_id="res-1", filters={"product_code": "DCL"}, q=None, - distinct=False, plain=True, language="english", - limit=100, offset=0, + distinct=False, + plain=True, + language="english", + limit=100, + offset=0, fields=["auction_id", "product_code"], sort=None, include_total=True, @@ -1446,7 +1447,8 @@ def test_search_returns_projection_schema_and_lazy_rows( assert rows == [(1, "DCL")] # Projected schema is what the writer needs to label columns. assert [f["name"] for f in result.schema["fields"]] == [ - "auction_id", "product_code", + "auction_id", + "product_code", ] @@ -1472,9 +1474,16 @@ def test_search_unfiltered_uses_cheap_row_count( result = backend.search( resource_id="res-1", - filters=None, q=None, distinct=False, plain=True, - language="english", limit=10, offset=0, - fields=None, sort=None, include_total=True, + filters=None, + q=None, + distinct=False, + plain=True, + language="english", + limit=10, + offset=0, + fields=None, + sort=None, + include_total=True, ) assert mock_client.query.call_count == 2 @@ -1489,9 +1498,7 @@ def test_search_unfiltered_uses_cheap_row_count( "FORMAT_TIMESTAMP('%Y-%m-%dT%H:%M:%S', `_updated_at`, 'UTC') " "AS `_updated_at` FROM `proj-1.ds-1.res-1` AS t" ) - assert sqls[1] == ( - "SELECT COUNT(*) AS n FROM `proj-1.ds-1.res-1`" - ) + assert sqls[1] == ("SELECT COUNT(*) AS n FROM `proj-1.ds-1.res-1`") # No filtered count subquery anywhere. assert not any("FROM (SELECT" in s for s in sqls) assert result.total == 42 @@ -1506,9 +1513,16 @@ def test_search_raises_not_found_for_undeclared_resource( with pytest.raises(NotFoundError, match="not found"): backend.search( resource_id="ghost", - filters=None, q=None, distinct=False, plain=True, - language="english", limit=10, offset=0, - fields=None, sort=None, include_total=False, + filters=None, + q=None, + distinct=False, + plain=True, + language="english", + limit=10, + offset=0, + fields=None, + sort=None, + include_total=False, ) mock_client.query.assert_not_called() @@ -1524,9 +1538,16 @@ def test_search_translates_builder_error_to_validation_error( with pytest.raises(ValidationError, match="unknown column"): backend.search( resource_id="res-1", - filters=None, q=None, distinct=False, plain=True, - language="english", limit=10, offset=0, - fields=["ghost"], sort=None, include_total=False, + filters=None, + q=None, + distinct=False, + plain=True, + language="english", + limit=10, + offset=0, + fields=["ghost"], + sort=None, + include_total=False, ) mock_client.query.assert_not_called() @@ -1580,7 +1601,9 @@ def test_delete_with_filters_binds_typed_parameters( backend = _backend_with_schema(mock_client, schema) backend.delete( - "res-1", filters={"auction_id": 144, "accepted": False}, fields=None, + "res-1", + filters={"auction_id": 144, "accepted": False}, + fields=None, ) sql_arg, kwargs = mock_client.query.call_args @@ -1622,12 +1645,9 @@ def test_delete_with_fields_drops_columns_and_refreshes_options( drop_sql = mock_client.query.call_args_list[0].args[0] opts_sql = mock_client.query.call_args_list[1].args[0] assert drop_sql == ( - "ALTER TABLE `proj-1.ds-1.res-1` " - "DROP COLUMN `extra`, DROP COLUMN `obsolete`" - ) - assert opts_sql.startswith( - "ALTER TABLE `proj-1.ds-1.res-1` SET OPTIONS(" + "ALTER TABLE `proj-1.ds-1.res-1` DROP COLUMN `extra`, DROP COLUMN `obsolete`" ) + assert opts_sql.startswith("ALTER TABLE `proj-1.ds-1.res-1` SET OPTIONS(") # Refreshed table metadata only references surviving fields + PK. assert '"primaryKey":["id"]' in opts_sql assert '"name":"id"' in opts_sql @@ -1731,8 +1751,8 @@ def test_search_sql_streams_rows_with_result_schema( backend = _ro_backend(mock_client) bq_schema = [ - MagicMock(name="day", field_type="DATE"), - MagicMock(name="avg", field_type="FLOAT64"), + MagicMock(name="day", field_type="DATE"), + MagicMock(name="avg", field_type="FLOAT64"), MagicMock(name="count", field_type="INT64"), ] # MagicMock binds `name` as kwarg-to-MagicMock-name, not to attr — @@ -1760,7 +1780,8 @@ def test_search_sql_streams_rows_with_result_schema( mock_client.query.side_effect = [count_job, data_job] result = backend.search_sql( - "SELECT day, avg, count FROM x LIMIT 100", limit=100, + "SELECT day, avg, count FROM x LIMIT 100", + limit=100, ) # Two queries fire. For this unfiltered plain SELECT the total comes @@ -1781,8 +1802,8 @@ def test_search_sql_streams_rows_with_result_schema( # Frictionless types come back from the BQ type map. assert result.schema == { "fields": [ - {"name": "day", "type": "date"}, - {"name": "avg", "type": "number"}, + {"name": "day", "type": "date"}, + {"name": "avg", "type": "number"}, {"name": "count", "type": "integer"}, ], } @@ -1879,7 +1900,8 @@ def test_search_sql_filtered_uses_count_subquery( mock_client.query.side_effect = [count_job, data_job] result = backend.search_sql( - "SELECT n FROM x WHERE n > 5 LIMIT 10", limit=10, + "SELECT n FROM x WHERE n > 5 LIMIT 10", + limit=10, ) count_sql = mock_client.query.call_args_list[0][0][0] @@ -1970,15 +1992,13 @@ def test_qualify_table_refs_prepends_project_dataset() -> None: from datastore.infrastructure.engines.bigquery.lib import ( qualify_table_refs, ) + out = qualify_table_refs( 'SELECT * FROM "c6153a74-43cb-4edf-8bdf-bb664feca937" LIMIT 10', project="my-project", dataset="my_dataset", ) - assert ( - "`my-project`.`my_dataset`.`c6153a74-43cb-4edf-8bdf-bb664feca937`" - in out - ) + assert "`my-project`.`my_dataset`.`c6153a74-43cb-4edf-8bdf-bb664feca937`" in out assert out.endswith("LIMIT 10") @@ -1987,9 +2007,11 @@ def test_qualify_table_refs_handles_joins() -> None: from datastore.infrastructure.engines.bigquery.lib import ( qualify_table_refs, ) + out = qualify_table_refs( 'SELECT a.id FROM "tbl_a" a JOIN "tbl_b" b ON a.id = b.id LIMIT 10', - project="p", dataset="d", + project="p", + dataset="d", ) assert "`p`.`d`.`tbl_a`" in out assert "`p`.`d`.`tbl_b`" in out @@ -2000,9 +2022,11 @@ def test_qualify_table_refs_skips_cte_aliases() -> None: from datastore.infrastructure.engines.bigquery.lib import ( qualify_table_refs, ) + out = qualify_table_refs( - 'WITH t AS (SELECT 1 AS a) SELECT * FROM t LIMIT 10', - project="p", dataset="d", + "WITH t AS (SELECT 1 AS a) SELECT * FROM t LIMIT 10", + project="p", + dataset="d", ) assert "`p`.`d`.`t`" not in out # CTE name `t` survives unqualified. @@ -2014,9 +2038,11 @@ def test_qualify_table_refs_leaves_already_qualified_refs_alone() -> None: from datastore.infrastructure.engines.bigquery.lib import ( qualify_table_refs, ) + out = qualify_table_refs( "SELECT * FROM other_project.other_dataset.tbl LIMIT 10", - project="p", dataset="d", + project="p", + dataset="d", ) assert "`p`.`d`.`other_project`" not in out assert "other_project" in out and "other_dataset" in out @@ -2064,8 +2090,14 @@ def test_search_passes_use_query_cache_true_on_data_and_count_jobs( backend.search( resource_id="res-1", filters={"product_code": "DCL"}, - q=None, distinct=False, plain=True, language="english", - limit=10, offset=0, fields=["auction_id"], sort=None, + q=None, + distinct=False, + plain=True, + language="english", + limit=10, + offset=0, + fields=["auction_id"], + sort=None, include_total=True, ) @@ -2095,7 +2127,8 @@ def test_search_sql_passes_use_query_cache_true_on_data_and_count_jobs( mock_client.query.side_effect = [count_job, data_job] backend.search_sql( - "SELECT n FROM res1 WHERE n > 0 LIMIT 10", limit=10, + "SELECT n FROM res1 WHERE n > 0 LIMIT 10", + limit=10, ) assert mock_client.query.call_count == 2 @@ -2111,7 +2144,8 @@ def test_info_count_rows_passes_use_query_cache_true( """`datastore_info` calls `_count_rows`, which issues `SELECT COUNT(*) FROM `. That SELECT must ride the cache.""" backend = _backend_with_schema( - mock_client, {"fields": [{"name": "id", "type": "integer"}]}, + mock_client, + {"fields": [{"name": "id", "type": "integer"}]}, ) count_row = MagicMock() count_row.__getitem__.side_effect = lambda k: 7 if k == "n" else None @@ -2134,7 +2168,8 @@ def test_use_query_cache_respects_config_opt_out( integration tests / freshness-sensitive deployments can force a fresh scan.""" backend = _backend_with_schema( - mock_client, {"fields": [{"name": "id", "type": "integer"}]}, + mock_client, + {"fields": [{"name": "id", "type": "integer"}]}, ) backend.config.BIGQUERY_USE_QUERY_CACHE = False count_row = MagicMock() diff --git a/tests/test_analytics.py b/tests/test_analytics.py index bfe2ac9..68024d3 100644 --- a/tests/test_analytics.py +++ b/tests/test_analytics.py @@ -15,10 +15,12 @@ import pytest from datastore import analytics +from datastore.analytics import action_name from datastore.api.context import RequestContext, get_auth_provider, get_ckan_client from datastore.auth.base import Decision from datastore.auth.ckan import Provider as CKANAuthProvider from datastore.core.config import get_config +from datastore.core.constants import API_BASE_PREFIX, API_PREFIX from datastore.infrastructure.cache import InMemoryCache from datastore.infrastructure.engines.bigquery import BigQueryBackend from datastore.main import create_app @@ -44,7 +46,7 @@ "group", } -SEARCH_URL = "/api/3/action/datastore_search" +SEARCH_URL = f"{API_PREFIX}/datastore_search" RESOURCE = "balancing_auction_results_2025" @@ -100,7 +102,7 @@ def test_a_post_carries_its_resource_in_the_body( ) -> None: """nginx cannot see a POST body; this is why the service records itself.""" client.post( - "/api/3/action/datastore_upsert", + f"{API_PREFIX}/datastore_upsert", json={"resource_id": RESOURCE, "force": True, "records": [{"a": 1}]}, ) @@ -119,7 +121,7 @@ async def fake_dump(self: BigQueryBackend, resource_id: str, fmt: str) -> list[s return [url] with patch.object(BigQueryBackend, "dump", fake_dump): - response = client.get(f"/datastore/dump/{RESOURCE}", follow_redirects=False) + response = client.get(f"{API_BASE_PREFIX}/dump/{RESOURCE}", follow_redirects=False) assert response.status_code == 302 event = recorded[0] @@ -131,7 +133,7 @@ async def fake_dump(self: BigQueryBackend, resource_id: str, fmt: str) -> list[s def test_a_sql_dump_is_recorded_under_its_own_name( client: TestClient, recorded: list[dict] ) -> None: - client.get("/datastore/dump/query", params={"sql": "SELECT 1"}) + client.get(f"{API_BASE_PREFIX}/dump/query", params={"sql": "SELECT 1"}) assert recorded[0]["action_type"] == "datastore_dump_query" @@ -170,7 +172,7 @@ def test_an_unmounted_action_is_recorded_as_its_status( client: TestClient, recorded: list[dict] ) -> None: """This service only mounts datastore actions; attempts still count.""" - client.get("/api/3/action/package_show") + client.get(f"{API_PREFIX}/package_show") assert recorded[0]["action_type"] == "package_show" assert recorded[0]["status_code"] == 404 @@ -310,3 +312,32 @@ def test_the_emitted_line_is_bare_json(caplog: pytest.LogCaptureFixture) -> None "action_type": "datastore_search", "status_code": 200, } + + +# --- the docs surface is not an action -------------------------------------- + + +@pytest.mark.parametrize( + "suffix", ["docs", "redoc", "openapi.json", "static/theme/theme.css"] +) +def test_the_docs_surface_is_not_recorded(suffix: str) -> None: + """Docs live *inside* the versioned prefix, so they match the action path + pattern. They are not API calls and must stay out of analytics.""" + assert action_name(f"{API_PREFIX}/{suffix}") is None + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + (f"{API_PREFIX}/datastore_search", "datastore_search"), + (f"{API_PREFIX}/datastore_create", "datastore_create"), + (f"{API_BASE_PREFIX}/dump/query", "datastore_dump_query"), + (f"{API_BASE_PREFIX}/dump/res-1", "datastore_dump"), + (f"{API_BASE_PREFIX}/health", None), + (f"{API_BASE_PREFIX}/ready", None), + ], +) +def test_action_name_matches_the_live_routes(path: str, expected: str | None) -> None: + """Pinned against the routing constants: if the namespace moves again, + this fails rather than analytics silently going dark.""" + assert action_name(path) == expected diff --git a/tests/test_cors.py b/tests/test_cors.py index 2815cb1..33b5292 100644 --- a/tests/test_cors.py +++ b/tests/test_cors.py @@ -18,9 +18,7 @@ @contextmanager -def _client_with_origins( - monkeypatch: pytest.MonkeyPatch, origins: str -) -> Iterator[TestClient]: +def _client_with_origins(monkeypatch: pytest.MonkeyPatch, origins: str) -> Iterator[TestClient]: monkeypatch.setenv("CORS_ORIGINS", origins) get_config.cache_clear() try: @@ -40,12 +38,8 @@ def test_wildcard_allows_any_origin(monkeypatch: pytest.MonkeyPatch) -> None: def test_specific_domain_allowed(monkeypatch: pytest.MonkeyPatch) -> None: origins = "https://data.example.org, https://app.example.org" with _client_with_origins(monkeypatch, origins) as client: - allowed = client.get( - "/datastore/api/health", headers={"Origin": "https://app.example.org"} - ) - denied = client.get( - "/datastore/api/health", headers={"Origin": "https://evil.example.org"} - ) + allowed = client.get("/datastore/api/health", headers={"Origin": "https://app.example.org"}) + denied = client.get("/datastore/api/health", headers={"Origin": "https://evil.example.org"}) assert allowed.headers["access-control-allow-origin"] == "https://app.example.org" assert "access-control-allow-origin" not in denied.headers @@ -53,7 +47,7 @@ def test_specific_domain_allowed(monkeypatch: pytest.MonkeyPatch) -> None: def test_preflight_options(monkeypatch: pytest.MonkeyPatch) -> None: with _client_with_origins(monkeypatch, "https://data.example.org") as client: r = client.options( - "/api/3/action/datastore_create", + "/datastore/api/v2/datastore_create", headers={ "Origin": "https://data.example.org", "Access-Control-Request-Method": "POST", diff --git a/tests/test_datastore_create.py b/tests/test_datastore_create.py index aec4a63..f94952f 100644 --- a/tests/test_datastore_create.py +++ b/tests/test_datastore_create.py @@ -16,7 +16,7 @@ from tests.conftest import FakeCKAN -CREATE_URL = "/api/3/action/datastore_create" +CREATE_URL = "/datastore/api/v2/datastore_create" def _valid_payload_with_resource_id() -> dict[str, Any]: @@ -92,9 +92,7 @@ def test_create_with_resource_dict_rejected_when_auth_type_is_not_ckan( from datastore.main import create_app app = create_app() - app.dependency_overrides[get_config] = lambda: Config( - AUTH_TYPE="anonymous", CKAN_URL="" - ) + app.dependency_overrides[get_config] = lambda: Config(AUTH_TYPE="anonymous", CKAN_URL="") app.dependency_overrides[get_ckan_client] = lambda: fake_ckan app.dependency_overrides[get_auth_provider] = lambda: AnonymousProvider() @@ -123,9 +121,7 @@ def test_create_with_resource_id_succeeds_under_anonymous_auth( from datastore.main import create_app app = create_app() - app.dependency_overrides[get_config] = lambda: Config( - AUTH_TYPE="anonymous", CKAN_URL="" - ) + app.dependency_overrides[get_config] = lambda: Config(AUTH_TYPE="anonymous", CKAN_URL="") app.dependency_overrides[get_ckan_client] = lambda: fake_ckan app.dependency_overrides[get_auth_provider] = lambda: AnonymousProvider() @@ -136,9 +132,7 @@ def test_create_with_resource_id_succeeds_under_anonymous_auth( response = c.post(CREATE_URL, json=_valid_payload_with_resource_id()) assert response.status_code == 200 - assert response.json()["result"]["resource_id"] == ( - "balancing_auction_results_2025" - ) + assert response.json()["result"]["resource_id"] == ("balancing_auction_results_2025") # 2. Missing required field ------------------------------------------------- @@ -187,9 +181,7 @@ def test_create_field_missing_id_returns_validation_error(client: TestClient) -> assert any("fields[0].id" in path for path in body["error"]["fields"]) -@pytest.mark.parametrize( - "bad_name", ["auction-id", "price (GBP)", "col;drop", 'a"b', "naïve"] -) +@pytest.mark.parametrize("bad_name", ["auction-id", "price (GBP)", "col;drop", 'a"b', "naïve"]) def test_create_field_id_with_special_chars_returns_validation_error( client: TestClient, bad_name: str ) -> None: @@ -263,9 +255,7 @@ def test_create_resource_id_with_denied_key_returns_403( assert body["error"]["__type"] == "Authorization Error" -def test_create_without_api_key_returns_403( - client: TestClient, fake_ckan: FakeCKAN -) -> None: +def test_create_without_api_key_returns_403(client: TestClient, fake_ckan: FakeCKAN) -> None: """Anonymous reads are allowed (CKAN decides on resource visibility), but writes always require an authenticated user — short-circuit with 403 before CKAN is even called.""" @@ -426,9 +416,7 @@ def test_create_with_resource_dict_tags_url_type_datastore( def test_create_on_readonly_resource_requires_force( client: TestClient, fake_ckan: FakeCKAN ) -> None: - fake_ckan.add_resource( - "ro-res", package_id="pkg-balancing-2025", url_type="upload" - ) + fake_ckan.add_resource("ro-res", package_id="pkg-balancing-2025", url_type="upload") payload = { "resource_id": "ro-res", "fields": [{"id": "auction_id", "type": "int4"}], @@ -446,9 +434,7 @@ def test_create_on_readonly_resource_requires_force( def test_create_on_readonly_resource_with_force_succeeds( client: TestClient, fake_ckan: FakeCKAN ) -> None: - fake_ckan.add_resource( - "ro-res", package_id="pkg-balancing-2025", url_type="upload" - ) + fake_ckan.add_resource("ro-res", package_id="pkg-balancing-2025", url_type="upload") payload = { "resource_id": "ro-res", "force": True, diff --git a/tests/test_datastore_delete.py b/tests/test_datastore_delete.py index 16cdfa2..2184e1e 100644 --- a/tests/test_datastore_delete.py +++ b/tests/test_datastore_delete.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `POST /api/3/action/datastore_delete`. +"""End-to-end tests for `POST /datastore/api/v2/datastore_delete`. Body accepts: resource_id / id (one required) — table to delete from @@ -24,24 +24,29 @@ from tests.conftest import FakeCKAN -DELETE_URL = "/api/3/action/datastore_delete" +DELETE_URL = "/datastore/api/v2/datastore_delete" _RESOURCE_ID = "balancing_auction_results_2025" # 1. Happy path ------------------------------------------------------------- + def test_delete_with_filters_echoes_them(client: TestClient) -> None: - response = client.post(DELETE_URL, json={ - "resource_id": _RESOURCE_ID, - "filters": {"product_code": "DCL", "accepted": False}, - }) + response = client.post( + DELETE_URL, + json={ + "resource_id": _RESOURCE_ID, + "filters": {"product_code": "DCL", "accepted": False}, + }, + ) assert response.status_code == 200 body = response.json() assert body["success"] is True assert body["result"]["resource_id"] == _RESOURCE_ID assert body["result"]["filters"] == { - "product_code": "DCL", "accepted": False, + "product_code": "DCL", + "accepted": False, } @@ -59,16 +64,20 @@ def test_delete_without_filters_drops_whole_table(client: TestClient) -> None: def test_force_flag_accepted(client: TestClient) -> None: """`force=True` is accepted (the placeholder doesn't enforce read-only; real BigQuery impl will check resource metadata).""" - response = client.post(DELETE_URL, json={ - "resource_id": _RESOURCE_ID, - "filters": {"x": 1}, - "force": True, - }) + response = client.post( + DELETE_URL, + json={ + "resource_id": _RESOURCE_ID, + "filters": {"x": 1}, + "force": True, + }, + ) assert response.status_code == 200 # 2. Aliases ---------------------------------------------------------------- + def test_id_alias_works(client: TestClient) -> None: """`id` is normalised to `resource_id` by the schema validator.""" response = client.post(DELETE_URL, json={"id": _RESOURCE_ID}) @@ -79,10 +88,13 @@ def test_id_alias_works(client: TestClient) -> None: def test_same_value_for_resource_id_and_id_accepted(client: TestClient) -> None: """Same value on both keys is the no-conflict legacy-echo case.""" - response = client.post(DELETE_URL, json={ - "resource_id": _RESOURCE_ID, - "id": _RESOURCE_ID, - }) + response = client.post( + DELETE_URL, + json={ + "resource_id": _RESOURCE_ID, + "id": _RESOURCE_ID, + }, + ) assert response.status_code == 200 assert response.json()["result"]["resource_id"] == _RESOURCE_ID @@ -90,10 +102,13 @@ def test_same_value_for_resource_id_and_id_accepted(client: TestClient) -> None: def test_conflicting_resource_id_and_id_rejected(client: TestClient) -> None: """Different `resource_id` vs `id` → 400. Silently preferring one would let a typo destroy the wrong resource.""" - response = client.post(DELETE_URL, json={ - "resource_id": _RESOURCE_ID, - "id": "different-value", - }) + response = client.post( + DELETE_URL, + json={ + "resource_id": _RESOURCE_ID, + "id": "different-value", + }, + ) assert response.status_code == 400 body = response.json() assert body["error"]["__type"] == "Validation Error" @@ -101,6 +116,7 @@ def test_conflicting_resource_id_and_id_rejected(client: TestClient) -> None: # 3. Validation ------------------------------------------------------------- + def test_missing_both_returns_validation_error(client: TestClient) -> None: response = client.post(DELETE_URL, json={}) @@ -111,10 +127,13 @@ def test_missing_both_returns_validation_error(client: TestClient) -> None: def test_extra_body_key_rejected(client: TestClient) -> None: """`extra='forbid'` blocks unknown keys to catch typos.""" - response = client.post(DELETE_URL, json={ - "resource_id": _RESOURCE_ID, - "filterz": {"x": 1}, # typo - }) + response = client.post( + DELETE_URL, + json={ + "resource_id": _RESOURCE_ID, + "filterz": {"x": 1}, # typo + }, + ) assert response.status_code == 400 assert response.json()["error"]["__type"] == "Validation Error" @@ -123,11 +142,14 @@ def test_extra_body_key_rejected(client: TestClient) -> None: def test_filters_and_fields_are_mutually_exclusive(client: TestClient) -> None: """Row delete (`filters`) and column drop (`fields`) are separate operations; sending both is ambiguous and rejected up front.""" - response = client.post(DELETE_URL, json={ - "resource_id": _RESOURCE_ID, - "filters": {"id": 1}, - "fields": ["label"], - }) + response = client.post( + DELETE_URL, + json={ + "resource_id": _RESOURCE_ID, + "filters": {"id": 1}, + "fields": ["label"], + }, + ) assert response.status_code == 400 body = response.json() @@ -138,10 +160,13 @@ def test_filters_and_fields_are_mutually_exclusive(client: TestClient) -> None: def test_empty_fields_list_rejected(client: TestClient) -> None: """`fields=[]` is ambiguous (column drop with no columns) — 400 rather than silently no-op.""" - response = client.post(DELETE_URL, json={ - "resource_id": _RESOURCE_ID, - "fields": [], - }) + response = client.post( + DELETE_URL, + json={ + "resource_id": _RESOURCE_ID, + "fields": [], + }, + ) assert response.status_code == 400 assert response.json()["error"]["__type"] == "Validation Error" @@ -149,6 +174,7 @@ def test_empty_fields_list_rejected(client: TestClient) -> None: # 4. Auth ------------------------------------------------------------------- + def test_unknown_resource_returns_404(client: TestClient) -> None: response = client.post(DELETE_URL, json={"resource_id": "does-not-exist"}) @@ -157,9 +183,7 @@ def test_unknown_resource_returns_404(client: TestClient) -> None: assert body["error"]["__type"] == "Not Found Error" -def test_denied_key_returns_403( - client: TestClient, fake_ckan: FakeCKAN -) -> None: +def test_denied_key_returns_403(client: TestClient, fake_ckan: FakeCKAN) -> None: fake_ckan.deny("test-token") response = client.post(DELETE_URL, json={"resource_id": _RESOURCE_ID}) @@ -174,9 +198,7 @@ def test_denied_key_returns_403( def test_delete_on_readonly_resource_requires_force( client: TestClient, fake_ckan: FakeCKAN ) -> None: - fake_ckan.add_resource( - "ro-res", package_id="pkg-balancing-2025", url_type="upload" - ) + fake_ckan.add_resource("ro-res", package_id="pkg-balancing-2025", url_type="upload") response = client.post(DELETE_URL, json={"resource_id": "ro-res"}) @@ -189,13 +211,9 @@ def test_delete_on_readonly_resource_requires_force( def test_delete_on_readonly_resource_with_force_succeeds( client: TestClient, fake_ckan: FakeCKAN ) -> None: - fake_ckan.add_resource( - "ro-res", package_id="pkg-balancing-2025", url_type="upload" - ) + fake_ckan.add_resource("ro-res", package_id="pkg-balancing-2025", url_type="upload") - response = client.post( - DELETE_URL, json={"resource_id": "ro-res", "force": True} - ) + response = client.post(DELETE_URL, json={"resource_id": "ro-res", "force": True}) assert response.status_code == 200 assert response.json()["success"] is True diff --git a/tests/test_datastore_dump.py b/tests/test_datastore_dump.py index 46b5096..ebfd822 100644 --- a/tests/test_datastore_dump.py +++ b/tests/test_datastore_dump.py @@ -1,4 +1,4 @@ -"""Tests for `GET /datastore/dump/{resource_id}`. +"""Tests for `GET /datastore/api/dump/{resource_id}`. The engine returns one signed URL (csv / gzip / ndjson shards are composed into a single object), so every format 302s. Only a sharded @@ -27,21 +27,24 @@ from tests.conftest import FakeCKAN -DUMP_URL = "/datastore/dump/balancing_auction_results_2025" +DUMP_URL = "/datastore/api/dump/balancing_auction_results_2025" def _patch_dump(urls_or_exc: list[str] | Exception): """Patch `BigQueryBackend.dump` to return URLs or raise.""" + async def fake(self: BigQueryBackend, resource_id: str, fmt: str) -> list[str]: if isinstance(urls_or_exc, Exception): raise urls_or_exc return urls_or_exc + return patch.object(BigQueryBackend, "dump", fake) def stub_signed_urls(client: TestClient, parts: dict[str, bytes]) -> None: """Serve `parts` (url → body) from `app.state.http`, the client the zip writer fetches the signed URLs with.""" + def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(200, content=parts[str(request.url)]) @@ -64,11 +67,14 @@ def test_single_shard_returns_302(client: TestClient) -> None: @pytest.mark.parametrize("fmt", ["csv", "gzip", "ndjson", "parquet"]) def test_each_format_supports_single_shard_redirect( - fmt: str, client: TestClient, + fmt: str, + client: TestClient, ) -> None: with _patch_dump([f"https://example/x.{fmt}"]): response = client.get( - DUMP_URL, params={"format": fmt}, follow_redirects=False, + DUMP_URL, + params={"format": fmt}, + follow_redirects=False, ) assert response.status_code == 302 @@ -132,7 +138,7 @@ def test_unknown_format_returns_validation_error(client: TestClient) -> None: def test_dump_for_unknown_resource_returns_404(client: TestClient) -> None: - response = client.get("/datastore/dump/missing-resource") + response = client.get("/datastore/api/dump/missing-resource") assert response.status_code == 404 @@ -140,7 +146,8 @@ def test_dump_for_unknown_resource_returns_404(client: TestClient) -> None: def test_dump_without_api_key_succeeds_when_public( - client: TestClient, fake_ckan: FakeCKAN, + client: TestClient, + fake_ckan: FakeCKAN, ) -> None: with _patch_dump(["https://example/a.csv?sig=1"]): client.headers.pop("Authorization", None) @@ -150,7 +157,8 @@ def test_dump_without_api_key_succeeds_when_public( def test_dump_with_denied_key_returns_403( - client: TestClient, fake_ckan: FakeCKAN, + client: TestClient, + fake_ckan: FakeCKAN, ) -> None: fake_ckan.deny("test-token") response = client.get(DUMP_URL) @@ -172,14 +180,8 @@ def test_build_export_select_iso_casts_timestamp_and_datetime() -> None: _bq_field("delivery_day", "DATE"), ] select = _export_select_list(schema, fmt="csv") - assert ( - "FORMAT_TIMESTAMP('%Y-%m-%dT%H:%M:%S', `delivery_start`, 'UTC')" - in select - ) - assert ( - "FORMAT_DATETIME('%Y-%m-%dT%H:%M:%S', `delivery_local`)" - in select - ) + assert "FORMAT_TIMESTAMP('%Y-%m-%dT%H:%M:%S', `delivery_start`, 'UTC')" in select + assert "FORMAT_DATETIME('%Y-%m-%dT%H:%M:%S', `delivery_local`)" in select # No `Z` suffix and no `%E*S` (which would re-introduce fractional seconds). assert "Z'," not in select assert "%E*S" not in select @@ -199,18 +201,20 @@ def test_build_export_select_parquet_casts_json_columns() -> None: _bq_field("delivery_start", "TIMESTAMP"), ] assert _export_select_list(schema, fmt="parquet") == ( - "`id`, TO_JSON_STRING(`bidder_metadata`) AS `bidder_metadata`, " - "`delivery_start`" + "`id`, TO_JSON_STRING(`bidder_metadata`) AS `bidder_metadata`, `delivery_start`" ) # --- helpers: too-large heuristic ----------------------------------------- -@pytest.mark.parametrize("message", [ - "Operation cannot be completed when exporting to a single URI", - "Cannot export more than 1 GB to a single URI; use the wildcard operator", -]) +@pytest.mark.parametrize( + "message", + [ + "Operation cannot be completed when exporting to a single URI", + "Cannot export more than 1 GB to a single URI; use the wildcard operator", + ], +) def test_too_large_marker_is_recognised(message: str) -> None: assert _is_export_too_large(RuntimeError(message)) is True @@ -320,12 +324,10 @@ def test_dump_cache_miss_submits_extract_then_returns_urls() -> None: # same one shard (nothing stale to delete on first dump ever). bucket_obj.list_blobs.side_effect = [[], [new_blob], [new_blob]] # The composite object's signed URL (csv always composes). - bucket_obj.blob.return_value.generate_signed_url.return_value = ( - "https://composed" - ) + bucket_obj.blob.return_value.generate_signed_url.return_value = "https://composed" # Job goes straight to DONE without errors. - job = MagicMock() # `job.result()` returns without raising + job = MagicMock() # `job.result()` returns without raising backend.client.query.return_value = job urls = _run_dump(backend, "res-1", "csv") @@ -347,7 +349,7 @@ def test_dump_gzip_exports_headerless_and_composes() -> None: bucket_obj = storage_client.bucket.return_value bucket_obj.list_blobs.side_effect = [[], [new_blob], [new_blob]] - job = MagicMock() # `job.result()` returns without raising + job = MagicMock() # `job.result()` returns without raising backend.client.query.return_value = job _run_dump(backend, "res-1", "gzip") @@ -355,7 +357,7 @@ def test_dump_gzip_exports_headerless_and_composes() -> None: sql = backend.client.query.call_args.args[0] assert "format='CSV'" in sql assert "compression='GZIP'" in sql - assert "header=false" in sql # header comes from the composed member + assert "header=false" in sql # header comes from the composed member assert "_*.csv.gz" in sql prefixes = {c.kwargs["prefix"] for c in bucket_obj.list_blobs.call_args_list} assert all(p.startswith("dumps/res-1/gzip/") for p in prefixes) @@ -371,7 +373,7 @@ def test_dump_parquet_export_uses_wildcard_uri() -> None: bucket_obj = storage_client.bucket.return_value bucket_obj.list_blobs.side_effect = [[], [new_blob], [new_blob]] - job = MagicMock() # `job.result()` returns without raising + job = MagicMock() # `job.result()` returns without raising backend.client.query.return_value = job urls = _run_dump(backend, "res-1", "parquet") @@ -419,8 +421,8 @@ def test_dump_ignores_an_attempt_without_success() -> None: urls = _run_dump(backend, "res-1", "parquet") assert urls == ["https://ok"] - assert backend.client.query.call_count == 1 # exported its own attempt - assert inflight.delete.call_count == 0 # never touched + assert backend.client.query.call_count == 1 # exported its own attempt + assert inflight.delete.call_count == 0 # never touched def test_dump_cache_key_changes_when_table_modified_advances() -> None: @@ -451,9 +453,7 @@ def test_dump_cache_key_changes_when_table_modified_advances() -> None: asyncio.run(backend.dump("res-1", "csv")) second_prefix = bucket_obj.list_blobs.call_args_list[-2].kwargs["prefix"] - assert first_prefix != second_prefix, ( - "table.modified change must produce a different cache key" - ) + assert first_prefix != second_prefix, "table.modified change must produce a different cache key" # --- test infrastructure -------------------------------------------------- diff --git a/tests/test_datastore_dump_sql.py b/tests/test_datastore_dump_sql.py index 1a231a7..af8eb4e 100644 --- a/tests/test_datastore_dump_sql.py +++ b/tests/test_datastore_dump_sql.py @@ -45,7 +45,7 @@ stub_signed_urls, ) -DUMP_SQL_URL = "/datastore/dump/query" +DUMP_SQL_URL = "/datastore/api/dump/query" _NOW = dt.datetime.now(dt.timezone.utc) @@ -99,9 +99,7 @@ def _expected_prefix(sql: str, fmt: str = "csv") -> str: modified 2026-01-01 UTC).""" qualified = qualify_table_refs(sql, project="proj-1", dataset="ds-1") qhash = hashlib.sha256(qualified.encode()).hexdigest()[:16] - us = int( - dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc).timestamp() * 1_000_000 - ) + us = int(dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc).timestamp() * 1_000_000) rev = hashlib.sha256(f"res1:{us}".encode()).hexdigest()[:16] return f"dumps/{qhash}/{fmt}/{rev}/" @@ -133,9 +131,7 @@ def test_cache_miss_dry_runs_then_exports() -> None: backend, storage_client = _engine_with_storage([]) bucket_obj = storage_client.bucket.return_value bucket_obj.list_blobs.side_effect = [[], [new_blob], [new_blob]] - bucket_obj.blob.return_value.generate_signed_url.return_value = ( - "https://composed" - ) + bucket_obj.blob.return_value.generate_signed_url.return_value = "https://composed" backend.client.query.side_effect = [_dry_job(), _export_job()] urls = _run(backend) @@ -160,9 +156,7 @@ def test_cache_miss_dry_runs_then_exports() -> None: def test_cache_prefix_matches_qhash_and_table_rev_scheme() -> None: """Pre-check prefix is `dumps// //`.""" - backend, storage_client = _engine_with_storage( - _attempt("dumps/h/csv/rev/", "data.csv") - ) + backend, storage_client = _engine_with_storage(_attempt("dumps/h/csv/rev/", "data.csv")) bucket_obj = storage_client.bucket.return_value sql = "SELECT * FROM res1" @@ -173,26 +167,20 @@ def test_cache_prefix_matches_qhash_and_table_rev_scheme() -> None: def test_cache_prefix_stable_across_identical_calls() -> None: - backend, storage_client = _engine_with_storage( - _attempt("dumps/h/csv/rev/", "data.csv") - ) + backend, storage_client = _engine_with_storage(_attempt("dumps/h/csv/rev/", "data.csv")) bucket_obj = storage_client.bucket.return_value _run(backend) _run(backend) - prefixes = { - c.kwargs["prefix"] for c in bucket_obj.list_blobs.call_args_list - } + prefixes = {c.kwargs["prefix"] for c in bucket_obj.list_blobs.call_args_list} assert len(prefixes) == 1 def test_rev_changes_when_any_referenced_table_changes() -> None: """Multi-table SQL: bumping either table's `modified` produces a new revision prefix; the other table alone can't satisfy the cache.""" - backend, storage_client = _engine_with_storage( - _attempt("dumps/h/csv/rev/", "data.csv") - ) + backend, storage_client = _engine_with_storage(_attempt("dumps/h/csv/rev/", "data.csv")) bucket_obj = storage_client.bucket.return_value t1, t2 = MagicMock(), MagicMock() @@ -223,7 +211,10 @@ def test_non_deterministic_sql_bypasses_cache() -> None: # No pre-check list per run — only post-export refresh + GC sweep. bucket_obj.list_blobs.side_effect = [[b1], [b1], [b2], [b2]] backend.client.query.side_effect = [ - _dry_job(), _export_job(), _dry_job(), _export_job(), + _dry_job(), + _export_job(), + _dry_job(), + _export_job(), ] _run(backend, function_names=["now"]) @@ -245,7 +236,10 @@ def test_table_modified_none_is_non_cacheable() -> None: b2 = _blob("y2", "https://two") bucket_obj.list_blobs.side_effect = [[b1], [b1], [b2], [b2]] backend.client.query.side_effect = [ - _dry_job(), _export_job(), _dry_job(), _export_job(), + _dry_job(), + _export_job(), + _dry_job(), + _export_job(), ] _run(backend) @@ -275,18 +269,18 @@ def test_gc_spares_young_attempts_and_reaps_old_revisions() -> None: young_sibling = _blob(f"{prefix}att2/part_000.parquet", age_hours=0.0) bucket_obj.list_blobs.side_effect = [ - [], # pre-check (cache miss) - [current], # post-export refresh - [current, old_rev, young_sibling], # GC sweep + [], # pre-check (cache miss) + [current], # post-export refresh + [current, old_rev, young_sibling], # GC sweep ] backend.client.query.side_effect = [_dry_job(), _export_job()] urls = _run(backend, sql=sql, fmt="parquet") assert urls == ["https://fresh"] - assert current.delete.call_count == 0 # our own attempt + assert current.delete.call_count == 0 # our own attempt assert young_sibling.delete.call_count == 0 # may still be exporting - assert old_rev.delete.call_count == 1 # past the URL expiry + assert old_rev.delete.call_count == 1 # past the URL expiry def test_gc_stale_blobs_no_age_gate_deletes_all_non_current() -> None: @@ -300,7 +294,9 @@ def test_gc_stale_blobs_no_age_gate_deletes_all_non_current() -> None: rw_gcs.list_blobs.return_value = [current, old, young] deleted = _delete_old_cache( - rw_gcs, sweep_prefix="dumps/h/csv/", keep_prefix=keep, + rw_gcs, + sweep_prefix="dumps/h/csv/", + keep_prefix=keep, min_age=None, ) @@ -322,7 +318,9 @@ def test_gc_stale_blobs_age_gate_keeps_young() -> None: rw_gcs.list_blobs.return_value = [current, old, young] deleted = _delete_old_cache( - rw_gcs, sweep_prefix="dumps/h/csv/", keep_prefix=keep, + rw_gcs, + sweep_prefix="dumps/h/csv/", + keep_prefix=keep, min_age=dt.timedelta(hours=1), ) @@ -435,9 +433,7 @@ def test_csv_export_iso_casts_timestamps_from_dry_run_schema() -> None: assert "format='CSV'" in export_sql # header=false: the single header is composed in as a separate member. assert "header=false" in export_sql - assert ( - "FORMAT_TIMESTAMP('%Y-%m-%dT%H:%M:%S', `ts`, 'UTC')" in export_sql - ) + assert "FORMAT_TIMESTAMP('%Y-%m-%dT%H:%M:%S', `ts`, 'UTC')" in export_sql def test_multi_shard_parquet_returns_every_url() -> None: @@ -491,11 +487,11 @@ def test_csv_multishard_composes_header_and_shards_into_one_url() -> None: composite = next(n for n in made if n.endswith("data.csv")) header = next(n for n in made if n.endswith("header.csv")) - assert urls == [f"url:{composite}"] # single URL + assert urls == [f"url:{composite}"] # single URL made[header].upload_from_string.assert_called_once() # header written made[composite].compose.assert_called_once() sources = made[composite].compose.call_args.args[0] - assert made[header] is sources[0] # header sorts first + assert made[header] is sources[0] # header sorts first assert shard0 in sources and shard1 in sources assert shard0.delete.called and shard1.delete.called # parts removed @@ -576,11 +572,11 @@ def test_attempt_without_success_is_invisible() -> None: urls = _run(backend, fmt="csv") - assert backend.client.query.call_count == 2 # exported its own + assert backend.client.query.call_count == 2 # exported its own composite = next(n for n in made if n.endswith("data.csv")) assert urls == [f"url:{composite}"] for b in inflight: - assert b.delete.call_count == 0 # left alone + assert b.delete.call_count == 0 # left alone def test_published_attempt_wins_over_an_inflight_one() -> None: @@ -618,8 +614,9 @@ def test_success_marker_is_never_composed_or_signed() -> None: sources = composite.compose.call_args.args[0] assert not any("_SUCCESS" in getattr(s, "name", "") for s in sources) assert not any("_SUCCESS" in u for u in urls) - made[next(n for n in made if n.endswith("_SUCCESS"))]\ - .upload_from_string.assert_called_once_with(b"") + made[ + next(n for n in made if n.endswith("_SUCCESS")) + ].upload_from_string.assert_called_once_with(b"") def test_compose_deletes_shards_and_header_but_not_the_marker() -> None: @@ -679,8 +676,8 @@ async def go() -> list: urls, pending = asyncio.run(go()) - assert urls # served without waiting - assert pending # …while cleanup was still running + assert urls # served without waiting + assert pending # …while cleanup was still running for shard in shards: assert shard.delete.call_count == 1 @@ -689,12 +686,14 @@ def test_leftover_shards_are_not_served_beside_the_composite() -> None: """A published attempt still holds its pre-compose shards until the background delete runs; only the composed object is handed out.""" base = "dumps/h/csv/rev/att1/" - backend, _ = _engine_with_storage([ - _blob(f"{base}data.csv", "https://composed"), - _blob(f"{base}part_000.csv", "https://part0"), - _blob(f"{base}header.csv", "https://header"), - _blob(f"{base}_SUCCESS"), - ]) + backend, _ = _engine_with_storage( + [ + _blob(f"{base}data.csv", "https://composed"), + _blob(f"{base}part_000.csv", "https://part0"), + _blob(f"{base}header.csv", "https://header"), + _blob(f"{base}_SUCCESS"), + ] + ) urls = _run(backend) @@ -716,10 +715,10 @@ def test_legacy_multishard_csv_hit_is_rebuilt_to_composite() -> None: backend, storage_client = _engine_with_storage([]) bucket_obj = storage_client.bucket.return_value bucket_obj.list_blobs.side_effect = [ - legacy, # pre-check (ro): invalid hit (no composite) - legacy, # re-list (rw) for the clear - [fresh], # post-export refresh - [], # GC sweep + legacy, # pre-check (ro): invalid hit (no composite) + legacy, # re-list (rw) for the clear + [fresh], # post-export refresh + [], # GC sweep ] backend.client.query.side_effect = [_dry_job(), _export_job()] factory, made = _distinct_blob_factory() @@ -728,10 +727,10 @@ def test_legacy_multishard_csv_hit_is_rebuilt_to_composite() -> None: urls = _run(backend, fmt="csv") for b in legacy: - assert b.delete.called # stale cache cleared - assert backend.client.query.call_count == 2 # re-exported + assert b.delete.called # stale cache cleared + assert backend.client.query.call_count == 2 # re-exported composite = next(n for n in made if n.endswith("data.csv")) - assert urls == [f"url:{composite}"] # single URL again + assert urls == [f"url:{composite}"] # single URL again def test_lone_raw_csv_shard_hit_is_rebuilt() -> None: @@ -757,9 +756,7 @@ def test_lone_raw_csv_shard_hit_is_rebuilt() -> None: def test_ndjson_single_shard_hit_is_valid() -> None: """ndjson needs no header member, so a published single shard is a perfectly good hit — served directly, no `data.json`.""" - backend, _ = _engine_with_storage( - _attempt("dumps/h/ndjson/rev/", "part_000.json") - ) + backend, _ = _engine_with_storage(_attempt("dumps/h/ndjson/rev/", "part_000.json")) urls = _run(backend, fmt="ndjson") @@ -798,9 +795,14 @@ def test_ndjson_multishard_hit_is_rebuilt() -> None: def test_placeholder_mode_returns_empty_list() -> None: backend = BigQueryBackend(mode="ro") - urls = asyncio.run(backend.dump_sql( - "SELECT 1", "csv", resource_ids=[], function_names=[], - )) + urls = asyncio.run( + backend.dump_sql( + "SELECT 1", + "csv", + resource_ids=[], + function_names=[], + ) + ) assert urls == [] @@ -808,9 +810,14 @@ def test_non_ro_mode_rejected() -> None: backend = BigQueryBackend(mode="rw") backend.client = MagicMock() with pytest.raises(ServerError, match="read-only"): - asyncio.run(backend.dump_sql( - "SELECT 1", "csv", resource_ids=[], function_names=[], - )) + asyncio.run( + backend.dump_sql( + "SELECT 1", + "csv", + resource_ids=[], + function_names=[], + ) + ) def test_bucket_unset_raises_server_error() -> None: @@ -819,9 +826,14 @@ def test_bucket_unset_raises_server_error() -> None: backend.config = MagicMock() backend.config.BIGQUERY_EXPORT_BUCKET = "" with pytest.raises(ServerError, match="BIGQUERY_EXPORT_BUCKET"): - asyncio.run(backend.dump_sql( - "SELECT 1", "csv", resource_ids=[], function_names=[], - )) + asyncio.run( + backend.dump_sql( + "SELECT 1", + "csv", + resource_ids=[], + function_names=[], + ) + ) def test_missing_table_raises_not_found() -> None: @@ -887,7 +899,7 @@ def test_zero_table_sql_exports_without_get_table() -> None: # ============================================================================= -# Endpoint: GET /datastore/dump/query +# Endpoint: GET /datastore/api/dump/query # ============================================================================= @@ -907,12 +919,14 @@ async def fake( resource_ids: list[str], function_names: list[str], ) -> list[str]: - calls.append({ - "sql": sql, - "fmt": fmt, - "resource_ids": resource_ids, - "function_names": function_names, - }) + calls.append( + { + "sql": sql, + "fmt": fmt, + "resource_ids": resource_ids, + "function_names": function_names, + } + ) if isinstance(urls_or_exc, Exception): raise urls_or_exc return urls_or_exc @@ -921,7 +935,8 @@ async def fake( def test_forwards_sql_and_names_verbatim( - client: TestClient, fake_ckan: FakeCKAN, + client: TestClient, + fake_ckan: FakeCKAN, ) -> None: """The engine receives the SQL exactly as sent (LIMIT intact) plus the schema-parsed table / function names.""" @@ -934,12 +949,14 @@ def test_forwards_sql_and_names_verbatim( follow_redirects=False, ) assert response.status_code == 302 - assert calls == [{ - "sql": sql, - "fmt": "ndjson", - "resource_ids": ["balancing_auction_results_2025"], - "function_names": ["count"], - }] + assert calls == [ + { + "sql": sql, + "fmt": "ndjson", + "resource_ids": ["balancing_auction_results_2025"], + "function_names": ["count"], + } + ] def test_format_defaults_to_csv(client: TestClient) -> None: @@ -981,9 +998,12 @@ def test_limit_above_search_cap_allowed(client: TestClient) -> None: def test_offset_without_limit_rejected(client: TestClient) -> None: - response = client.get(DUMP_SQL_URL, params={ - "sql": "SELECT 1 OFFSET 10", - }) + response = client.get( + DUMP_SQL_URL, + params={ + "sql": "SELECT 1 OFFSET 10", + }, + ) assert response.status_code == 400 body = response.json() assert body["error"]["__type"] == "Validation Error" @@ -991,15 +1011,19 @@ def test_offset_without_limit_rejected(client: TestClient) -> None: def test_bogus_format_rejected(client: TestClient) -> None: - response = client.get(DUMP_SQL_URL, params={ - "sql": "SELECT 1 LIMIT 5", "format": "xml", - }) + response = client.get( + DUMP_SQL_URL, + params={ + "sql": "SELECT 1 LIMIT 5", + "format": "xml", + }, + ) assert response.status_code == 400 assert response.json()["error"]["__type"] == "Validation Error" def test_missing_sql_names_the_field(client: TestClient) -> None: - """`/datastore/dump/query` resolves to the SQL route (declared before + """`/datastore/api/dump/query` resolves to the SQL route (declared before `/{resource_id}`, so `query` is a reserved resource name) — a missing `sql` param is a validation error on this endpoint, not a 404 dump of a table called 'query'.""" @@ -1035,16 +1059,17 @@ def test_multi_file_parquet_streams_one_zip(client: TestClient) -> None: patcher, _ = _patch_dump_sql(list(parts)) with patcher: - response = client.get(DUMP_SQL_URL, params={ - "sql": "SELECT 1 LIMIT 5", "format": "parquet", - }) + response = client.get( + DUMP_SQL_URL, + params={ + "sql": "SELECT 1 LIMIT 5", + "format": "parquet", + }, + ) assert response.status_code == 200 assert response.headers["content-type"] == "application/zip" - assert ( - response.headers["content-disposition"] - == 'attachment; filename="query.zip"' - ) + assert response.headers["content-disposition"] == 'attachment; filename="query.zip"' archive = zipfile.ZipFile(io.BytesIO(response.content)) assert archive.testzip() is None @@ -1057,9 +1082,13 @@ def test_payload_too_large_error_maps_to_413(client: TestClient) -> None: PayloadTooLargeError("exported as multiple parquet shards"), ) with patcher: - response = client.get(DUMP_SQL_URL, params={ - "sql": "SELECT 1 LIMIT 5", "format": "parquet", - }) + response = client.get( + DUMP_SQL_URL, + params={ + "sql": "SELECT 1 LIMIT 5", + "format": "parquet", + }, + ) assert response.status_code == 413 assert response.json()["error"]["__type"] == "Payload Too Large" @@ -1069,34 +1098,46 @@ def test_disallowed_function_rejected_before_engine( ) -> None: """The function allow-list applies here too — the service raises before any engine/export work.""" - response = client.get(DUMP_SQL_URL, params={ - "sql": "SELECT pg_read_file('/etc/passwd') LIMIT 1", - }) + response = client.get( + DUMP_SQL_URL, + params={ + "sql": "SELECT pg_read_file('/etc/passwd') LIMIT 1", + }, + ) assert response.status_code == 400 assert "pg_read_file" in response.json()["error"]["message"].lower() def test_unknown_table_returns_404( - client: TestClient, fake_ckan: FakeCKAN, + client: TestClient, + fake_ckan: FakeCKAN, ) -> None: - response = client.get(DUMP_SQL_URL, params={ - "sql": 'SELECT * FROM "does-not-exist" LIMIT 10', - }) + response = client.get( + DUMP_SQL_URL, + params={ + "sql": 'SELECT * FROM "does-not-exist" LIMIT 10', + }, + ) assert response.status_code == 404 def test_denied_key_returns_403( - client: TestClient, fake_ckan: FakeCKAN, + client: TestClient, + fake_ckan: FakeCKAN, ) -> None: fake_ckan.deny("test-token") - response = client.get(DUMP_SQL_URL, params={ - "sql": 'SELECT * FROM "balancing_auction_results_2025" LIMIT 10', - }) + response = client.get( + DUMP_SQL_URL, + params={ + "sql": 'SELECT * FROM "balancing_auction_results_2025" LIMIT 10', + }, + ) assert response.status_code == 403 def test_join_authorizes_each_table( - client: TestClient, fake_ckan: FakeCKAN, + client: TestClient, + fake_ckan: FakeCKAN, ) -> None: fake_ckan.add_resource("other_table", package_id="pkg-balancing-2025") before = fake_ckan.authorize_calls @@ -1119,8 +1160,11 @@ def test_join_authorizes_each_table( def test_unconfigured_engine_returns_500(client: TestClient) -> None: """Placeholder engine (no BQ creds in the test env) exports nothing; the endpoint refuses to serve an empty file and 500s explicitly.""" - response = client.get(DUMP_SQL_URL, params={ - "sql": "SELECT 1 LIMIT 5", - }) + response = client.get( + DUMP_SQL_URL, + params={ + "sql": "SELECT 1 LIMIT 5", + }, + ) assert response.status_code == 500 assert "not configured" in response.json()["error"]["message"] diff --git a/tests/test_datastore_info.py b/tests/test_datastore_info.py index 42a3be3..187de1e 100644 --- a/tests/test_datastore_info.py +++ b/tests/test_datastore_info.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `GET /api/3/action/datastore_info`. +"""End-to-end tests for `GET /datastore/api/v2/datastore_info`. Single `resource_id` query parameter; the response envelope's `result` holds `meta` (free-form dict) + `fields` (column schema list). @@ -19,12 +19,13 @@ from tests.conftest import FakeCKAN -INFO_URL = "/api/3/action/datastore_info" +INFO_URL = "/datastore/api/v2/datastore_info" _RESOURCE_ID = "balancing_auction_results_2025" # 1. Happy path ------------------------------------------------------------- + def test_basic_info_succeeds(client: TestClient) -> None: response = client.get(INFO_URL, params={"resource_id": _RESOURCE_ID}) @@ -55,6 +56,7 @@ def test_response_shape(client: TestClient) -> None: # 2. Validation + aliases --------------------------------------------------- + def test_id_alias_works(client: TestClient) -> None: """`id` is a CKAN-style alias for `resource_id`; either is accepted.""" response = client.get(INFO_URL, params={"id": _RESOURCE_ID}) @@ -68,10 +70,13 @@ def test_id_alias_works(client: TestClient) -> None: def test_same_value_for_resource_id_and_id_accepted(client: TestClient) -> None: """Same value on both `resource_id` and `id` is the no-conflict case (legacy clients echoing both keys); accepted as `resource_id`.""" - response = client.get(INFO_URL, params={ - "resource_id": _RESOURCE_ID, - "id": _RESOURCE_ID, - }) + response = client.get( + INFO_URL, + params={ + "resource_id": _RESOURCE_ID, + "id": _RESOURCE_ID, + }, + ) assert response.status_code == 200 assert response.json()["result"]["meta"]["resource_id"] == _RESOURCE_ID @@ -80,10 +85,13 @@ def test_conflicting_resource_id_and_id_rejected(client: TestClient) -> None: """Different values for `resource_id` and `id` → 400. Silently preferring one would let CKAN-style legacy params mask a real client bug, so the request must be unambiguous.""" - response = client.get(INFO_URL, params={ - "resource_id": _RESOURCE_ID, - "id": "different-value", - }) + response = client.get( + INFO_URL, + params={ + "resource_id": _RESOURCE_ID, + "id": "different-value", + }, + ) assert response.status_code == 400 body = response.json() assert body["error"]["__type"] == "Validation Error" @@ -100,10 +108,13 @@ def test_missing_both_returns_validation_error(client: TestClient) -> None: def test_extra_query_param_rejected(client: TestClient) -> None: """`extra='forbid'` — only `resource_id` / `id` are allowed.""" - response = client.get(INFO_URL, params={ - "resource_id": _RESOURCE_ID, - "verbose": "true", - }) + response = client.get( + INFO_URL, + params={ + "resource_id": _RESOURCE_ID, + "verbose": "true", + }, + ) assert response.status_code == 400 body = response.json() @@ -112,6 +123,7 @@ def test_extra_query_param_rejected(client: TestClient) -> None: # 3. Auth ------------------------------------------------------------------- + def test_unknown_resource_returns_404(client: TestClient) -> None: response = client.get(INFO_URL, params={"resource_id": "does-not-exist"}) @@ -121,9 +133,7 @@ def test_unknown_resource_returns_404(client: TestClient) -> None: assert "does-not-exist" in body["error"]["message"] -def test_denied_key_returns_403( - client: TestClient, fake_ckan: FakeCKAN -) -> None: +def test_denied_key_returns_403(client: TestClient, fake_ckan: FakeCKAN) -> None: fake_ckan.deny("test-token") response = client.get(INFO_URL, params={"resource_id": _RESOURCE_ID}) diff --git a/tests/test_datastore_search.py b/tests/test_datastore_search.py index b6567a4..642e1ac 100644 --- a/tests/test_datastore_search.py +++ b/tests/test_datastore_search.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `GET /api/3/action/datastore_search`. +"""End-to-end tests for `GET /datastore/api/v2/datastore_search`. `datastore_search` is GET with query parameters. Complex types are encoded: @@ -37,7 +37,7 @@ from tests.conftest import FakeCKAN -SEARCH_URL = "/api/3/action/datastore_search" +SEARCH_URL = "/datastore/api/v2/datastore_search" _RESOURCE_ID = "balancing_auction_results_2025" @@ -246,7 +246,8 @@ def test_denied_key_returns_403(client: TestClient, fake_ckan: FakeCKAN) -> None def test_anonymous_read_calls_ckan_and_succeeds( - client: TestClient, fake_ckan: FakeCKAN, + client: TestClient, + fake_ckan: FakeCKAN, ) -> None: """No Authorization header on a read → we still call CKAN's `datastore_authorize`. CKAN itself decides based on resource @@ -392,9 +393,7 @@ def fake_search(self: BigQueryBackend, **kwargs: Any) -> SearchResult: return consumed -def test_objects_format_streams_rows( - client: TestClient, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_objects_format_streams_rows(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: consumed = _install_mock_search(monkeypatch) response = client.get(SEARCH_URL, params=_params()) @@ -427,9 +426,7 @@ def test_lists_format_streams_positional_arrays( ] -def test_csv_format_streams_data_rows( - client: TestClient, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_csv_format_streams_data_rows(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: """`records_format=csv` — JSON envelope, `records` is one CSV string of data rows only. Column names live on `result.fields` (no header in the records string).""" @@ -440,14 +437,10 @@ def test_csv_format_streams_data_rows( assert response.status_code == 200 assert response.headers["content-type"].startswith("application/json") body = response.json() - assert body["result"]["records"] == ( - "144,DCL,47.82\n" "145,DCH,51.1\n" "146,FFR,32.4\n" - ) + assert body["result"]["records"] == ("144,DCL,47.82\n145,DCH,51.1\n146,FFR,32.4\n") -def test_tsv_format_streams_data_rows( - client: TestClient, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_tsv_format_streams_data_rows(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: """`records_format=tsv` — JSON envelope, `records` is one TSV string of data rows only.""" _install_mock_search(monkeypatch) @@ -457,9 +450,7 @@ def test_tsv_format_streams_data_rows( assert response.status_code == 200 assert response.headers["content-type"].startswith("application/json") body = response.json() - assert body["result"]["records"] == ( - "144\tDCL\t47.82\n" "145\tDCH\t51.1\n" "146\tFFR\t32.4\n" - ) + assert body["result"]["records"] == ("144\tDCL\t47.82\n145\tDCH\t51.1\n146\tFFR\t32.4\n") def test_csv_quotes_values_with_special_chars( @@ -489,9 +480,7 @@ def test_csv_quotes_values_with_special_chars( assert response.status_code == 200 body = response.json() assert body["result"]["records"] == ( - "plain,ordinary value\n" - '"with,comma","with""quote"\n' - '"with\nnewline",tab\there\n' + 'plain,ordinary value\n"with,comma","with""quote"\n"with\nnewline",tab\there\n' ) @@ -505,7 +494,7 @@ def test_search_objects_response_includes_links(client: TestClient) -> None: assert response.status_code == 200 links = response.json()["result"]["_links"] assert set(links) == {"start", "page_size", "page", "total_pages"} - assert links["start"].startswith("http://testserver/api/3/action/datastore_search") + assert links["start"].startswith("http://testserver/datastore/api/v2/datastore_search") assert "offset" not in links["start"] assert f"resource_id={_RESOURCE_ID}" in links["start"] assert links["page_size"] == 100 # default limit diff --git a/tests/test_datastore_search_sql.py b/tests/test_datastore_search_sql.py index 867dd20..f763838 100644 --- a/tests/test_datastore_search_sql.py +++ b/tests/test_datastore_search_sql.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `GET /api/3/action/datastore_search_sql`. +"""End-to-end tests for `GET /datastore/api/v2/datastore_search_sql`. Only `sql` is accepted as a query parameter; the response reuses the `datastore_search` envelope shape (same writer, same `_links` / `limit` / @@ -29,11 +29,12 @@ from tests.conftest import FakeCKAN -SQL_URL = "/api/3/action/datastore_search_sql" +SQL_URL = "/datastore/api/v2/datastore_search_sql" # 1. Happy path ------------------------------------------------------------- + def test_basic_sql_succeeds(client: TestClient) -> None: response = client.get(SQL_URL, params={"sql": "SELECT 1 LIMIT 10"}) @@ -47,9 +48,9 @@ def test_basic_sql_succeeds(client: TestClient) -> None: def test_with_cte_succeeds(client: TestClient) -> None: """`WITH ... SELECT` (CTE) is allowed alongside plain SELECT.""" - response = client.get(SQL_URL, params={ - "sql": "WITH t AS (SELECT 1 AS a) SELECT * FROM t LIMIT 10" - }) + response = client.get( + SQL_URL, params={"sql": "WITH t AS (SELECT 1 AS a) SELECT * FROM t LIMIT 10"} + ) assert response.status_code == 200 @@ -59,9 +60,7 @@ def test_trailing_semicolon_allowed(client: TestClient) -> None: def test_leading_comment_then_select_allowed(client: TestClient) -> None: - response = client.get(SQL_URL, params={ - "sql": "-- a note\nSELECT 1 LIMIT 10" - }) + response = client.get(SQL_URL, params={"sql": "-- a note\nSELECT 1 LIMIT 10"}) assert response.status_code == 200 @@ -78,9 +77,12 @@ def test_missing_limit_rejected(client: TestClient) -> None: def test_limit_above_max_rejected(client: TestClient) -> None: """LIMIT must be <= `SEARCH_RESULT_ROWS_MAX` (default 32000). Above the cap → 400 with a 'paginate with OFFSET' hint.""" - response = client.get(SQL_URL, params={ - "sql": "SELECT 1 LIMIT 50000", - }) + response = client.get( + SQL_URL, + params={ + "sql": "SELECT 1 LIMIT 50000", + }, + ) assert response.status_code == 400 body = response.json() assert body["error"]["__type"] == "Validation Error" @@ -89,18 +91,23 @@ def test_limit_above_max_rejected(client: TestClient) -> None: # 2. Response envelope shape ------------------------------------------------ + def test_response_shape_matches_datastore_search(client: TestClient) -> None: """Same envelope as `datastore_search` so clients can share a parser. `limit` / `offset` come from the SQL's LIMIT / OFFSET literals.""" - response = client.get(SQL_URL, params={ - "sql": "SELECT 1 LIMIT 50 OFFSET 100" - }) + response = client.get(SQL_URL, params={"sql": "SELECT 1 LIMIT 50 OFFSET 100"}) assert response.status_code == 200 assert response.headers["content-type"].startswith("application/json") result = response.json()["result"] assert set(result) >= { - "resource_id", "schema", "fields", "records", "limit", "offset", "_links", + "resource_id", + "schema", + "fields", + "records", + "limit", + "offset", + "_links", } # Both column shapes are present: canonical `schema` + legacy `fields`. assert isinstance(result["schema"], dict) @@ -129,10 +136,7 @@ def test_response_echoes_original_sql(client: TestClient) -> None: """`result.sql` echoes the request SQL verbatim. Useful when `_links.next` rewrites the OFFSET — clients can still see what actually ran on this page.""" - sql = ( - 'SELECT auction_id FROM "balancing_auction_results_2025" ' - 'LIMIT 5 OFFSET 10' - ) + sql = 'SELECT auction_id FROM "balancing_auction_results_2025" LIMIT 5 OFFSET 10' response = client.get(SQL_URL, params={"sql": sql}) assert response.status_code == 200 assert response.json()["result"]["sql"] == sql @@ -144,9 +148,7 @@ def test_pagination_links_rewrite_sql_offset(client: TestClient) -> None: string with OFFSET advanced by LIMIT. Verify the URL builder rewrites OFFSET on the `start` link from the current offset back to 0.""" - response = client.get(SQL_URL, params={ - "sql": "SELECT 1 LIMIT 50 OFFSET 200" - }) + response = client.get(SQL_URL, params={"sql": "SELECT 1 LIMIT 50 OFFSET 200"}) assert response.status_code == 200 links = response.json()["result"]["_links"] # `start` resets to OFFSET 0 — `prev` lands at max(0, 200-50) = 150. @@ -158,6 +160,7 @@ def test_pagination_links_rewrite_sql_offset(client: TestClient) -> None: # 3. SQL validation --------------------------------------------------------- + def test_missing_sql_returns_validation_error(client: TestClient) -> None: response = client.get(SQL_URL, params={}) @@ -204,9 +207,9 @@ def test_unparseable_sql_rejected(client: TestClient) -> None: `_extract_sql_references` raises ValueError → 400. Real safety still sits at the engine credential layer; this is just fail-fast UX.""" for sql in ( - "SELECT $$$ random", # tokenizer error - "SELECT FROM WHERE", # bare FROM - "SELECT * FROM", # missing table + "SELECT $$$ random", # tokenizer error + "SELECT FROM WHERE", # bare FROM + "SELECT * FROM", # missing table ): response = client.get(SQL_URL, params={"sql": sql}) assert response.status_code == 400, f"expected 400 for: {sql}" @@ -217,6 +220,7 @@ def test_unparseable_sql_rejected(client: TestClient) -> None: # 4. Extra params rejected -------------------------------------------------- + def test_extra_query_param_rejected(client: TestClient) -> None: """`extra='forbid'` — only `sql` is allowed on this endpoint.""" response = client.get(SQL_URL, params={"sql": "SELECT 1", "limit": 10}) @@ -228,26 +232,28 @@ def test_extra_query_param_rejected(client: TestClient) -> None: # 5. sqlglot extraction (unit tests on parse_sql_references) --------------- -@pytest.mark.parametrize("sql,tables,functions", [ - # tables only - ('SELECT * FROM "abc-def" WHERE title LIKE \'jones\'', - ["abc-def"], []), - # functions, no table - ("SELECT COUNT(*), pg_read_file('/etc/passwd')", - [], ["count", "pg_read_file"]), - # aggregate + date function - ("SELECT AVG(price), DATE_TRUNC('day', d) FROM auctions GROUP BY 2", - ["auctions"], ["avg", "date_trunc"]), - # CTE aliases are NOT external tables - ("WITH t AS (SELECT 1 AS a) SELECT * FROM t", - [], []), - # JOIN — multiple tables, deduped - ("SELECT u.id FROM users u JOIN orders o ON u.id = o.user_id", - ["orders", "users"], []), - # CASE WHEN is syntactic, not a function - ("SELECT CASE WHEN x > 1 THEN 'big' ELSE 'small' END FROM t", - ["t"], []), -]) + +@pytest.mark.parametrize( + "sql,tables,functions", + [ + # tables only + ("SELECT * FROM \"abc-def\" WHERE title LIKE 'jones'", ["abc-def"], []), + # functions, no table + ("SELECT COUNT(*), pg_read_file('/etc/passwd')", [], ["count", "pg_read_file"]), + # aggregate + date function + ( + "SELECT AVG(price), DATE_TRUNC('day', d) FROM auctions GROUP BY 2", + ["auctions"], + ["avg", "date_trunc"], + ), + # CTE aliases are NOT external tables + ("WITH t AS (SELECT 1 AS a) SELECT * FROM t", [], []), + # JOIN — multiple tables, deduped + ("SELECT u.id FROM users u JOIN orders o ON u.id = o.user_id", ["orders", "users"], []), + # CASE WHEN is syntactic, not a function + ("SELECT CASE WHEN x > 1 THEN 'big' ELSE 'small' END FROM t", ["t"], []), + ], +) def test_parse_sql_references_extracts_names( sql: str, tables: list[str], functions: list[str] ) -> None: @@ -264,11 +270,15 @@ def test_parse_sql_references_rejects_unparseable() -> None: # 6. Function allow-list ---------------------------------------------------- + def test_disallowed_function_returns_validation_error(client: TestClient) -> None: """`pg_read_file` isn't in `ALLOWED_SQL_FUNCTIONS` → 400.""" - response = client.get(SQL_URL, params={ - "sql": "SELECT pg_read_file('/etc/passwd') LIMIT 1", - }) + response = client.get( + SQL_URL, + params={ + "sql": "SELECT pg_read_file('/etc/passwd') LIMIT 1", + }, + ) assert response.status_code == 400 body = response.json() assert body["error"]["__type"] == "Validation Error" @@ -283,64 +293,74 @@ def test_allowed_function_succeeds(client: TestClient) -> None: # 7. Per-table authorization ------------------------------------------------ -def test_unknown_table_returns_404( - client: TestClient, fake_ckan: FakeCKAN -) -> None: + +def test_unknown_table_returns_404(client: TestClient, fake_ckan: FakeCKAN) -> None: """Each referenced table is authorized via CKAN — unknown → 404.""" - response = client.get(SQL_URL, params={ - "sql": 'SELECT * FROM "does-not-exist" LIMIT 10', - }) + response = client.get( + SQL_URL, + params={ + "sql": 'SELECT * FROM "does-not-exist" LIMIT 10', + }, + ) assert response.status_code == 404 body = response.json() assert body["error"]["__type"] == "Not Found Error" -def test_existing_table_authorized( - client: TestClient, fake_ckan: FakeCKAN -) -> None: +def test_existing_table_authorized(client: TestClient, fake_ckan: FakeCKAN) -> None: """Referenced table that exists in CKAN clears auth → 200.""" - response = client.get(SQL_URL, params={ - "sql": 'SELECT * FROM "balancing_auction_results_2025" LIMIT 10', - }) + response = client.get( + SQL_URL, + params={ + "sql": 'SELECT * FROM "balancing_auction_results_2025" LIMIT 10', + }, + ) assert response.status_code == 200 -def test_denied_api_key_returns_403( - client: TestClient, fake_ckan: FakeCKAN -) -> None: +def test_denied_api_key_returns_403(client: TestClient, fake_ckan: FakeCKAN) -> None: """Auth gate uses the same path as datastore_search — denial returns 403.""" fake_ckan.deny("test-token") - response = client.get(SQL_URL, params={ - "sql": 'SELECT * FROM "balancing_auction_results_2025" LIMIT 10', - }) + response = client.get( + SQL_URL, + params={ + "sql": 'SELECT * FROM "balancing_auction_results_2025" LIMIT 10', + }, + ) assert response.status_code == 403 assert response.json()["error"]["__type"] == "Authorization Error" -def test_each_table_authorized_once_for_joins( - client: TestClient, fake_ckan: FakeCKAN -) -> None: +def test_each_table_authorized_once_for_joins(client: TestClient, fake_ckan: FakeCKAN) -> None: """A JOIN over two existing tables calls authorize twice.""" fake_ckan.add_resource("other_table", package_id="pkg-balancing-2025") before = fake_ckan.authorize_calls - response = client.get(SQL_URL, params={ - "sql": ( - 'SELECT a.id FROM "balancing_auction_results_2025" a ' - 'JOIN "other_table" b ON a.id = b.id LIMIT 10' - ), - }) + response = client.get( + SQL_URL, + params={ + "sql": ( + 'SELECT a.id FROM "balancing_auction_results_2025" a ' + 'JOIN "other_table" b ON a.id = b.id LIMIT 10' + ), + }, + ) assert response.status_code == 200 assert fake_ckan.authorize_calls - before == 2 -# 8. Download param moved to /datastore/dump/query -------------------------- +# 8. Download param moved to /datastore/api/dump/query -------------------------- + def test_download_param_no_longer_accepted(client: TestClient) -> None: - """SQL downloads live at `GET /datastore/dump/query?sql=&format=` now; + """SQL downloads live at `GET /datastore/api/dump/query?sql=&format=` now; `extra="forbid"` rejects the retired `download` param here.""" - response = client.get(SQL_URL, params={ - "sql": "SELECT 1 LIMIT 10", "download": "csv", - }) + response = client.get( + SQL_URL, + params={ + "sql": "SELECT 1 LIMIT 10", + "download": "csv", + }, + ) assert response.status_code == 400 assert response.json()["error"]["__type"] == "Validation Error" @@ -361,7 +381,8 @@ def test_parse_sql_pagination_optional_limit_present() -> None: """A LIMIT/OFFSET present in the SQL is honored as written even when not required.""" assert parse_sql_pagination( - "SELECT 1 LIMIT 5 OFFSET 2", require_limit=False, + "SELECT 1 LIMIT 5 OFFSET 2", + require_limit=False, ) == (5, 2) diff --git a/tests/test_datastore_upsert.py b/tests/test_datastore_upsert.py index 2c9139a..66aadc9 100644 --- a/tests/test_datastore_upsert.py +++ b/tests/test_datastore_upsert.py @@ -1,4 +1,4 @@ -"""End-to-end tests for `POST /api/3/action/datastore_upsert`. +"""End-to-end tests for `POST /datastore/api/v2/datastore_upsert`. Covers: 1. all three methods (upsert, insert, update) and the default method @@ -17,7 +17,7 @@ from tests.conftest import FakeCKAN -UPSERT_URL = "/api/3/action/datastore_upsert" +UPSERT_URL = "/datastore/api/v2/datastore_upsert" _RESOURCE_ID = "balancing_auction_results_2025" @@ -38,6 +38,7 @@ def _payload(**overrides: Any) -> dict[str, Any]: # 1. Methods ----------------------------------------------------------------- + def test_upsert_method_succeeds(client: TestClient) -> None: response = client.post(UPSERT_URL, json=_payload(method="upsert")) @@ -75,6 +76,7 @@ def test_default_method_is_upsert(client: TestClient) -> None: # 2. Optional flags --------------------------------------------------------- + def test_include_records_echoes_records(client: TestClient) -> None: payload = _payload(include_records=True) @@ -106,6 +108,7 @@ def test_default_omits_optional_fields(client: TestClient) -> None: # 3. Records optional -------------------------------------------------------- + def test_records_optional(client: TestClient) -> None: payload = _payload() payload.pop("records") @@ -118,6 +121,7 @@ def test_records_optional(client: TestClient) -> None: # 4. Validation -------------------------------------------------------------- + def test_missing_resource_id_returns_validation_error(client: TestClient) -> None: payload = _payload() payload.pop("resource_id") @@ -150,10 +154,9 @@ def test_extra_field_rejected(client: TestClient) -> None: # 5. Auth -------------------------------------------------------------------- + def test_unknown_resource_id_returns_404(client: TestClient) -> None: - response = client.post( - UPSERT_URL, json=_payload(resource_id="does-not-exist") - ) + response = client.post(UPSERT_URL, json=_payload(resource_id="does-not-exist")) assert response.status_code == 404 body = response.json() @@ -161,9 +164,7 @@ def test_unknown_resource_id_returns_404(client: TestClient) -> None: assert "does-not-exist" in body["error"]["message"] -def test_denied_key_returns_403( - client: TestClient, fake_ckan: FakeCKAN -) -> None: +def test_denied_key_returns_403(client: TestClient, fake_ckan: FakeCKAN) -> None: fake_ckan.deny("test-token") # conftest sets this header on the client response = client.post(UPSERT_URL, json=_payload()) @@ -179,9 +180,7 @@ def test_denied_key_returns_403( def test_upsert_on_readonly_resource_requires_force( client: TestClient, fake_ckan: FakeCKAN ) -> None: - fake_ckan.add_resource( - "ro-res", package_id="pkg-balancing-2025", url_type="upload" - ) + fake_ckan.add_resource("ro-res", package_id="pkg-balancing-2025", url_type="upload") response = client.post(UPSERT_URL, json=_payload(resource_id="ro-res")) @@ -194,13 +193,9 @@ def test_upsert_on_readonly_resource_requires_force( def test_upsert_on_readonly_resource_with_force_succeeds( client: TestClient, fake_ckan: FakeCKAN ) -> None: - fake_ckan.add_resource( - "ro-res", package_id="pkg-balancing-2025", url_type="upload" - ) + fake_ckan.add_resource("ro-res", package_id="pkg-balancing-2025", url_type="upload") - response = client.post( - UPSERT_URL, json=_payload(resource_id="ro-res", force=True) - ) + response = client.post(UPSERT_URL, json=_payload(resource_id="ro-res", force=True)) assert response.status_code == 200 assert response.json()["success"] is True diff --git a/tests/test_health.py b/tests/test_health.py index 7ebb45b..6c4b38f 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -1,7 +1,7 @@ -"""End-to-end tests for `GET /`, `GET /health`, `GET /ready`. +"""End-to-end tests for the health probes. Covers: - 1. / — welcome envelope + 1. / — no route; the service has no landing endpoint 2. /health — always 200 while the process is up 3. /ready — 200 when both engines pass healthcheck; 503 with a Service Unavailable envelope when either fails @@ -25,26 +25,23 @@ def _clean_engine_cache() -> Iterator[None]: reset_engine_cache() -# 1. Welcome ---------------------------------------------------------------- +# 1. No landing endpoint ---------------------------------------------------- -def test_welcome_returns_envelope(client: TestClient) -> None: - response = client.get("/") - assert response.status_code == 200 - body = response.json() - assert body["success"] is True - assert isinstance(body["result"]["message"], str) +def test_root_is_not_routed(client: TestClient) -> None: + """There is no welcome/landing endpoint — every route lives under the + versioned API prefix.""" + assert client.get("/").status_code == 404 -def test_welcome_not_mounted_under_action_prefix(client: TestClient) -> None: - """Welcome is root-only — `/api/3/action/` is the CKAN action - namespace and shouldn't echo a generic landing message.""" - response = client.get("/api/3/action/") - assert response.status_code == 404 +def test_action_prefix_root_is_not_routed(client: TestClient) -> None: + """The action namespace itself isn't a route either.""" + assert client.get("/datastore/api/v2/").status_code == 404 # 2. /health ---------------------------------------------------------------- + def test_health_returns_ok(client: TestClient) -> None: """Liveness — always 200 while the process is up.""" response = client.get("/datastore/api/health") @@ -56,6 +53,7 @@ def test_health_returns_ok(client: TestClient) -> None: # 3. /ready ----------------------------------------------------------------- + def test_ready_503_when_engine_unhealthy(client: TestClient) -> None: """Default test env has `bigquery` engine + no BIGQUERY_PROJECT, so the client is never built and healthcheck returns False. Both modes @@ -89,9 +87,7 @@ def test_ready_200_when_engines_healthy( assert body["result"]["status"] == "ready" -def test_ready_503_when_only_rw_fails( - client: TestClient, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_ready_503_when_only_rw_fails(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: """If rw fails but ro passes, /ready still 503s — pod isn't really 'ready' until both modes are reachable. Envelope stays in StatusResponse shape (`result.status` = "not_ready").""" @@ -117,12 +113,11 @@ def test_ready_handles_engine_construction_error( ) -> None: """If building the engine raises (bad credentials, missing module), /ready returns 503 in StatusResponse shape instead of bubbling a 500.""" + def boom(*args: object, **kwargs: object) -> object: raise RuntimeError("engine construction failed") - monkeypatch.setattr( - "datastore.api.endpoints.health.get_datastore_engine", boom - ) + monkeypatch.setattr("datastore.api.endpoints.health.get_datastore_engine", boom) response = client.get("/datastore/api/ready") diff --git a/tests/test_openapi.py b/tests/test_openapi.py index 7ae98c6..3164f94 100644 --- a/tests/test_openapi.py +++ b/tests/test_openapi.py @@ -10,17 +10,20 @@ from __future__ import annotations +from pathlib import Path from typing import Any import pytest -from datastore.core.config import get_config +from datastore.api import docs as docs_module +from datastore.api.docs import api_description +from datastore.core.config import Config, get_config +from datastore.core.constants import API_PREFIX, API_VERSION, DEFAULT_API_URL from datastore.main import create_app from fastapi.testclient import TestClient +from pydantic import ValidationError -def _build_schema( - monkeypatch: pytest.MonkeyPatch, auth_type: str -) -> dict[str, Any]: +def _build_schema(monkeypatch: pytest.MonkeyPatch, auth_type: str) -> dict[str, Any]: monkeypatch.setenv("AUTH_TYPE", auth_type) get_config.cache_clear() return create_app().openapi() @@ -42,14 +45,17 @@ def _operations(schema: dict[str, Any]) -> list[dict[str, Any]]: # 1. ckan --------------------------------------------------------------------- + def test_ckan_scheme_describes_api_key( monkeypatch: pytest.MonkeyPatch, ) -> None: + """Under CKAN auth the scheme describes an API token, and says nothing + about JWTs — the two providers' wording must not leak into each other.""" schema = _build_schema(monkeypatch, "ckan") scheme = _scheme(schema) assert scheme is not None - assert "CKAN API key" in scheme["description"] + assert "API token" in scheme["description"] assert "JWT" not in scheme["description"] @@ -60,7 +66,8 @@ def test_ckan_operations_reference_the_scheme( schema = _build_schema(monkeypatch, "ckan") secured = [ - op for op in _operations(schema) + op + for op in _operations(schema) if any("Authorization" in req for req in op.get("security", [])) ] assert secured, "no operation references the Authorization scheme" @@ -68,6 +75,7 @@ def test_ckan_operations_reference_the_scheme( # 2. jwt ---------------------------------------------------------------------- + def test_jwt_scheme_describes_bearer_token( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -82,6 +90,7 @@ def test_jwt_scheme_describes_bearer_token( # 3. anonymous ---------------------------------------------------------------- + def test_anonymous_has_no_security_scheme( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -101,19 +110,369 @@ def test_anonymous_operations_carry_no_security( assert "security" not in operation +# 3b. description tracks AUTH_TYPE ------------------------------------------ + + +@pytest.mark.parametrize( + ("auth_type", "expected", "forbidden"), + [ + ("ckan", "CKAN API token", "JWT"), + ("jwt", "signed JWT", "CKAN API token"), + ("anonymous", "no credentials are required", "CKAN API token"), + # matched case-insensitively below, so rewording the sentence's + # opening doesn't break the assertion + ], +) +def test_description_describes_the_active_provider( + monkeypatch: pytest.MonkeyPatch, + auth_type: str, + expected: str, + forbidden: str, +) -> None: + """The description is the first thing a reader sees, so telling them to + paste a CKAN token when the service runs on JWT would mislead.""" + schema = _build_schema(monkeypatch, auth_type) + + description = schema["info"]["description"] + assert expected.lower() in description.lower() + assert forbidden.lower() not in description.lower() + + +def test_description_falls_back_for_unknown_provider() -> None: + """A third-party provider under `datastore/auth//` must not be + handed another provider's instructions.""" + description = api_description("some-third-party-provider") + + assert "Send your credentials" in description + assert "CKAN API token" not in description + assert "signed JWT" not in description + + +def test_description_placeholder_is_always_substituted() -> None: + """The auth slot is spliced with a plain replace (the text contains + literal braces, so `str.format` would raise) — make sure no raw + placeholder can reach the page.""" + for auth_type in ("ckan", "jwt", "anonymous", "unknown"): + assert "%(auth)s" not in api_description(auth_type) + + +def test_description_survives_literal_braces( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A path like `/dump/{resource_id}` in the description must survive. + + `str.format`/%-formatting would read those braces as a field and raise at + startup, so the auth slot is spliced with a plain replace. Asserted against + a description that actually contains braces, so the guarantee holds even if + the shipped prose currently has none. + """ + monkeypatch.setattr( + docs_module, + "API_DESCRIPTION", + "See `/dump/{resource_id}`.\n\n%(auth)s", + ) + + rendered = api_description("ckan") + + assert "{resource_id}" in rendered + assert "CKAN API token" in rendered + + +# 3c. contract version, not build version ----------------------------------- + + +def test_info_version_is_the_api_contract_version(client: TestClient) -> None: + """`info.version` describes the API contract, not the installed build. + + The two are independent: the package can ship any number of releases + without the contract changing. A build number here would tell a client + nothing about which request/response shapes it is looking at. + """ + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + assert schema["info"]["version"] == API_VERSION + + +def test_info_version_matches_the_url_prefix(client: TestClient) -> None: + """The documented contract version and the one in the URL are the same + value, so a schema can never advertise a version its routes don't serve.""" + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + documented = schema["info"]["version"] + assert any(path.startswith(f"/datastore/api/{documented}/") for path in schema["paths"]), ( + f"no route served under the documented version {documented!r}" + ) + + +# 3d. `help` deep-links into the docs --------------------------------------- + + +def test_help_deep_links_to_the_operation(client: TestClient) -> None: + """`help` points at the endpoint's own entry in Swagger, not back at the + URL the caller just requested.""" + body = client.get(f"{API_PREFIX}/datastore_search?resource_id=x").json() + + assert body["help"].endswith(f"{API_PREFIX}/docs#/Datastore/datastore_search") + + +def test_help_is_present_on_errors(client: TestClient) -> None: + """The error envelope carries the same link — that is when a caller is + most likely to want the docs.""" + response = client.get(f"{API_PREFIX}/datastore_search") + + assert response.status_code == 400 + assert response.json()["help"].endswith("#/Datastore/datastore_search") + + +def test_help_anchors_resolve_in_the_schema(client: TestClient) -> None: + """Every `#//` anchor must name a real operation. + + A link that 'works' but lands nowhere is worse than no link, so this + checks the pairs against the published schema rather than trusting the + string format. + """ + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + for path, item in schema["paths"].items(): + for method, operation in item.items(): + for tag in operation.get("tags", []): + assert operation["operationId"], f"{method} {path} has no operationId" + assert tag in {t["name"] for t in schema["tags"]}, ( + f"{method} {path} tagged {tag!r}, which is not a declared tag" + ) + + +def test_operation_ids_are_the_handler_names(client: TestClient) -> None: + """Clean operationIds keep the deep-link anchors readable and give + generated clients sane method names.""" + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + ids = {op["operationId"] for it in schema["paths"].values() for op in it.values()} + + assert "datastore_search" in ids + assert not any("_get" in i or "_post" in i for i in ids), ( + f"FastAPI's auto-generated ids leaked through: {sorted(ids)}" + ) + + +# 3e. schema examples are absolute and consistent --------------------------- + + +def test_error_example_url_matches_the_route_prefix(client: TestClient) -> None: + """The example `help` is built from `API_PREFIX`, so it can't drift out of + sync with the paths the service actually serves.""" + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + example = schema["components"]["schemas"]["ErrorEnvelope"]["example"] + + assert example["help"] == (f"{DEFAULT_API_URL}{API_PREFIX}/docs#/Datastore/datastore_search") + + +def test_api_url_sets_the_example_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`API_URL` from the environment becomes the example's host, so a + deployed service's docs don't advertise `example.com`.""" + monkeypatch.setenv("API_URL", "https://data.example.org") + get_config.cache_clear() + + try: + with TestClient(create_app()) as configured: + schema = configured.get(f"{API_PREFIX}/openapi.json").json() + finally: + get_config.cache_clear() + + example = schema["components"]["schemas"]["ErrorEnvelope"]["example"] + assert example["help"] == ( + f"https://data.example.org{API_PREFIX}/docs#/Datastore/datastore_search" + ) + + +def test_api_url_trailing_slash_does_not_double_up( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A trailing slash in config must not yield `host//path`.""" + monkeypatch.setenv("API_URL", "https://data.example.org/") + get_config.cache_clear() + + try: + with TestClient(create_app()) as configured: + schema = configured.get(f"{API_PREFIX}/openapi.json").json() + finally: + get_config.cache_clear() + + example = schema["components"]["schemas"]["ErrorEnvelope"]["example"] + assert "//datastore" not in example["help"] + + +def test_example_help_is_absolute(client: TestClient) -> None: + """The published example must be a full URL, not the relative path + `schemas/` declares — a reader copying it should see a real response's + shape.""" + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + example = schema["components"]["schemas"]["ErrorEnvelope"]["example"] + assert example["help"].startswith("https://") + + +def test_error_example_anchor_names_a_real_operation(client: TestClient) -> None: + """The example deep-links at an operation that exists — an example that + points nowhere teaches the reader the wrong URL shape.""" + schema = client.get(f"{API_PREFIX}/openapi.json").json() + + example = schema["components"]["schemas"]["ErrorEnvelope"]["example"] + _, _, anchor = example["help"].partition("#/") + tag, _, operation_id = anchor.partition("/") + + ids = { + op["operationId"] + for item in schema["paths"].values() + for op in item.values() + if tag in op.get("tags", []) + } + assert operation_id in ids, f"{anchor!r} names no operation tagged {tag!r}" + + +def test_live_help_is_derived_from_the_request(client: TestClient) -> None: + """Runtime `help` comes from the incoming request, not `API_URL` — so it + stays correct behind a proxy or on any host.""" + body = client.get(f"{API_PREFIX}/datastore_search?resource_id=x").json() + + assert body["help"].startswith("http://testserver/") + + # 4. Swagger UI page ------------------------------------------------------------ + def test_docs_page_renders_swagger_ui(client: TestClient) -> None: - response = client.get("/datastore/api/docs") + response = client.get("/datastore/api/v2/docs") assert response.status_code == 200 assert "SwaggerUIBundle" in response.text - assert "/datastore/api/openapi.json" in response.text + assert "/datastore/api/v2/openapi.json" in response.text + + +def test_docs_page_serves_vendored_assets(client: TestClient) -> None: + """Swagger UI is vendored, not pulled from a CDN, so `/docs` renders + in air-gapped deployments.""" + response = client.get("/datastore/api/v2/docs") + + assert "cdn.jsdelivr.net" not in response.text + assert "/datastore/api/v2/static/swagger-ui/swagger-ui-bundle.js" in response.text + assert "/datastore/api/v2/static/theme/theme.css" in response.text + + for asset in ( + "/datastore/api/v2/static/swagger-ui/swagger-ui.css", + "/datastore/api/v2/static/swagger-ui/swagger-ui-bundle.js", + "/datastore/api/v2/static/theme/theme.css", + ): + assert client.get(asset).status_code == 200, asset + + +def test_docs_page_widens_authorize_input() -> None: + """The Authorize modal's token input is ~230px stock — too short to see + a pasted JWT/API key. The theme widens the modal and the input.""" + css = ( + Path(__file__).resolve().parent.parent / "datastore/api/static/theme/theme.css" + ).read_text() + assert "max-width: 900px" in css + assert ".swagger-ui .auth-container input" in css + + +def test_docs_page_applies_theme_config(monkeypatch: pytest.MonkeyPatch) -> None: + """`DOCS_*` env vars reach the page's CSS custom properties and header, + so a deployment rebrands without shipping CSS.""" + monkeypatch.setenv("DOCS_PRIMARY_COLOR", "#7A3864") + monkeypatch.setenv("DOCS_HEADER_COLOR", "#123456") + monkeypatch.setenv("DOCS_SITE_TITLE", "NESO Datastore API") + monkeypatch.setenv("DOCS_LOGO_URL", "/static/logo.png") + get_config.cache_clear() + + try: + with TestClient(create_app()) as themed_client: + body = themed_client.get("/datastore/api/v2/docs").text + finally: + get_config.cache_clear() + + assert "--docs-primary: #7A3864;" in body + assert "--docs-header-bg: #123456;" in body + assert "NESO Datastore API" in body + assert ' None: + """`DOCS_PRIMARY_COLOR` alone must brand the whole page. + + With `DOCS_HEADER_COLOR` unset no `--docs-header-bg` is emitted, so the + stylesheet's `--docs-header-bg: var(--docs-primary)` fallback applies and + the header takes the brand colour. A non-empty default here would pin the + bar to a fixed grey and make branding look broken. + """ + monkeypatch.setenv("DOCS_PRIMARY_COLOR", "#7A3864") + get_config.cache_clear() + + try: + assert Config().DOCS_HEADER_COLOR == "" + with TestClient(create_app()) as themed_client: + body = themed_client.get("/datastore/api/v2/docs").text + finally: + get_config.cache_clear() + + assert "--docs-primary: #7A3864;" in body + assert "--docs-header-bg" not in body + + +def test_docs_page_rejects_non_css_color(monkeypatch: pytest.MonkeyPatch) -> None: + """The colour lands inside a `