diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index e14eaee..2e6b8d3 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -29,7 +29,19 @@ jobs: path: deployment_assets/bin/linux-x86_64/* build-macos-intel: - runs-on: macos-13 # Intel runner + # `macos-15-intel`, not `macos-13`. + # + # The macOS 13 image was retired in December 2025. Every tagged release + # since has hung here: the job sat queued for the full 24-hour limit + # waiting for a runner that no longer exists, while linux, macos-arm and + # windows finished in 8-15 minutes, and `create-release` was skipped + # because it `needs:` all four. That is why v0.4.0 through v0.6.0 produced + # no GitHub release and no binaries - the workflow had never once + # succeeded. + # + # `macos-15-intel` is GitHub's documented replacement for workflows that + # genuinely need x86_64. + runs-on: macos-15-intel steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -98,8 +110,13 @@ jobs: steps: - uses: actions/download-artifact@v4 - name: Create Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: + body: | + See [CHANGELOG.md](https://github.com/AlexMercedCoder/Pangolin/blob/main/CHANGELOG.md) + for the full list of changes, and + [SECURITY.md](https://github.com/AlexMercedCoder/Pangolin/blob/main/SECURITY.md) + for the security advisory and upgrade steps. files: | linux-binaries/* macos-intel-binaries/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c66eba9..063a4c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,14 @@ env: CARGO_TERM_COLOR: always # Keep the debug artefacts small; CI does not need line tables. CARGO_PROFILE_TEST_DEBUG: 0 - RUSTFLAGS: -D warnings + # No `RUSTFLAGS: -D warnings` here. It used to be, and it was a trap rather + # than a gate: the tree still carries a warning backlog, so every job that + # compiles Rust had to override it with `RUSTFLAGS: ""` - and the two that + # forgot (`guardrails` and `mongo-replica-set`) failed at *compile*, so the + # authz matrix, the parity suite and every MongoDB test they were added to + # run had never once executed. A guard that cannot run is worse than no + # guard, because it reads as protection. `clippy-ratchet` is the real gate on + # warnings; it counts them and fails if the count grows. defaults: run: @@ -55,8 +62,6 @@ jobs: # `clippy-ratchet` below — and it flips to blocking once the backlog is # cleared (Phase 2.2). - run: cargo clippy --workspace --all-targets - env: - RUSTFLAGS: "" clippy-ratchet: name: clippy warning count @@ -71,8 +76,6 @@ jobs: workspaces: pangolin - run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev - name: Count warnings and compare against the budget - env: - RUSTFLAGS: "" run: | # `--message-format short` emits one `path:line:col: warning: ...` # line per finding, plus per-crate summary lines that start with @@ -125,14 +128,11 @@ jobs: done - name: cargo test --workspace - env: - RUSTFLAGS: "" run: cargo test --workspace --no-fail-fast - name: cargo test --workspace (against live databases) if: matrix.services env: - RUSTFLAGS: "" PANGOLIN_TEST_POSTGRES_URL: postgresql://testuser:testpass@localhost:5432/testdb PANGOLIN_TEST_MONGO_URL: mongodb://testuser:testpass@localhost:27017 # MongoDB parity is still incomplete; see the Known limitations section @@ -145,10 +145,86 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - uses: rustsec/audit-check@v2.0.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: pangolin + - run: cargo install cargo-audit --locked + # Run from `pangolin/`, not through `rustsec/audit-check`. + # + # That action invokes `cargo audit --file pangolin/Cargo.lock` from the + # repository root, and `cargo audit` reads its config from + # `.cargo/audit.toml` relative to the *current directory* only - it does + # not walk up the tree. So the exceptions in `pangolin/.cargo/audit.toml` + # were silently not applied and the job failed on advisories that had + # already been reviewed and justified. + # + # Duplicating the config at the repository root would fix CI and leave a + # developer running `cargo audit` in `pangolin/` - where the workspace + # is - seeing a different answer from the one CI sees. This job now runs + # the identical command in the identical directory instead (the + # workflow's `defaults.run.working-directory` is `pangolin`). + - run: cargo audit + + # Every optional feature must compile. + # + # None of them did. `cargo build` and `cargo test` run with default features, + # and no job ever passed `--features`, so the entire cloud-credential surface + # - `aws-sts`, `azure-oauth`, `gcp-oauth` and the `cloud-credentials` bundle + # that unions them - had rotted into code that could not be built at all: + # parameters bound as `_name` and then referenced as `name` inside the `cfg` + # block, a missing `anyhow!` import, an `expiration()` that returns a + # DateTime being fed to an RFC3339 string parser. For a catalog whose job + # includes vending scoped cloud credentials, that is the feature set, and + # nothing was watching it. + features: + name: optional features build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + feature: + - aws-sts + - azure-oauth + - gcp-oauth + - cloud-credentials + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: pangolin + - run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev + - run: cargo check -p pangolin_api --features ${{ matrix.feature }} --all-targets + - name: The store's own optional backends must build too + if: matrix.feature == 'cloud-credentials' + run: cargo check -p pangolin_store --features azure,gcp --all-targets + + # The declared MSRV must be true, not aspirational. + # + # `rust-version` is a promise to consumers, and every other job runs + # `stable`, so nothing checked it. It was 1.92 and is now 1.94 - raised + # deliberately to pick up the AWS SDK releases that carry fixed `aws-lc-sys` + # and `rustls-webpki`, which is what cleared the certificate-validation + # advisories. + msrv: + name: minimum supported rust version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Read rust-version from the workspace manifest + id: msrv + run: | + v=$(grep -m1 '^rust-version' Cargo.toml | sed 's/.*"\(.*\)".*/\1/') + echo "version=$v" >> "$GITHUB_OUTPUT" + echo "declared MSRV: $v" + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.msrv.outputs.version }} + - uses: Swatinem/rust-cache@v2 with: - token: ${{ secrets.GITHUB_TOKEN }} - working-directory: pangolin + workspaces: pangolin + - run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev + - run: cargo check --workspace --all-targets helm: name: helm lint @@ -198,6 +274,44 @@ jobs: tags: pangolin-api:ci cache-from: type=gha cache-to: type=gha,mode=max + # The server must outlive its own shutdown grace period. + # + # `PANGOLIN_SHUTDOWN_GRACE_SECS` is meant to bound the *drain* - the + # window between SIGTERM and giving up on in-flight requests. It was + # briefly implemented as `timeout(grace, serve)`, which bounds the whole + # server instead: the process exited cleanly `grace` seconds after + # startup, with no signal, and every container would have crash-looped. + # Nothing caught it, because no test ran the binary for longer than the + # default 25s. This runs the real image with a short grace and asserts it + # is still serving well past it. + - name: The server must not exit when its grace period elapses + run: | + set -euo pipefail + docker run -d --name grace-probe -p 8181:8080 \ + -e PANGOLIN_JWT_SECRET=ci-only-secret-of-adequate-length-000000 \ + -e PANGOLIN_STORAGE_TYPE=memory \ + -e PANGOLIN_SHUTDOWN_GRACE_SECS=5 \ + pangolin-api:ci + for _ in $(seq 1 30); do + if curl -sf http://localhost:8181/health/ready >/dev/null 2>&1; then break; fi + sleep 1 + done + curl -sf http://localhost:8181/health/ready >/dev/null + echo "serving; now waiting 20s, four times the grace period" + sleep 20 + if ! docker ps --filter name=grace-probe --filter status=running -q | grep -q .; then + echo "::error::the server exited on its own $(echo 20)s after startup with a 5s grace" + docker logs grace-probe | tail -20 + exit 1 + fi + curl -sf http://localhost:8181/health/ready >/dev/null + echo "still serving after 20s" + # And it must still stop promptly when actually told to. + start=$(date +%s) + docker stop -t 30 grace-probe >/dev/null + echo "stopped in $(( $(date +%s) - start ))s" + docker rm grace-probe >/dev/null + - name: The container must not run as root run: | user=$(docker inspect --format '{{.Config.User}}' pangolin-api:ci) @@ -206,3 +320,413 @@ jobs: echo "::error::the image runs as root" exit 1 fi + + # Roadmap improvements #0 and #1: the two suites that make whole classes of + # finding *testable* rather than reviewable. + # + # * the permission matrix drives each sensitive route as Root / + # TenantAdmin / TenantUser / a foreign tenant's admin and asserts the + # expected 200 or 403. A handler that forgets `check_permission` is + # indistinguishable from one that calls it until something asserts the + # 403 - which is how the B0a-B0m cluster survived a security release. + # * the cross-backend parity suite runs the same assertions against all + # four stores, which is the only thing that catches one backend quietly + # disagreeing with the others. + # + # Called out as their own job so a failure here reads as "authorization or + # backend parity regressed", not as an anonymous test failure. + guardrails: + name: authz matrix + backend parity + runs-on: ubuntu-latest + services: + # The parity suite is only meaningful against real backends. Running it + # on memory and SQLite alone is what let four cross-backend defects reach + # a release. + postgres: + image: postgres:15-alpine + env: + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + POSTGRES_DB: testdb + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U testuser" + --health-interval 5s --health-timeout 5s --health-retries 10 + mongo: + image: mongo:7.0 + env: + MONGO_INITDB_ROOT_USERNAME: testuser + MONGO_INITDB_ROOT_PASSWORD: testpass + ports: ["27017:27017"] + options: >- + --health-cmd "mongosh --eval 'db.adminCommand(\"ping\")'" + --health-interval 5s --health-timeout 5s --health-retries 10 + # MinIO is not a service container - see the step that starts it below. + env: + S3_ENDPOINT: http://localhost:9000 + AWS_ENDPOINT_URL: http://localhost:9000 + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_REGION: us-east-1 + AWS_ALLOW_HTTP: "true" + # Without this the S3 client probes 169.254.169.254 for instance + # credentials and burns the retry budget before failing. + AWS_EC2_METADATA_DISABLED: "true" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + # The store compliance tests exercise file IO. Without an object store + # they fall through to the EC2 instance-metadata endpoint and spend ~11s + # timing out before failing, which reads as a code failure rather than a + # missing service. + # + # This is a step rather than a `services:` entry because `minio/minio` + # needs the `server /data` argument and service containers cannot + # override a command. The image that does not - `bitnami/minio`, whose + # MINIO_DEFAULT_BUCKETS created the buckets for free - was withdrawn from + # Docker Hub, so the service definition that used to be here no longer + # resolves. + - name: Object storage for the file-IO tests + run: | + set -euo pipefail + docker run -d --name minio -p 9000:9000 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data + for _ in $(seq 1 30); do + if curl -sf http://localhost:9000/minio/health/live >/dev/null; then break; fi + sleep 2 + done + curl -sf http://localhost:9000/minio/health/live >/dev/null + # `warehouse` is the application's bucket; the other two are test + # fixtures. A missing one fails as `NoSuchBucket`, which the + # compliance test reports as "File IO should be supported". + docker run --rm --network host --entrypoint sh minio/mc -c ' + mc alias set m http://localhost:9000 minioadmin minioadmin + for b in warehouse bucket test-bucket; do mc mb --ignore-existing "m/$b"; done + ' + + - name: Permission matrix + run: cargo test -p pangolin_api --lib authz_matrix_tests -- --nocapture + - name: Cross-backend parity (all four backends) + env: + PANGOLIN_TEST_POSTGRES_URL: postgresql://testuser:testpass@localhost:5432/testdb + # A standalone mongod supports neither transactions nor retryable + # writes; the code degrades for the former and this disables the + # latter. A replica set would exercise both paths and is what + # production should use - see docs/operations/backend-parity.md. + PANGOLIN_TEST_MONGO_URL: mongodb://testuser:testpass@localhost:27017/?retryWrites=false + run: cargo test -p pangolin_store --test store_integration + + - name: The parity suite must not have skipped a backend + env: + PANGOLIN_TEST_POSTGRES_URL: postgresql://testuser:testpass@localhost:5432/testdb + PANGOLIN_TEST_MONGO_URL: mongodb://testuser:testpass@localhost:27017/?retryWrites=false + run: | + # The suite skips a backend with a printed note when its URL is + # unset, which passes. That is right for a developer laptop and wrong + # for CI: a silently skipped backend is how Postgres and MongoDB went + # untested through an entire security release. Assert all four ran. + out=$(cargo test -p pangolin_store --test store_integration 2>&1) + echo "$out" + for backend in memory sqlite postgres mongo; do + if ! grep -q "test test_${backend}_store_regression \.\.\. ok" <<<"$out"; then + echo "::error::the ${backend} backend did not run" + exit 1 + fi + done + if grep -qiE "^skipping:" <<<"$out"; then + echo "::error::a backend was skipped" + exit 1 + fi + echo "all four backends ran" + + # A standalone `mongod` supports neither transactions nor retryable writes, so + # the `guardrails` job above exercises only MongoDB's *degraded* paths. That + # is the topology in which `delete_catalog`'s fallback was found to be broken + # - and the transactional branch it falls back *from* has correspondingly + # never run. Testing one branch and not the other is the same gap one layer in. + # + # A single-node replica set is enough: a set of one supports transactions and + # retryable writes, which is the whole distinction. It is not a model of + # production redundancy and is not meant to be. + mongo-replica-set: + name: mongodb (replica set) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Start a single-node replica set + run: | + # Not a `services:` container: the set has to be initiated after the + # process is listening, and a service container's healthcheck has no + # hook for that. + docker run -d --name mongo-rs -p 27018:27018 mongo:7.0 \ + mongod --replSet rs0 --bind_ip_all --port 27018 + + for i in $(seq 1 30); do + if docker exec mongo-rs mongosh --port 27018 --quiet --eval \ + "rs.initiate({_id:'rs0',members:[{_id:0,host:'localhost:27018'}]})" \ + >/dev/null 2>&1; then + break + fi + sleep 2 + done + + # Wait for a primary. Accepting connections is not the same as being + # able to serve a write, and a client that connects in between fails + # in ways that look like application bugs. + for i in $(seq 1 30); do + state=$(docker exec mongo-rs mongosh --port 27018 --quiet \ + --eval "try { rs.status().myState } catch (e) { 0 }" 2>/dev/null | tr -d '[:space:]') + if [ "$state" = "1" ]; then + echo "replica set has a primary" + exit 0 + fi + sleep 2 + done + echo "::error::the replica set never elected a primary" + docker logs mongo-rs | tail -50 + exit 1 + + - name: The set must actually support transactions + run: | + # Otherwise this job silently re-tests the standalone paths and the + # whole point of it is lost. + docker exec mongo-rs mongosh --port 27018 --quiet --eval ' + const s = db.getMongo().startSession(); + s.startTransaction(); + s.getDatabase("probe").c.insertOne({ok: 1}); + s.commitTransaction(); + print("transactions available"); + ' + + - name: MongoDB suites against a replica set + env: + PANGOLIN_TEST_MONGO_URL: mongodb://localhost:27018/?replicaSet=rs0&directConnection=true + PANGOLIN_TEST_MONGO_DB: pangolin_rs_test + run: | + cargo test -p pangolin_store --test store_integration test_mongo + cargo test -p pangolin_store --test mongo_comprehensive_tests + cargo test -p pangolin_store --test mongo_audit_tests + cargo test -p pangolin_store --test mongo_tests + cargo test -p pangolin_store --test mongo_uuid_round_trip_tests + + - name: The suites must not have skipped MongoDB + env: + PANGOLIN_TEST_MONGO_URL: mongodb://localhost:27018/?replicaSet=rs0&directConnection=true + PANGOLIN_TEST_MONGO_DB: pangolin_rs_test + run: | + out=$(cargo test -p pangolin_store --test store_integration test_mongo 2>&1) + echo "$out" + grep -q "test test_mongo_store_regression \.\.\. ok" <<<"$out" || { + echo "::error::the MongoDB backend did not run" + exit 1 + } + + # Deployment artefacts drift from the code that reads them, and nothing + # noticed. `PANGOLIN_STORE_TYPE` sat in both compose files for several + # releases while the server read `PANGOLIN_STORAGE_TYPE` (B9); the quick-start + # compose file could not start the API at all because it set no signing secret + # (B8); and the release compose file pinned an image four versions old and ran + # a verification script that did not exist (B10). All three are the same + # failure: config that is never validated. + config-drift: + name: config drift + runs-on: ubuntu-latest + defaults: + run: + working-directory: . + steps: + - uses: actions/checkout@v4 + + - name: Every compose file must be valid + env: + PANGOLIN_JWT_SECRET: ci-only-secret-of-adequate-length-0000000 + # docker-compose.release.yml takes RELEASE_* rather than PANGOLIN_* + # so that a developer's local .env cannot feed the release + # verification harness. They are required (`:?`), so `config` cannot + # render the file without them. + RELEASE_JWT_SECRET: ci-only-secret-of-adequate-length-0000000 + RELEASE_ROOT_PASSWORD: ci-only-root-password-not-a-placeholder + run: | + for f in docker-compose*.yml; do + echo "validating $f" + docker compose -f "$f" config > /dev/null + done + + - name: The quick start must fail loudly without a signing secret + run: | + # `docker compose config` must *fail* here: the `:?` form on + # PANGOLIN_JWT_SECRET is what turns a crash-looping container into an + # immediate, explicable error. + if docker compose -f docker-compose.yml config > /dev/null 2>&1; then + echo "::error::docker-compose.yml accepted an unset PANGOLIN_JWT_SECRET" + exit 1 + fi + echo "compose correctly refuses to start without a signing secret" + + - name: No compose file may set a variable the server does not read + run: | + # PANGOLIN_STORE_TYPE is the specific name that survived for releases; + # the general check below catches its successors. + # Comment lines are excluded: two compose files carry the offending + # name verbatim while explaining why it is wrong. The previous + # `grep -v 'not read'` was meant to do this and matched neither + # comment ("was renamed", "which nothing reads"), so this guard + # failed on its own documentation. + if grep -rn 'PANGOLIN_STORE_TYPE' docker-compose*.yml deployment_assets 2>/dev/null \ + | grep -vE ":[[:space:]]*#"; then + echo "::error::PANGOLIN_STORE_TYPE is not read by the server; use PANGOLIN_STORAGE_TYPE" + exit 1 + fi + # Comment lines are stripped first, for the same reason as above: + # several compose comments name a variable precisely to explain that + # it is *not* read, and the guard would flag its own documentation. + # + # Two names are legitimately not read by the server: + # PANGOLIN_VERSION - a compose-level image tag, not an app setting. + # PANGOLIN_API_URL - set on the `tests` service in + # docker-compose.release.yml and read by + # scripts/release_smoke_test.py, which is a Python script rather + # than the Rust server this guard greps. (It previously needed an + # exemption for a worse reason: nothing read it at all, because + # the script that service ran could not run in that container.) + exempt='PANGOLIN_VERSION|PANGOLIN_API_URL' + names=$(grep -rhE '^[^#]*PANGOLIN_[A-Z0-9_]+' docker-compose*.yml \ + | sed 's/#.*//' | grep -ohE 'PANGOLIN_[A-Z0-9_]+' | sort -u) + for name in $names; do + if echo "$name" | grep -qE "^($exempt)$"; then continue; fi + if ! grep -rq "$name" pangolin/pangolin_api/src pangolin/pangolin_store/src; then + echo "::error::$name appears in a compose file but no code reads it" + exit 1 + fi + done + echo "every PANGOLIN_* name in the compose files is read by the server" + + - name: The environment-variable reference must match the code + working-directory: pangolin + run: ./scripts/check_env_var_docs.sh + + - name: Every artifact must carry the same version + run: | + # Improvement #8. The "one version everywhere" property introduced in + # 0.6.0 had already drifted a day later in two places, because it was + # maintained by hand across five files. + version=$(grep -oE '^version = "[0-9]+\.[0-9]+\.[0-9]+[^"]*"' pangolin/Cargo.toml \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+[^"]*') + echo "workspace version: $version" + ./scripts/bump_version.sh "$version" --check + + # Improvement #2: the SDK and UI had no CI at all - only Rust, Helm and + # Docker were covered. That is exactly where the 0.6.0 "no CI, silent rot" + # lesson was repeating: the SDK's test suite could not even *collect* + # (B41) and the UI's vitest config broke every suite that imported the API + # client (B46), and nothing noticed either. + sdk: + name: python sdk + runs-on: ubuntu-latest + defaults: + run: + working-directory: pypangolin + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e '.[iceberg]' + # Pinned deliberately: `pip install ruff` picked up 0.16, whose + # wider default rule set produced 316 findings in a package that had + # opted into ruff's defaults as they stood. Adopting the new rules is + # a separate, deliberate change - not something a runner should decide + # on the day it happens to run. + pip install pytest 'ruff==0.15.19' + - name: The package must import without the Iceberg extra + run: | + # B41 regression: `pip install pypangolin` then `import pypangolin` + # must not require the whole Iceberg stack. + pip uninstall -y pyiceberg + python -c "import pypangolin; print(pypangolin.__version__)" + - name: The version must match the package metadata + run: | + # B38: `__version__` was hardcoded to 0.1.0 against a 0.6.0 package. + python - <<'PY' + from importlib.metadata import version + import pypangolin + assert pypangolin.__version__ == version("pypangolin"), ( + f"__version__ {pypangolin.__version__} != metadata {version('pypangolin')}" + ) + print("version is single-sourced") + PY + - run: ruff check src + - run: pytest -q + + ui: + name: management ui + runs-on: ubuntu-latest + defaults: + run: + working-directory: pangolin_ui + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - run: npm ci + - name: The UI must build + run: npm run build + + # This job built the UI and never ran its tests, so the suite drifted to + # 40 failures across 14 files without anything noticing - including tests + # that could not pass at all because the global setup mocked the very + # module they were testing. + - name: The UI test suite must pass + run: npm test + + # `svelte-check` reports a backlog inherited from before this job + # existed, so it is a ratchet rather than a gate - the same arrangement as + # `clippy-ratchet` above, and it flips to blocking once the backlog is + # cleared. + - name: Count type errors and compare against the budget + run: | + count=$(npm run check 2>&1 | grep -cE "^Error:" || true) + budget=$(cat svelte-check-budget.txt) + echo "svelte-check errors: $count (budget: $budget)" + if [ "$count" -gt "$budget" ]; then + echo "::error::svelte-check errors rose from $budget to $count." + exit 1 + fi + if [ "$count" -lt "$budget" ]; then + echo "::notice::svelte-check errors fell to $count; lower svelte-check-budget.txt." + fi + + - name: No raw /api/v1 fetches may reappear + run: | + # B32: a relative `fetch('/api/v1/...')` works under the dev proxy and + # 404s in production, while also skipping the tenant header. Every + # call must go through apiClient, which is not something a type check + # can express. + # Comment lines are excluded: several carry the offending pattern + # verbatim while explaining why it is wrong. + if grep -rnE "fetch\((['\`\"])/api/v1" src/ | grep -vE ":[[:space:]]*(//|\*|/\*)"; then + echo "::error::raw /api/v1 fetch found; route it through apiClient" + exit 1 + fi + echo "no raw /api/v1 fetches" + - name: Only PUBLIC_API_URL may name the API base URL + run: | + # B31: four different spellings coexisted and none agreed, so every + # deployed build called the visitor's own localhost. + if grep -rn "VITE_API_URL" src/ .env.example vitest.config.ts 2>/dev/null \ + | grep -vE ":[[:space:]]*(//|#|\*|/\*)"; then + echo "::error::VITE_API_URL is not read by SvelteKit; use PUBLIC_API_URL" + exit 1 + fi + echo "PUBLIC_API_URL is the only spelling" diff --git a/.gitignore b/.gitignore index fc9a6d0..0f4410d 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,14 @@ check_log.txt test_results.txt debug_output*.txt :memory: + +# Editor backup copies of source files (B45). +# +# `pangolin_store/src/memory.rs.bak` and `mongo.rs.bak` sat in the tree as ~4k +# lines of divergent, dead query copies - a grep trap that returned plausible +# but stale code for anyone searching the store layer. +*.bak +*.rs.bak + +# Logs live under logs/ directories that were previously tracked. +logs/ diff --git a/AUDIT_EXECUTION_PLAN.md b/AUDIT_EXECUTION_PLAN.md index 913c7be..d029a5a 100644 --- a/AUDIT_EXECUTION_PLAN.md +++ b/AUDIT_EXECUTION_PLAN.md @@ -24,8 +24,48 @@ Three findings define the current state: The good news is that the fixes are tractable and mostly independent. The critical security items are each a few dozen lines. The correctness items are contained in one file. The hygiene items are largely mechanical. **The single highest-leverage action is standing up CI**, because without it every fix below is one refactor away from silently regressing — which is precisely how the current state was reached. +> ## Status as of 2026-08-11 — read this first +> +> This document is the **original audit of 2026-08-09**, kept as the historical +> record of what was found. Much of it has since been fixed. Where this file and +> the reconciled status below disagree, **the status below is correct**. +> +> Current state is tracked in: +> +> - [`STATUS.md`](STATUS.md) — the single reconciled view of done vs. outstanding +> - [`CHANGELOG.md`](CHANGELOG.md) — what changed in each release, and why +> - [`SECURITY.md`](SECURITY.md) — the advisory and the remaining security gaps +> - [`docs/operations/`](docs/operations/) — backend parity, encryption, backup +> and recovery, performance, multiple replicas +> +> The scorecard immediately below has been updated in place; every other section +> of this document is left as it was written on 2026-08-09. + ### Readiness scorecard +**Updated 2026-08-11.** Ratings in parentheses are the original 2026-08-09 +assessment, kept so the direction of travel is visible. + +| Dimension | Rating | One-line justification | +|---|---|---| +| Feature breadth | **Strong** (Strong) | Four backends, three clouds, branching/merge, RBAC, audit, SSO, SDK, UI | +| Iceberg REST correctness | **Good** (Weak) | Requirements and updates enforced; `registerTable`, `listViews`, `viewExists`, `dropView` added. `commitTransaction` deliberately absent — see below | +| Security (authn/authz) | **Adequate** (Critical) | OAuth exfiltration, default JWT secret, bypass path and the 0.7.0 authorization cluster all fixed; rate limiting and credential encryption added. OIDC is still not OIDC | +| Error handling | **Adequate** (Weak) | Iceberg error envelope conforms; `unwrap()` counts unchanged in non-Iceberg paths | +| Observability | **Good** (Absent) | `/metrics` with latency histograms, request tracing, `RUST_LOG` honoured, `/health/live` and `/health/ready` | +| Reliability | **Good** (Weak) | Graceful shutdown, timeouts, body and concurrency limits, readiness that probes the store | +| Data integrity | **Good** (Weak) | Postgres and SQLite wrap catalog delete, branch delete, merge, and branch-create-by-copy. MongoDB wraps the cascade where a session exists | +| Test coverage | **Good** (Critical) | 63 targets / 415 tests green against live PostgreSQL, MongoDB and MinIO; 19 CI jobs including an authz matrix and a four-backend parity suite | +| Code hygiene | **Adequate** (Weak) | `rustfmt` clean; clippy at a ratcheted budget of 30, down from 314 | +| Enterprise readiness | **Partial** (Partial) | Credentials encrypted at rest, backup/restore drilled and measured, multi-replica constraints documented. No HA proof, no tamper-evident audit, no OIDC | +| Deployment | **Good** (Partial) | Helm lints and templates; container runs non-root; release pipeline actually produces a release (it never had) | +| Documentation | **Strong** (Strong user / Absent contributor) | CONTRIBUTING, SECURITY, CHANGELOG, and an operations set covering parity, encryption, backup, performance and replicas | + +**Still weak, stated plainly:** OIDC (no PKCE, no JWKS, no `id_token` +validation), no tamper-evident audit trail, no point-in-time recovery, +multi-replica operation untested under load, and `commitTransaction` absent +because the store cannot commit several tables atomically. + | Dimension | Rating | One-line justification | |---|---|---| | Feature breadth | **Strong** | Four backends, three clouds, branching/merge, RBAC, audit, SSO, SDK, UI | diff --git a/CHANGELOG.md b/CHANGELOG.md index fab1c57..54df9ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,679 @@ From 0.6.0 the server, both CLIs, the Python SDK, the UI and the Helm chart all carry the same version number. Before that they had drifted to five different values and there was no way to tell which combination had been tested together. +## [Unreleased] + +Bucket 2 of the production-readiness work. + +### Added — operations: replicas, backup, performance + +**The token-cleanup job never ran.** `start_token_cleanup_job` was defined, the +module was declared, and nothing anywhere called it — so `revoked_tokens` grew +for the life of every deployment, and the revocation check reads that table on +every authenticated request. It now runs, and staggers its own start within the +interval so replicas from one rolling deploy do not sweep in lockstep. Jitter +rather than leader election: the sweep is a `DELETE ... WHERE expires_at < now` +and is therefore safe to run concurrently, so a lock table and a lease would be +complexity spent serialising something that does not need it. + +**Backup and recovery, drilled rather than described.** +`scripts/backup_restore_drill.sh` dumps, **destroys the schema**, restores, and +verifies — a canary row, matching row counts, and a populated +`_sqlx_migrations` (without which the next startup re-runs every migration and +fails). Measured on a laptop against PostgreSQL 15 with 1,345 rows: 7s backup, +53s restore. Writing it surfaced two real traps: `pg_dump` refuses to dump a +newer server *after* you think you have a backup, and `PANGOLIN_ENCRYPTION_KEY` +is not in the dump, so a team that backs up the database religiously and never +records the key restores a catalog full of unreadable credentials. + +**A load harness, and a correction.** The first version used +`urllib.request.urlopen` per request and reported ~33ms and 504 req/s against a +server whose own histogram said 29 **microseconds**. It was measuring Python. +With one keep-alive connection per worker the same server reports 5086 req/s — +a 10× difference that came entirely from the client. The harness now prints the +server's own means alongside its own, and says that a large gap means the +generator is the bottleneck. Publishing the first set of numbers would have +understated the catalog by roughly 1000×. + +Measured: `/health/ready` 0.018ms server-side, `/v1/config` 0.023ms, an +authenticated catalog list 0.060ms — so authorization roughly triples +server-side cost and is still 60 microseconds. + +New operations documentation: `running-multiple-replicas.md` (what works, what +needs session affinity, and what has not been tested), `backup-and-recovery.md`, +`performance.md`. + +### Documentation — the audit documents are reconciled + +`AUDIT_EXECUTION_PLAN.md` and `roadmap_aug10.md` are historical records and were +being read as to-do lists. Both now carry a status header, the readiness +scorecard is updated in place with the original ratings kept alongside for +direction of travel, and every recommended improvement and feature carries its +real state. `STATUS.md` is the single reconciled view; README and SECURITY point +at it. + +### Added — the missing Iceberg REST operations (A-5) + +`listViews`, `viewExists`, `dropView` and `registerTable` had no route. A client +calling them got a routing `404`, which is indistinguishable from a catalog +answering "no such view" — so `SHOW VIEWS` returned nothing, `DROP VIEW` +appeared to succeed against a catalog that had never heard of the operation, and +a view created through the Iceberg API could not be removed through it at all. + +- **`GET .../views`** — views are stored as assets with `kind: View`, so this is + a filtered `list_assets` rather than a second source of truth that can fall + out of step. Namespace-scoped `Read`, matching `listTables`: knowing which + views exist is itself information about the data. +- **`HEAD .../views/{view}`** — authorized identically to `loadView`, because + answering "does this exist" to a caller who may not read it still discloses + that it exists. +- **`DELETE .../views/{view}`** — refuses to delete a table addressed as a view. + Without that check a caller with view-drop rights could remove a table. +- **`POST .../namespaces/{ns}/register`** — how an engine adopts a table whose + metadata already sits in storage: a migration from another catalog, a restore, + a table written directly by a job. It refuses a metadata location it cannot + read, because registering a location that is not there leaves a table whose + every subsequent `loadTable` fails, far from the request that caused it. It + also refuses to shadow an existing table unless `overwrite` is set. + +`loadNamespaceMetadata` and `namespaceExists`, also listed under A-5, were +already wired during the 0.7.0 work. + +**`commitTransaction` is still absent, deliberately.** The spec's +`POST /v1/{prefix}/transactions/commit` is a multi-table *atomic* commit; the +commit path does compare-and-swap per table and there is no cross-table +transaction behind it. Routing it and committing tables one at a time would be +worse than leaving it unrouted: an engine that sees the endpoint relies on the +atomicity the spec promises, and a partial failure would leave half a +multi-table change applied with no way to detect it. A `404` makes clients fall +back to per-table commits, which is what happens today, and is honest about it. +A test pins that decision so it stays a choice rather than becoming an +oversight. + +### Added — MongoDB index management + +MongoDB had two indexes, `commits(parent_id)` and `active_tokens(user_id)`, +created at startup with their errors discarded by `.ok()`. Everything else was a +collection scan: every catalog lookup, every asset resolution on the Iceberg +commit path, and — worst — the role and permission reads that run on *every* +authenticated request. + +The full set is now created at startup, derived from the filters the code +actually issues rather than from what seemed likely. Each entry records why it +exists, so a future reader can tell which are safe to drop. + +Several are `unique`, which is the constraint the SQL backends express as a +primary key. MongoDB previously accepted two catalogs with the same name in one +tenant and returned an arbitrary one on lookup — a correctness difference from +the other three backends, not a performance one. Catalogs, warehouses, branches, +tags, and one business-metadata record per asset are now enforced. + +Failures are reported rather than swallowed. A unique index cannot be created +over a collection that already holds duplicates, and that is worth an error +naming the collection and saying the constraint is not in force — startup +continues, because refusing to boot over a missing index would turn a +performance problem into an outage. + +A unit test asserts the kebab-case collections are spelled as stored: an index +on `user_id` where the field is `user-id` indexes nothing and silently does +nothing, which is the same spelling trap that made every role assignment +unreadable. + +### Fixed — creating a branch by copy is atomic, and no longer lies (A-24) + +Two defects, the second worse than the first. + +The branch row and its copied assets were written by independent statements, so +a failure between them left a branch that existed holding an arbitrary subset of +its assets, with no rollback and no repair tool. + +And the copy error was **logged and discarded** — `Err(e) => +tracing::error!(...)` — after which the handler returned `200`. The caller was +told the branch was ready when it was empty. The per-asset path did the same +thing more quietly: `if let Ok(_)` around each create, and `continue` past any +name it could not parse. + +`create_branch_with_assets` now does both in one transaction on PostgreSQL and +SQLite. PostgreSQL copies with a single `INSERT ... SELECT`, so there is no +window in which some rows exist and others do not and a large branch need not +fit in memory. A malformed asset name now fails the whole operation rather than +silently copying nothing. + +MongoDB has no atomic version, and says so rather than pretending: the trait +default is an error, the API takes the sequential fallback deliberately, logs a +warning that a partial failure will leave the branch incomplete, and — the part +that matters — returns a `500` naming the branch instead of a `200`. + +The rollback test was verified to be load-bearing by committing the branch row +before the failure point: it fails with "the branch row survived a failed +create" and passes again when the transaction is restored. + +### Added — warehouse credentials encrypted at rest (C-11) + +A warehouse holds the credentials Pangolin uses to reach a customer's object +storage. They were plaintext JSON in the catalog database, so anything that +could read one row of `warehouses` — a backup, a replica, a snapshot, an analyst +with `SELECT` — held every tenant's cloud keys. + +Credential fields are now sealed with AES-256-GCM and a fresh 96-bit nonce per +value, stored as `enc:v1:`. Only credentials are sealed; bucket, region, +endpoint and account name stay readable, because the object-store factory +compares and concatenates them and they are not secrets. + +Deliberate choices worth knowing about: + +- **Off unless `PANGOLIN_ENCRYPTION_KEY` is set**, and the server warns loudly + at startup when it is not. Requiring it would break every existing deployment + on upgrade; doing nothing silently is the failure mode this audit keeps + finding, so it is said out loud instead. +- **Reads tolerate plaintext**, so a database written before this exists keeps + working. Those rows stay unsealed until something rewrites them — + `docs/operations/encryption.md` explains how to force that and how to find + what still needs it. +- **The wrong key fails loudly.** GCM authenticates, so a mismatched key gives + an error naming `PANGOLIN_ENCRYPTION_KEY` rather than returning rubbish. +- **This protects a stolen database, not a compromised host.** The key is in the + server's environment. That limit is documented rather than implied away. + +PostgreSQL, SQLite and MongoDB seal on write and open on read, on both the +create and the update paths — sealing only on create would protect the first +credential and leak every rotation, which is worse than not doing it at all, +because the table would look encrypted. The memory backend is excluded on +purpose: it loses everything on restart, so it has no "at rest". + +`warehouse_encryption_tests.rs` reads the raw stored bytes through its own +database connection rather than through the store, and asserts the plaintext is +absent. Asking the store to read back its own writes would pass just as happily +if `seal` were never called. + +### Added — OpenID Connect (C-2/C-3) + +What the OAuth flow did before this was *authorization*, not authentication. It +exchanged a code for an access token, called the provider's userinfo endpoint, +and believed the response — which is sufficient only if the access token could +not have come from anywhere else, and establishing that is exactly what OIDC is +for. + +Now, for every provider that supports it: + +- **PKCE (S256).** An attacker who intercepts the authorization code — a + referrer header, a proxy log, shell history on a shared machine — cannot + redeem it without the verifier. The verifier is held **server-side**, never in + `state`: `state` travels through the browser in the same URL as the code, so + putting the verifier there would hand it to exactly the attacker PKCE exists + to stop. +- **`id_token` signature validation** against the provider's JWKS, so identity + comes from something the provider signed rather than an HTTP response any + holder of some access token could have elicited. +- **`aud`** must contain our `client_id` — without it, a token minted for a + different application at the same provider logs its holder in here, which is + the classic confused deputy. +- **`iss`**, checked against a discovery document whose own `issuer` is verified + to match where it was fetched — otherwise `iss` validation is circular. +- **`exp`** with 60s leeway for clock skew. +- **`nonce`**, bound to the login, so an `id_token` seen in one flow cannot be + replayed into another. +- **Asymmetric algorithms only.** `alg` is attacker-controlled; accepting HS256 + would let anyone holding the provider's *public* key forge a token, because + for HMAC that is also the verification key. + +Discovery and JWKS are cached for an hour. An unknown `kid` — what key rotation +looks like — triggers one refetch, rate-limited to once a minute per provider: +without the refetch a rotation breaks every login until the cache expires; +without the limit, a stream of junk `kid`s becomes a denial-of-service against +the provider and against our own latency. + +**GitHub is not an OIDC provider** and is not treated as one. It issues no +`id_token`, so its logins still use the userinfo endpoint, and the code says so +rather than reporting validation it did not perform. +`PANGOLIN_OIDC_REQUIRE=true` refuses any provider that cannot be +OIDC-validated; it is off by default because enabling it would break a working +GitHub deployment on upgrade. + +Configuration: `PANGOLIN_OIDC_REQUIRE`, and `PANGOLIN__ISSUER` to +point at a self-hosted Keycloak, Auth0, private Okta or internal IdP. Google, +Microsoft and Okta issuers are derived automatically. + +Tested against a `wiremock` provider serving a real discovery document and JWKS, +with tokens signed by a real 2048-bit RSA key — nothing mocked at the crypto +layer, because the properties under test *are* the crypto. The suite was checked +for being load-bearing: disabling audience validation makes the confused-deputy +test fail, and weakening the nonce comparison makes the replay test fail. The +first attempt at that check was inconclusive for the audience case, because +`jsonwebtoken` also rejects a token carrying an `aud` when none is configured; +the validation errors now name which check fired, which both improves the logs +and makes the test able to tell. + +### Added — rate limiting on the authentication endpoints (C-5) + +The login endpoint had no throttle of any kind and was brute-forceable. There +were global concurrency and body limits and a request timeout, but nothing made +the thousandth password guess cost more than the first. Bcrypt slowed each +attempt, which raises the price of a broad campaign and does nothing against a +targeted guess at one weak password - while making the endpoint an efficient way +to burn the server's CPU. + +Throttled on two keys, because either alone has a blind spot: + +* **by source address** — bounds one attacker working through many accounts; +* **by account** — bounds many addresses working on one account, which is the + shape of a credential-stuffing run and which a per-address limit cannot see. + +Both are checked before any password verification, so a refused attempt costs a +cache lookup rather than a bcrypt round. A successful login clears the account's +counter, so mistyping a password twice and then getting it right does not leave +you near the limit. Refusals answer `429` with `Retry-After` and increment +`pangolin_auth_throttled_total`, which is worth alerting on. + +`X-Forwarded-For` is honoured **only** when `PANGOLIN_TRUST_FORWARDED_FOR=true`. +Trusting it unconditionally would let a caller set a fresh value per request and +bypass the per-address half entirely - protection that reads as protection and +is not. + +Configuration: `PANGOLIN_AUTH_RATE_LIMIT` (default 10, 0 disables), +`PANGOLIN_AUTH_RATE_WINDOW_SECS` (default 60), `PANGOLIN_TRUST_FORWARDED_FOR` +(default false). + +Known limitation, stated rather than buried: the counters are in-process, so the +limit is **per replica**. With N replicas an attacker gets N times the budget. + +`main` now serves through `into_make_service_with_connect_info`, without which +the peer address is not available and every attempt would share one bucket. + +## [0.7.0] — 2026-08-10 + +Implements `roadmap_aug10.md`, the full-repo audit of 2026-08-10. **This is a +security release.** Five of the fixes below are exploitable by any +authenticated principal, including the lowest-privilege tenant user and any +service-user API key. If you run 0.6.0, upgrade and rotate tokens. + +### Security + +- **Any authenticated caller could mint a `Root` JWT for any tenant (B0a).** + `POST /api/v1/tokens` took no session at all and mapped a body-supplied + `roles: ["Root"]` straight into signed claims. Since `check_permission` + short-circuits for `Root`, this was a total privilege escalation reachable by + a tenant user. Minting is now restricted to `Root`, or to a `TenantAdmin` + within its own tenant and never above its own rank. +- **Any tenant member could vend read+write cloud credentials for the whole + warehouse (B0b).** The credential endpoint performed no authorization, never + looked the table up, and hardcoded `["read", "write"]` - so it issued + credentials for tables the caller had no rights to and that need not exist. + It now resolves the asset, requires `Read`, and adds `"write"` only when + `Write` is actually held. +- **Logout did not revoke anything (B0j).** Revocation is keyed by the token's + `jti`; the handler revoked `session.user_id`, which no token ever carries as + its `jti`. Logout returned 200 and the token kept working for its full + 24-hour lifetime. `UserSession` now carries the `jti`. +- **An expired service user could renew indefinitely (B0g).** The Iceberg OAuth + token endpoint checked `active` but not expiry, so an expired API key could + still exchange `client_credentials` for a fresh JWT - bypassing key expiry + entirely, and renewably. +- **`PANGOLIN_DEV_MODE` waived the `NO_AUTH` public-bind guard (B0h).** The two + flags are routinely set together in compose and dev setups, and together they + started a server on `0.0.0.0` that treated every anonymous request as + `TenantAdmin`. Dev mode now relaxes secret strength only, never exposure. +- **A tenant-wide grant applied across tenants (B0i).** `PermissionScope::Tenant` + matched without comparing the grant's tenant to the resource's. +- **OAuth linked accounts by unverified email (B0l).** Anyone who could set a + matching address on any configured provider - GitHub reports unverified ones - + logged in as that Pangolin user, including the seeded tenant admin. Identity + is now `(provider, subject)`; email linking needs a verified address and an + operator domain allowlist. +- **The OAuth login flow could not complete (B0k).** `POST /api/v1/oauth/exchange` + was not in the public-path allowlist, so the endpoint whose job is to issue + the first token demanded one. The browser landed with a `?code=` it could + never redeem. +- Missing authorization on `rename_table` (B0c), `update_namespace_properties` + (B0d), view create/read (B0e), `perform_maintenance` (B0f), `rebase_branch` + and `delete_business_metadata`. `perform_maintenance` additionally ran + destructive snapshot expiry against a hardcoded `"default"` catalog rather + than the one in the path. +- A caller-supplied `expires_in_hours` could panic token issuance and abort the + connection task (B0m); clamped, plus a `CatchPanicLayer`. +- A malformed or absent `jti` skipped the revocation check entirely, making such + tokens unrevocable for their lifetime (B0o). +- The admin CLI and Python SDK wrote their auth tokens world-readable, and + `generate-code` / `get-token` echoed live JWTs into copy-paste output + (B_cli7, B_sdk4). +- A live PyPI API token was removed from the repo-root `.env` (B44). **It was + present in plaintext and must be rotated.** + +### Fixed + +- **Storage backends disagreed with each other in ten ways (B1-B7, B17-B30).** + A cross-tenant audit read on Mongo, a revocation that was a silent no-op on + Mongo, a SQLite branch delete that orphaned its assets and referenced a + column that does not exist, a Postgres search that panicked on any hit, a + Mongo compare-and-swap that lost Iceberg snapshots, a memory index that + broke another tenant's lookups, and all three persistent backends silently + rewriting 15 of the 17 asset types to `IcebergTable`. Plus pagination that + could repeat or skip rows everywhere, and four different answers to the same + search. +- **Two defects the new parity suite found on its first run.** `SqliteStore` + had no inherent `revoke_token`/`is_token_revoked`, so the trait delegations + called themselves - revoking a token on SQLite recursed until the stack was + exhausted and *aborted the process*. And the SQLite `audit_logs` table still + declared its original column set while the code inserted the full entry, so + every audit write failed and the backend kept no audit trail at all. +- **Iceberg metadata was not spec-conformant (B11-B16o).** `default-spec-id` + was written under the wrong name, the required `last-partition-id` was + absent, schemas omitted `"type": "struct"`, and `metadata-log` was never + appended - so metadata Pangolin wrote could not be read as v2 metadata by an + external engine. On the commit path: nested namespaces registered under one + key and looked up under another (every commit to one 404'd), a client could + jump the sequence counter to `i64::MAX` and overflow the next commit, a + feature-branch commit moved `main`, `last-updated-ms` only advanced on + snapshots, `-1` resolved against the whole list rather than what the commit + added, `create_table` returned the table directory as `metadata-location` + and dropped every complex-typed column from the schema, and lost + compare-and-swaps orphaned metadata files. +- **`docker compose up` could not start the API (B8-B10).** No signing secret + was set and the server has refused to start without one since 0.6.0. Both + compose files also set a storage variable nothing reads, and the release + compose file pinned an image four versions old and ran a script that does + not exist. +- **The management UI was disconnected from the server (B31-B37).** Four + spellings of the API base URL coexisted and none agreed, so every deployed + build called the visitor's own localhost; ~13 raw `fetch('/api/v1/...')` + calls 404'd outside the dev proxy and skipped the tenant header; three + endpoints the UI called did not exist; nothing handled a 401, so an expired + token left a permanently broken session; tag-filtered search 400'd end to + end; there was no way to create a catalog from the catalogs page; and the + tenant switcher was dead code referencing an unimported store. +- **Both CLIs and the SDK called endpoints that do not exist (B_cli1-8, + B_sdk1-5).** Roughly 35 sites: wrong paths, wrong field names, wrong types, + commands that were `Ok(())` stubs reporting success. All of it survived + because every command swallowed its error and exited 0. + +### Fixed — found by running the suites against live databases + +The parity suite was written against memory and SQLite, the two backends CI +could run without a service container. Pointing it at a live PostgreSQL and +MongoDB for the first time failed on both. None were regressions; all had been +present for as long as the code had. + +- **PostgreSQL: asset search was broken outright.** No migration ever created + `business_metadata`, while `search_assets` joined it — so every search failed + with `relation "business_metadata" does not exist`, a hard SQL error rather + than an empty result. The three CRUD methods were unimplemented, so the + trait's "Operation not supported by this store" default answered them. Added + the migration and the implementation. +- **MongoDB: role assignments were unreadable.** `bson::to_document` writes a + `Uuid` as a string while the deserializer expects BSON Binary, so + `assign_role` wrote documents that `get_user_roles` could never match and + that could not be deserialized at all. Every role-derived permission silently + vanished: **a user holding an admin role was authorized as though they held + none.** The same asymmetry caused B1 and B2 in two other collections; one + helper now covers all of them. +- **MongoDB: `get_metadata_location` had no fallback** to the asset's own + `location`, unlike the other three backends. A table created with a location + but no explicit metadata-location property reported none, so its metadata + could not be loaded and its commits compared against a different value than + the read path returned. +- **MongoDB: the "no transaction support" fallback was unreachable.** + `start_transaction` is a local call in the Rust driver and cannot fail for + want of a replica set; the error arrives on the first operation *inside* the + transaction and was propagated rather than caught. `delete_catalog` failed + outright on any standalone `mongod` instead of degrading as its own comment + promised. +- **SQLite: the `audit_logs` fix did not reach existing databases.** The schema + file is written with `CREATE TABLE IF NOT EXISTS`, which does nothing when the + table already exists — so fresh installs got the corrected columns and every + upgraded database kept the broken ones, with a bumped version number now + claiming otherwise. Added a real v1→v2 migration, keyed off table + introspection rather than the recorded version, with the old table preserved + as `audit_logs_pre_v2`. + +### Fixed — the MongoDB UUID encoding audit + +The string/Binary asymmetry above had by then been fixed four times, in four +collections, each time as its own bug. Auditing every collection at once — with +a round-trip test per entity rather than per feature — found four more, and a +second encoding disagreement nobody had noticed. + +There are three ways this codebase converts a `Uuid` to BSON and they all differ: +`to_bson_uuid` gives Binary with the generic subtype, `doc! { "k": uuid }` gives +Binary with the *UUID* subtype, and `bson::to_document` gives a string. Reads +disagree too: a typed `Collection` demands binary, `bson::from_bson` demands a +string. A write and a read chosen independently agree only by luck, and when they +do not, nothing fails loudly — the filter just matches nothing. + +- **Every service-user method was a no-op.** `create_service_user` let Mongo + generate an `ObjectId` while the four by-id methods filtered on + `{"_id": }`, which matched nothing; the tenant listing and the + API-key lookup used snake_case field names for a kebab-case struct; and + `update_service_user_last_used` wrote to a field no reader looks at. The + consequence that matters: **API-key authentication could never resolve a + service user on MongoDB.** It fails closed, so this was an outage of + service-user auth rather than a bypass. Changing a role also wrote the Rust + variant name instead of its serde form, making the record unreadable + afterwards. +- **Business metadata could be written but never read.** Only `asset-id` was + rewritten as Binary; `id`, `created-by` and `updated-by` kept the string form, + so `get_business_metadata` failed on the first of them. Writing metadata made + an asset's metadata permanently unreadable. +- **Listing active tokens failed outright.** `store_token` writes timestamps as + BSON DateTime, which `bson::from_bson::>` rejects — chrono wants + an RFC3339 string. The `created_at` arm swallowed the same error and + substituted `now()`, so even without the hard failure every token would have + reported the listing time as its creation time. +- **A branch with a head commit could not be read.** `create_branch` writes the + head through `doc!` (Binary, UUID subtype) and the reader accepted only a + string. A freshly created branch has no head, so this only bit once a branch + had been committed to — which is why it survived every existing test. + +`from_bson_uuid` now accepts all three encodings, so records already written by +any of them still load, while writes go through `to_bson_uuid` alone. +`mongo_uuid_round_trip_tests.rs` covers all 21 collections and runs against both +MongoDB topologies in CI. + +### Fixed — test environment drift + +- **`docker-compose.db-test.yml` had no object store.** The store compliance + tests exercise file IO; with no S3 they fall through to the EC2 + instance-metadata endpoint, hang for eleven seconds and fail with a + credentials error that names nothing relevant. CI had MinIO and the documented + local workflow did not, so the two disagreed about what it takes to run the + suite. +- **The MinIO image CI pulled no longer exists.** `bitnami/minio:latest` was + withdrawn from Docker Hub and now fails with `manifest unknown`. Both CI and + the compose file use `minio/minio` with an explicit bucket-creation step — + `warehouse` for the application, `bucket` and `test-bucket` for the compliance + tests, whose absence surfaces as `NoSuchBucket`. + +### Fixed — the management UI + +The UI job in CI built the app and never ran its tests, so the suite had drifted +to **40 failures across 14 files**. Most could not have passed: the global test +setup replaces `$lib/api/catalogs`, `$lib/api/warehouses`, `$lib/stores/auth`, +`$lib/stores/tenant` and `$lib/stores/notifications` with stubs, so the unit +tests *for those modules* were asserting against the stub rather than the code. +Others were asserting behaviour the app no longer had. Running them turned up +real defects underneath: + +- **A root user could not create another root user.** The role option's value + was `Root`; the server's `UserRole` is kebab-case, so the request was + rejected. The same PascalCase leftovers meant a tenant admin was shown an Edit + control for root users (`row.role !== 'Root'` never matched), and the role + badge colours on the users page keyed off `Root`/`TenantAdmin`, which the API + never returns. +- **A warehouse created in the UI showed no bucket in the UI.** The list page + read `s3.bucket`/`azure.container`; the create form writes plain + `bucket`/`container`. Both conventions are now accepted on read, as the server + already does. The warehouse table also rendered its Type column twice, in + place of the Region column its own template already had a branch for. +- **`production` is not a branch type.** The API knows exactly `ingest` and + `experimental`, but the UI's types declared `'experimental' | 'production'` + and the green badge keyed off `production` - so every branch rendered as + though it were experimental, and `ingest`, the type that carries a distinct + permission, had no representation at all. +- **A branch with no recorded parent was displayed as branching from `main`**, + claiming a lineage the data does not contain. +- **A local catalog could be created with no storage location and no + warehouse**, leaving it with nowhere to write its tables. The server accepts + it (the field is optional there, for federated catalogs), so nothing rejected + it. +- **The role select on the user edit page had no accessible name** - the label + named nothing - and its fallback value, `TenantUser`, matched none of the + options, so a user record without a role showed an empty select. +- **`logout()` could throw**, skipping the caller's redirect and stranding the + user on a page they were no longer authenticated for, if the token-revocation + call misbehaved. Its `void`/`.catch` pair only covered a rejected promise. +- Every unparameterised list call left a bare `?` on the end of its URL. + +`DataTable` moved from `createEventDispatcher` to callback props, with its six +consumers. That is the Svelte 5 idiom, and it is what makes the component +testable at all: `component.$on(...)` was removed in Svelte 5, so the row-click +test had been left as a stub that asserted nothing. + +CI now runs `npm test`, and carries a `svelte-check` error budget on the same +ratchet as the clippy one. `svelte-check` errors fell from 166 to 150 - mostly by +giving `StorageConfig` the index signature the server's free-form +`HashMap` always implied. + +Note that the app runs in Svelte 5's **legacy mode**: none of its 90 components +use runes. That is supported and works; converting them is a separate piece of +work and has not been done here. + +### Fixed — the cloud-credential features had never compiled + +Found while bumping dependencies: `cargo check -p pangolin_api --features +cloud-credentials` fails, and had been failing at every version. So did each of +`aws-sts`, `azure-oauth` and `gcp-oauth` individually. Nothing built with +`--features`, so nothing noticed. + +For a catalog whose job includes vending scoped, time-limited cloud +credentials, that is the feature set. Every deployment using it was running +without it. + +The errors were the kind that only appear when a `cfg` block is never +type-checked: + +- parameters bound as `_duration`, `_resource_path`, `_permissions` to silence + unused warnings in the default build, then referenced as `duration`, + `resource_path`, `permissions` inside the feature block — three files, five + bindings; +- `anyhow!` used in `gcp_signer.rs` with only `anyhow::Result` imported; +- `creds.expiration()` fed to `chrono::DateTime::parse_from_rfc3339`, but it + returns an `aws_smithy_types::DateTime`, not text — so the STS credential + expiry was parsed from a value that was never a string. + +`aws-sdk-sts` was also pinned to an exact `=1.50.0` with no comment. It was +protecting nothing — the feature failed identically at that version — and it +blocked `aws-config` from reaching a release that drops the second, vulnerable +TLS stack. Relaxed to `1.109`. + +A `features` CI job now builds each optional feature, and `pangolin_store`'s +`azure` and `gcp` backends alongside them. + +### Changed — minimum supported Rust version is now 1.94 + +Raised from 1.92 to pick up the AWS SDK releases carrying fixed `aws-lc-sys` +and `rustls-webpki`. That is what cleared the certificate-validation +advisories — two of them high severity, on the path Pangolin uses to reach S3, +Azure Blob Storage and GCS. + +`rust-version` is a promise to consumers and nothing verified it: every job ran +`stable`. An `msrv` job now reads the declared version out of the workspace +manifest and builds with exactly that toolchain, so the floor is checked rather +than asserted. `Dockerfile`, `README.md`, `CONTRIBUTING.md` and the deployment +guide are all in step. + +### Security — dependency advisories + +`cargo audit` reported 26 vulnerabilities, the one job still red after the CI +repair. Now zero, by a combination of upgrades and eight deliberate, +individually justified exceptions in `.cargo/audit.toml`. + +Cleared by upgrading: the `aws-lc-sys` cluster (including two high-severity +certificate-validation bypasses and a PKCS7 signature-validation bypass), +`rustls-webpki` name-constraint and CRL-parsing defects, `quinn-proto`, +`hickory-proto`, `bytes`, `crossbeam-epoch`, `time`. + +Accepted, with the reason recorded against each ID: `rsa`'s Marvin attack +(never compiled — `sqlx-mysql` is an optional dependency this workspace does +not enable), `quick-xml` (held by `object_store` 0.11 and by `azure_core` 0.20, +which is behind an optional feature), the second `rustls-webpki` copy that +arrives via the AWS SDK's `rustls` 0.21, `http-types`, and `rand`'s +custom-logger unsoundness. The ignore list names specific advisory IDs, so a +new advisory — including a new one against these same crates — still fails CI. + +### Fixed — a regression caught by the release gate + +Found by running the new release smoke test against the built 0.7.0 image, and +fixed before 0.7.0 shipped. + +- **The server exited 25 seconds after startup, having received no signal.** + The B16n change meant to bound the shutdown *drain* was written as + `tokio::time::timeout(shutdown_grace, serve)`. `serve` is the whole server, + not the drain, so it bounded the lifetime of the process: + `PANGOLIN_SHUTDOWN_GRACE_SECS` became a countdown to a clean exit rather than + a limit on how long draining may take. Every container would have + crash-looped. The deadline is now armed inside `shutdown_signal`, only once a + signal has actually been seen. + + All 18 CI jobs passed with this present, as did the full workspace suite: + nothing ran the binary for longer than the 25-second default. The `docker` + job now starts the built image with a 5-second grace, waits 20 seconds, and + fails if it is no longer serving - then checks it still stops promptly when + told to. + +- **Release verification inherited the developer's `.env`.** Compose auto-loads + it, so a local `PANGOLIN_ROOT_USER` / `PANGOLIN_ROOT_PASSWORD` fed the + harness - and where that password is a placeholder, the server's config guard + refuses to start, so verification failed for reasons having nothing to do + with the artifact. The harness now takes `RELEASE_*` names that cannot + collide with a real deployment's. + +- **The release stack shared a Compose project with the development database + stack**, both deriving `pangolin` from the directory name, so `up` and + `down -v` in one stopped containers belonging to the other. It now declares + `name: pangolin-release`, and its MinIO no longer publishes host ports it + never used. + +### Added + +- **A permission matrix test (improvement #0).** Drives each sensitive route as + Root / tenant admin / ungranted tenant user / foreign tenant admin and + asserts the expected 200 or 403. Every bug in the authorization cluster was + invisible to CI precisely because nothing asserted this. +- **A cross-backend parity suite (improvement #1).** Runs the same assertions + against memory, SQLite, Postgres and Mongo. Nearly half the storage findings + were one backend diverging from the others, which no per-backend test can + see. +- `#[serde(deny_unknown_fields)]` on all 34 server request structs, so a client + sending the wrong field name gets a 422 naming it rather than a 200 for a + request the server silently emptied. It caught five such payloads in the + project's own tests immediately. +- CI jobs for the Python SDK, the UI, configuration drift, and the two + guardrail suites above (improvements #1-#3). Previously CI covered only Rust, + Helm and Docker - which is exactly where the SDK and UI rot happened. +- `loadNamespaceMetadata`, `namespaceExists`, `DELETE /api/v1/branches/{name}` + and `GET /api/v1/oauth/providers`; spec pagination (`pageToken`/`pageSize` + and `next-page-token`) and the spec error envelope across the Iceberg + handlers. +- `scripts/check_env_var_docs.sh`, which regenerates the environment-variable + reference check from `config.rs`. The old page documented three variables + that do not exist and omitted 34 that do (B43). +- `scripts/bump_version.sh`, which sets the version across all five artifacts + and their inter-crate requirements, with a `--check` mode wired into CI + (improvement #8). The "one version everywhere" property had already drifted + in two places a day after the release that introduced it. +- CI now runs the parity suite against live PostgreSQL and MongoDB service + containers, and **fails if any backend was skipped**. A skipped backend + passing silently is how two of them went untested through a security release. +- An upgrade guide at `docs/upgrading/0.6-to-0.7.md`, and a security advisory + in `SECURITY.md`. + +### Removed + +- 18 unused UI runtime dependencies, ~260 KB of tracked debug output, two + ~4k-line `.bak` store monoliths that any grep of the storage layer would hit, + 27 `console.log` calls (one logging a bearer-token prefix), and an + unauthenticated file-read route in the UI with no callers and a + blacklist-based traversal guard. + ## [0.6.0] — 2026-08-09 **This is a security release.** If you are running any earlier version, upgrade diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc1b77a..15f1881 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ recently, nothing at all for contributors — this file closes that gap. ## From clone to green -You need Rust 1.92 or newer and Docker (only for the database-backed tests). +You need Rust 1.94 or newer and Docker (only for the database-backed tests). ```bash git clone https://github.com/AlexMercedCoder/pangolin diff --git a/README.md b/README.md index 00902bd..066e9c1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Finally, pangolins are rare and specialized. They exist for a specific purpose a ## 🚀 Quick Start ### Prerequisites -- Rust 1.92+ +- Rust 1.94+ - Docker (optional, for MinIO) ### Running Locally @@ -160,27 +160,46 @@ could silently fork snapshot lineage under concurrent writers. See ### Known limitations +> For the reconciled view of what is done and what is not — across both audit +> documents and every release — see **[STATUS.md](STATUS.md)**. + + Stated plainly rather than buried: - **Administrative multi-statement operations are only partly transactional.** - As of 0.6.0 PostgreSQL wraps a cascading catalog delete, a branch delete and a - branch merge in a transaction, and MongoDB wraps a cascading catalog delete - where the deployment supports a session — a standalone `mongod` cannot. - **Creating a branch by copying assets is still issued as independent - statements**, so a failure partway through leaves the catalog partially - applied, with no rollback and no repair tool. Take a backup before large - administrative operations. + PostgreSQL wraps a cascading catalog delete, a branch delete, a branch merge + and — from 0.8.0 — creating a branch by copying assets. SQLite wraps the same + branch-by-copy path. MongoDB wraps a cascading catalog delete where the + deployment supports a session; a standalone `mongod` cannot, and MongoDB has + no atomic branch-by-copy, so the API falls back to sequential statements and + says so in the logs. On that path a failure partway through leaves the branch + incomplete — but the caller now gets a `500` naming the branch, rather than + the `200` it used to get. Take a backup before large administrative + operations. (The Iceberg table-commit path *is* safe — it uses compare-and-swap with requirement enforcement.) -- **No rate limiting.** There are global concurrency and body limits and a - request timeout, but no per-IP or per-account throttle, so the login endpoint - is brute-forceable. -- **OAuth is not full OIDC.** No PKCE, no `id_token` validation, no JWKS, no - discovery. Users are matched on provider-supplied email with no - `email_verified` check. See [docs/operations/oidc.md](docs/operations/oidc.md). -- **Warehouse cloud credentials are stored unencrypted** in the catalog database, - and the in-process warehouse cache is node-local, so a rotated credential can - be served by a peer for up to the cache TTL (5s by default). +- **Rate limiting is per replica.** The authentication endpoints are throttled + per source address *and* per account (`PANGOLIN_AUTH_RATE_LIMIT`, default 10 + per `PANGOLIN_AUTH_RATE_WINDOW_SECS`, default 60). The counters are + in-process, so with N replicas the effective limit is N times the configured + one. Set `PANGOLIN_TRUST_FORWARDED_FOR=true` **only** behind a proxy that + overwrites `X-Forwarded-For`; trusting it otherwise lets a caller set the + header per request and bypass the per-address half entirely. +- **OIDC is implemented for providers that support it** (Google, Microsoft, + Okta, and any IdP via `PANGOLIN__ISSUER`): PKCE, `id_token` + signature validation against the provider's JWKS, and `iss`/`aud`/`exp`/ + `nonce` checks. **GitHub is not an OIDC provider** — it issues no `id_token` + — so a GitHub login still relies on the userinfo endpoint; + `PANGOLIN_OIDC_REQUIRE=true` refuses it. The PKCE verifier is held in process, + so OAuth needs session affinity across replicas. See + [docs/operations/oidc.md](docs/operations/oidc.md). +- **Warehouse cloud credentials are encrypted at rest only if you configure a + key.** Set `PANGOLIN_ENCRYPTION_KEY` (`openssl rand -base64 32`); without it + they are stored in plaintext and the server says so at startup. See + [docs/operations/encryption.md](docs/operations/encryption.md), which is also + honest about what envelope encryption does not protect against. The + in-process warehouse cache is still node-local, so a rotated credential can be + served by a peer for up to the cache TTL (5s by default). - **Running more than one replica works but is unproven.** The background token cleanup job runs in every replica with no coordination, and the OAuth nonce store is in-process, so OAuth needs session affinity. @@ -217,10 +236,20 @@ than ignored. `add-sort-order`, `set-default-sort-order`, `remove-snapshots`. An unrecognised update returns `501` rather than a false `200 OK`. -**Not implemented:** `loadNamespaceMetadata` (GET on a namespace), -`namespaceExists` (HEAD), `registerTable`, `commitTransaction` (multi-table -atomic commits), and most of the view API — no list, drop, replace, exists or -rename. +**Implemented since 0.8.0:** `loadNamespaceMetadata`, `namespaceExists`, +`registerTable` (adopting a table whose metadata already exists in storage), and +the view API's `listViews`, `viewExists` and `dropView`. + +**Still not implemented:** + +- `commitTransaction` (multi-table atomic commits). This is **deliberate**, not + an oversight. The spec promises that either every table in the transaction + moves or none does; Pangolin's commit path does compare-and-swap per table + with no cross-table transaction behind it. Routing the endpoint and committing + tables one at a time would be worse than leaving it absent — an engine that + sees it will rely on atomicity that is not there. Clients currently fall back + to per-table commits, which is what actually happens. +- `replaceView` and `renameView`. --- diff --git a/SECURITY.md b/SECURITY.md index 454d965..9364535 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -41,8 +41,130 @@ security fixes. | Version | Supported | |---|---| -| 0.6.x | Yes | -| 0.5.x and earlier | No — upgrade to 0.6.x | +| 0.7.x | Yes | +| 0.6.x | No — upgrade to 0.7.x, see the advisory below | +| 0.5.x and earlier | No — upgrade to 0.7.x | + +## Fixed in 0.7.0 + +**Every version before 0.7.0 is affected by a privilege-escalation +vulnerability and should not be run.** Upgrade to 0.7.0 and rotate every issued +token. The findings below come from a full-repo audit conducted the day after +the 0.6.0 release; the authorization cluster was outside that release's scope +and is new information, not a regression. + +Note which version you are actually running. The published container image +`alexmerced/pangolin-api` was last pushed at **0.5.1** — no 0.6.0 image was +ever released — so most deployments are on 0.5.1, which carries everything +described here *and* everything 0.6.0 fixed. The affected range is +`< 0.7.0`, not `0.6.0` alone. + +### Exploitable by any authenticated principal + +Every issue in this table needs nothing but a valid credential — including the +lowest-privilege `tenant-user` account, or any service-user API key. + +| ID | Issue | Impact | +|---|---|---| +| B0a | `POST /api/v1/tokens` took no session and mapped a body-supplied `roles: ["Root"]` straight into signed claims | **Full privilege escalation.** Any authenticated caller could mint a valid `Root` token for any tenant. `check_permission` short-circuits for `Root`, so the resulting token bypasses every subsequent authorization check in the system | +| B0b | The credential-vending endpoint performed no authorization, never resolved the table, and hardcoded `["read", "write"]` | **Cloud storage credential disclosure.** Any tenant member obtained read+write credentials for the entire warehouse, naming a table they had no rights to and which need not exist | +| B0j | Logout revoked `session.user_id`; revocation is keyed by the token's `jti`, which no token carries as its `user_id` | **Logout did nothing.** The token stayed valid for its full 24-hour lifetime. On a shared machine, "signing out" left a working credential behind | +| B0g | The Iceberg OAuth token endpoint checked `active` but not expiry, unlike the API-key path | **Expired credentials were renewable indefinitely.** An expired service user could exchange `client_credentials` for a fresh JWT, repeatedly | +| B0h | The `NO_AUTH` public-bind guard read `no_auth && !dev_mode && !is_loopback(..)` | **Unauthenticated tenant-admin access.** `PANGOLIN_NO_AUTH=true` with `PANGOLIN_DEV_MODE=true` — routinely set together in compose and development setups — started happily on `0.0.0.0` and treated every anonymous request as `TenantAdmin` | +| B0l | OAuth matched existing users on email with no `email_verified` check and no provider binding | **Account takeover.** Anyone able to set a matching address on any configured provider — GitHub permits unverified addresses — logged in as that Pangolin user, including the seeded tenant admin | +| B0i | `PermissionScope::Tenant` matched without comparing the grant's tenant to the resource's | A tenant-wide grant issued in one tenant satisfied authorization for another tenant's resources | +| B0c–B0f | `rename_table`, `update_namespace_properties`, view create/read and `perform_maintenance` performed no authorization check at all | Any tenant member could move any table (an effective delete), rewrite namespace properties including `location`, read any view's SQL, and trigger snapshot expiry and orphan-file deletion. `perform_maintenance` additionally ran against a hardcoded `"default"` catalog rather than the one addressed | +| B0m | `expires_in_hours` reached `chrono::Duration::hours` unclamped, and no catch-panic layer was installed | **Remote panic.** A large value aborted the connection task, taking other in-flight requests on that connection with it | +| B0o | A token whose `jti` was absent or unparseable skipped the revocation check entirely | Such tokens were unrevocable for their full lifetime | + +### Availability + +| ID | Issue | Impact | +|---|---|---| +| — | `SqliteStore` had no inherent `revoke_token`/`is_token_revoked`, so the trait implementations called themselves | **Remote crash on the SQLite backend.** Revoking a token — which logout does — recursed until the thread stack was exhausted and aborted the process | +| B2 | On MongoDB the revocation write and the revocation check used different field names *and* different types | Revocation could never match. Revoked tokens, including after logout, stayed valid | + +### Confidentiality and integrity of records + +| ID | Issue | Impact | +|---|---|---| +| B1 | MongoDB's `get_audit_event` discarded the `tenant_id` parameter | **Cross-tenant audit disclosure.** Any tenant holding an audit-event UUID could read another tenant's record: username, IP address, resource names and metadata | +| — | The SQLite `audit_logs` table declared different columns than the code inserted | **No audit trail at all on SQLite.** Every audit write failed at runtime. If you run SQLite, assume you have no audit history prior to 0.7.0 | +| — | The admin CLI and Python SDK wrote their stored bearer tokens at the process umask, typically `0644` | Any local account could read the token. Now `0600` under a `0700` directory | +| — | `generate-code` and `get-token` printed live JWTs into copy-paste output | Tokens landed in shell scrollback, session transcripts and pasted snippets | +| B44 | A live PyPI API publish token was committed to the repository working tree in `.env` | Never tracked by Git, but readable by any local tool and passed into containers by `docker compose`. **The token has been removed and must be treated as compromised** | + +### Data loss + +Not security boundaries, but silent corruption is worth the same attention: + +* **B5** MongoDB's `update_metadata_location` ignored the compare-and-swap + entirely. Two concurrent Iceberg commits both reported success and one + snapshot was lost. Memory, Postgres and SQLite all enforced it. +* **B3** SQLite's `delete_branch` referenced a column that does not exist and + was not transactional: the branch was committed away and its assets orphaned + permanently, while the caller received an error suggesting nothing happened. +* **B7** All three persistent backends stored the `Debug` spelling of + `AssetType` and parsed only two of seventeen variants. A `DeltaTable`, + `MlModel` or `Lance` asset round-tripped as an Iceberg table. +* **B4** Postgres decoded a `TEXT[]` column as `String`; `sqlx::Row::get` + panics on a decode failure, so any search returning at least one hit panicked + the request. + +### Credential vending could not be built + +The `aws-sts`, `azure-oauth` and `gcp-oauth` features - and the +`cloud-credentials` bundle that unions them - did not compile, at any version. +`cargo build` and `cargo test` run with default features and no job ever passed +`--features`, so the entire cloud-credential surface had rotted into code that +could not be built at all: parameters bound as `_name` and referenced as `name` +inside the `cfg` block, a missing macro import, and an STS expiry parsed from a +`DateTime` as though it were an RFC3339 string. + +This is not an exploitable defect - unbuildable code ships in no binary. It +matters because it means **STS-based credential vending was not running +anywhere**, so any deployment that believed it was handing out scoped, +time-limited credentials was not. Check what your warehouses are actually +configured with: if `use_sts` is set but the server was built without the +feature (which is to say, always), the static-credential fallback in +`S3Signer::generate_credentials` is what answered - vending your long-lived +warehouse keys, with no expiry, instead of a scoped session token. Where no +static keys were configured either, the call failed with "AWS credentials not +configured", which at least failed closed. + +Fixed in 0.7.0, with a CI job that builds every optional feature so it cannot +recur silently. + +### Upgrading to 0.7.0 + +1. **Rotate every issued token.** B0a means any account may have minted a + `Root` token, and B0j means logging out never invalidated anything. Rotating + `PANGOLIN_JWT_SECRET` invalidates all existing sessions at once and is the + fastest way to be sure. +2. **Rotate service-user API keys**, for the same reason: B0g allowed expired + keys to keep issuing fresh JWTs. +3. **Rotate any cloud storage credentials** reachable through a warehouse that + an untrusted tenant member could name. B0b vended them to anyone. +4. **Audit for the escalation.** `POST /api/v1/tokens` calls from non-admin + principals, and credential-vending calls for tables the caller had no grant + on, are the two signals. Note that on SQLite there is no audit history to + check, and on MongoDB action-filtered queries returned nothing (B23) — so a + clean audit log is not evidence of absence on those backends. +5. **Check `PANGOLIN_NO_AUTH` and `PANGOLIN_DEV_MODE`.** If both were set on a + non-loopback bind, treat the deployment as having been open to anonymous + tenant-admin access for that period. The server now refuses to start in that + configuration. +6. **Set `PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS`** if you rely on OAuth accounts + being matched to existing local users by email. Without it, identity is + `(provider, subject)` only and an address never adopts an existing account — + which is the safe default, and a behaviour change. +7. **On SQLite, upgrading migrates the `audit_logs` table** to schema version 2. + The previous table is preserved as `audit_logs_pre_v2`; it is empty in + practice, because nothing could ever write to it. +8. **Third-party API clients may need changes.** Server request bodies now + reject unknown fields with `422` instead of silently ignoring them. If a + client sent a misspelled or obsolete field, it was already being discarded — + the request was never doing what it appeared to. ## Fixed in 0.6.0 @@ -112,19 +234,40 @@ Before exposing Pangolin to anything you care about: ## Known gaps +The full reconciled list, including non-security work, is in +[STATUS.md](STATUS.md). + + Stated plainly, because a checklist that hides its limits is worse than none: -* **No rate limiting on authentication endpoints.** The login endpoint is - brute-forceable. There are global concurrency and body limits, and a request - timeout, but no per-IP or per-account throttle. -* **OAuth is not full OIDC.** No PKCE, no `id_token` signature validation, no - JWKS, no discovery document. Users are matched on the email a provider - returns, with no `email_verified` check. +* **Rate limiting is in-process, so it is per replica.** The authentication + endpoints are throttled from 0.7.0, per source address and per account, but + the counters are not shared between replicas: with N replicas an attacker + gets N times the configured budget. A shared limiter needs Redis or + equivalent, which this project does not otherwise require. +* **OIDC applies only to providers that support it.** Google, Microsoft, Okta + and any IdP configured via `PANGOLIN__ISSUER` get PKCE, `id_token` + signature validation via JWKS, and `iss`/`aud`/`exp`/`nonce` checks. GitHub + issues no `id_token`, so its logins still rest on the userinfo endpoint; set + `PANGOLIN_OIDC_REQUIRE=true` to refuse any provider that cannot be + OIDC-validated. No back-channel logout, no refresh-token handling, and the + PKCE verifier is held in process so OAuth needs session affinity. * **Symmetric HS256 JWTs with no key rotation.** Rotating the secret invalidates every session at once. -* **Warehouse credentials are stored unencrypted** in the catalog database. +* **Warehouse credentials are encrypted at rest only when a key is set.** + `PANGOLIN_ENCRYPTION_KEY` enables AES-256-GCM sealing of the credential + fields; unset, they are plaintext and the server warns at startup. The key + lives in the server environment, so this protects a stolen database and not a + compromised host. * **Audit records are not tamper-evident.** They live in the same database as application data, with no hash chaining and no WORM option. * **No MFA, password policy, or account lockout.** +* **Eight dependency advisories are accepted rather than fixed**, each with its + reasoning recorded in `.cargo/audit.toml`. None is reachable in a default + build, but they are exceptions rather than absences: `rsa` (never compiled), + `quick-xml` (held back by `object_store` 0.11 and `azure_core` 0.20), + a second `rustls-webpki` arriving via the AWS SDK's `rustls` 0.21, + `http-types`, and `rand`'s custom-logger unsoundness. Re-check the list when + dependencies move. These are tracked in `AUDIT_EXECUTION_PLAN.md` (items C-2 through C-20). diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 0000000..e0aa868 --- /dev/null +++ b/STATUS.md @@ -0,0 +1,144 @@ +# Status + +**Updated 2026-08-11.** One reconciled view of what is done and what is not. + +Two audit documents exist and are kept as historical records — +[`AUDIT_EXECUTION_PLAN.md`](AUDIT_EXECUTION_PLAN.md) (2026-08-09) and +[`roadmap_aug10.md`](roadmap_aug10.md) (2026-08-10). Both carry status headers +pointing here. **Where they disagree with this file, this file is correct.** + +## Verification standard + +Everything marked done below is verified by tests that run against live +PostgreSQL, MongoDB and MinIO, not by inspection: + +- **63 test targets, 415 tests, zero failures** +- **19 CI jobs green**, including an authorization matrix, a four-backend parity + suite, both MongoDB topologies, an MSRV check, and a build of every optional + feature +- `cargo audit` clean; clippy at a ratcheted budget of 30 (from 314) + +That standard exists because this project has repeatedly had things that +compiled, passed CI, and did not work. Twice in the 0.8.0 work alone: the +cloud-credential features had never compiled at any version, and a server that +exited 25 seconds after startup passed all 18 CI jobs and the full suite. + +## Done + +### Security + +| Item | Where | +|---|---| +| Authorization bypasses (any principal could mint a `Root` JWT; any tenant member could vend warehouse credentials) | 0.7.0 | +| OAuth token exfiltration, decorative `state` nonce, default JWT secret, auth-bypass path suffix | 0.6.0 | +| Rate limiting on the authentication endpoints, per source address **and** per account | 0.8.0 | +| Warehouse cloud credentials encrypted at rest (AES-256-GCM) | 0.8.0 | +| Dependency advisories: 26 → 0, with eight justified exceptions | 0.7.0 | +| OIDC: PKCE, `id_token` signature validation via JWKS, `iss`/`aud`/`exp`/`nonce` checks, discovery, rate-limited key rotation | 0.8.0 | + +### Correctness + +| Item | Where | +|---|---| +| Iceberg commit requirements and updates enforced rather than silently dropped | 0.6.0 / 0.7.0 | +| Four-backend parity: tenant scoping, branch scoping, serde formats, CAS, pagination | 0.7.0 | +| The MongoDB UUID encoding audit — one bug wearing eight hats across 21 collections | 0.7.0 | +| Transactional branch-create-by-copy; the API no longer returns `200` on a failed copy | 0.8.0 | +| MongoDB index management, including the uniqueness the SQL backends get from primary keys | 0.8.0 | +| `registerTable`, `listViews`, `viewExists`, `dropView` | 0.8.0 | + +### Operations + +| Item | Where | +|---|---| +| CI that actually runs — 19 jobs | 0.7.0 / 0.8.0 | +| A release pipeline that produces a release (it never had; `macos-13` was retired and hung every tag for 24h) | 0.7.0 | +| A release gate that verifies the published image over HTTP | 0.7.0 | +| Token-cleanup sweep that **runs** (it was dead code) and staggers across replicas | 0.8.0 | +| Backup/restore drilled and measured: 7s backup, 53s restore, 1345 rows | 0.8.0 | +| Load harness with measured figures, reporting client **and** server-side latency | 0.8.0 | +| Operations docs: backend parity, encryption, backup and recovery, performance, multiple replicas | 0.8.0 | + +## Not done + +Ordered by how much it would block a production deployment. + +### 1. GitHub logins cannot be OIDC-validated + +OIDC is implemented and applies to Google, Microsoft, Okta and any IdP given +`PANGOLIN__ISSUER`. **GitHub is not an OIDC provider** — it issues no +`id_token` and publishes no JWKS — so a GitHub login still rests on the userinfo +endpoint and on GitHub's own token scoping. + +`PANGOLIN_OIDC_REQUIRE=true` refuses any provider that cannot be +OIDC-validated. It is off by default because turning it on would break a working +GitHub deployment on upgrade with no warning; an operator who wants every login +validated should set it. + +Also outstanding on this path: no back-channel logout, no refresh-token +handling, no per-tenant provider configuration, and the PKCE verifier is held in +process — so OAuth needs session affinity across replicas. + +### 2. Multi-replica is constrained and unproven + +Works with PostgreSQL, one caveat each way: + +- OAuth requires session affinity (the nonce and code-exchange stores are + in-process) +- Rate limiting is per replica, so N replicas give N× the budget +- A rotated warehouse credential can be served by a peer for up to the cache TTL + (5s) + +Documented in [`docs/operations/running-multiple-replicas.md`](docs/operations/running-multiple-replicas.md). +**Not load tested and not soaked.** + +### 3. `commitTransaction` is absent, deliberately + +The spec promises multi-table atomicity; the store commits per table with +compare-and-swap and has no cross-table transaction. Routing the endpoint and +committing tables one at a time would be worse than leaving it out — an engine +that sees it relies on atomicity that is not there. A test pins the decision. + +### 4. Revocation fails open + +If the revocation check errors, the request proceeds (A-13). During a database +blip, every revoked token is accepted again. Watch +`pangolin_token_revocation_check_errors_total`. + +### 5. Smaller gaps + +- No tamper-evident audit trail, no SIEM export +- Symmetric HS256 JWTs with no rotation; rotating invalidates every session +- No MFA, password policy, or account lockout +- No point-in-time recovery — dump and restore only +- Session tokens are stored at rest in plaintext, not hashed +- MongoDB has no versioned schema migrations +- Management API error envelope is still flat `{"error": "..."}` +- `replaceView` and `renameView` not implemented +- The UI's 90 components are all Svelte 4 style in Svelte 5 legacy mode +- Eight accepted dependency advisories to re-check when dependencies move +- clippy 30 and svelte-check 150 backlogs, both ratcheted + +## Not shipped + +**0.7.0 and 0.8.0 are not published.** The work is merged to the branch and CI +is green, but the merge, tag, Docker push and PyPI upload have not been made. +The most recent published artifact is `alexmerced/pangolin-api:0.5.1` from +2025-12-30 — so **anything running Pangolin today is on 0.5.1**, which predates +every security fix listed above. + +The `SECURITY.md` advisory covers `< 0.7.0` for that reason. + +The PyPI token has been rotated. Still requiring a person: decide whether to +publish a GHSA once a fixed version actually exists. + +## If you are deciding whether to run this + +The honest summary: the security holes found in the audits are fixed and there +is now CI that would catch them coming back. For untrusted multi-tenant use, the +remaining gaps are multi-replica being unproven under load, GitHub logins not +being OIDC-validatable, and revocation failing open. + +The smallest credible posture today: PostgreSQL, one replica, OAuth disabled, +network-restricted, `PANGOLIN_ENCRYPTION_KEY` set, a backup you have actually +restored once, and the drill script run against a copy of your own data. diff --git a/deployment_assets/demo/evaluate_single_tenant/docker-compose.yml b/deployment_assets/demo/evaluate_single_tenant/docker-compose.yml index e22d972..fede60c 100644 --- a/deployment_assets/demo/evaluate_single_tenant/docker-compose.yml +++ b/deployment_assets/demo/evaluate_single_tenant/docker-compose.yml @@ -44,7 +44,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=memory + - PANGOLIN_STORAGE_TYPE=memory - PANGOLIN_NO_AUTH=true - PANGOLIN_ROOT_USER=root - PANGOLIN_ROOT_PASSWORD=rootpass diff --git a/deployment_assets/helm/pangolin/Chart.yaml b/deployment_assets/helm/pangolin/Chart.yaml index 87a62bc..9443db0 100644 --- a/deployment_assets/helm/pangolin/Chart.yaml +++ b/deployment_assets/helm/pangolin/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pangolin description: A Helm chart for deploying the Pangolin Lakehouse Catalog type: application -version: 0.6.0 -appVersion: "0.6.0" +version: 0.7.0 +appVersion: "0.7.0" keywords: - iceberg - lakehouse diff --git a/deployment_assets/production/azure_mongo/docker-compose.yml b/deployment_assets/production/azure_mongo/docker-compose.yml index 4dbc972..25da39a 100644 --- a/deployment_assets/production/azure_mongo/docker-compose.yml +++ b/deployment_assets/production/azure_mongo/docker-compose.yml @@ -16,7 +16,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=mongo + - PANGOLIN_STORAGE_TYPE=mongo - DATABASE_URL=mongodb://mongo:27017 - MONGO_DB_NAME=${MONGO_DB_NAME:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} diff --git a/deployment_assets/production/azure_postgres/docker-compose.yml b/deployment_assets/production/azure_postgres/docker-compose.yml index 50f21fe..d83a633 100644 --- a/deployment_assets/production/azure_postgres/docker-compose.yml +++ b/deployment_assets/production/azure_postgres/docker-compose.yml @@ -25,7 +25,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=postgres + - PANGOLIN_STORAGE_TYPE=postgres - DATABASE_URL=postgres://${POSTGRES_USER:-pangolin}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} - PANGOLIN_ROOT_PASSWORD=${PANGOLIN_ROOT_PASSWORD:-change-me-in-prod} diff --git a/deployment_assets/production/gcp_mongo/docker-compose.yml b/deployment_assets/production/gcp_mongo/docker-compose.yml index 796b3f4..12679fe 100644 --- a/deployment_assets/production/gcp_mongo/docker-compose.yml +++ b/deployment_assets/production/gcp_mongo/docker-compose.yml @@ -16,7 +16,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=mongo + - PANGOLIN_STORAGE_TYPE=mongo - DATABASE_URL=mongodb://mongo:27017 - MONGO_DB_NAME=${MONGO_DB_NAME:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} diff --git a/deployment_assets/production/gcp_postgres/docker-compose.yml b/deployment_assets/production/gcp_postgres/docker-compose.yml index 8499367..14bb187 100644 --- a/deployment_assets/production/gcp_postgres/docker-compose.yml +++ b/deployment_assets/production/gcp_postgres/docker-compose.yml @@ -25,7 +25,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=postgres + - PANGOLIN_STORAGE_TYPE=postgres - DATABASE_URL=postgres://${POSTGRES_USER:-pangolin}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} - PANGOLIN_ROOT_PASSWORD=${PANGOLIN_ROOT_PASSWORD:-change-me-in-prod} diff --git a/deployment_assets/production/s3_mongo/docker-compose.yml b/deployment_assets/production/s3_mongo/docker-compose.yml index bce15dd..ba9a147 100644 --- a/deployment_assets/production/s3_mongo/docker-compose.yml +++ b/deployment_assets/production/s3_mongo/docker-compose.yml @@ -16,7 +16,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=mongo + - PANGOLIN_STORAGE_TYPE=mongo - DATABASE_URL=mongodb://mongo:27017 - MONGO_DB_NAME=${MONGO_DB_NAME:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} diff --git a/deployment_assets/production/s3_postgres/docker-compose.yml b/deployment_assets/production/s3_postgres/docker-compose.yml index 11cd71a..b9b5378 100644 --- a/deployment_assets/production/s3_postgres/docker-compose.yml +++ b/deployment_assets/production/s3_postgres/docker-compose.yml @@ -25,7 +25,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=postgres + - PANGOLIN_STORAGE_TYPE=postgres - DATABASE_URL=postgres://${POSTGRES_USER:-pangolin}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} - PANGOLIN_ROOT_PASSWORD=${PANGOLIN_ROOT_PASSWORD:-change-me-in-prod} diff --git a/deployment_assets/test/localstack/docker-compose.yml b/deployment_assets/test/localstack/docker-compose.yml index 92ec71e..40d87d0 100644 --- a/deployment_assets/test/localstack/docker-compose.yml +++ b/deployment_assets/test/localstack/docker-compose.yml @@ -21,7 +21,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=memory + - PANGOLIN_STORAGE_TYPE=memory - PANGOLIN_NO_AUTH=true - AWS_ACCESS_KEY_ID=test - AWS_SECRET_ACCESS_KEY=test diff --git a/docker-compose.db-test.yml b/docker-compose.db-test.yml index 1dc184a..f8870d3 100644 --- a/docker-compose.db-test.yml +++ b/docker-compose.db-test.yml @@ -42,3 +42,48 @@ services: interval: 5s timeout: 5s retries: 5 + + # Object storage for the file-IO half of the store compliance suite. + # + # This file used to provide only the databases, so `test_*_store_compliance` + # had no S3 to talk to: it fell through to the EC2 instance-metadata endpoint, + # hung for eleven seconds, and failed with a credentials error that looked + # nothing like "you have no object store". CI has always had one, so the + # documented local workflow and CI disagreed about what it took to run the + # suite. + # + # `warehouse` is the bucket the application itself expects; `bucket` and + # `test-bucket` are fixtures the compliance tests write to. Creating only + # `warehouse` (as `docker-compose.yml` does, for the app) leaves the suite + # failing with `NoSuchBucket`. + # + # The obvious image for this is `bitnami/minio`, whose MINIO_DEFAULT_BUCKETS + # makes the init container unnecessary. Bitnami withdrew their public Docker + # Hub images, so `bitnami/minio:latest` no longer resolves at all - it fails + # with `manifest unknown` rather than anything that names the cause. + minio: + image: minio/minio + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + command: server /data --console-address ":9001" + ports: + - "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 20 + + createbuckets: + image: minio/mc + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set m http://minio:9000 minioadmin minioadmin; + for b in warehouse bucket test-bucket; do /usr/bin/mc mb --ignore-existing m/$$b; done; + exit 0; + " diff --git a/docker-compose.mongo-rs.yml b/docker-compose.mongo-rs.yml new file mode 100644 index 0000000..0f9421b --- /dev/null +++ b/docker-compose.mongo-rs.yml @@ -0,0 +1,41 @@ +# A single-node MongoDB replica set, for exercising the paths a standalone +# `mongod` cannot reach. +# +# `docker-compose.db-test.yml` runs MongoDB standalone, which is the *degraded* +# topology: no transactions and no retryable writes. Everything verified against +# it therefore exercised the fallback paths only — and the fallback in +# `delete_catalog` was itself broken for as long as the code had existed, +# precisely because nothing ever ran the other branch. +# +# A single node is enough: a replica set of one supports transactions and +# retryable writes, which is what distinguishes the two code paths. It is not a +# model of production redundancy and is not meant to be. +# +# docker compose -f docker-compose.mongo-rs.yml up -d +# export PANGOLIN_TEST_MONGO_URL='mongodb://localhost:27018/?replicaSet=rs0&directConnection=true' +# cargo test -p pangolin_store +# +# Note the port: 27018, so this can run alongside the standalone on 27017 and +# the two topologies can be tested in one sitting. +# +# No authentication here. A replica set with auth needs a shared keyfile, which +# is meaningful setup for a throwaway test fixture; the standalone compose file +# covers the authenticated path. + +services: + mongo-rs: + image: mongo:7.0 + command: ["mongod", "--replSet", "rs0", "--bind_ip_all", "--port", "27018"] + ports: + - "27018:27018" + healthcheck: + # Health is "the replica set has a primary", not "the process started". + # `mongod` accepts connections long before the set is initiated, and a + # client that connects in between fails in confusing ways. + test: >- + mongosh --port 27018 --quiet --eval + "try { rs.status().ok } catch (e) { rs.initiate({_id:'rs0',members:[{_id:0,host:'localhost:27018'}]}).ok }" + interval: 5s + timeout: 10s + retries: 20 + start_period: 10s diff --git a/docker-compose.release.yml b/docker-compose.release.yml index e2b8cda..2a4cf9c 100644 --- a/docker-compose.release.yml +++ b/docker-compose.release.yml @@ -1,11 +1,23 @@ version: '3.8' +# An explicit project name, separate from every other compose file here. +# +# Compose derives the project name from the directory, so this file, +# docker-compose.yml and docker-compose.db-test.yml all shared the project +# `pangolin` - and `up`, `down -v` and orphan cleanup in one would stop and +# remove containers belonging to another. Verifying a release while a +# development database stack is running is a completely ordinary thing to do, +# and it made the API container exit mid-verification for reasons that had +# nothing to do with the release. +name: pangolin-release + services: minio: image: minio/minio - ports: - - "9000:9000" - - "9001:9001" + # No host port publication. The API and the smoke test both reach MinIO + # over the compose network, so claiming 9000/9001 on the host bought + # nothing and made this file unrunnable whenever a development stack was + # already using them - which is exactly when someone verifies a release. environment: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin @@ -34,18 +46,45 @@ services: " pangolin-api: - image: alexmerced/pangolin-api:0.2.0 + # B10: this pinned 0.2.0 - four releases behind the workspace - so the + # "release verification" compose file verified an image nobody ships. + # Parameterised so a release can be tested by setting PANGOLIN_VERSION, + # with the current version as the default. + image: alexmerced/pangolin-api:${PANGOLIN_VERSION:-0.7.0} + # Published so a human can poke at the server under test. Override + # RELEASE_API_PORT if 8080 is taken. ports: - - "8080:8080" + - "${RELEASE_API_PORT:-8080}:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=memory + # B9: the server reads PANGOLIN_STORAGE_TYPE; PANGOLIN_STORE_TYPE was + # silently ignored. + - PANGOLIN_STORAGE_TYPE=memory + # B8: the server refuses to start without a JWT secret. + # RELEASE_* rather than PANGOLIN_*, deliberately. + # + # Docker Compose auto-loads `.env` from the project directory, so a + # developer's local PANGOLIN_ROOT_USER / PANGOLIN_ROOT_PASSWORD were + # silently feeding this file. In this working copy that meant the server + # refused to start at all - `.env` carries a placeholder password, and the + # config guard rejects it - so the release verification could not run, + # for a reason having nothing to do with the artifact being verified. + # + # A verification harness that inherits ambient local config is not + # reproducible. These names cannot collide with a real deployment's. + - PANGOLIN_JWT_SECRET=${RELEASE_JWT_SECRET:?set it first, e.g. export RELEASE_JWT_SECRET=$(openssl rand -base64 48)} - AWS_ACCESS_KEY_ID=minioadmin - AWS_SECRET_ACCESS_KEY=minioadmin - AWS_REGION=us-east-1 - AWS_ENDPOINT_URL=http://minio:9000 - AWS_ALLOW_HTTP=true - PANGOLIN_NO_AUTH=${PANGOLIN_NO_AUTH:-false} + # Root basic auth, so the smoke test can exercise a real authenticated + # write against the released image. `PANGOLIN_NO_AUTH` cannot be used for + # this: the server refuses to start with it on a non-loopback bind, which + # is every container, so that branch was unreachable here. + - PANGOLIN_ROOT_USER=${RELEASE_ROOT_USER:-releaseroot} + - PANGOLIN_ROOT_PASSWORD=${RELEASE_ROOT_PASSWORD:?set it first, e.g. export RELEASE_ROOT_PASSWORD=$(openssl rand -base64 24)} depends_on: minio: condition: service_healthy @@ -62,10 +101,23 @@ services: - AWS_ACCESS_KEY_ID=minioadmin - AWS_SECRET_ACCESS_KEY=minioadmin - AWS_REGION=us-east-1 - - TEST_MODE=${TEST_MODE:-no-auth} + # Mirrors the server's own setting rather than a separate TEST_MODE that + # nothing read, so the smoke test knows whether the authenticated surface + # is reachable. + - PANGOLIN_NO_AUTH=${PANGOLIN_NO_AUTH:-false} + - PANGOLIN_ROOT_USER=${RELEASE_ROOT_USER:-releaseroot} + - PANGOLIN_ROOT_PASSWORD=${RELEASE_ROOT_PASSWORD} depends_on: - pangolin-api - command: sh -c "pip install requests pyiceberg pyarrow && python scripts/test_release_v0.2.0.py" + # B10 pointed this at scripts/test_release_v0.2.0.py, which does not exist, + # then at scripts/integration_test.py - which builds and starts its own + # server with `cargo run`, and so cannot run in a python:3.11-slim container + # with only ./scripts mounted. Either way the release was never verified. + # + # release_smoke_test.py talks to the already-running container over HTTP, + # which is both the only thing possible here and the more useful check: it + # exercises the published image rather than a rebuild of the source. + command: sh -c "pip install requests && python scripts/release_smoke_test.py" volumes: minio_data_release: diff --git a/docker-compose.yml b/docker-compose.yml index 62c3d12..20e3a48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,17 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=memory + # B8: since 0.6.0 the server refuses to start without a JWT secret, and + # refuses PANGOLIN_NO_AUTH on the default 0.0.0.0 bind - so the documented + # quick start produced a crash-looping container with no explanation. The + # `:?` form fails the `docker compose up` itself with this message, which + # is a far better failure than a restart loop. + - PANGOLIN_JWT_SECRET=${PANGOLIN_JWT_SECRET:?set it first, e.g. export PANGOLIN_JWT_SECRET=$(openssl rand -base64 48)} + # B9: the server reads PANGOLIN_STORAGE_TYPE. This said + # PANGOLIN_STORE_TYPE, which nothing reads - so anyone editing it to + # `postgres` silently kept the in-memory backend and lost their data on + # every restart. + - PANGOLIN_STORAGE_TYPE=memory - AWS_ACCESS_KEY_ID=minioadmin - AWS_SECRET_ACCESS_KEY=minioadmin - AWS_REGION=us-east-1 @@ -57,7 +67,11 @@ services: ports: - "3000:3000" environment: - - VITE_API_URL=http://localhost:8080 + # B31: SvelteKit's dynamic public env requires the PUBLIC_ prefix. The + # client reads PUBLIC_API_URL; this used to pass VITE_API_URL, which + # nothing read, so every deployed build fell back to the *end user's* + # localhost:8080. + - PUBLIC_API_URL=http://localhost:8080 - ORIGIN=http://localhost:3000 depends_on: - pangolin-api diff --git a/docs/best-practices/deployment.md b/docs/best-practices/deployment.md index e15c4dc..3c55224 100644 --- a/docs/best-practices/deployment.md +++ b/docs/best-practices/deployment.md @@ -101,7 +101,7 @@ export PANGOLIN_JWT_SECRET=$(aws secretsmanager get-secret-value \ **Production Dockerfile** ```dockerfile -FROM rust:1.92 as builder +FROM rust:1.94 as builder WORKDIR /app COPY . . RUN cargo build --release --bin pangolin_api diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 60ed622..f510f2f 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -1,6 +1,13 @@ # Environment Variables Reference -This document provides a comprehensive reference for all environment variables used by Pangolin API. +This document provides a comprehensive reference for all environment variables +used by the Pangolin API. + +> **This page is checked against the code.** `pangolin/scripts/check_env_var_docs.sh` +> re-derives the set of `PANGOLIN_*` variables the server actually reads and +> fails if this file documents one that nothing reads, or omits one that +> something does. It runs in CI. Before that check existed, this page listed +> three variables that did not exist and omitted thirty-four that did. ## Table of Contents @@ -108,6 +115,44 @@ export JWT_SECRET=$(openssl rand -base64 32) --- +### `PANGOLIN_AUTH_RATE_LIMIT` + +**Required:** No +**Type:** Integer +**Default:** `10` +**Description:** Failed authentication attempts allowed per window, counted per +source address **and** separately per account. `0` disables throttling. The +counters are in-process, so with N replicas the effective limit is N times this. + +### `PANGOLIN_AUTH_RATE_WINDOW_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Default:** `60` +**Description:** The window those attempts are counted over. + +### `PANGOLIN_TRUST_FORWARDED_FOR` + +**Required:** No +**Type:** Boolean +**Default:** `false` +**Description:** Honour `X-Forwarded-For` when identifying the client for rate +limiting. **Only set this behind a proxy that overwrites the header.** Trusting +it otherwise lets a caller set a fresh value per request and bypass the +per-address limit entirely - protection that reads as protection and is not. + +### `PANGOLIN_ENCRYPTION_KEY` + +**Required:** No, but strongly recommended +**Type:** base64, exactly 32 bytes (`openssl rand -base64 32`) +**Description:** Encrypts warehouse cloud credentials at rest with AES-256-GCM. +Unset, they are stored in plaintext and the server warns at startup. **Not +included in a database dump** - back it up separately, or a restore produces a +catalog whose every warehouse credential is unreadable. Losing it is +unrecoverable. See [operations/encryption.md](operations/encryption.md). + +--- + ## Object Storage (S3/MinIO) These variables configure access to S3-compatible object storage for Iceberg table metadata and data files. @@ -234,34 +279,208 @@ export DATABASE_URL="/var/lib/pangolin/pangolin.db" ## Server Configuration -### `PANGOLIN_HOST` +### `PANGOLIN_BIND_ADDRESS` -**Required:** No -**Type:** String (IP address) -**Default:** `0.0.0.0` +**Required:** No +**Type:** String (IP address) +**Default:** `0.0.0.0` **Description:** IP address to bind the server to. +Note: if `PANGOLIN_NO_AUTH=true`, this **must** be a loopback address. The +server refuses to start otherwise, unconditionally - `PANGOLIN_DEV_MODE` does +not waive the check. + ```bash # Listen on all interfaces -export PANGOLIN_HOST="0.0.0.0" +export PANGOLIN_BIND_ADDRESS="0.0.0.0" -# Listen only on localhost -export PANGOLIN_HOST="127.0.0.1" +# Listen only on localhost (required when PANGOLIN_NO_AUTH=true) +export PANGOLIN_BIND_ADDRESS="127.0.0.1" ``` -### `PANGOLIN_PORT` or `PORT` +### `PORT` -**Required:** No -**Type:** Integer -**Default:** `8080` +**Required:** No +**Type:** Integer +**Default:** `8080` **Description:** Port number for the API server. ```bash -export PANGOLIN_PORT=3000 -# or export PORT=3000 ``` +### `PANGOLIN_MAX_BODY_BYTES` + +**Required:** No +**Type:** Integer (bytes) +**Default:** 10 MiB +**Description:** Largest request body the server will buffer. + +### `PANGOLIN_REQUEST_TIMEOUT_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Default:** `30` +**Description:** Deadline for a request, **including** time spent queued behind +the concurrency limiter. + +### `PANGOLIN_MAX_CONCURRENT_REQUESTS` + +**Required:** No +**Type:** Integer +**Default:** `512` +**Description:** Requests admitted concurrently; the rest queue, bounded by the +request timeout above. + +### `PANGOLIN_SHUTDOWN_GRACE_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Description:** Upper bound on the drain after SIGTERM. Readiness fails +immediately; in-flight requests then have this long to finish before the +process exits regardless. Set it below your orchestrator's termination grace +period. + +### `PANGOLIN_CORS_ALLOWED_ORIGINS` + +**Required:** No +**Type:** Comma-separated list of origins +**Default:** any origin +**Description:** Restricts CORS to the listed origins. + +### `PANGOLIN_WAREHOUSE_CACHE_TTL_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Default:** `5` +**Description:** TTL of the warehouse cache. Entries hold cloud storage +credentials and the cache is node-local, so with more than one replica a +rotated credential can still be vended by peers for up to this long. + +### `PANGOLIN_METRICS_ENABLED` + +**Required:** No +**Type:** Boolean +**Description:** Serve Prometheus metrics at `/metrics`. + +--- + +## Identity and Access + +### `PANGOLIN_JWT_SECRET` + +**Required:** **Yes**, unless `PANGOLIN_DEV_MODE` or `PANGOLIN_NO_AUTH` is set +**Type:** String (32+ characters) +**Description:** Signing key for session JWTs. The server refuses to start +without it, and rejects known-weak values. + +```bash +export PANGOLIN_JWT_SECRET="$(openssl rand -base64 48)" +``` + +### `PANGOLIN_ROOT_USER` / `PANGOLIN_ROOT_PASSWORD` + +**Required:** No +**Type:** String +**Description:** Credentials for the root basic-auth principal. Compared in +constant time. A weak password is refused outside dev mode. + +### `PANGOLIN_SEED_ADMIN`, `PANGOLIN_ADMIN_USER`, `PANGOLIN_ADMIN_PASSWORD` + +**Required:** No +**Type:** Boolean / String / String +**Description:** Seed a first tenant administrator on an empty database. + +### `PANGOLIN_SESSION_TTL_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Description:** Lifetime of an issued session token. + +### `PANGOLIN_DEV_MODE` + +**Required:** No +**Type:** Boolean +**Default:** `false` +**Description:** Relaxes *secret strength* requirements for local development. +It does **not** relax network exposure: the loopback requirement on +`PANGOLIN_NO_AUTH` applies regardless. + +### `PANGOLIN_ALLOW_LEGACY_API_KEYS` + +**Required:** No +**Type:** Boolean +**Default:** `false` +**Description:** Accept API keys minted before the key-id format. Off by +default because a legacy key costs a bcrypt verification per candidate. + +### `PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI` + +**Required:** No +**Type:** Boolean +**Default:** `false` +**Description:** Accept JWTs carrying no `jti`. Such a token can never be +revoked, so this exists only as a migration window. + +--- + +## OAuth / OIDC + +Each provider is enabled by setting its client id and secret. The redirect URI +defaults to `/oauth/callback/`. + +| Provider | Variables | +|-----------|-----------| +| Google | `PANGOLIN_GOOGLE_CLIENT_ID`, `PANGOLIN_GOOGLE_CLIENT_SECRET`, `PANGOLIN_GOOGLE_REDIRECT_URI` | +| GitHub | `PANGOLIN_GITHUB_CLIENT_ID`, `PANGOLIN_GITHUB_CLIENT_SECRET`, `PANGOLIN_GITHUB_REDIRECT_URI` | +| Microsoft | `PANGOLIN_MICROSOFT_CLIENT_ID`, `PANGOLIN_MICROSOFT_CLIENT_SECRET`, `PANGOLIN_MICROSOFT_REDIRECT_URI`, `PANGOLIN_MICROSOFT_TENANT_ID` | +| Okta | `PANGOLIN_OKTA_CLIENT_ID`, `PANGOLIN_OKTA_CLIENT_SECRET`, `PANGOLIN_OKTA_REDIRECT_URI`, `PANGOLIN_OKTA_DOMAIN` | + +### `FRONTEND_URL` + +**Required:** No +**Type:** URL +**Default:** `http://localhost:5173` +**Description:** Where the UI lives. Always an acceptable OAuth landing page. + +### `PANGOLIN_OAUTH_REDIRECT_URIS` + +**Required:** No +**Type:** Comma-separated list of exact URLs +**Description:** Additional URLs an OAuth flow may hand control back to. +Anything not listed (and not `FRONTEND_URL`) is refused. + +### `PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS` + +**Required:** No +**Type:** Comma-separated list of domains +**Default:** empty +**Description:** Domains whose **verified** addresses may adopt a pre-existing +local account. With no allowlist, identity is `(provider, subject)` only and an +email address never links an account - which is what stops someone setting a +matching address on any configured provider and logging in as that user. + +### `PANGOLIN_OIDC_REQUIRE` + +**Required:** No +**Type:** Boolean +**Default:** `false` +**Description:** Refuse any provider that cannot be OIDC-validated. GitHub +issues no `id_token` and publishes no JWKS, so with this set a GitHub login is +rejected rather than falling back to the userinfo endpoint. Off by default +because enabling it would break a working GitHub deployment on upgrade with no +warning. + +### `PANGOLIN__ISSUER` + +**Required:** No +**Type:** URL +**Description:** Override the OIDC issuer for a provider, e.g. +`PANGOLIN_GOOGLE_ISSUER`. Needed for a self-hosted Keycloak, Auth0, a private +Okta, or any internal IdP. Google, Microsoft and Okta issuers are derived +automatically from the variables above. The discovery document's own `issuer` +must match this value, or the login is refused. + --- ## Logging @@ -316,7 +535,7 @@ export AWS_ACCESS_KEY_ID="AKIAIOSFODNN7EXAMPLE" export AWS_SECRET_ACCESS_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" export AWS_REGION="us-east-1" export RUST_LOG=info -export PANGOLIN_PORT=8080 +export PORT=8080 ./pangolin_api ``` @@ -379,8 +598,10 @@ services: ### ❌ Using the wrong variable names ```bash -# WRONG - These are not used -export PANGOLIN_STORE_TYPE=mongo +# WRONG - This one is not read by anything. It appeared in the compose files +# and in this document for several releases, so editing it to `postgres` +# silently left you on the in-memory backend. +export PANGOLIN_STORE_TYPE=mongo # export MONGODB_URI=mongodb://localhost:27017 export MONGODB_DATABASE=pangolin @@ -433,7 +654,7 @@ When multiple variables could configure the same thing: 1. **Storage Backend:** `DATABASE_URL` (only this is used) 2. **MongoDB Database:** `MONGO_DB_NAME` (only this is used) 3. **S3 Endpoint:** `S3_ENDPOINT` or `AWS_ENDPOINT_URL` (both work) -4. **Server Port:** `PANGOLIN_PORT` or `PORT` (PANGOLIN_PORT takes precedence) +4. **Server Port:** `PORT` --- diff --git a/docs/getting-started/env_vars.md b/docs/getting-started/env_vars.md index 32ae9cf..b9ace1a 100644 --- a/docs/getting-started/env_vars.md +++ b/docs/getting-started/env_vars.md @@ -1,63 +1,12 @@ # Environment Variables -Pangolin is configured via environment variables. This guide lists all available options. +**This page has moved to [../environment-variables.md](../environment-variables.md).** -## 🚀 Core API Configuration +Two hand-maintained environment-variable references existed side by side and +both had drifted from the code (B43). Keeping one of them was not a fix - the +problem was that either could drift silently. -| Variable | Description | Default | -|----------|-------------|---------| -| `PORT` | The port the API server will listen on. | `8080` | -| `RUST_LOG` | Log level (`error`, `warn`, `info`, `debug`, `trace`). | `info` | - -## 💾 Metadata Persistence - -Pangolin stores its own metadata (tenants, users, catalogs) in a backend database. - -| Variable | Description | Default | -|----------|-------------|---------| -| `DATABASE_URL` | Connection string for Postgres, MongoDB, or SQLite. | (None) | -| `PANGOLIN_STORAGE_TYPE` | Storage driver if `DATABASE_URL` is missing. (`memory`, `postgres`, `mongo`, `sqlite`). | `memory` | -| `MONGO_DB_NAME` | Database name (when using MongoDB). | `pangolin` | - -> [!NOTE] -> If `DATABASE_URL` is provided, the driver is automatically inferred from the URI scheme (e.g., `postgresql://`, `mongodb://`, `sqlite://`). - -## 🛡️ Authentication & Security - -| Variable | Description | Default | -|----------|-------------|---------| -| `PANGOLIN_NO_AUTH` | **Evaluation Mode**. Auto-provisions a default tenant and admin user. Set to `"true"` to enable. | `false` | -| `PANGOLIN_JWT_SECRET` | Secret key for JWT signing. **MUST** be changed in production. | `default_secret` | -| `PANGOLIN_SEED_ADMIN` | Auto-provision a Tenant Admin even if `NO_AUTH` is false. Set to `"true"`. | `false` | -| `PANGOLIN_ADMIN_USER` | Username for seed admin. | `tenant_admin` | -| `PANGOLIN_ADMIN_PASSWORD` | Password for seed admin. | `password123` | -| `PANGOLIN_ROOT_USER` | Initial Root user for multi-tenant bootstrapping. | `admin` | -| `PANGOLIN_ROOT_PASSWORD` | Password for Root user. | `password` | - -## 🌐 OAuth 2.0 (External Providers) - -Required for enabling "Login with Google/GitHub/Microsoft" in the UI. - -| Variable | Provider | -|----------|----------| -| `OAUTH_GOOGLE_CLIENT_ID` / `_SECRET` | Google | -| `OAUTH_MICROSOFT_CLIENT_ID` / `_SECRET` | Microsoft | -| `OAUTH_GITHUB_CLIENT_ID` / `_SECRET` | GitHub | - -## ☁️ Cloud Provider Features - -When building with cloud features (`--features aws-sts`, etc.), these standard variables are used by the underlying SDKs for the **Signer** logic. - -| Variable | Description | -|----------|-------------| -| `AWS_ACCESS_KEY_ID` | AWS Credentials for STS / Signer. | -| `AWS_SECRET_ACCESS_KEY` | AWS Credentials for STS / Signer. | -| `AWS_REGION` | Default AWS region. | -| `AWS_ENDPOINT_URL` | Override for MinIO or custom S3 backends. | - -## 🚦 Security Checklist - -1. **Disable NO_AUTH**: Ensure `PANGOLIN_NO_AUTH` is unset or `false` in production. -2. **Rotate JWT Secret**: Use a 32+ character random string. -3. **Strong Root Password**: Change the default `admin/password` immediately. -4. **Use DATABASE_URL**: Avoid the `memory` store for any non-trivial use case. +The surviving reference is checked against the source in CI by +`pangolin/scripts/check_env_var_docs.sh`, which fails the build if it documents +a `PANGOLIN_*` variable nothing reads, or omits one something does. This page is +a redirect so existing links keep working. diff --git a/docs/operations/backend-parity.md b/docs/operations/backend-parity.md index 5a498e8..8bd1773 100644 --- a/docs/operations/backend-parity.md +++ b/docs/operations/backend-parity.md @@ -8,6 +8,13 @@ on which backend. **Recommended for production: PostgreSQL.** +Every ✅ below is now backed by the cross-backend parity suite +(`cargo test -p pangolin_store --test store_integration`) run against a live +instance of each backend, not by inspection. That distinction matters: until +0.7.0 the Postgres and MongoDB tests had never actually been executed, and this +table asserted several capabilities that failed at the first request — see +"What the first live run found" below. + | Capability | Memory | SQLite | PostgreSQL | MongoDB | |---|:--:|:--:|:--:|:--:| | Tenants, catalogs, namespaces, assets | ✅ | ✅ | ✅ | ✅ | @@ -52,15 +59,119 @@ a `pg_advisory_lock` so that concurrent replicas do not race. > also carried a shape the code could not write to, so every audit write failed. > Both are fixed in 0.6.0; see `migrations/20260809000000_repair_orphaned_schema.sql`. -**MongoDB.** Functional for core catalog operations. Known gaps: +**MongoDB.** Functional for core catalog operations. The RBAC and audit-log +failures listed here before 0.7.0 are fixed, along with four more found by +auditing every collection at once: they were all one bug wearing eight hats, and +"The MongoDB UUID encoding rule" below is what it takes not to add a ninth. Note +in particular that the ✅ against *service users and API keys* was wrong until +0.7.0 — every method in that module was a silent no-op, so an API key could not +authenticate on MongoDB at all. Remaining gaps: + +* **Indexes are managed; schema migrations are not.** From 0.8.0 the backend + creates the full index set at startup, derived from the filters the code + actually issues, and reports anything it could not create rather than + discarding the error. Several are `unique`, which is the constraint the SQL + backends get from a primary key — so MongoDB now rejects a duplicate catalog, + warehouse, branch or tag name in a tenant, where before it accepted them and + returned an arbitrary one. There is still no versioned migration chain: a + field added to a struct changes what new documents look like and nothing + rewrites the old ones. + + If a unique index cannot be created because the collection already holds + duplicates, startup logs an error naming the collection and continues — that + uniqueness is then *not* enforced until you deduplicate and restart. +* **A replica set is strongly recommended.** On a standalone `mongod`: + * transactions are unavailable, so `delete_catalog` degrades to a sequential, + non-atomic cascade. It now degrades with a warning; before 0.7.0 the + degradation path was unreachable and the delete failed outright. + * retryable writes are unavailable — add `retryWrites=false` to your + connection string or single-document writes will be rejected. + + Both topologies are tested. CI runs the MongoDB suites twice: once against a + standalone (the `guardrails` job) and once against a single-node replica set + (`mongo-replica-set`), because the two exercise *different branches* of + `delete_catalog` and testing one says nothing about the other. Locally: + + ```bash + # standalone — the degraded paths + docker compose -f docker-compose.db-test.yml up -d mongo + export PANGOLIN_TEST_MONGO_URL='mongodb://testuser:testpass@localhost:27017/?retryWrites=false' + + # replica set — transactions and retryable writes + docker compose -f docker-compose.mongo-rs.yml up -d + export PANGOLIN_TEST_MONGO_URL='mongodb://localhost:27018/?replicaSet=rs0&directConnection=true' + ``` + + They use different ports so both can run at once. +* Multi-statement operations other than `delete_catalog` are still not atomic. + +## What the first live run found + +The parity suite was written against the memory and SQLite backends, which are +the two that CI could run without a service container. The first time it was +pointed at a live PostgreSQL and MongoDB it failed on both, for reasons no +amount of code reading had surfaced: + +| Backend | Defect | +|---|---| +| PostgreSQL | **`business_metadata` was never created by any migration**, while `search_assets` joined it. Every asset search failed with `relation "business_metadata" does not exist` — a hard SQL error, not an empty result. The three CRUD methods were unimplemented, so the trait's "not supported" default answered them. | +| MongoDB | **`get_metadata_location` had no fallback to the asset's own `location`**, unlike the other three backends. A table created with a location but no explicit metadata-location property reported none, so its metadata could not be loaded and its commits compared against a different value than the read path returned. | +| MongoDB | **Role assignments were written by serde and queried as BSON Binary.** `bson::to_document` writes a `Uuid` as a string; the deserializer expects Binary. So `get_user_roles` never matched, every role-derived permission silently vanished, and a user holding an admin role was authorized as though they held none. The same asymmetry caused B1 (audit) and B2 (token revocation). | +| MongoDB | **`delete_catalog`'s "fall back when transactions are unavailable" path was unreachable.** `start_transaction` is a local call in the Rust driver, so it cannot fail for want of a replica set; the error arrives on the first operation *inside* the transaction and was propagated instead of caught. | +| MongoDB | Once that fallback *did* run, it carried B21 — the cascade deleted every matching child row before checking the catalog existed, then reported "not found" to a caller with every reason to believe nothing had happened. The same shape had been fixed for SQLite during the roadmap work; it survived here because this branch had never executed. | + +Postgres, notably, is the only backend that makes the orphaned-child state +*unreachable*: it carries foreign keys from namespaces to catalogs and from +assets to namespaces, so a mis-ordered cascade has nothing to destroy. The other +three permit orphans, which is why the parity suite asserts the ordering there +and records the asymmetry rather than skipping quietly. + +None of these were regressions. They had been present for as long as the code +had, and a per-backend test could not have found the first three: each is one +backend disagreeing with the others, which is only visible when something asserts +they agree. -* No schema or index management at all — indexes must be created by hand. -* No transactions: `MongoStore` opens no sessions, so multi-statement - operations are not atomic. -* RBAC aggregation and audit-log filtering have failing tests - (`test_mongo_rbac_operations`, `test_mongo_list_user_permissions_aggregation`, - `test_mongo_audit_log_filtering`, `test_mongo_store_regression`). Treat - MongoDB as beta. +## The MongoDB UUID encoding rule + +If you write a MongoDB backend method, this is the one thing to get right. + +There are three ways this codebase converts a `Uuid` into BSON, and they produce +three different values: + +| Route | Produces | +|---|---| +| `to_bson_uuid(id)` | `Binary`, generic subtype | +| `doc! { "k": id }` | `Binary`, **UUID** subtype | +| `bson::to_document(&value)` | a `String` | + +Reads disagree as well. A typed `Collection` deserializes non-human-readably +and requires binary; `bson::from_bson` requires a string. So a write and a read +chosen independently agree only by luck. + +When they disagree the code does not fail loudly. The document is written, the +filter matches nothing, and the caller gets an empty result that is +indistinguishable from "there is nothing there". That is how a user holding an +admin role came to be authorized as though they held none, how token revocation +became a silent no-op, and how every service-user method became an unremarked +no-op including the API-key lookup. + +The rule: + +* **Write** UUIDs with `to_bson_uuid`, or with `with_binary_uuids` when the + document came from `bson::to_document`. Pass the *serialized* field names — + most of these structs are `rename_all = "kebab-case"`, so it is `"tenant-id"`, + not `"tenant_id"`. Getting that wrong is its own silent no-op. +* **Read** UUIDs with `from_bson_uuid` (or `read_optional_uuid` for an optional + field), which accepts all three encodings so records already in a deployed + database still load. +* **Add a case to `mongo_uuid_round_trip_tests.rs`** for any new collection. It + runs against both topologies in CI and is the only thing that checks the write + and the read agree. + +Timestamps have the same shape of problem in the opposite direction: serde +writes a `DateTime` as an RFC3339 string, `doc!` writes a BSON DateTime, +and chrono's `Deserialize` accepts only the string. Match whatever the +collection already uses. ## Transactions @@ -75,7 +186,10 @@ rollback and no repair tooling. What exists today: -* SQLite uses a transaction in one place. +* SQLite uses transactions for branch deletion and the catalog cascade. +* MongoDB uses a transaction for the catalog cascade where the deployment + supports one, and degrades to a sequential cascade with a warning where it + does not. * PostgreSQL serialises schema setup with an advisory lock, so concurrent replicas cannot corrupt the schema. * The Iceberg table-commit path uses compare-and-swap on the metadata pointer diff --git a/docs/operations/backup-and-recovery.md b/docs/operations/backup-and-recovery.md new file mode 100644 index 0000000..f69c4c7 --- /dev/null +++ b/docs/operations/backup-and-recovery.md @@ -0,0 +1,118 @@ +# Backup and recovery + +C-16. This page exists because the previous documentation described a backup +procedure that, as far as the repository showed, nobody had ever restored from. +A backup procedure that has not been executed is a hypothesis. + +`scripts/backup_restore_drill.sh` runs the whole cycle — dump, **destroy**, +restore, verify — against a real database and fails loudly if anything does not +come back. Run it before you need it. + +## What must be backed up + +Two things, and losing either one loses your catalog. + +| What | How | If you lose it | +|---|---|---| +| The catalog database | `pg_dump` | Everything: catalogs, tables, branches, users, permissions, audit history | +| `PANGOLIN_ENCRYPTION_KEY` | Your secret manager | The database restores fine and **every warehouse credential is unreadable** | + +The second one is the trap. The key is not in the dump — it is deliberately not +in the dump — so a team that backs up the database religiously and never records +the key has a restore that produces a working catalog full of warehouses nobody +can authenticate to. Back the key up wherever you keep break-glass secrets, and +verify you can actually retrieve it. + +`PANGOLIN_JWT_SECRET` is worth recording too, though losing it is milder: every +session is invalidated and users log in again. + +Object storage is not backed up by any of this. Pangolin stores *pointers* to +Iceberg metadata; the metadata and data files live in your bucket and are +covered by whatever versioning and retention you have configured there. + +## Taking a backup + +```bash +pg_dump --format=custom --no-owner --no-privileges \ + --file=pangolin-$(date +%Y%m%d-%H%M%S).dump "$DATABASE_URL" +``` + +Use a `pg_dump` whose major version is **at least** the server's. An older +client refuses, and it refuses after you have already decided a backup exists. +The drill script checks this up front for that reason. + +## Restoring + +```bash +psql "$DATABASE_URL" -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" +pg_restore --dbname="$DATABASE_URL" --no-owner --no-privileges \ + --exit-on-error pangolin-20260811-120000.dump +``` + +`--exit-on-error` matters. Without it `pg_restore` reports success while +skipping objects it could not create, which is exactly the failure a restore +drill is looking for. + +Then start the server and check `/health/ready`, which probes the store rather +than just answering `200`. + +## Verifying a restore + +The drill checks three things beyond "the command exited zero": + +1. **A canary row survives.** Row counts alone would compare equal if the + restore silently produced an empty database. +2. **Row counts match** across catalogs, warehouses and assets. +3. **`_sqlx_migrations` is populated.** If the migration ledger does not + restore, the next server start tries to re-run every migration against a + database that already has the objects, and fails with + `relation "tenants" already exists`. This is not hypothetical — it happened + during development when migrations were applied by hand. + +## Measured figures + +From an actual run of the drill on a developer laptop, against PostgreSQL 15 in +Docker with 1,345 catalog/warehouse/asset rows: + +| Phase | Time | +|---|---| +| Backup | 7s | +| Restore | 53s | +| **Mechanical RTO** | **~60s** | + +**These numbers describe a laptop, not your production hardware**, and a dataset +that is small. Treat the shape as useful — restore dominates backup by roughly +8× — and re-measure on your own infrastructure. Run the drill against a copy of +production-sized data if you want a figure you can put in an SLA. + +Your real RTO is that 60 seconds plus how long it takes to notice, decide, and +get someone with credentials to a terminal. That is usually the dominant term +and it is not something a script can measure. + +**RPO is your dump schedule and nothing else.** Pangolin has no continuous +archiving or point-in-time recovery of its own. If you dump nightly, your RPO is +24 hours. If you need better, use PostgreSQL WAL archiving or your cloud +provider's managed backups — both are outside Pangolin and both work fine with +it. + +## Running the drill + +```bash +DATABASE_URL=postgres://user:pass@host/pangolin_test scripts/backup_restore_drill.sh +``` + +It refuses to run against a database whose name does not contain `test`, +`drill`, `scratch` or `tmp`, because it drops the schema. That guard is +deliberate and you should not remove it; point it at a restored copy of +production instead, which also gives you a more honest number. + +## What is not covered + +- **No point-in-time recovery.** Dump-and-restore only. +- **No automated backup scheduling.** Use cron, a Kubernetes CronJob, or your + provider's managed backups. +- **No tested MongoDB or SQLite drill.** The script is PostgreSQL-only. + `mongodump`/`mongorestore` and copying the SQLite file are the equivalents, + but neither has been drilled here, so neither is written up as though it had + been. +- **Multi-region failover is out of scope.** This covers restoring one database. diff --git a/docs/operations/encryption.md b/docs/operations/encryption.md new file mode 100644 index 0000000..060ce2b --- /dev/null +++ b/docs/operations/encryption.md @@ -0,0 +1,126 @@ +# Encrypting warehouse credentials at rest + +A warehouse holds the credentials Pangolin uses to reach your object storage: +AWS secret access keys, Azure account keys, GCP service account JSON. Before +0.8.0 these were stored in the catalog database as plaintext JSON, so anything +that could read one row of the `warehouses` table held every tenant's cloud +credentials — a backup, a read replica, a snapshot, an analyst with `SELECT`. + +From 0.8.0 they are encrypted with AES-256-GCM when you configure a key. + +## Turning it on + +```bash +openssl rand -base64 32 +``` + +Set the result as `PANGOLIN_ENCRYPTION_KEY` and restart. The server logs a +warning at startup while it is unset, because a security control that silently +does nothing is worse than one that is visibly absent. + +```yaml +environment: + - PANGOLIN_ENCRYPTION_KEY=${PANGOLIN_ENCRYPTION_KEY:?generate with: openssl rand -base64 32} +``` + +The key belongs in a secret manager, not in `.env` and not in the image. It is +the only thing standing between a database dump and your customers' cloud +accounts. + +## What it protects, and what it does not + +| Threat | Protected | +|---|:--:| +| Stolen database backup or snapshot | ✅ | +| Read replica, or an operator with `SELECT` | ✅ | +| SQL injection elsewhere in the application | ✅ | +| Full compromise of a running server | ❌ | +| Compromise of the secret manager holding the key | ❌ | + +The key lives in the server's environment, so an attacker who owns the running +process can read it and decrypt everything. That is the normal limit of envelope +encryption without an HSM or a cloud KMS, and it is stated here rather than +implied away. What this buys you is that the *database* is no longer sufficient +on its own. + +## Existing warehouses + +Turning encryption on does not rewrite anything. Reads tolerate plaintext, so +every existing warehouse keeps working — an upgrade must not be an outage. But +those rows stay in plaintext until something writes them again. + +To seal them, update each warehouse once. Any update does it, including one that +changes nothing meaningful: + +```bash +pangolin-admin warehouse update --use-sts false +``` + +Or through the API, re-submitting the storage config: + +```bash +curl -X PUT "$PANGOLIN_URL/api/v1/warehouses/$NAME" \ + -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"storage_config": { ... }}' +``` + +To find what still needs it, look for rows whose credential fields do not begin +with `enc:v1:`: + +```sql +-- PostgreSQL +SELECT tenant_id, name +FROM warehouses +WHERE storage_config->>'secret_access_key' NOT LIKE 'enc:v1:%' + OR storage_config->>'account_key' NOT LIKE 'enc:v1:%' + OR storage_config->>'client_secret' NOT LIKE 'enc:v1:%'; +``` + +## Losing the key + +There is no recovery. The credentials are gone and the warehouses must be +recreated with fresh credentials from your cloud provider. Back the key up +wherever you back up your other break-glass secrets, and treat it with the same +care as `PANGOLIN_JWT_SECRET`. + +If you start the server with the *wrong* key, reads fail loudly rather than +returning rubbish — GCM authenticates the ciphertext — with an error naming +`PANGOLIN_ENCRYPTION_KEY` as the likely cause. + +## Rotating the key + +There is no online rotation yet. To change keys: decrypt with the old key by +running with it set, re-submit every warehouse's storage config to bring the +values into memory, stop, set the new key, and re-submit again. For a small +number of warehouses this is minutes of work; for a large estate, wait for +proper rotation support rather than scripting this. + +## What is encrypted + +Only the credential-bearing entries of `storage_config`. The bucket, container, +region, endpoint and account name stay readable: they are not secrets, the +object-store factory compares and concatenates them, and encrypting them would +break storage access for no gain. + +Covered keys, in both the dotted and undotted spellings that appear in real +configurations: + +`secret_access_key` · `access_key_id` · `session_token` · `account_key` · +`client_secret` · `service_account_json` · `external_id` + +If you use a storage backend whose credential is not on that list, it is **not** +being encrypted. Say so in an issue and it will be added; the list is an +allowlist precisely so that non-secrets stay usable, and the cost of that choice +is that a new secret has to be added deliberately. + +## Backends + +PostgreSQL, SQLite and MongoDB all seal on write and open on read. The memory +backend does not: it keeps everything in a `DashMap` and loses it on restart, so +there is no "at rest" to protect. + +`pangolin_store/tests/warehouse_encryption_tests.rs` reads the raw stored bytes +through its own database connection — not through the store — and asserts the +plaintext is absent. A backend added later that forgets to call `secrets::seal` +fails that test rather than quietly storing credentials in the clear. diff --git a/docs/operations/oidc.md b/docs/operations/oidc.md index 4aa4992..16dd6c9 100644 --- a/docs/operations/oidc.md +++ b/docs/operations/oidc.md @@ -1,121 +1,123 @@ -# OAuth / SSO configuration +# OpenID Connect + +Before 0.8.0 this page described a gap. Pangolin's "OAuth" login was +*authorization*, not authentication: it exchanged a code for an access token, +called the provider's userinfo endpoint, and believed the response. That is only +sufficient if the access token could not have come from anywhere else — which is +precisely what OIDC exists to establish. + +From 0.8.0 the flow does real OIDC where the provider supports it. + +## What is verified now + +| Check | What it prevents | +|---|---| +| **PKCE (S256)** | An attacker who intercepts the authorization code — from a referrer header, a proxy log, shell history on a shared machine — cannot redeem it without the verifier | +| **`id_token` signature** against the provider's JWKS | Identity comes from something the provider signed, not from an HTTP response any holder of some access token could have elicited | +| **`aud`** must contain our `client_id` | A token minted for a *different* application at the same provider logging its holder in here — the classic confused deputy | +| **`iss`** must match the discovery document | A token from any other issuer being accepted | +| **`exp`** with 60s leeway | Replay of an expired assertion; the leeway is for clock skew, which otherwise produces logins that fail for one second and nobody can diagnose | +| **`nonce`** must match this login | An `id_token` observed in one flow being replayed into another | +| **Asymmetric `alg` only** | An attacker setting `alg: HS256` and signing with the provider's *public* key, which for HMAC is also the verification key | + +Retained from 0.6.0/0.7.0: signed single-use `state` (CSRF), an allowlisted +redirect resolved to an index so the URL never travels inside `state`, the +session token never placed in a redirect URL, and `(provider, subject)` as the +identity — email only adopts a pre-existing account when the provider says it is +verified *and* its domain is operator-allowlisted. -**Read this before upgrading to 0.6.0 if you use OAuth.** The token delivery -mechanism changed, and existing clients will break. +## Configuration -## What changed, and why +Nothing new is required. If a provider has a known issuer, OIDC applies +automatically. -Before 0.6.0 the callback base64-decoded the `state` parameter, read -`redirect_uri` out of it, and appended the freshly minted session JWT to that -URL as a query parameter: +| Variable | Purpose | +|---|---| +| `PANGOLIN_OIDC_REQUIRE` | `true` refuses any provider that is not OIDC-capable. Off by default | +| `PANGOLIN__ISSUER` | Override the issuer URL — needed for self-hosted Keycloak, Auth0, a private Okta, or any internal IdP | -```rust -let base_url = frontend_url.unwrap_or_else(|| env::var("FRONTEND_URL")…); -let redirect_url = format!("{}?token={}", base_url, token); -Redirect::to(&redirect_url) -``` +Known issuers, derived automatically: -`state` was plain base64 JSON — unsigned, unencrypted, never stored server-side -— and there was no allowlist on the destination. Sending someone an authorize -link whose `state` decoded to `{"redirect_uri":"https://evil.example/"}` caused -Pangolin to 302 their browser to the attacker's host with a valid token for -their real identity in the URL. Full account takeover, no credential theft -(A-8). Separately, the nonce embedded in `state` was generated and never -verified, which is login CSRF (A-9). - -From 0.6.0: - -* `state` is HMAC-SHA256 signed with the server's secret and carries an expiry. -* The nonce inside it is registered server-side and consumed exactly once, so a - captured callback cannot be replayed. -* `state` is bound to the provider it was issued for. -* `redirect_uri` is validated against an operator-configured allowlist by exact - match, and only an *index* into that allowlist travels inside `state`. -* **The token is never in a URL.** The callback parks it and redirects with a - short-lived, single-use `code`, which the client exchanges over POST. +- **Google** — `https://accounts.google.com` +- **Microsoft** — from `PANGOLIN_MICROSOFT_TENANT_ID` +- **Okta** — from `PANGOLIN_OKTA_DOMAIN` -## Configuration +## GitHub is not an OIDC provider -```bash -FRONTEND_URL=https://app.example.com/oauth/callback +GitHub's OAuth issues no `id_token` and publishes no JWKS. There is no honest +way to validate a GitHub login the way the table above describes. -# Every additional URL an OAuth flow may return to, comma-separated. -# FRONTEND_URL is always allowed and does not need repeating. -PANGOLIN_OAUTH_REDIRECT_URIS=https://app.example.com/oauth/callback,https://admin.example.com/oauth/callback +So a GitHub login still uses the userinfo endpoint, and the code says so rather +than reporting validation it did not perform. `PANGOLIN_OIDC_REQUIRE=true` +refuses GitHub outright — which is the right behaviour for an operator who has +decided every login must be OIDC-validated, and the reason the setting exists. -PANGOLIN_GOOGLE_CLIENT_ID=… -PANGOLIN_GOOGLE_CLIENT_SECRET=… -PANGOLIN_GOOGLE_REDIRECT_URI=https://catalog.example.com/oauth/callback/google -``` +GitHub also does not report `email_verified`, so its addresses are treated as +unverified and can never adopt a pre-existing account. -Supported providers: `google`, `microsoft` (also needs -`PANGOLIN_MICROSOFT_TENANT_ID`), `github`, `okta` (also needs -`PANGOLIN_OKTA_DOMAIN`). +## Turning on strict mode -A `redirect_uri` that is not in the allowlist gets `400` from -`/oauth/authorize/{provider}` with a message naming the variable to add. Matching -is exact — `https://app.example.com` does not match -`https://app.example.com/`. +```bash +PANGOLIN_OIDC_REQUIRE=true +``` -## The flow +With this set: -``` - 1. Browser → GET /oauth/authorize/google?redirect_uri= - 2. Pangolin → 302 to the provider, carrying signed single-use state - 3. Provider → 302 to /oauth/callback/google?code=…&state=… - 4. Pangolin verifies the signature, expiry, provider binding and nonce - exchanges the code, fetches user info, mints a session - 5. Pangolin → 302 to ?code= - 6. Client → POST /api/v1/oauth/exchange {"code": ""} - 7. Pangolin → 200 {"token": "", "token_type": "Bearer"} -``` +- a provider with no issuer is refused at authorize **and** at callback; +- a discovery failure fails the login rather than silently proceeding without + PKCE. -## Client migration +It is off by default because turning it on would break a working GitHub +deployment on upgrade with no warning. That is a deliberate choice about +upgrades, not a judgement that GitHub logins are fine. -The redirect now carries `code`, not `token`. +## Key rotation -```js -// Before -const token = new URLSearchParams(location.search).get('token'); +JWKS documents are cached for an hour. When a token arrives with a `kid` that is +not in the cache — which is what key rotation looks like — the JWKS is refetched +once, rate-limited to one forced refetch a minute per provider. -// After -const code = new URLSearchParams(location.search).get('code'); -const res = await fetch('/api/v1/oauth/exchange', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ code }), -}); -const { token } = await res.json(); -``` +Both halves matter. Without the refetch, a provider rotating keys breaks every +login until the hour expires. Without the rate limit, a stream of tokens +carrying junk `kid`s becomes a denial-of-service against the provider's JWKS +endpoint and against our own latency, since every such request would block on an +outbound fetch. + +## Multi-replica -The code is single-use and expires in two minutes. Redeem it immediately, and -strip it from the URL afterwards (`history.replaceState`). +The PKCE verifier and OIDC nonce are held **in process**, keyed by the state +nonce. If the callback lands on a different replica than the one that started +the flow, the login fails. -## Limitations +The verifier is deliberately *not* carried in `state`: `state` travels through +the browser in the same URL as the authorization code, so anyone positioned to +steal the code would also hold the verifier, and PKCE would protect nothing in +the one situation it exists for. -Pangolin implements the OAuth 2.0 authorization-code grant followed by a call to -the provider's userinfo endpoint. It is **not** a full OIDC relying party: +So OAuth still requires session affinity. See +[running-multiple-replicas.md](running-multiple-replicas.md). Moving this to the +database would remove the constraint and is not done. -* No PKCE. -* No `id_token` validation: no JWKS fetch, no signature check, no `iss`/`aud` - verification, no `nonce` binding to the ID token. -* No discovery document, so onboarding an arbitrary enterprise IdP needs a code - change — four providers are hardcoded. -* No SAML, SCIM provisioning, or group-to-role mapping. -* Users are found or created by the **email** the provider returns, with no - `email_verified` check and no domain allowlist. With a provider that permits - unverified emails this is an account-takeover path. Looking users up by - `(provider, subject)` is the correct fix and is not done yet. +## What is still not done -These are tracked as C-2 and C-3 in `AUDIT_EXECUTION_PLAN.md` (Phase 3.1). +- **No `email_verified` enforcement at the point of account creation.** A + verified email is required to *adopt an existing* account; a new account is + created from the provider's subject regardless. +- **No back-channel logout** (RP-initiated or front-channel). +- **No refresh-token handling.** Sessions are Pangolin JWTs with their own + lifetime; when one expires the user logs in again. +- **No dynamic client registration**, and no per-tenant provider configuration — + providers are process-wide environment variables. +- **The nonce and verifier stores are in-process**, as above. -**Until then:** only enable providers that verify email addresses, and restrict -sign-in at the identity provider rather than relying on Pangolin to do it. +## Testing -## Multi-replica note +`pangolin_api/tests/oidc_validation_tests.rs` stands up a provider with +`wiremock`, serving a real discovery document and JWKS, and signs tokens with a +real 2048-bit RSA key. Nothing is mocked at the crypto layer, because the +properties under test *are* the crypto — a test that stubbed signature checking +would pass against code that skipped it. -The OAuth nonce and exchange-code stores are in-process. With more than one -replica and no session affinity, a callback that lands on a different pod than -the authorize request will be rejected as an unknown nonce. Enable sticky -sessions on your ingress for `/oauth/*`, or run a single replica for the OAuth -path, until these move to the shared store. +Each case names the attack it prevents, and the suite was checked for being +load-bearing: disabling audience validation makes the confused-deputy test fail, +and weakening the nonce comparison makes the replay test fail. diff --git a/docs/operations/performance.md b/docs/operations/performance.md new file mode 100644 index 0000000..7af6d06 --- /dev/null +++ b/docs/operations/performance.md @@ -0,0 +1,95 @@ +# Performance + +Nobody had measured this. The audit could say "API-key authentication is +O(tenants × service users) bcrypt calls per request" by reading the code, but +not what any of it costs — and a performance claim nobody has measured is a +guess with a number attached. + +`scripts/load_test.py` is the harness. It reports **both** client-observed +latency and the server's own histogram, for a reason explained below. + +## Measured figures + +Developer laptop, Docker, memory backend, 16 concurrent keep-alive connections, +2000 requests per scenario after 200 warm-up: + +| Scenario | client p50 | client p95 | client p99 | client req/s | **server mean** | +|---|--:|--:|--:|--:|--:| +| `/health/ready` | 2.4ms | 7.5ms | 13.5ms | 5086 | **0.018ms** | +| `/v1/config` | 2.7ms | 11.6ms | 18.1ms | 3858 | **0.023ms** | +| list catalogs (authenticated) | 4.5ms | 15.0ms | 21.6ms | 2631 | **0.060ms** | + +**Read the two columns together.** The client figures include the Python load +generator and the loopback stack; the server-side means come from the server's +own histogram and include only its handling. The gap is roughly 100×, which +means at this scale **the harness is the bottleneck, not the catalog**. + +## Why both numbers are reported + +The first version of this harness used `urllib.request.urlopen` per request — a +fresh TCP connection each time, dispatched through a thread pool. It reported +~33ms per request and ~500 req/s against a server whose own histogram said 29 +**microseconds**. + +Those numbers would have been published as "the catalog does 500 req/s". They +were measuring Python. Switching to one keep-alive connection per worker moved +the same server from an apparent 504 req/s to 5086 — a 10× difference that came +entirely from the client. + +So the harness now prints the server's own means alongside its own, and says +plainly that a large gap means the generator is the limit. A load test that +cannot tell you which side is slow is not a load test. + +## What this does and does not tell you + +**Does:** authorization costs something real but small. Adding an authenticated +permission check takes the server-side mean from 0.018ms to 0.060ms — roughly +3×, and still 60 microseconds. Whatever limits throughput in a deployment, it is +not the permission check at this scale. + +**Does not:** + +- **These are laptop numbers on a memory backend.** PostgreSQL adds a network + round trip and real I/O per request. Re-measure on your own hardware with your + own backend before putting anything in an SLA. +- **Writes are not measured.** No commit path, no concurrent-writer contention. + A load test that creates thousands of tables leaves a database full of them, + and the interesting write property — commit conflicts under contention — needs + a harness that coordinates writers on one table. That does not exist yet. +- **API-key authentication is not measured**, which is unfortunate because it is + the path the audit flagged. It enumerates every tenant and every service user + per request; the bcrypt cost is now bounded to one round by a key-id prefix, + but the *database enumeration* is not. With a handful of service users this is + invisible. It has not been measured with thousands. +- **No sustained soak.** Two thousand requests is a burst, not an hour. Nothing + here would catch a slow leak or a cache that degrades over time. +- **Single replica.** Multi-replica behaviour is untested; see + [running-multiple-replicas.md](running-multiple-replicas.md). + +## Running it + +```bash +python3 scripts/load_test.py --url http://localhost:8080 \ + --user root --password "$PANGOLIN_ROOT_PASSWORD" \ + --concurrency 16 --requests 2000 +``` + +Add `--catalog X --namespace Y --table Z` to include a table read, which is the +path an engine actually exercises and the only one that touches object storage. + +Warm-up is not optional. The first authenticated request fills the warehouse +cache and the connection pool; folding it into the sample puts a cold-start +outlier straight into the p99. + +## Known cost centres, from reading the code + +Unmeasured, listed so they are not forgotten: + +1. **API-key authentication enumerates tenants and service users** on every + request (`auth_middleware.rs`). The bcrypt cost is bounded; the enumeration + is not. +2. **The revocation check reads `revoked_tokens` on every authenticated + request.** It is indexed on MongoDB from 0.8.0, and the sweep that keeps the + table small never actually ran before 0.8.0 either. +3. **Every Iceberg table load reads its metadata file from object storage.** + That is inherent to the format, and it is why the metadata cache exists. diff --git a/docs/operations/running-multiple-replicas.md b/docs/operations/running-multiple-replicas.md new file mode 100644 index 0000000..b443964 --- /dev/null +++ b/docs/operations/running-multiple-replicas.md @@ -0,0 +1,104 @@ +# Running more than one replica + +Short version: **you can, with PostgreSQL, if you configure session affinity and +accept the caveats below.** Each one is stated with what actually goes wrong, +not just that it is "unsupported". + +## What works without any special handling + +**The Iceberg commit path.** This is the operation most likely to race, and it +is safe. Table commits use compare-and-swap on the metadata pointer with +requirement enforcement, so two replicas committing to the same table cannot +both win — one gets a commit conflict and retries. That property does not depend +on there being one replica. + +**Schema migrations.** PostgreSQL applies its migration chain under a +`pg_advisory_lock`, so replicas starting simultaneously cannot race each other +into a corrupted schema. + +**The revocation sweep.** Every replica runs it. `cleanup_expired_tokens` is a +`DELETE ... WHERE expires_at < now`, so concurrent sweeps are idempotent — the +second deletes nothing. Each replica staggers its own start within the interval +so that replicas from one rolling deploy do not sweep in lockstep forever. + +> Before 0.8.0 this job **never ran at all**. It was defined, the module was +> declared, and nothing called it, so `revoked_tokens` grew for the life of the +> deployment — and the revocation check reads that table on every authenticated +> request. + +## What needs configuration + +### OAuth requires session affinity + +The OAuth `state` nonce and the one-time code exchange are held **in process**. +If the browser's callback lands on a different replica than the one that started +the flow, the nonce is not found and the login fails with an invalid-state +error. + +Configure sticky sessions on your load balancer for the OAuth endpoints, or +disable OAuth and use password or API-key authentication. + +```yaml +# Kubernetes Service +spec: + sessionAffinity: ClientIP +``` + +This is a known limitation, not a design choice, and it is on the list for the +OIDC work — which needs server-side state for PKCE anyway, and will move both +stores into the database. + +### Rate limiting is per replica + +The authentication throttle counts in process. With N replicas an attacker +distributing attempts across them gets N times the configured budget. Set +`PANGOLIN_AUTH_RATE_LIMIT` accordingly, and put a limiter in front of the +service if you need a hard number. + +### A rotated warehouse credential is served briefly by peers + +Warehouses are cached in process for `PANGOLIN_WAREHOUSE_CACHE_TTL_SECS` +(default 5). Invalidation is local, so after you rotate a credential on replica +A, replica B can keep vending the old one for up to the TTL. Five seconds is +deliberately short for exactly this reason. If you need rotation to be +immediate, restart the replicas after rotating. + +## What to watch + +`pangolin_auth_throttled_total` — a sustained non-zero rate is an attack or a +misconfigured client. + +`pangolin_token_revocation_check_errors_total` — revocation currently **fails +open**: if the check errors, the request proceeds. During a database blip every +revoked token is accepted again. This is a known gap (A-13) and this metric is +how you see it happening. + +`pangolin_table_commit_cas_retries_total` — rising retries mean replicas are +contending on the same tables. Correct, but it is the signal that you are +scaling writes to one table rather than across tables. + +## Recommended configuration + +| Setting | Value | Why | +|---|---|---| +| Backend | PostgreSQL | The only backend with a migration chain and full transactional support | +| Replicas | 2–3 to start | Enough for availability; small enough to observe | +| `sessionAffinity` | `ClientIP` | Required if OAuth is enabled | +| `PANGOLIN_JWT_SECRET` | Same across replicas | Otherwise each replica rejects the others' tokens | +| `PANGOLIN_ENCRYPTION_KEY` | Same across replicas | A warehouse sealed by one replica must be readable by all | +| `PANGOLIN_WAREHOUSE_CACHE_TTL_SECS` | 5 (default) | Bounds how long a rotated credential can be served by a peer | + +Two of those are worth repeating because getting them wrong fails in confusing +ways: **`PANGOLIN_JWT_SECRET` and `PANGOLIN_ENCRYPTION_KEY` must be identical on +every replica.** A different signing secret makes each replica reject sessions +issued by the others, which looks like random logouts. A different encryption +key makes warehouses written by one replica unreadable by the others, which +looks like intermittent credential-vending failures. + +## What has not been tested + +Multi-replica operation has **not** been load tested or run for an extended +period. The properties above are established by reading the code and by the test +suite, not by having run three replicas under production traffic for a week. +Treat this page as "the known constraints", not as "this configuration is +proven". diff --git a/docs/upgrading/0.6-to-0.7.md b/docs/upgrading/0.6-to-0.7.md new file mode 100644 index 0000000..5388d27 --- /dev/null +++ b/docs/upgrading/0.6-to-0.7.md @@ -0,0 +1,226 @@ +# Upgrading from 0.6.x to 0.7.0 + +**0.7.0 is a security release.** 0.6.0 is affected by a privilege escalation +that any authenticated principal could exploit. Read +[SECURITY.md](../../SECURITY.md#fixed-in-070) first, then come back here for the +compatibility detail. + +This page covers only what changes for *you*. The full list of fixes is in the +[changelog](../../CHANGELOG.md). + +--- + +## Do these before anything else + +1. **Rotate `PANGOLIN_JWT_SECRET`.** Any account could have minted a `Root` + token (B0a), and logging out never invalidated anything (B0j). Rotating the + signing secret invalidates every outstanding session in one step, which is + the only way to be certain none survive. +2. **Rotate service-user API keys.** An expired key could keep exchanging + itself for fresh JWTs (B0g). +3. **Rotate cloud storage credentials** on any warehouse an untrusted tenant + member could reach. Credential vending had no authorization at all (B0b). + +--- + +## Breaking changes + +### Unknown request fields are now rejected + +Server request bodies carry `#[serde(deny_unknown_fields)]`. A field the server +does not recognise is a `422` naming the field, where it used to be silently +discarded and the request would return `200`. + +**Nothing that was working stops working.** If a client sent a field the server +did not have, that field was already being thrown away — the request was never +doing what it appeared to. What changes is that you now find out. + +Real examples from Pangolin's own clients, all of which looked like they worked: + +| Sent | Server field | What actually happened | +|---|---|---| +| `warehouse` | `warehouse_name` | The catalog was created with **no warehouse**, and the existence check was skipped | +| `type: "pangea"` | `catalog_type` | Defaulted to `Local` | +| `expires_in_days: 30` | `expires_in_hours` | Every token was 24 hours | +| `motivation` | `reason` | Access requests reached reviewers with no justification | + +If you maintain a third-party client, run it against 0.7.0 in a staging +environment and fix whatever 422s. Each one is a call that was not doing what +you thought. + +### A commit to a non-`main` branch no longer moves `main` + +Previously, committing to any branch set `current_snapshot_id` and fabricated a +`main` ref pointing at the new snapshot. A `dev`-branch commit therefore changed +what `main` readers resolved (B16). + +`current_snapshot_id` and `snapshot_log` now describe `main` alone, and a +feature-branch commit updates only its own ref. + +**If you depended on the old behaviour** — for instance a pipeline that wrote to +a branch and expected `main` to follow — you now need an explicit merge or a +`set-snapshot-ref` on `main`. + +### Tokens without a `jti` are rejected + +A JWT carrying no `jti` cannot be revoked, and was previously exempt from the +revocation check entirely (B0o). Such tokens are now refused. + +Set `PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI=true` for a migration window if you have +long-lived tokens minted before this. Re-issue them and unset it. + +### OAuth no longer links accounts by email + +Identity is `(provider, subject)`. An email address adopts a pre-existing local +account only when the provider reports it as **verified** *and* its domain is +listed in `PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS` (B0l). + +**If your users sign in with OAuth against accounts created locally**, set that +variable or those users will get a *new* account on next login: + +```bash +export PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS=yourcompany.com,subsidiary.example +``` + +Leaving it unset is the safe default and the right choice if you do not rely on +this. + +### `PANGOLIN_DEV_MODE` no longer waives the `NO_AUTH` bind guard + +`PANGOLIN_NO_AUTH=true` now requires a loopback bind address unconditionally. +Dev mode relaxes secret *strength*, never network exposure (B0h). + +A deployment setting both flags on `0.0.0.0` **will now refuse to start**. That +configuration served every anonymous request as `TenantAdmin`. + +### `PANGOLIN_STORE_TYPE` is gone from the compose files + +It never did anything — the server reads `PANGOLIN_STORAGE_TYPE` (B9). If you +copied a compose file and edited `PANGOLIN_STORE_TYPE` to `postgres`, **you have +been running the in-memory backend**, losing all catalog metadata on every +restart. Check before you upgrade. + +### The UI reads `PUBLIC_API_URL` + +`VITE_API_URL` was never read by the client (B31). If you set it, rename it: + +```diff +- VITE_API_URL=https://pangolin.internal ++ PUBLIC_API_URL=https://pangolin.internal +``` + +Leave it empty for a same-origin reverse-proxy deployment. + +--- + +## Python SDK + +Four signatures changed, each because the old one could not work: + +```python +# permissions.grant - the old scope shape had no matching server field +- client.permissions.grant(user_id, action, scope_type, scope_id) ++ client.permissions.grant( ++ user_id=..., actions=["read"], scope_type="catalog", catalog_id=... ++ ) + +# business_metadata.delete - the `key` was ignored; this deletes everything +- client.business_metadata.delete(asset_id, key) ++ client.business_metadata.delete(asset_id) + +# request_access - `motivation` was dropped on the floor +- client.business_metadata.request_access(asset_id, motivation="...") ++ client.business_metadata.request_access(asset_id, reason="...") + +# branches.rebase - omitted a required field and always 422'd +- client.branches.rebase(branch_name, base_branch, catalog_name=...) ++ client.branches.rebase(branch_name, catalog_name=...) + +# tokens.generate - three of its four arguments were ignored +- client.tokens.generate(name, user_id, tenant_id, expires_in_days=30) ++ client.tokens.generate(username=..., tenant_id=..., expires_in_hours=720) +``` + +Also: + +* **`pyiceberg` is now an optional extra.** Install `pypangolin[iceberg]` if you + use `get_iceberg_catalog`. Importing the package no longer pulls in the whole + Iceberg stack (B41). +* **Minimum Python is 3.9.** The declared `>=3.8` was unsatisfiable with the + package's own dependencies (B40). +* **Every CLI command now exits non-zero on failure.** Scripts that relied on a + zero exit code will start failing — correctly. Commands were reporting + success for calls that raised `TypeError` internally. +* **`pangolin` is now on `PATH`** after install, via a real console script. + +## Rust CLIs + +Roughly 35 call sites targeted endpoints that do not exist. Commands that +previously printed an error and exited `0` now exit non-zero. If you have +scripts that ignored the exit code, they were ignoring failures. + +`pangolin-admin grant-permission` changed shape, because the old arguments +matched no server field: + +```diff +- pangolin-admin grant-permission --username alice --action read --resource cat ++ pangolin-admin grant-permission --user-id --action read \ ++ --scope-type catalog --catalog-id +``` + +`pangolin-admin update-user --username` now errors instead of silently doing +nothing: the server has no rename operation. + +--- + +## Storage backends + +### SQLite + +Upgrading migrates `audit_logs` to schema version 2 automatically on startup. +The previous table is preserved as `audit_logs_pre_v2` and can be dropped once +you are satisfied — it is empty in practice, because the old column set meant +**every audit write failed**. Assume you have no SQLite audit history before +0.7.0. + +If you keep backups, take one before upgrading, as with any schema change. + +### MongoDB + +**A replica set is strongly recommended.** On a standalone `mongod`: + +* transactions are unavailable, so `delete_catalog` falls back to a + sequential, non-atomic cascade (it now degrades with a warning instead of + failing outright, which it did before); +* retryable writes are unavailable — add `retryWrites=false` to your connection + string. + +Revocation records written before 0.7.0 are unreadable by the fixed code path, +because the old write used a different field name *and* type (B2). They never +worked as revocations either. Rotating the signing secret, which you should do +anyway, makes this moot. + +### Postgres + +No schema change. `list_catalogs` now returns the real `catalog_type` and +`federated_config` instead of hardcoding `Local`/`None` (B24) — if you have code +branching on a listing's `catalog_type`, it will now take the federated path +where it should have all along. + +--- + +## Verifying the upgrade + +```bash +# every artifact carries the same version +scripts/bump_version.sh 0.7.0 --check + +# the authorization matrix and cross-backend parity suites +cd pangolin +cargo test -p pangolin_api --lib authz_matrix_tests +cargo test -p pangolin_store --test store_integration +``` + +The second command runs against whichever backends you configure via +`PANGOLIN_TEST_POSTGRES_URL` and `PANGOLIN_TEST_MONGO_URL`, and skips the rest +with a note rather than passing silently. diff --git a/pangolin/.cargo/audit.toml b/pangolin/.cargo/audit.toml new file mode 100644 index 0000000..3ccadb4 --- /dev/null +++ b/pangolin/.cargo/audit.toml @@ -0,0 +1,64 @@ +# Advisories accepted deliberately, each with the reason it cannot be fixed by +# upgrading and what the exposure actually is. +# +# This file is not a way to make `cargo audit` quiet. Every entry is a specific +# advisory ID, so a *new* advisory - including a new one against these same +# crates - still fails CI. Re-check this list whenever dependencies move; the +# right outcome for most of these is deletion once upstream releases. +# +# Reviewed: 2026-08-10, against Cargo.lock at pangolin 0.7.0. + +[advisories] +ignore = [ + # ---- No fixed version exists at any release ---- + + # RUSTSEC-2023-0071 - Marvin Attack: RSA key recovery through timing side + # channels. The `rsa` crate has no fixed release; the maintainers' guidance + # is that a constant-time rewrite is required. + # + # It is never compiled into Pangolin. Its only route in is `sqlx-mysql`, + # which is an *optional* dependency of `sqlx`: this workspace builds sqlx + # with `postgres` and `sqlite` only, and there is no MySQL `CatalogStore`. + # `cargo tree -i rsa --target all` finds no path to it. It appears here + # solely because `cargo audit` scans Cargo.lock, which records optional + # dependencies whether or not their feature is enabled. + "RUSTSEC-2023-0071", + + # ---- Held back by an upstream that has not released ---- + + # RUSTSEC-2026-0194 / -0195 - quick-xml quadratic attribute checking and + # unbounded namespace allocation, both DoS. Two copies are in the tree and + # neither is ours to move: + # * 0.31.0 via azure_core 0.20, which is behind the OPTIONAL `azure` + # feature and absent from a default build; + # * 0.37.5 via object_store 0.11, which is not optional. + # object_store 0.14 carries a fixed quick-xml but is three minor versions + # ahead with breaking API changes across the IO layer. That upgrade is + # worth doing on its own terms, not as a drive-by inside a security fix. + # Exposure: both parse XML from the object store's own responses, so an + # attacker needs control of the storage endpoint's replies. + "RUSTSEC-2026-0194", + "RUSTSEC-2026-0195", + + # RUSTSEC-2026-0098 / -0099 / -0104 - rustls-webpki name-constraint and CRL + # defects. The fixed 0.103.13 IS in the tree and is what rustls 0.23 uses. + # The flagged 0.101.7 comes in alongside it via rustls 0.21, pulled by + # aws-smithy-http-client 1.2.0. No release of the AWS SDK has dropped that + # second TLS stack yet. Pangolin's own TLS goes through the 0.23 path. + "RUSTSEC-2026-0098", + "RUSTSEC-2026-0099", + "RUSTSEC-2026-0104", + + # RUSTSEC-2026-0174 - http-types can build Authorization and + # WwwAuthenticate header values that violate ASCII invariants. Via + # azure_core 0.20, so behind the optional `azure` feature. http-types is + # unmaintained and azure_core 0.20 is the last release of that line before + # the Azure SDK restructure; moving off it is an Azure-support project. + "RUSTSEC-2026-0174", + + # RUSTSEC-2026-0097 - `rand` is unsound when a custom logger calls + # `rand::rng()` re-entrantly. Pangolin installs no custom logger that draws + # randomness; `tracing-subscriber`'s formatters do not. Both 0.8 and 0.9 are + # in the tree transitively and neither has a fixed release. + "RUSTSEC-2026-0097", +] diff --git a/pangolin/.test_output.txt b/pangolin/.test_output.txt deleted file mode 100644 index 643cc43..0000000 --- a/pangolin/.test_output.txt +++ /dev/null @@ -1,391 +0,0 @@ -warning: unused import: `std::collections::HashMap` - --> pangolin_core/src/business_metadata.rs:3:5 - | -3 | use std::collections::HashMap; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: `pangolin_core` (lib) generated 1 warning (run `cargo fix --lib -p pangolin_core` to apply 1 suggestion) -warning: unused import: `CatalogType` - --> pangolin_store/src/memory.rs:7:14 - | -7 | ...g, CatalogType, N... - | ^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: unused import: `BusinessMetadata` - --> pangolin_store/src/memory.rs:16:40 - | -16 | ...::{BusinessMetadata, A... - | ^^^^^^^^^^^^^^^^ - -warning: unused import: `PermissionGrant` - --> pangolin_store/src/postgres.rs:9:51 - | -9 | ...n, PermissionGrant, U... - | ^^^^^^^^^^^^^^^ - -warning: unused import: `DateTime` - --> pangolin_store/src/postgres.rs:16:14 - | -16 | ...::{DateTime, U... - | ^^^^^^^^ - -warning: unused imports: `OAuthProvider` and `UserRole as CoreUserRole` - --> pangolin_store/src/mongo.rs:13:33 - | -13 | ...r, UserRole as CoreUserRole, OAuthProvider}; - | ^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ - -warning: unused import: `PermissionGrant` - --> pangolin_store/src/mongo.rs:14:51 - | -14 | ...n, PermissionGrant, U... - | ^^^^^^^^^^^^^^^ - -warning: unused import: `RequestStatus` - --> pangolin_store/src/mongo.rs:15:55 - | -15 | ...t, RequestStatus}; - | ^^^^^^^^^^^^^ - -warning: unused import: `PermissionGrant` - --> pangolin_store/src/sqlite.rs:5:51 - | -5 | ...n, PermissionGrant, U... - | ^^^^^^^^^^^^^^^ - -warning: unused import: `DateTime` - --> pangolin_store/src/sqlite.rs:12:14 - | -12 | ...::{DateTime, U... - | ^^^^^^^^ - -warning: unused import: `crate::memory::MemoryStore` - --> pangolin_store/src/tests/multi_cloud.rs:1:5 - | -1 | use crate::memory::MemoryStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `crate::CatalogStore` - --> pangolin_store/src/tests/multi_cloud.rs:2:5 - | -2 | use crate::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^ - -warning: unused imports: `Credentials` and `Signer` - --> pangolin_store/src/tests/multi_cloud.rs:3:21 - | -3 | ...::{Signer, Credentials}; - | ^^^^^^ ^^^^^^^^^^^ - -warning: unused imports: `Tenant`, `VendingStrategy`, and `Warehouse` - --> pangolin_store/src/tests/multi_cloud.rs:4:28 - | -4 | ...::{Warehouse, VendingStrategy, Tenant}; - | ^^^^^^^^^ ^^^^^^^^^^^^^^^ ^^^^^^ - -warning: unused import: `std::collections::HashMap` - --> pangolin_store/src/tests/multi_cloud.rs:5:5 - | -5 | use std::collections::HashMap; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `uuid::Uuid` - --> pangolin_store/src/tests/multi_cloud.rs:6:5 - | -6 | use uuid::Uuid; - | ^^^^^^^^^^ - -warning: unused imports: `AuditAction`, `AuditLogEntry`, `AuditLogFilter`, `AuditResult`, and `ResourceType` - --> pangolin_store/src/tests/audit_tests.rs:1:28 - | - 1 | ...::{AuditAction, AuditLogEntry, AuditLogFilter, AuditResult, ResourceType}; - | ^^^^^^^^^^^ ^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ ^^^^^^^^^^^ ^^^^^^^^^^^^ - | -help: if this is a test module, consider adding a `#[cfg(test)]` to the containing module - --> pangolin_store/src/tests/mod.rs:154:1 - | -154 | pub mod audit_tests; - | ^^^^^^^^^^^^^^^^^^^^ - -warning: unused imports: `CatalogStore` and `MemoryStore` - --> pangolin_store/src/tests/audit_tests.rs:2:13 - | - 2 | ...::{CatalogStore, MemoryStore}; - | ^^^^^^^^^^^^ ^^^^^^^^^^^ - | -help: if this is a test module, consider adding a `#[cfg(test)]` to the containing module - --> pangolin_store/src/tests/mod.rs:154:1 - | -154 | pub mod audit_tests; - | ^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `uuid::Uuid` - --> pangolin_store/src/tests/audit_tests.rs:3:5 - | - 3 | use uuid::Uuid; - | ^^^^^^^^^^ - | -help: if this is a test module, consider adding a `#[cfg(test)]` to the containing module - --> pangolin_store/src/tests/mod.rs:154:1 - | -154 | pub mod audit_tests; - | ^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `chrono::Utc` - --> pangolin_store/src/tests/audit_tests.rs:4:5 - | - 4 | use chrono::Utc; - | ^^^^^^^^^^^ - | -help: if this is a test module, consider adding a `#[cfg(test)]` to the containing module - --> pangolin_store/src/tests/mod.rs:154:1 - | -154 | pub mod audit_tests; - | ^^^^^^^^^^^^^^^^^^^^ - -warning: unused imports: `Duration` and `Utc` - --> pangolin_store/src/azure_signer.rs:2:14 - | -2 | ...::{Duration, Utc}; - | ^^^^^^^^ ^^^ - -warning: unused import: `azure_storage_blobs::prelude::*` - --> pangolin_store/src/azure_signer.rs:4:5 - | -4 | use azure_storage_blobs::prelude::*; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused imports: `DateTime` and `Utc` - --> pangolin_store/src/gcp_signer.rs:2:14 - | -2 | ...::{DateTime, Utc}; - | ^^^^^^^^ ^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_store/src/object_store_factory.rs:4:5 - | -4 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `TimeZone` - --> pangolin_store/src/postgres.rs:16:29 - | -16 | ...c, TimeZone}; - | ^^^^^^^^ - -warning: unused variable: `now` - --> pangolin_store/src/memory.rs:1184:13 - | -1184 | ...et now = ... - | ^^^ help: if this is intentional, prefix it with an underscore: `_now` - | - = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `client` - --> pangolin_store/src/gcp_signer.rs:52:13 - | -52 | ...et client = ... - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` - -warning: value assigned to `param_count` is never read - --> pangolin_store/src/postgres.rs:1168:17 - | -1168 | ... param_count += 1; - | ^^^^^^^^^^^^^^^^ - | - = help: maybe it is overwritten before being read? - = note: `#[warn(unused_assignments)]` (part of `#[warn(unused)]`) on by default - -warning: value assigned to `param_count` is never read - --> pangolin_store/src/postgres.rs:1312:17 - | -1312 | ... param_count += 1; - | ^^^^^^^^^^^^^^^^ - | - = help: maybe it is overwritten before being read? - -warning: unused variable: `tenant_id` - --> pangolin_store/src/mongo.rs:178:38 - | -178 | ...f, tenant_id: U... - | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_tenant_id` - -warning: variable does not need to be mutable - --> pangolin_store/src/mongo.rs:357:13 - | -357 | ...et mut filter = ... - | ----^^^^^^ - | | - | help: remove this `mut` - | - = note: `#[warn(unused_mut)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `p` - --> pangolin_store/src/mongo.rs:362:21 - | -362 | ...me(p) = pa... - | ^ help: if this is intentional, prefix it with an underscore: `_p` - -warning: unused variable: `s3_nested` - --> pangolin_store/src/mongo.rs:1662:18 - | -1662 | ...et s3_nested = ... - | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_s3_nested` - -warning: field `signer` is never read - --> pangolin_store/src/memory.rs:37:5 - | -22 | pub struct MemoryStore { - | ----------- field in this struct -... -37 | signer: crate::signer... - | ^^^^^^ - | - = note: `MemoryStore` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default - -warning: field `key` is never read - --> pangolin_store/src/signer.rs:41:5 - | -40 | pub struct SignerImpl { - | ---------- field in this struct -41 | key: String, - | ^^^ - | - = note: `SignerImpl` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - -warning: methods `revoke_token`, `is_token_revoked`, and `cleanup_expired_tokens` are never used - --> pangolin_store/src/postgres.rs:1809:14 - | -1750 | impl PostgresStore { - | ------------------ methods in this implementation -... -1809 | async fn revoke_token(&self, token... - | ^^^^^^^^^^^^ -... -1821 | async fn is_token_revoked(&self, t... - | ^^^^^^^^^^^^^^^^ -... -1831 | async fn cleanup_expired_tokens(&s... - | ^^^^^^^^^^^^^^^^^^^^^^ - -warning: field `client` is never read - --> pangolin_store/src/mongo.rs:27:5 - | -26 | pub struct MongoStore { - | ---------- field in this struct -27 | client: Client, - | ^^^^^^ - | - = note: `MongoStore` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - -warning: methods `catalogs`, `branches`, `tags`, `commits`, and `audit_logs` are never used - --> pangolin_store/src/mongo.rs:63:8 - | -31 | impl MongoStore { - | --------------- methods in this implementation -... -63 | fn catalogs(&sel... - | ^^^^^^^^ -... -75 | fn branches(&sel... - | ^^^^^^^^ -... -79 | fn tags(&self) -... - | ^^^^ -... -83 | fn commits(&self... - | ^^^^^^^ -... -87 | fn audit_logs(&s... - | ^^^^^^^^^^ - -warning: methods `revoke_token`, `is_token_revoked`, and `cleanup_expired_tokens` are never used - --> pangolin_store/src/sqlite.rs:2003:14 - | -1941 | impl SqliteStore { - | ---------------- methods in this implementation -... -2003 | async fn revoke_token(&self, token... - | ^^^^^^^^^^^^ -... -2018 | async fn is_token_revoked(&self, t... - | ^^^^^^^^^^^^^^^^ -... -2029 | async fn cleanup_expired_tokens(&s... - | ^^^^^^^^^^^^^^^^^^^^^^ - -warning: function `test_azure_path_parsing` is never used - --> pangolin_store/src/tests/multi_cloud.rs:129:4 - | -129 | fn test_azure_path_parsing() { - | ^^^^^^^^^^^^^^^^^^^^^^^ - -warning: fields `account_name` and `account_key` are never read - --> pangolin_store/src/azure_signer.rs:7:5 - | -6 | pub struct AzureSigner { - | ----------- fields in this struct -7 | account_name: String, - | ^^^^^^^^^^^^ -8 | account_key: String, - | ^^^^^^^^^^^ - -warning: `pangolin_store` (lib) generated 40 warnings (run `cargo fix --lib -p pangolin_store` to apply 29 suggestions) -warning: unused import: `CatalogStore` - --> pangolin_store/tests/store_integration.rs:2:58 - | -2 | ...e, CatalogStore, - | ^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: `pangolin_store` (test "store_integration") generated 1 warning (run `cargo fix --test "store_integration" -p pangolin_store` to apply 1 suggestion) - Finished `test` profile [unoptimized + debuginfo] target(s) in 0.32s - Running tests/store_integration.rs (target/debug/deps/store_integration-9aac1c19d52cf927) - -running 4 tests -test test_memory_store_regression ... ok -test test_sqlite_store_regression ... FAILED -test test_postgres_store_regression ... FAILED -test test_mongo_store_regression ... FAILED - -failures: - ----- test_sqlite_store_regression stdout ---- - -thread 'test_sqlite_store_regression' (314071) panicked at /home/alexmerced/development/personal/Personal/2026/pangolin/pangolin/pangolin_store/src/tests/mod.rs:51:90: -Failed to create asset: error returned from database: (code: 1) no such table: assets - -Caused by: - (code: 1) no such table: assets -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - ----- test_postgres_store_regression stdout ---- - -thread 'test_postgres_store_regression' (314070) panicked at /home/alexmerced/development/personal/Personal/2026/pangolin/pangolin/pangolin_store/src/tests/mod.rs:51:90: -Failed to create asset: error returned from database: insert or update on table "assets" violates foreign key constraint "assets_tenant_id_fkey" - -Caused by: - insert or update on table "assets" violates foreign key constraint "assets_tenant_id_fkey" - ----- test_mongo_store_regression stdout ---- - -thread 'test_mongo_store_regression' (314069) panicked at /home/alexmerced/development/personal/Personal/2026/pangolin/pangolin/pangolin_store/src/tests/mod.rs:56:5: -assertion `left == right` failed: Initial metadata location mismatch - left: None - right: Some("s3://bucket/path/v1.json") - - -failures: - test_mongo_store_regression - test_postgres_store_regression - test_sqlite_store_regression - -test result: FAILED. 1 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.87s - -error: test failed, to rerun pass `-p pangolin_store --test store_integration` diff --git a/pangolin/Cargo.lock b/pangolin/Cargo.lock index 57cd327..09f3913 100644 --- a/pangolin/Cargo.lock +++ b/pangolin/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -44,18 +44,18 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -68,15 +68,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -103,9 +103,18 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] [[package]] name = "assert-json-diff" @@ -160,11 +169,11 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ - "event-listener 5.4.1", + "event-listener 5.4.2", "event-listener-strategy", "pin-project-lite", ] @@ -182,16 +191,16 @@ dependencies = [ "async-task", "blocking", "cfg-if", - "event-listener 5.4.1", + "event-listener 5.4.2", "futures-lite 2.6.1", "rustix", ] [[package]] name = "async-signal" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c070bbf59cd3570b6b2dd54cd772527c7c3620fce8be898406dd3ed6adc64c" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" dependencies = [ "async-io", "async-lock", @@ -224,7 +233,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -235,13 +244,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -261,15 +270,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.5.10" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" +checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -277,17 +286,18 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.60.12", - "aws-smithy-json 0.60.7", + "aws-smithy-http", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "hex", - "http 0.2.12", - "ring", + "http 1.5.0", + "sha1 0.10.7", "time", "tokio", "tracing", @@ -297,9 +307,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.11" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -309,9 +319,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.15.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ "aws-lc-sys", "zeroize", @@ -319,35 +329,39 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.35.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] name = "aws-runtime" -version = "1.5.17" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d81b5b2898f6798ad58f484856768bca817e3cd9de0974c24ae0f1113fe88f1b" +checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", - "aws-smithy-http 0.62.6", + "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "bytes-utils", + "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "http-body 0.4.6", + "http-body 1.1.0", "percent-encoding", "pin-project-lite", "tracing", @@ -356,127 +370,141 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.118.0" +version = "1.141.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e6b7079f85d9ea9a70643c9f89f50db70f5ada868fa9cfe08c1ffdf51abc13" +checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-sigv4", "aws-smithy-async", "aws-smithy-checksums", "aws-smithy-eventstream", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", + "http 1.5.0", + "http-body 1.1.0", "lru", "percent-encoding", "regex-lite", - "sha2", + "sha2 0.11.0", "tracing", "url", ] [[package]] name = "aws-sdk-sso" -version = "1.91.0" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee6402a36f27b52fe67661c6732d684b2635152b676aa2babbfb5204f99115d" +checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.93.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45a7f750bbd170ee3677671ad782d90b894548f4e4ae168302c57ec9de5cb3e" +checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.50.0" +version = "1.110.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" +checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.60.12", - "aws-smithy-json 0.60.7", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", + "fastrand 2.5.0", "http 0.2.12", - "once_cell", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.3.7" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e523e1c4e8e7e8ff219d732988e22bfeae8a1cafdbe6d9eca1546fa080be7c" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", - "aws-smithy-http 0.62.6", + "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint 0.5.5", + "crypto-bigint", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "p256", "percent-encoding", - "ring", - "sha2", + "sha2 0.11.0", "subtle", "time", "tracing", @@ -485,9 +513,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.7" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee19095c7c4dda59f1697d028ce704c24b2d33c6718790c7f1d5a3015b4107c" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -496,29 +524,30 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.63.12" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87294a084b43d649d967efe58aa1f9e0adc260e13a6938eb904c0ae9b45824ae" +checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307" dependencies = [ - "aws-smithy-http 0.62.6", + "aws-smithy-http", "aws-smithy-types", "bytes", "crc-fast", "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "md-5 0.11.0", "pin-project-lite", - "sha1", - "sha2", + "sha1 0.11.0", + "sha2 0.11.0", "tracing", ] [[package]] name = "aws-smithy-eventstream" -version = "0.60.14" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc12f8b310e38cad85cf3bef45ad236f470717393c613266ce0a89512286b650" +checksum = "5a9381123ab62d20c13082b151f30f962a3b112b727345394536dfa39a482944" dependencies = [ "aws-smithy-types", "bytes", @@ -527,29 +556,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.12" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http" -version = "0.62.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -558,9 +567,9 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", "percent-encoding", "pin-project-lite", "pin-utils", @@ -569,89 +578,87 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.5" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e62db736db19c488966c8d787f52e6270be565727236fd5579eaa301e7bc4a" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.12", + "h2 0.4.15", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.8.1", + "hyper 1.11.0", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.7", + "hyper-rustls 0.27.9", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.35", + "rustls 0.23.43", "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", - "tower 0.5.2", + "tower 0.5.3", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-json" -version = "0.61.9" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f616c3f2260612fe44cede278bafa18e73e6479c4e393e2c4518cf2a9a228a" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.9" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5d689cf437eae90460e944a58b5668530d433b4ff85789e69d2f2a556e057d" +checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.9.6" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fda37911905ea4d3141a01364bc5509a0f32ae3f3b22d6e330c0abfb62d247" +checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" dependencies = [ "aws-smithy-async", - "aws-smithy-http 0.62.6", + "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "http-body 0.4.6", - "http-body 1.0.1", + "http-body 1.1.0", + "http-body-util", "pin-project-lite", "pin-utils", "tokio", @@ -660,35 +667,58 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0d43d899f9e508300e587bf582ba54c27a452dd0a9ea294690669138ae14a2" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "pin-project-lite", "tokio", "tracing", "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + [[package]] name = "aws-smithy-types" -version = "1.3.5" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905cb13a9895626d49cf2ced759b062d913834c7482c38e49557eac4e6193f01" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "http-body 0.4.6", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "itoa", "num-integer", @@ -703,22 +733,26 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.13" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.11" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" +checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "rustc_version", "tracing", @@ -734,10 +768,10 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.11.0", "hyper-util", "itoa", "matchit", @@ -752,7 +786,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.2", "tokio", - "tower 0.5.2", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", @@ -767,8 +801,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "mime", "pin-project-lite", @@ -790,19 +824,19 @@ dependencies = [ "bytes", "dyn-clone", "futures", - "getrandom 0.2.16", - "hmac", + "getrandom 0.2.17", + "hmac 0.12.1", "http-types", "once_cell", "paste", "pin-project", "quick-xml 0.31.0", - "rand 0.8.5", - "reqwest 0.12.26", + "rand 0.8.7", + "reqwest 0.12.28", "rustc_version", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "time", "tracing", "url", @@ -888,9 +922,9 @@ dependencies = [ [[package]] name = "base16ct" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" @@ -922,9 +956,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.6.0" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bcrypt" @@ -934,7 +968,7 @@ checksum = "e65938ed058ef47d92cf8b346cc76ef48984572ade631927e9937b5ffc7662c7" dependencies = [ "base64 0.22.1", "blowfish", - "getrandom 0.2.16", + "getrandom 0.2.17", "subtle", "zeroize", ] @@ -956,18 +990,18 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "serde_core", ] [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -984,6 +1018,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blocking" version = "1.6.2" @@ -1017,13 +1060,13 @@ dependencies = [ "base64 0.22.1", "bitvec", "chrono", - "getrandom 0.2.16", + "getrandom 0.2.17", "getrandom 0.3.4", "hex", "indexmap", "js-sys", "once_cell", - "rand 0.9.2", + "rand 0.9.5", "serde", "serde_bytes", "serde_json", @@ -1033,9 +1076,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "byteorder" @@ -1045,9 +1088,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1061,9 +1104,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.49" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -1085,15 +1128,26 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -1109,15 +1163,15 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] [[package]] name = "clap" -version = "4.5.53" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -1125,9 +1179,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.53" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -1137,21 +1191,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] name = "clap_lex" -version = "0.7.6" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "clipboard-win" @@ -1164,18 +1218,34 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.56" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b042e5d8a74ae91bb0961acd039822472ec99f8ab0948cbf6d1369588f8be586" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] [[package]] name = "concurrent-queue" @@ -1205,6 +1275,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1220,16 +1296,16 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] [[package]] name = "const_fn" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" +checksum = "413d67b29ef1021b4d60f4aa1e925ca031751e213832b4b1d588fae623c05c60" [[package]] name = "convert_case" @@ -1275,6 +1351,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -1286,21 +1371,18 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc-fast" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "crc", - "digest", - "rand 0.9.2", - "regex", - "rustversion", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1320,36 +1402,36 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1359,9 +1441,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" -version = "0.4.9" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", "rand_core 0.6.4", @@ -1370,23 +1452,22 @@ dependencies = [ ] [[package]] -name = "crypto-bigint" -version = "0.5.5" +name = "crypto-common" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "rand_core 0.6.4", - "subtle", + "generic-array", + "typenum", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] @@ -1410,11 +1491,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ "darling_core", "darling_macro", @@ -1422,27 +1512,26 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -1460,9 +1549,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1474,9 +1563,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.9.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "deadpool" @@ -1496,34 +1585,23 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid", - "zeroize", -] - [[package]] name = "der" version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -1535,40 +1613,40 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "derive-where" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "derive_more" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "convert_case", "proc-macro2", "quote", "rustc_version", - "syn 2.0.111", + "syn 2.0.119", "unicode-xid", ] @@ -1591,12 +1669,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "directories" version = "5.0.1" @@ -1641,13 +1731,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -1670,39 +1760,41 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" -version = "0.14.8" +version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.6.1", + "der", + "digest 0.10.7", "elliptic-curve", "rfc6979", - "signature 1.6.4", + "signature", + "spki", ] [[package]] name = "either" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" dependencies = [ "serde", ] [[package]] name = "elliptic-curve" -version = "0.12.3" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", - "crypto-bigint 0.4.9", - "der 0.6.1", - "digest", + "crypto-bigint", + "digest 0.10.7", "ff", "generic-array", "group", - "pkcs8 0.9.0", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", "sec1", "subtle", @@ -1730,18 +1822,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.111", -] - [[package]] name = "env_logger" version = "0.10.2" @@ -1796,11 +1876,10 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -1811,7 +1890,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener 5.4.1", + "event-listener 5.4.2", "pin-project-lite", ] @@ -1826,9 +1905,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "fd-lock" @@ -1843,9 +1922,9 @@ dependencies = [ [[package]] name = "ff" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ "rand_core 0.6.4", "subtle", @@ -1853,15 +1932,15 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", @@ -1875,7 +1954,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.9", ] [[package]] @@ -1890,6 +1969,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1928,9 +2013,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -1943,9 +2028,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -1953,15 +2038,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -1981,9 +2066,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-lite" @@ -2006,7 +2091,7 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" dependencies = [ - "fastrand 2.3.0", + "fastrand 2.5.0", "futures-core", "futures-io", "parking", @@ -2015,32 +2100,32 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -2050,30 +2135,30 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] [[package]] name = "gcp_auth" -version = "0.12.5" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24fd9357c4d0ae97d90b38e8fabeb07f9590f6944f4f65f50b5110d7c3882b6" +checksum = "26d27dbcc645b60b8e7f6e2868a9d7102ece97d1bb49c1288b5321fcc67f7260" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", - "http 1.4.0", + "http 1.5.0", "http-body-util", - "hyper 1.8.1", - "hyper-rustls 0.27.7", + "hyper 1.11.0", + "hyper-rustls 0.27.9", "hyper-util", "ring", + "rustls 0.23.43", "rustls-pki-types", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.20", "tokio", "tracing", "tracing-futures", @@ -2088,6 +2173,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2103,9 +2189,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "js-sys", @@ -2123,11 +2209,25 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + [[package]] name = "google-cloud-auth" version = "0.16.0" @@ -2140,7 +2240,7 @@ dependencies = [ "google-cloud-token", "home", "jsonwebtoken", - "reqwest 0.12.26", + "reqwest 0.12.28", "serde", "serde_json", "thiserror 1.0.69", @@ -2156,7 +2256,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d901aeb453fd80e51d64df4ee005014f6cf39f2d736dd64f7239c132d9d39a6a" dependencies = [ - "reqwest 0.12.26", + "reqwest 0.12.28", "thiserror 1.0.69", "tokio", ] @@ -2179,14 +2279,14 @@ dependencies = [ "hex", "once_cell", "percent-encoding", - "pkcs8 0.10.2", + "pkcs8", "regex", - "reqwest 0.12.26", + "reqwest 0.12.28", "reqwest-middleware", "ring", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "time", "tokio", @@ -2205,9 +2305,9 @@ dependencies = [ [[package]] name = "group" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", "rand_core 0.6.4", @@ -2235,16 +2335,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.12" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http 1.5.0", "indexmap", "slab", "tokio", @@ -2266,7 +2366,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -2274,6 +2374,17 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" @@ -2303,47 +2414,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "hickory-proto" -version = "0.25.2" +name = "hickory-net" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" dependencies = [ "async-trait", "cfg-if", "data-encoding", - "enum-as-inner", "futures-channel", "futures-io", "futures-util", + "hickory-proto", "idna", "ipnet", + "jni", + "rand 0.10.2", + "thiserror 2.0.20", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", "once_cell", - "rand 0.9.2", + "prefix-trie", + "rand 0.10.2", "ring", - "thiserror 2.0.17", + "thiserror 2.0.20", "tinyvec", - "tokio", "tracing", "url", ] [[package]] name = "hickory-resolver" -version = "0.25.2" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" dependencies = [ "cfg-if", "futures-util", + "hickory-net", "hickory-proto", "ipconfig", + "ipnet", + "jni", "moka", + "ndk-context", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.10.2", "resolv-conf", "smallvec", - "thiserror 2.0.17", + "system-configuration 0.7.0", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -2354,7 +2489,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -2363,16 +2498,25 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] name = "home" -version = "0.5.9" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2388,9 +2532,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -2409,24 +2553,24 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.4.0", + "http 1.5.0", ] [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", - "http 1.4.0", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "pin-project-lite", ] @@ -2464,9 +2608,18 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] [[package]] name = "hyper" @@ -2494,22 +2647,21 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.12", - "http 1.4.0", - "http-body 1.0.1", + "h2 0.4.15", + "http 1.5.0", + "http-body 1.1.0", "httparse", "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -2532,16 +2684,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.0", - "hyper 1.8.1", + "http 1.5.0", + "hyper 1.11.0", "hyper-util", - "rustls 0.23.35", + "rustls 0.23.43", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", "tower-service", @@ -2568,7 +2719,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.1", + "hyper 1.11.0", "hyper-util", "native-tls", "tokio", @@ -2578,24 +2729,23 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", - "futures-core", "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "hyper 1.8.1", + "http 1.5.0", + "http-body 1.1.0", + "hyper 1.11.0", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", - "system-configuration", + "socket2 0.6.5", + "system-configuration 0.7.0", "tokio", "tower-service", "tracing", @@ -2604,9 +2754,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2628,12 +2778,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -2641,9 +2792,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -2654,9 +2805,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2668,15 +2819,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -2688,15 +2839,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -2726,9 +2877,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -2736,12 +2887,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -2772,29 +2923,23 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ - "socket2 0.5.10", + "socket2 0.6.5", "widestring", - "windows-sys 0.48.0", - "winreg", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", ] [[package]] name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.9" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" dependencies = [ - "memchr", "serde", ] @@ -2826,27 +2971,77 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] @@ -2871,30 +3066,31 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.9", ] [[package]] name = "libc" -version = "0.2.178" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.11" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" +checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "libc", - "redox_syscall 0.6.0", + "plain", + "redox_syscall 0.9.1", ] [[package]] @@ -2916,15 +3112,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -2937,17 +3133,17 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru" -version = "0.12.5" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -2965,7 +3161,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -2979,7 +3175,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -2990,7 +3186,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -3001,7 +3197,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -3026,14 +3222,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" @@ -3063,9 +3269,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -3074,16 +3280,16 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.12" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" dependencies = [ "async-lock", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", "equivalent", - "event-listener 5.4.1", + "event-listener 5.4.2", "futures-util", "parking_lot", "portable-atomic", @@ -3094,9 +3300,9 @@ dependencies = [ [[package]] name = "mongocrypt" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" +checksum = "8426a875ded61430d4a811dbfda7633b6b8af0225c547fc6c28b8b0aa7d79a13" dependencies = [ "bson", "mongocrypt-sys", @@ -3106,18 +3312,18 @@ dependencies = [ [[package]] name = "mongocrypt-sys" -version = "0.1.5+1.15.1" +version = "0.1.6+1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224484c5d09285a7b8cb0a0c117e847ebd14cb6e4470ecf68cdb89c503b0edb9" +checksum = "851fac73f7fe22f6a3ab87f720ce509cae7c9fd08e7dd27866cc232dee07ccf4" [[package]] name = "mongodb" -version = "3.4.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f5c20217413bed97c613714e6d6dfe39ef59dd79a68999f1043b0566192975" +checksum = "b814038f367d212f55de0a630cb35102a9b8ca23785a86955d62c0087c93846d" dependencies = [ "base64 0.22.1", - "bitflags 2.10.0", + "bitflags 2.13.1", "bson", "derive-where", "derive_more", @@ -3125,54 +3331,54 @@ dependencies = [ "futures-io", "futures-util", "hex", + "hickory-net", "hickory-proto", "hickory-resolver", - "hmac", + "hmac 0.13.0", "macro_magic", - "md-5", + "md-5 0.11.0", "mongocrypt", "mongodb-internal-macros", "pbkdf2", "percent-encoding", - "rand 0.9.2", + "rand 0.9.5", "rustc_version_runtime", - "rustls 0.23.35", - "rustversion", + "rustls 0.23.43", "serde", "serde_bytes", "serde_with", - "sha1", - "sha2", - "socket2 0.6.1", + "sha1 0.11.0", + "sha2 0.11.0", + "socket2 0.6.5", "stringprep", "strsim", "take_mut", - "thiserror 2.0.17", + "thiserror 2.0.20", "tokio", "tokio-rustls 0.26.4", "tokio-util", "typed-builder", "uuid", - "webpki-roots 1.0.4", + "webpki-roots 1.0.9", ] [[package]] name = "mongodb-internal-macros" -version = "3.4.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20033442aa13664e70bc9f8be1bacabebf6a31b6d4bb5608ceb99c4ec96e9951" +checksum = "f736d2fbc56e0011a341fbb9172bd822fda75c5f93b82fae1c7aab1e2613c810" dependencies = [ "macro_magic", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "native-tls" -version = "0.2.14" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ "libc", "log", @@ -3180,11 +3386,17 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework 2.11.1", + "security-framework", "security-framework-sys", "tempfile", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3200,7 +3412,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases 0.1.1", "libc", @@ -3217,9 +3429,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -3236,16 +3448,16 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.7", "smallvec", "zeroize", ] [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -3258,11 +3470,10 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -3304,13 +3515,13 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.16", + "getrandom 0.2.17", "http 0.2.12", - "rand 0.8.5", + "rand 0.8.7", "serde", "serde_json", "serde_path_to_error", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "url", ] @@ -3328,14 +3539,14 @@ dependencies = [ "futures", "httparse", "humantime", - "hyper 1.8.1", + "hyper 1.11.0", "itertools", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest 0.12.26", + "rand 0.8.7", + "reqwest 0.12.28", "ring", "rustls-pemfile 2.2.0", "serde", @@ -3349,9 +3560,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" dependencies = [ "critical-section", "portable-atomic", @@ -3365,11 +3576,11 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "onig" -version = "6.5.1" +version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "libc", "once_cell", "onig_sys", @@ -3377,9 +3588,9 @@ dependencies = [ [[package]] name = "onig_sys" -version = "69.9.1" +version = "69.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" dependencies = [ "cc", "pkg-config", @@ -3387,15 +3598,14 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -3408,20 +3618,20 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "openssl-probe" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -3443,18 +3653,19 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "p256" -version = "0.11.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", "elliptic-curve", - "sha2", + "primeorder", + "sha2 0.10.9", ] [[package]] name = "pangolin_api" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "async-trait", @@ -3471,20 +3682,21 @@ dependencies = [ "chrono", "dotenvy", "gcp_auth", - "hmac", - "hyper 1.8.1", + "hmac 0.12.1", + "hyper 1.11.0", "jsonwebtoken", "moka", "pangolin_api", "pangolin_core", "pangolin_store", - "rand 0.8.5", + "rand 0.8.7", "reqwest 0.11.27", + "ring", "serde", "serde_json", "serde_yaml", "serial_test", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "tokio", "tower 0.4.13", @@ -3500,7 +3712,7 @@ dependencies = [ [[package]] name = "pangolin_cli_admin" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "async-trait", @@ -3522,7 +3734,7 @@ dependencies = [ [[package]] name = "pangolin_cli_common" -version = "0.6.0" +version = "0.7.0" dependencies = [ "async-trait", "chrono", @@ -3540,7 +3752,7 @@ dependencies = [ [[package]] name = "pangolin_cli_user" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "clap", @@ -3558,7 +3770,7 @@ dependencies = [ [[package]] name = "pangolin_core" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "chrono", @@ -3573,7 +3785,7 @@ dependencies = [ [[package]] name = "pangolin_store" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "async-trait", @@ -3583,10 +3795,11 @@ dependencies = [ "aws-sdk-sts", "azure_core", "azure_storage_blobs", + "base64 0.22.1", "bson", "bytes", "chrono", - "dashmap 6.1.0", + "dashmap 6.2.1", "futures", "google-cloud-auth", "google-cloud-storage", @@ -3596,9 +3809,11 @@ dependencies = [ "pangolin_core", "pangolin_store", "regex", - "reqwest 0.12.26", + "reqwest 0.12.28", + "ring", "serde", "serde_json", + "serial_test", "sqlx", "thiserror 1.0.69", "tokio", @@ -3643,11 +3858,11 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pbkdf2" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -3677,29 +3892,29 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -3709,34 +3924,24 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" dependencies = [ "atomic-waker", - "fastrand 2.3.0", + "fastrand 2.5.0", "futures-io", ] [[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der 0.7.10", - "pkcs8 0.10.2", - "spki 0.7.3", -] - -[[package]] -name = "pkcs8" -version = "0.9.0" +name = "pkcs1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.6.1", - "spki 0.6.0", + "der", + "pkcs8", + "spki", ] [[package]] @@ -3745,25 +3950,31 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.10", - "spki 0.7.3", + "der", + "spki", ] [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plain" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "plist" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" +checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" dependencies = [ "base64 0.22.1", "indexmap", - "quick-xml 0.38.4", + "quick-xml 0.41.0", "serde", "time", ] @@ -3784,15 +3995,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -3812,6 +4023,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "prettytable-rs" version = "0.10.0" @@ -3826,6 +4048,15 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3852,9 +4083,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -3881,28 +4112,28 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", ] [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", - "cfg_aliases 0.2.1", + "cfg_aliases 0.2.2", "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.35", - "socket2 0.6.1", - "thiserror 2.0.17", + "rustls 0.23.43", + "socket2 0.6.5", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -3910,20 +4141,21 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.2", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", - "rustls 0.23.35", + "rustls 0.23.43", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -3931,23 +4163,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ - "cfg_aliases 0.2.1", + "cfg_aliases 0.2.2", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.5", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -3958,6 +4190,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -3989,9 +4227,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -4000,12 +4238,23 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -4035,7 +4284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -4053,18 +4302,24 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", ] [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_hc" version = "0.2.0" @@ -4074,22 +4329,31 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", ] [[package]] name = "redox_syscall" -version = "0.6.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96166dafa0886eb81fe1c0a388bece180fbef2135f97c1e2cf8302e74b43b5" +checksum = "07507be7b4a5f9f26eeb41eeaebb1f5a7ff29dfb29739facc21d35bf8b11c21e" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", ] [[package]] @@ -4098,16 +4362,16 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "libredox", "thiserror 1.0.69", ] [[package]] name = "regex" -version = "1.12.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -4117,9 +4381,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -4128,15 +4392,15 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -4167,7 +4431,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tower-service", @@ -4180,21 +4444,21 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.26" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2 0.4.12", - "http 1.4.0", - "http-body 1.0.1", + "h2 0.4.15", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", - "hyper 1.8.1", - "hyper-rustls 0.27.7", + "hyper 1.11.0", + "hyper-rustls 0.27.9", "hyper-tls 0.6.0", "hyper-util", "js-sys", @@ -4205,7 +4469,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.43", "rustls-native-certs", "rustls-pki-types", "serde", @@ -4216,8 +4480,8 @@ dependencies = [ "tokio-native-tls", "tokio-rustls 0.26.4", "tokio-util", - "tower 0.5.2", - "tower-http 0.6.8", + "tower 0.5.3", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", @@ -4234,8 +4498,8 @@ checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" dependencies = [ "anyhow", "async-trait", - "http 1.4.0", - "reqwest 0.12.26", + "http 1.5.0", + "reqwest 0.12.28", "serde", "thiserror 1.0.69", "tower-service", @@ -4249,13 +4513,12 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "rfc6979" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "crypto-bigint 0.4.9", - "hmac", - "zeroize", + "hmac 0.12.1", + "subtle", ] [[package]] @@ -4266,7 +4529,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -4274,29 +4537,29 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", "pkcs1", - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", - "signature 2.2.0", - "spki 0.7.3", + "signature", + "spki", "subtle", "zeroize", ] [[package]] name = "rust-embed" -version = "8.9.0" +version = "8.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947d7f3fad52b283d261c4c99a084937e2fe492248cb9a68a8435a861b8798ca" +checksum = "e9e7760e252aaba7b09f4be00e36476cf585bdb68a53552ac954cdf504ab4bc9" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -4305,32 +4568,33 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "8.9.0" +version = "8.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa2c8c9e8711e10f9c4fd2d64317ef13feaab820a4c51541f1a8c8e2e851ab2" +checksum = "3bcfc4d6f53af43755f7a723e4b6b8794fcce052a178dd8c6c1dadc5f5343097" dependencies = [ + "mime_guess", "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.111", + "syn 2.0.119", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "8.9.0" +version = "8.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b161f275cb337fe0a44d924a5f4df0ed69c2c39519858f931ce61c779d3475" +checksum = "42ffa149f6aa81b58a5b3011d01a857c4ed12c7a732d2c51947a4c7c692185f0" dependencies = [ - "sha2", + "sha2 0.11.0", "walkdir", ] [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -4353,11 +4617,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -4378,30 +4642,30 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework", ] [[package]] @@ -4424,9 +4688,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -4444,9 +4708,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -4456,9 +4720,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "rustyline" @@ -4466,7 +4730,7 @@ version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "clipboard-win", "fd-lock", @@ -4484,9 +4748,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "same-file" @@ -4499,9 +4763,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -4524,38 +4788,25 @@ dependencies = [ [[package]] name = "sec1" -version = "0.3.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der 0.6.1", + "der", "generic-array", - "pkcs8 0.9.0", + "pkcs8", "subtle", "zeroize", ] [[package]] name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -4564,9 +4815,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -4574,15 +4825,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -4600,36 +4851,36 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -4668,9 +4919,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.16.1" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" dependencies = [ "serde_core", "serde_with_macros", @@ -4678,14 +4929,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.16.1" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -4723,18 +4974,29 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -4744,8 +5006,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -4765,68 +5038,75 @@ checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" -version = "1.4.7" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] [[package]] name = "signature" -version = "1.6.4" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] [[package]] -name = "signature" -version = "2.2.0" +name = "simd-adler32" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" dependencies = [ - "digest", - "rand_core 0.6.4", + "rustc_version", + "simdutf8", ] [[package]] -name = "simd-adler32" -version = "0.3.8" +name = "simdutf8" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "simple_asn1" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.17", + "thiserror 2.0.20", "time", ] [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" dependencies = [ "serde", ] @@ -4849,7 +5129,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -4864,32 +5144,28 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] [[package]] -name = "spki" -version = "0.6.0" +name = "spin" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -4898,7 +5174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.10", + "der", ] [[package]] @@ -4926,7 +5202,7 @@ dependencies = [ "crc", "crossbeam-queue", "either", - "event-listener 5.4.1", + "event-listener 5.4.2", "futures-core", "futures-intrusive", "futures-io", @@ -4938,12 +5214,12 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.35", + "rustls 0.23.43", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -4962,7 +5238,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -4980,12 +5256,12 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.111", + "syn 2.0.119", "tokio", "url", ] @@ -4998,12 +5274,12 @@ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.10.0", + "bitflags 2.13.1", "byteorder", "bytes", "chrono", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -5013,22 +5289,22 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", - "rand 0.8.5", + "rand 0.8.7", "rsa", "serde", - "sha1", - "sha2", + "sha1 0.10.7", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.20", "tracing", "uuid", "whoami", @@ -5042,7 +5318,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.10.0", + "bitflags 2.13.1", "byteorder", "chrono", "crc", @@ -5053,21 +5329,21 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", - "rand 0.8.5", + "rand 0.8.7", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.20", "tracing", "uuid", "whoami", @@ -5093,7 +5369,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror 2.0.20", "tracing", "url", "uuid", @@ -5140,9 +5416,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -5172,7 +5459,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -5191,7 +5478,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.20", "walkdir", "yaml-rust", ] @@ -5204,7 +5491,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.9.4", + "system-configuration-sys 0.6.0", ] [[package]] @@ -5217,6 +5515,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -5237,12 +5545,12 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "fastrand 2.3.0", - "getrandom 0.3.4", + "fastrand 2.5.0", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -5279,11 +5587,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.20", ] [[package]] @@ -5294,58 +5602,57 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] [[package]] name = "time" -version = "0.3.44" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "itoa", "js-sys", "libc", "num-conv", "num_threads", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -5362,9 +5669,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -5372,9 +5679,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -5387,9 +5694,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -5397,20 +5704,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.5", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -5439,15 +5746,15 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.35", + "rustls 0.23.43", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -5456,12 +5763,10 @@ dependencies = [ [[package]] name = "tokio-test" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" dependencies = [ - "async-stream", - "bytes", "futures-core", "tokio", "tokio-stream", @@ -5469,14 +5774,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", + "futures-util", + "libc", "pin-project-lite", "tokio", ] @@ -5500,9 +5807,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -5520,10 +5827,11 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "bytes", - "http 1.4.0", - "http-body 1.0.1", + "futures-util", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "pin-project-lite", "tokio", @@ -5534,20 +5842,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "iri-string", + "http 1.5.0", + "http-body 1.1.0", "pin-project-lite", - "tower 0.5.2", + "tower 0.5.3", "tower-layer", "tower-service", + "url", ] [[package]] @@ -5564,9 +5872,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -5582,14 +5890,14 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "tracing-core" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -5628,9 +5936,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -5670,14 +5978,14 @@ checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "tz-rs" @@ -5690,9 +5998,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" [[package]] name = "unicode-bidi" @@ -5702,9 +6010,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization" @@ -5723,9 +6031,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -5759,14 +6067,15 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -5809,7 +6118,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.111", + "syn 2.0.119", "uuid", ] @@ -5831,11 +6140,11 @@ dependencies = [ [[package]] name = "uuid" -version = "1.19.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "js-sys", "serde_core", "wasm-bindgen", @@ -5904,9 +6213,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] @@ -5919,9 +6228,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -5932,22 +6241,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5955,22 +6261,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -5990,9 +6296,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -6014,14 +6320,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.4", + "webpki-roots 1.0.9", ] [[package]] name = "webpki-roots" -version = "1.0.4" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] @@ -6094,7 +6400,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -6105,7 +6411,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -6170,15 +6476,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -6212,30 +6509,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -6248,12 +6528,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6266,12 +6540,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6284,24 +6552,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6314,12 +6570,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6332,12 +6582,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6350,12 +6594,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6368,12 +6606,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winreg" version = "0.50.0" @@ -6394,9 +6626,9 @@ dependencies = [ "base64 0.22.1", "deadpool", "futures", - "http 1.4.0", + "http 1.5.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.11.0", "hyper-util", "log", "once_cell", @@ -6409,15 +6641,15 @@ dependencies = [ [[package]] name = "wit-bindgen" -version = "0.46.0" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wyz" @@ -6445,9 +6677,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -6456,68 +6688,68 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -6526,9 +6758,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -6537,13 +6769,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -6557,3 +6789,9 @@ dependencies = [ "crossbeam-utils", "flate2", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/pangolin/Cargo.toml b/pangolin/Cargo.toml index 1ca930d..efa54be 100644 --- a/pangolin/Cargo.toml +++ b/pangolin/Cargo.toml @@ -16,9 +16,9 @@ members = [ # UI 0.5.0 and the Helm chart 0.1.0 - with no way to tell which combination was # ever tested together (C-23). Every crate now inherits from here. [workspace.package] -version = "0.6.0" +version = "0.7.0" edition = "2021" -rust-version = "1.92" +rust-version = "1.94" license = "MIT" repository = "https://github.com/AlexMercedCoder/pangolin" authors = ["Alex Merced"] @@ -63,7 +63,7 @@ async-trait = "0.1" dashmap = "6.0" futures = "0.3" tower = { version = "0.4", features = ["limit", "util"] } -tower-http = { version = "0.5", features = ["cors", "trace", "timeout", "limit"] } +tower-http = { version = "0.5", features = ["cors", "trace", "timeout", "limit", "catch-panic"] } reqwest = { version = "0.11", features = ["json"] } url = "2.5" bytes = "1.5" diff --git a/pangolin/Dockerfile b/pangolin/Dockerfile index 476a30e..83780d8 100644 --- a/pangolin/Dockerfile +++ b/pangolin/Dockerfile @@ -4,8 +4,9 @@ # Build stage # --------------------------------------------------------------------------- # The base image is pinned to the Rust version the README requires. It used to -# be 1.88 while the README asked for 1.92+ (A-36). -FROM rust:1.92-slim-bookworm AS builder +# be 1.88 while the README asked for 1.92+ (A-36). Kept in step with +# `rust-version` in Cargo.toml, which the `msrv` CI job verifies. +FROM rust:1.94-slim-bookworm AS builder WORKDIR /usr/src/pangolin diff --git a/pangolin/clippy-warning-budget.txt b/pangolin/clippy-warning-budget.txt index 7facc89..64bb6b7 100644 --- a/pangolin/clippy-warning-budget.txt +++ b/pangolin/clippy-warning-budget.txt @@ -1 +1 @@ -36 +30 diff --git a/pangolin/logs/api_log.txt b/pangolin/logs/api_log.txt deleted file mode 100644 index 5e1615a..0000000 --- a/pangolin/logs/api_log.txt +++ /dev/null @@ -1,417 +0,0 @@ -warning: unused import: `std::collections::HashMap` - --> pangolin_store/src/postgres.rs:10:5 - | -10 | use std::collections::HashMap; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `target` - --> pangolin_store/src/postgres.rs:637:13 - | -637 | let target = self.get_branch(tenant_id, catalog_name, target_branch.clone()).await? - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_target` - | - = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `location` - --> pangolin_store/src/postgres.rs:697:31 - | -697 | async fn read_file(&self, location: &str) -> Result> { - | ^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_location` - -warning: unused variable: `bytes` - --> pangolin_store/src/postgres.rs:703:48 - | -703 | async fn write_file(&self, location: &str, bytes: Vec) -> Result<()> { - | ^^^^^ help: if this is intentional, prefix it with an underscore: `_bytes` - -warning: unused variable: `location` - --> pangolin_store/src/postgres.rs:703:32 - | -703 | async fn write_file(&self, location: &str, bytes: Vec) -> Result<()> { - | ^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_location` - -warning: unused variable: `branch` - --> pangolin_store/src/postgres.rs:820:83 - | -820 | ...d: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String, expected_location: Option, new_loc... - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `tenant_id` - --> pangolin_store/src/mongo.rs:121:38 - | -121 | async fn create_warehouse(&self, tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { - | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_tenant_id` - -warning: variable does not need to be mutable - --> pangolin_store/src/mongo.rs:284:13 - | -284 | let mut filter = doc! { - | ----^^^^^^ - | | - | help: remove this `mut` - | - = note: `#[warn(unused_mut)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `p` - --> pangolin_store/src/mongo.rs:289:21 - | -289 | if let Some(p) = parent { - | ^ help: if this is intentional, prefix it with an underscore: `_p` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:364:68 - | -364 | ...d: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result> { - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:401:70 - | -401 | async fn list_assets(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec) -> Result> { - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:432:71 - | -432 | ...d: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result<()> { - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:443:71 - | -443 | ...d: Uuid, catalog_name: &str, branch: Option, source_namespace: Vec, source_name: String, dest_namespace: Vec, ... - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:714:83 - | -714 | ...d: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String, expected_location: Option, new_loc... - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: field `signer` is never read - --> pangolin_store/src/memory.rs:32:5 - | -17 | pub struct MemoryStore { - | ----------- field in this struct -... -32 | signer: crate::signer::SignerImpl, - | ^^^^^^ - | - = note: `MemoryStore` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default - -warning: field `key` is never read - --> pangolin_store/src/signer.rs:25:5 - | -24 | pub struct SignerImpl { - | ---------- field in this struct -25 | key: String, - | ^^^ - | - = note: `SignerImpl` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - -warning: field `client` is never read - --> pangolin_store/src/mongo.rs:18:5 - | -17 | pub struct MongoStore { - | ---------- field in this struct -18 | client: Client, - | ^^^^^^ - | - = note: `MongoStore` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - -warning: multiple methods are never used - --> pangolin_store/src/mongo.rs:38:8 - | -22 | impl MongoStore { - | --------------- methods in this implementation -... -38 | fn catalogs(&self) -> Collection { - | ^^^^^^^^ -... -42 | fn namespaces(&self) -> Collection { - | ^^^^^^^^^^ -... -46 | fn assets(&self) -> Collection { - | ^^^^^^ -... -50 | fn branches(&self) -> Collection { - | ^^^^^^^^ -... -54 | fn tags(&self) -> Collection { - | ^^^^ -... -58 | fn commits(&self) -> Collection { - | ^^^^^^^ -... -62 | fn audit_logs(&self) -> Collection { - | ^^^^^^^^^^ - -warning: `pangolin_store` (lib) generated 18 warnings (run `cargo fix --lib -p pangolin_store` to apply 14 suggestions) -warning: unused import: `pangolin_store::memory::MemoryStore` - --> pangolin_api/src/lib.rs:4:5 - | -4 | use pangolin_store::memory::MemoryStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: unused import: `Any` - --> pangolin_api/src/lib.rs:6:35 - | -6 | use tower_http::cors::{CorsLayer, Any}; - | ^^^ - -warning: unused imports: `Request` and `body::Body` - --> pangolin_api/src/iceberg_handlers.rs:5:43 - | -5 | http::{StatusCode, HeaderMap, Method, Request}, - | ^^^^^^^ -6 | body::Body, - | ^^^^^^^^^^ - -warning: unused import: `pangolin_store::memory::MemoryStore` - --> pangolin_api/src/iceberg_handlers.rs:12:5 - | -12 | use pangolin_store::memory::MemoryStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `UserRole` - --> pangolin_api/src/iceberg_handlers.rs:1021:27 - | -1021 | use pangolin_core::user::{UserRole, UserSession}; - | ^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/tenant_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `std::collections::HashMap` - --> pangolin_api/src/tenant_handlers.rs:9:5 - | -9 | use std::collections::HashMap; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/warehouse_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/asset_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `HeaderMap` - --> pangolin_api/src/auth.rs:3:24 - | -3 | http::{StatusCode, HeaderMap}, - | ^^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/signing_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `Json` - --> pangolin_api/src/oauth_handlers.rs:5:5 - | -5 | Json, - | ^^^^ - -warning: unused import: `Serialize` - --> pangolin_api/src/oauth_handlers.rs:7:26 - | -7 | use serde::{Deserialize, Serialize}; - | ^^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/business_metadata_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused imports: `MergeConflict` and `MergeOperation` - --> pangolin_api/src/merge_handlers.rs:7:48 - | -7 | use pangolin_core::model::{ConflictResolution, MergeConflict, MergeOperation, ResolutionStrategy}; - | ^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ - -warning: unused import: `Serialize` - --> pangolin_api/src/permission_handlers.rs:8:26 - | -8 | use serde::{Deserialize, Serialize}; - | ^^^^^^^^^ - -warning: unused import: `Serialize` - --> pangolin_api/src/service_user_handlers.rs:10:26 - | -10 | use serde::{Deserialize, Serialize}; - | ^^^^^^^^^ - -warning: variable does not need to be mutable - --> pangolin_api/src/auth.rs:92:21 - | -92 | let mut validation = Validation::new(Algorithm::HS256); - | ----^^^^^^^^^^ - | | - | help: remove this `mut` - | - = note: `#[warn(unused_mut)]` (part of `#[warn(unused)]`) on by default - -warning: variable does not need to be mutable - --> pangolin_api/src/oauth_handlers.rs:110:14 - | -110 | Some(mut u) => { - | ----^ - | | - | help: remove this `mut` - -warning: variable does not need to be mutable - --> pangolin_api/src/permission_handlers.rs:100:10 - | -100 | Json(mut role): Json, - | ----^^^^ - | | - | help: remove this `mut` - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/business_metadata_handlers.rs:9:5 - | -9 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/tenant_handlers.rs:10:5 - | -10 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/warehouse_handlers.rs:9:5 - | -9 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/asset_handlers.rs:9:5 - | -9 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/signing_handlers.rs:9:5 - | -9 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused variable: `prefix` - --> pangolin_api/src/iceberg_handlers.rs:286:5 - | -286 | prefix: Option>, - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_prefix` - | - = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `timestamp` - --> pangolin_api/src/iceberg_handlers.rs:656:24 - | -656 | if let Ok(timestamp) = timestamp_str.parse::() { - | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_timestamp` - -warning: unused variable: `session` - --> pangolin_api/src/iceberg_handlers.rs:854:15 - | -854 | Extension(session): Extension, - | ^^^^^^^ help: if this is intentional, prefix it with an underscore: `_session` - -warning: unused variable: `session` - --> pangolin_api/src/iceberg_handlers.rs:982:15 - | -982 | Extension(session): Extension, - | ^^^^^^^ help: if this is intentional, prefix it with an underscore: `_session` - -warning: unused variable: `params` - --> pangolin_api/src/pangolin_handlers.rs:66:11 - | -66 | Query(params): Query, - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_params` - -warning: unused variable: `tenant_uuid` - --> pangolin_api/src/token_handlers.rs:34:9 - | -34 | let tenant_uuid = match Uuid::parse_str(&payload.tenant_id) { - | ^^^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_tenant_uuid` - -warning: fields `identifier` and `requirements` are never read - --> pangolin_api/src/iceberg_handlers.rs:175:5 - | -174 | pub struct CommitTableRequest { - | ------------------ fields in this struct -175 | identifier: Option, - | ^^^^^^^^^^ -176 | requirements: Vec, - | ^^^^^^^^^^^^ - | - = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default - -warning: field `removals` is never read - --> pangolin_api/src/iceberg_handlers.rs:968:5 - | -967 | pub struct UpdateNamespacePropertiesRequest { - | -------------------------------- field in this struct -968 | removals: Option>, - | ^^^^^^^^ - -warning: fields `name` and `catalog` are never read - --> pangolin_api/src/pangolin_handlers.rs:30:5 - | -29 | pub struct ListBranchParams { - | ---------------- fields in this struct -30 | name: Option, - | ^^^^ -31 | catalog: Option, - | ^^^^^^^ - -warning: `pangolin_api` (lib) generated 34 warnings (run `cargo fix --lib -p pangolin_api` to apply 26 suggestions) -warning: unused import: `std::env` - --> pangolin_api/src/main.rs:3:5 - | -3 | use std::env; - | ^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: unused imports: `Any` and `CorsLayer` - --> pangolin_api/src/main.rs:4:24 - | -4 | use tower_http::cors::{CorsLayer, Any}; - | ^^^^^^^^^ ^^^ - -warning: unused imports: `HeaderValue` and `Method` - --> pangolin_api/src/main.rs:5:18 - | -5 | use axum::http::{HeaderValue, Method}; - | ^^^^^^^^^^^ ^^^^^^ - -warning: unused variable: `storage_type` - --> pangolin_api/src/main.rs:17:9 - | -17 | let storage_type = std::env::var("PANGOLIN_STORAGE_TYPE").unwrap_or_else(|_| "memory".to_string()); - | ^^^^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_storage_type` - | - = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default - -warning: `pangolin_api` (bin "pangolin_api") generated 4 warnings (run `cargo fix --bin "pangolin_api" -p pangolin_api` to apply 4 suggestions) - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.25s - Running `target/debug/pangolin_api` -2025-12-14T22:11:31.649619Z  INFO pangolin_api: Using Memory Storage -2025-12-14T22:11:31.650363Z  INFO pangolin_api: Created default tenant for testing: 00000000-0000-0000-0000-000000000000 -2025-12-14T22:11:31.653384Z  INFO pangolin_api: listening on 0.0.0.0:8080 diff --git a/pangolin/pangolin_api/Cargo.toml b/pangolin/pangolin_api/Cargo.toml index 1d2ae27..88855a1 100644 --- a/pangolin/pangolin_api/Cargo.toml +++ b/pangolin/pangolin_api/Cargo.toml @@ -15,8 +15,8 @@ categories = ["database"] workspace = true [dependencies] -pangolin_core = { path = "../pangolin_core", version = "0.6.0" } -pangolin_store = { path = "../pangolin_store", version = "0.6.0" } +pangolin_core = { path = "../pangolin_core", version = "0.7.0" } +pangolin_store = { path = "../pangolin_store", version = "0.7.0" } axum = { workspace = true } tokio = { workspace = true } serde = { workspace = true } @@ -42,11 +42,16 @@ utoipa-swagger-ui = { version = "6", features = ["axum"] } async-trait = { workspace = true } thiserror = { workspace = true } hmac = "0.12" +# OIDC: PKCE verifiers and nonces need a CSPRNG. `ring` is already compiled into +# the tree via jsonwebtoken and rustls, so this declares an existing dependency +# rather than adding one - and it is a better source than `rand`, which carries +# an open unsoundness advisory (RUSTSEC-2026-0097). +ring = "0.17" sha2 = "0.10" # Cloud provider SDKs (optional features) aws-config = { version = "1.0", optional = true } -aws-sdk-sts = { version = "=1.50.0", optional = true } +aws-sdk-sts = { version = "1.109", optional = true } aws-credential-types = { version = "1.0", optional = true } aws-smithy-types = { version = "1.0", optional = true } azure_identity = { version = "0.20", optional = true } @@ -68,6 +73,6 @@ cloud-credentials = ["aws-sts", "azure-oauth", "gcp-oauth"] [dev-dependencies] hyper = { version = "1.0", features = ["full"] } pangolin_api = { path = ".", features = ["test-fixtures"] } -pangolin_store = { path = "../pangolin_store", version = "0.6.0" } +pangolin_store = { path = "../pangolin_store", version = "0.7.0" } serial_test = "2.0" wiremock = "0.6" diff --git a/pangolin/pangolin_api/src/asset_handlers.rs b/pangolin/pangolin_api/src/asset_handlers.rs index 9e378f5..0f055da 100644 --- a/pangolin/pangolin_api/src/asset_handlers.rs +++ b/pangolin/pangolin_api/src/asset_handlers.rs @@ -1,7 +1,7 @@ use crate::auth::TenantId; use crate::authz::check_permission; -use crate::iceberg::parse_table_identifier; use crate::iceberg::AppState; +use crate::iceberg::{parse_namespace, parse_table_identifier}; use axum::{ extract::{Extension, Path, Query, State}, http::StatusCode, @@ -17,6 +17,7 @@ use utoipa::ToSchema; use uuid::Uuid; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateViewRequest { pub name: String, pub sql: String, @@ -66,6 +67,7 @@ impl From for ViewResponse { pub async fn create_view( State(store): State, Extension(tenant): Extension, + Extension(session): Extension, Path((prefix, namespace)): Path<(String, String)>, Json(payload): Json, ) -> impl IntoResponse { @@ -73,10 +75,35 @@ pub async fn create_view( let catalog_name = prefix; let (view_name, branch_from_name) = parse_table_identifier(&payload.name); - let branch = branch_from_name.unwrap_or("main".to_string()); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .unwrap_or_else(|| "main".to_string()); - // Parse namespace - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_view: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + // B0e: this handler took no session and performed no authorization at all, + // so any tenant member could create a view in any namespace. It now mirrors + // `create_table`'s namespace-scoped Create check. + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Create, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_view: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } let mut properties = payload.properties.unwrap_or_default(); properties.insert("sql".to_string(), payload.sql); @@ -104,7 +131,10 @@ pub async fn create_view( .await { Ok(_) => (StatusCode::CREATED, Json(ViewResponse::from(asset))).into_response(), - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_view: failed to create asset"); + (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + } } } @@ -128,18 +158,274 @@ pub async fn create_view( pub async fn get_view( State(store): State, Extension(tenant): Extension, + Extension(session): Extension, Path((prefix, namespace, view)): Path<(String, String, String)>, ) -> impl IntoResponse { let tenant_id = tenant.0; let catalog_name = prefix; let (view_name, branch_from_name) = parse_table_identifier(&view); - let branch = branch_from_name.unwrap_or("main".to_string()); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .unwrap_or_else(|| "main".to_string()); - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "get_view: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; - match store + let asset = match store .get_asset( + tenant_id, + &catalog_name, + Some(branch), + namespace_parts.clone(), + view_name, + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return (StatusCode::NOT_FOUND, "View not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "get_view: failed to load asset"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + if asset.kind != AssetType::View { + return (StatusCode::NOT_FOUND, "Asset is not a view").into_response(); + } + + // B0e: a view's `properties["sql"]` is its whole definition - business + // logic, column names, sometimes filter predicates that reveal the data. + // Reading it used to require nothing beyond being authenticated. + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + asset_id: asset.id, + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (StatusCode::OK, Json(ViewResponse::from(asset))).into_response(), + Ok(false) => (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "get_view: permission check failed"); + (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response() + } + } +} + +/// `GET /v1/{prefix}/namespaces/{namespace}/views` — the spec's `listViews`. +/// +/// A-5. Only `createView` and `loadView` were routed, so an engine could make a +/// view and read one it already knew the name of, and had no way to discover +/// what existed. Spark's `SHOW VIEWS` and every catalog browser call this. +/// +/// Views are stored as assets with `kind: View`, so this is `list_assets` +/// filtered - there is no separate view collection to fall out of step. +pub async fn list_views( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path((prefix, namespace)): Path<(String, String)>, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = prefix; + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_ns.unwrap_or_else(|| "main".to_string()); + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "list_views: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + // Namespace-scoped Read, matching `list_tables`. Knowing which views exist + // is itself information about the data. + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "list_views: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } + + let assets = match store + .list_assets( + tenant_id, + &catalog_name, + Some(branch), + namespace_parts.clone(), + None, + ) + .await + { + Ok(a) => a, + Err(e) => { + tracing::error!(error = %e, "list_views: failed to list assets"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + // The spec's shape: `{"identifiers": [{"namespace": [...], "name": "..."}]}`. + let identifiers: Vec = assets + .into_iter() + .filter(|a| a.kind == AssetType::View) + .map(|a| { + serde_json::json!({ + "namespace": namespace_parts.clone(), + "name": a.name, + }) + }) + .collect(); + + ( + StatusCode::OK, + Json(serde_json::json!({ "identifiers": identifiers })), + ) + .into_response() +} + +/// `HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}` — `viewExists`. +/// +/// A bare existence check, with no body. Clients use it before a create to +/// decide between create and replace; without it they have to issue a full +/// `loadView` and read the status, which transfers the view's SQL to answer a +/// yes/no question. +pub async fn view_exists( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path((prefix, namespace, view)): Path<(String, String, String)>, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = prefix; + let (view_name, branch_from_name) = parse_table_identifier(&view); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .unwrap_or_else(|| "main".to_string()); + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return StatusCode::NOT_FOUND, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR, + }; + + let asset = match store + .get_asset( + tenant_id, + &catalog_name, + Some(branch), + namespace_parts.clone(), + view_name, + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return StatusCode::NOT_FOUND, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR, + }; + + if asset.kind != AssetType::View { + return StatusCode::NOT_FOUND; + } + + // Authorized like `loadView`. Answering "does this exist" to a caller who + // may not read it still leaks that it exists, so the check is the same one. + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + asset_id: asset.id, + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => StatusCode::NO_CONTENT, + Ok(false) => StatusCode::FORBIDDEN, + Err(_) => StatusCode::INTERNAL_SERVER_ERROR, + } +} + +/// `DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}` — `dropView`. +/// +/// Without this a view could be created and never removed through the Iceberg +/// API at all; the only way out was the Pangolin-native asset endpoint, which +/// an Iceberg client does not know about. +pub async fn drop_view( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path((prefix, namespace, view)): Path<(String, String, String)>, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = prefix; + let (view_name, branch_from_name) = parse_table_identifier(&view); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .unwrap_or_else(|| "main".to_string()); + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "drop_view: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + let asset = match store + .get_asset( + tenant_id, + &catalog_name, + Some(branch.clone()), + namespace_parts.clone(), + view_name.clone(), + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return (StatusCode::NOT_FOUND, "View not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "drop_view: failed to load asset"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + // Refuse to delete a table through the view endpoint. Without this check a + // client with view-drop rights could remove a table by addressing it as a + // view. + if asset.kind != AssetType::View { + return (StatusCode::NOT_FOUND, "Asset is not a view").into_response(); + } + + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + asset_id: asset.id, + }; + match check_permission(&store, &session, &Action::Delete, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "drop_view: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } + + match store + .delete_asset( tenant_id, &catalog_name, Some(branch), @@ -148,19 +434,16 @@ pub async fn get_view( ) .await { - Ok(Some(asset)) => { - if asset.kind == AssetType::View { - (StatusCode::OK, Json(ViewResponse::from(asset))).into_response() - } else { - (StatusCode::NOT_FOUND, "Asset is not a view").into_response() - } + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => { + tracing::error!(error = %e, "drop_view: failed to delete asset"); + (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() } - Ok(None) => (StatusCode::NOT_FOUND, "View not found").into_response(), - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), } } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct RegisterAssetRequest { pub name: String, pub kind: AssetType, @@ -487,6 +770,7 @@ pub async fn list_assets( // Filter assets based on permissions let filtered = crate::authz_utils::filter_assets( + tenant_id, assets_with_metadata, &permissions, session.role, @@ -613,6 +897,7 @@ mod tests { tenant_id: Some(tenant_id), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), issued_at: chrono::Utc::now(), + token_id: None, }; let payload = RegisterAssetRequest { @@ -646,6 +931,7 @@ mod tests { tenant_id: Some(tenant_id), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), issued_at: chrono::Utc::now(), + token_id: None, }; let payload = RegisterAssetRequest { @@ -680,6 +966,7 @@ mod tests { tenant_id: Some(tenant_id), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), issued_at: chrono::Utc::now(), + token_id: None, }; let payload = RegisterAssetRequest { @@ -713,6 +1000,7 @@ mod tests { tenant_id: Some(tenant_id), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), issued_at: chrono::Utc::now(), + token_id: None, }; let asset_types = vec![ diff --git a/pangolin/pangolin_api/src/audit_tests.rs b/pangolin/pangolin_api/src/audit_tests.rs index cbe6a5b..11981a9 100644 --- a/pangolin/pangolin_api/src/audit_tests.rs +++ b/pangolin/pangolin_api/src/audit_tests.rs @@ -77,7 +77,8 @@ mod tests { .header("Authorization", format!("Bearer {}", root_token)) .header("Content-Type", "application/json") .body(Body::from( - json!({"name":"audit-tenant", "organization":"org"}).to_string(), + json!({"name":"audit-tenant", "properties": {"organization":"org"}}) + .to_string(), )) .unwrap(), ) @@ -106,7 +107,8 @@ mod tests { .header("Authorization", format!("Bearer {}", user_token)) .header("Content-Type", "application/json") .body(Body::from( - json!({"name":"cat","warehouse_name":"wh","type":"pangolin"}).to_string(), + json!({"name":"cat","warehouse_name":"wh","catalog_type":"Local"}) + .to_string(), )) .unwrap(), ) diff --git a/pangolin/pangolin_api/src/auth.rs b/pangolin/pangolin_api/src/auth.rs index 50592a3..b13700f 100644 --- a/pangolin/pangolin_api/src/auth.rs +++ b/pangolin/pangolin_api/src/auth.rs @@ -47,6 +47,14 @@ impl Claims { .ok_or("Invalid issued_at timestamp")?, expires_at: DateTime::from_timestamp(self.exp, 0) .ok_or("Invalid expires_at timestamp")?, + // Carried through so logout can revoke the token that is actually + // presented rather than the user id (B0j). + token_id: self + .jti + .as_ref() + .map(|id| Uuid::parse_str(id)) + .transpose() + .map_err(|e| e.to_string())?, }) } } diff --git a/pangolin/pangolin_api/src/auth_middleware.rs b/pangolin/pangolin_api/src/auth_middleware.rs index b8f3a65..4ca76eb 100644 --- a/pangolin/pangolin_api/src/auth_middleware.rs +++ b/pangolin/pangolin_api/src/auth_middleware.rs @@ -57,6 +57,12 @@ pub fn create_session( role, issued_at: now, expires_at, + // Sessions built here are not derived from a presented JWT: they are + // either about to have a token minted *from* them (login), or backed by + // an API key / root basic auth, which have nothing to revoke. Sessions + // that do come from a bearer token get their `jti` in + // `Claims::to_session`. + token_id: None, } } @@ -223,6 +229,18 @@ pub async fn auth_middleware( // tests and tools can exercise a specific identity even in NO_AUTH mode. } + // Public endpoints, matched structurally rather than by suffix (A-11). + // + // B0o: this check used to sit *below* the API-key branch, which returned + // unconditionally. A client that sets `X-API-Key` globally - the normal way + // to configure an HTTP client - therefore could not reach `/v1/config`, + // `/health` or the OAuth token endpoint at all, the opposite of the + // documented ordering. Resolving public paths first also keeps an + // unauthenticated `/health` probe away from bcrypt. + if crate::public_paths::is_public_path(&path) { + return next.run(req).await; + } + // Service-user API key. if let Some(api_key_header) = req.headers().get("X-API-Key") { let Ok(api_key) = api_key_header.to_str() else { @@ -271,11 +289,6 @@ pub async fn auth_middleware( } } - // Public endpoints, matched structurally rather than by suffix (A-11). - if crate::public_paths::is_public_path(&path) { - return next.run(req).await; - } - let auth_header = req .headers() .get(header::AUTHORIZATION) @@ -369,8 +382,19 @@ pub async fn auth_middleware( // Revocation must fail *closed*. Previously an error from the store was // logged and ignored, so during a database blip every revoked token was // accepted again (A-13). - if let Some(ref jti_str) = claims.jti { - if let Ok(token_id) = uuid::Uuid::parse_str(jti_str) { + // + // B0o: the two nested `if let`s also failed open on the *shape* of the + // claim. A token whose `jti` was present but not a UUID skipped the + // revocation check entirely and was unrevocable for its whole lifetime, and + // a token minted with no `jti` at all was likewise exempt. A malformed `jti` + // is now a hard rejection, and the no-`jti` case is accepted only when the + // operator has explicitly opted into legacy tokens. + match claims.jti.as_deref() { + Some(jti_str) => { + let Ok(token_id) = uuid::Uuid::parse_str(jti_str) else { + tracing::warn!(jti = %jti_str, "rejected a token with a malformed jti"); + return (StatusCode::UNAUTHORIZED, "Invalid token").into_response(); + }; match store.is_token_revoked(token_id).await { Ok(true) => { tracing::warn!(jti = %jti_str, "revoked token presented"); @@ -390,6 +414,15 @@ pub async fn auth_middleware( } } } + None => { + if !crate::config::allow_tokens_without_jti() { + tracing::warn!( + "rejected a token with no jti; it could never be revoked. Re-issue the \ + token, or set PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI=true during migration" + ); + return (StatusCode::UNAUTHORIZED, "Invalid token").into_response(); + } + } } let session = match claims.to_session() { diff --git a/pangolin/pangolin_api/src/authz.rs b/pangolin/pangolin_api/src/authz.rs index adcc747..a3397f0 100644 --- a/pangolin/pangolin_api/src/authz.rs +++ b/pangolin/pangolin_api/src/authz.rs @@ -183,6 +183,7 @@ mod tests { username: "test_user".to_string(), issued_at: chrono::Utc::now(), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + token_id: None, }; // Target Action/Scope @@ -267,6 +268,7 @@ mod tests { username: "test_user".to_string(), issued_at: chrono::Utc::now(), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + token_id: None, }; // --- Branch Permission Test --- diff --git a/pangolin/pangolin_api/src/authz_matrix_tests.rs b/pangolin/pangolin_api/src/authz_matrix_tests.rs new file mode 100644 index 0000000..e20e4d0 --- /dev/null +++ b/pangolin/pangolin_api/src/authz_matrix_tests.rs @@ -0,0 +1,686 @@ +//! Permission matrix: who is allowed to call what. +//! +//! Roadmap improvement #0, the highest-leverage addition in the August audit. +//! +//! Every finding in the B0a-B0m cluster was invisible to CI. The code compiled, +//! it was formatted, it was lint-clean, and the tests passed - because nothing +//! asserted *who is allowed to call what*. A handler that simply forgot to call +//! `check_permission` looks identical, to every existing check, to one that +//! calls it correctly. `POST /api/v1/tokens` minting a `Root` JWT for any +//! authenticated caller sat there through a full security release. +//! +//! This module drives each sensitive route as several principals and asserts +//! the outcome for each. A missing authorization check fails here as a `200` +//! where a `403` was expected, which is the only signal that distinguishes the +//! two shapes of handler from the outside. +//! +//! Deliberately end-to-end through the real router: a unit test of +//! `check_permission` proves the function works, not that the handler calls it, +//! and "the handler does not call it" is the entire bug class. + +use crate::tests_common::EnvGuard; +use axum::{ + body::{to_bytes, Body}, + http::{Request, StatusCode}, + Router, +}; +use pangolin_store::memory::MemoryStore; +use serde_json::json; +use serial_test::serial; +use std::sync::Arc; +use tower::ServiceExt; +use uuid::Uuid; + +fn app() -> Router { + let store = Arc::new(MemoryStore::new()); + crate::app(store) +} + +/// The principals every route is driven as. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Principal { + /// Global superuser. + Root, + /// Administrator of the tenant that owns the resource. + TenantAdmin, + /// Ordinary member of the tenant with no explicit grants. + TenantUser, + /// Administrator of a *different* tenant. + ForeignTenantAdmin, +} + +struct Fixture { + app: Router, + root_token: String, + admin_token: String, + user_token: String, + foreign_admin_token: String, + tenant_id: Uuid, + foreign_tenant_id: Uuid, +} + +impl Fixture { + fn token(&self, who: Principal) -> &str { + match who { + Principal::Root => &self.root_token, + Principal::TenantAdmin => &self.admin_token, + Principal::TenantUser => &self.user_token, + Principal::ForeignTenantAdmin => &self.foreign_admin_token, + } + } + + fn tenant_header(&self, who: Principal) -> Uuid { + match who { + Principal::ForeignTenantAdmin => self.foreign_tenant_id, + _ => self.tenant_id, + } + } + + async fn request( + &self, + who: Principal, + method: &str, + uri: &str, + body: Option, + ) -> StatusCode { + let mut builder = Request::builder() + .method(method) + .uri(uri) + .header("Authorization", format!("Bearer {}", self.token(who))) + .header("Content-Type", "application/json") + .header("X-Pangolin-Tenant", self.tenant_header(who).to_string()); + + // Root has no tenant of its own, so it addresses one explicitly. + if who == Principal::Root { + builder = builder.header("X-Pangolin-Tenant", self.tenant_id.to_string()); + } + + let body = match body { + Some(v) => Body::from(v.to_string()), + None => Body::empty(), + }; + + self.app + .clone() + .oneshot(builder.body(body).unwrap()) + .await + .unwrap() + .status() + } +} + +async fn login(app: &Router, username: &str, password: &str, tenant_id: Option) -> String { + let mut body = json!({ "username": username, "password": password }); + if let Some(tid) = tenant_id { + body.as_object_mut() + .unwrap() + .insert("tenant-id".to_string(), json!(tid)); + } + + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/users/login") + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::OK, + "login for {username} should succeed" + ); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + json["token"].as_str().unwrap().to_string() +} + +async fn create_tenant(app: &Router, root_token: &str, name: &str) -> Uuid { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/tenants") + .header("Authorization", format!("Bearer {root_token}")) + .header("Content-Type", "application/json") + .body(Body::from(json!({ "name": name }).to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + Uuid::parse_str(json["id"].as_str().unwrap()).unwrap() +} + +async fn create_user( + app: &Router, + token: &str, + tenant_id: Uuid, + username: &str, + password: &str, + role: &str, +) { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/users") + .header("Authorization", format!("Bearer {token}")) + .header("X-Pangolin-Tenant", tenant_id.to_string()) + .header("Content-Type", "application/json") + .body(Body::from( + json!({ + "username": username, + "password": password, + "email": format!("{username}@example.test"), + "role": role, + "tenant_id": tenant_id, + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::CREATED, + "creating {username} should succeed" + ); +} + +/// Two tenants, each with an admin, plus one ordinary user with no grants. +async fn fixture() -> Fixture { + let app = app(); + let root_token = login(&app, "admin", "password", None).await; + + let tenant_id = create_tenant(&app, &root_token, "matrix-tenant").await; + let foreign_tenant_id = create_tenant(&app, &root_token, "matrix-other-tenant").await; + + // Root creates the tenant admins; root is deliberately *not* allowed to + // create tenant users (it has no tenant context of its own), so the admin + // creates those. + create_user( + &app, + &root_token, + tenant_id, + "matrix_admin", + "password", + "tenant-admin", + ) + .await; + create_user( + &app, + &root_token, + foreign_tenant_id, + "matrix_foreign_admin", + "password", + "tenant-admin", + ) + .await; + + let admin_token = login(&app, "matrix_admin", "password", Some(tenant_id)).await; + create_user( + &app, + &admin_token, + tenant_id, + "matrix_user", + "password", + "tenant-user", + ) + .await; + + Fixture { + user_token: login(&app, "matrix_user", "password", Some(tenant_id)).await, + admin_token, + foreign_admin_token: login( + &app, + "matrix_foreign_admin", + "password", + Some(foreign_tenant_id), + ) + .await, + app, + root_token, + tenant_id, + foreign_tenant_id, + } +} + +fn root_env() -> (EnvGuard, EnvGuard, EnvGuard) { + ( + EnvGuard::new("PANGOLIN_ROOT_USER", "admin"), + EnvGuard::new("PANGOLIN_ROOT_PASSWORD", "password"), + EnvGuard::new( + "PANGOLIN_JWT_SECRET", + "authz-matrix-test-secret-of-adequate-length-000000", + ), + ) +} + +/// **B0a.** Token minting is the single most dangerous endpoint in the API: a +/// `Root` token is a global bypass, because `check_permission` short-circuits +/// for `Root`. This endpoint took no session at all, so any authenticated +/// principal - down to a `TenantUser` - could mint one for any tenant. +#[tokio::test] +#[serial] +async fn token_minting_is_not_open_to_everyone() { + let _env = root_env(); + let f = fixture().await; + + // A TenantUser must not be able to mint anything at all. + let status = f + .request( + Principal::TenantUser, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.tenant_id, "roles": ["Root"] })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "a TenantUser must not be able to mint a Root token (B0a)" + ); + + // Nor a token for itself. + let status = f + .request( + Principal::TenantUser, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.tenant_id })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "a TenantUser must not be able to mint tokens (B0a)" + ); + + // A TenantAdmin may mint within its own tenant, but not above its rank. + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.tenant_id, "roles": ["Root"] })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "a TenantAdmin must not be able to mint a Root token (B0a)" + ); + + // ...and not for somebody else's tenant. + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.foreign_tenant_id, "roles": ["tenant-admin"] })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "a TenantAdmin must not be able to mint a token for another tenant (B0a)" + ); + + // The legitimate case still works. + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.tenant_id, "roles": ["tenant-user"] })), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a TenantAdmin must still be able to mint a lesser token for its own tenant" + ); +} + +/// **B0m.** `expires_in_hours` is caller-controlled and used to reach +/// `chrono::Duration::hours` unclamped, which panics on a large enough value - +/// aborting the connection task, since nothing caught it. +#[tokio::test] +#[serial] +async fn an_absurd_token_lifetime_does_not_take_the_server_down() { + let _env = root_env(); + let f = fixture().await; + + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/tokens", + Some(json!({ + "tenant_id": f.tenant_id, + "roles": ["tenant-user"], + "expires_in_hours": u64::MAX, + })), + ) + .await; + + // Clamped rather than panicking; either a clamp (200) or a refusal (400) is + // acceptable, a dropped connection is not. + assert!( + status == StatusCode::OK || status == StatusCode::BAD_REQUEST, + "an absurd expires_in_hours must be handled, not panic (B0m); got {status}" + ); +} + +/// **Improvement #0.** Request structs reject fields they do not have. +/// +/// This is what turns client drift from a silent no-op into an error. Before +/// it, a CLI sending `warehouse` instead of `warehouse_name` got a `201` for a +/// catalog created without the warehouse it named. +#[tokio::test] +#[serial] +async fn unknown_request_fields_are_rejected() { + let _env = root_env(); + let f = fixture().await; + + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/catalogs", + Some(json!({ "name": "drift", "warehouse": "wh", "type": "pangea" })), + ) + .await; + + assert_eq!( + status, + StatusCode::UNPROCESSABLE_ENTITY, + "a request naming fields the server does not have must be refused, not \ + silently emptied (improvement #0)" + ); +} + +/// **B0e.** View creation and reading had no authorization at all. A view's +/// `properties["sql"]` is its whole definition. +#[tokio::test] +#[serial] +async fn view_endpoints_require_permission() { + let _env = root_env(); + let f = fixture().await; + + // Setup: an admin creates the catalog and namespace. + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/warehouses", + Some(json!({ + "name": "wh", + "use_sts": false, + "storage_config": { "type": "filesystem", "root": "/tmp/pangolin-authz" } + })), + ) + .await, + StatusCode::CREATED + ); + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/catalogs", + Some(json!({ "name": "cat", "warehouse_name": "wh", "catalog_type": "Local" })), + ) + .await, + StatusCode::CREATED + ); + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/v1/cat/namespaces", + Some(json!({ "namespace": ["ns"] })), + ) + .await, + StatusCode::OK + ); + + // A TenantUser with no grants must not be able to create a view. + let status = f + .request( + Principal::TenantUser, + "POST", + "/v1/cat/namespaces/ns/views", + Some(json!({ "name": "v", "sql": "SELECT 1" })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "creating a view must require Create on the namespace (B0e)" + ); +} + +/// **B0f.** Maintenance expires snapshots and deletes orphan files. It had no +/// session and no check, and ran against a hardcoded `"default"` catalog. +#[tokio::test] +#[serial] +async fn maintenance_requires_permission() { + let _env = root_env(); + let f = fixture().await; + + let status = f + .request( + Principal::TenantUser, + "POST", + "/v1/cat/namespaces/ns/tables/t/maintenance", + Some(json!({ "job_type": "expire_snapshots" })), + ) + .await; + + assert!( + status == StatusCode::FORBIDDEN || status == StatusCode::NOT_FOUND, + "destructive maintenance must never be reachable by an ungranted user \ + (B0f); got {status}" + ); + assert_ne!( + status, + StatusCode::OK, + "maintenance ran for a user with no grants (B0f)" + ); +} + +/// **B0b.** Credential vending hands out real cloud-storage credentials. It +/// performed no authorization, never looked the table up, and hardcoded +/// read+write. +#[tokio::test] +#[serial] +async fn credential_vending_requires_permission() { + let _env = root_env(); + let f = fixture().await; + + let status = f + .request( + Principal::TenantUser, + "GET", + "/v1/cat/namespaces/ns/tables/does_not_exist/credentials", + None, + ) + .await; + + assert_ne!( + status, + StatusCode::OK, + "credentials were vended for a table that does not exist, to a user \ + with no grants (B0b)" + ); +} + +/// **B0i.** A tenant admin must not reach another tenant's resources, even +/// though `check_permission` short-circuits on the role. +#[tokio::test] +#[serial] +async fn a_tenant_admin_cannot_reach_another_tenants_catalogs() { + let _env = root_env(); + let f = fixture().await; + + // Tenant A's admin creates a catalog. + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/warehouses", + Some(json!({ + "name": "wh", + "use_sts": false, + "storage_config": { "type": "filesystem", "root": "/tmp/pangolin-authz-2" } + })), + ) + .await, + StatusCode::CREATED + ); + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/catalogs", + Some(json!({ "name": "private", "warehouse_name": "wh", "catalog_type": "Local" })), + ) + .await, + StatusCode::CREATED + ); + + // Tenant B's admin must not see it. Their requests are scoped to their own + // tenant, so the catalog must simply not be there. + let status = f + .request( + Principal::ForeignTenantAdmin, + "GET", + "/api/v1/catalogs/private", + None, + ) + .await; + + assert_ne!( + status, + StatusCode::OK, + "a tenant admin reached another tenant's catalog (B0i)" + ); +} + +/// **B0j.** Logout must actually revoke: the token has to stop working. +/// +/// It used to revoke `session.user_id`, which no token carries as its `jti`, so +/// the revocation check could never match and logout was cosmetic. +#[tokio::test] +#[serial] +async fn logout_revokes_the_presented_token() { + let _env = root_env(); + let f = fixture().await; + + // The token works before logout. + assert_eq!( + f.request(Principal::TenantAdmin, "GET", "/api/v1/catalogs", None) + .await, + StatusCode::OK + ); + + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/auth/revoke", + Some(json!({ "reason": "logout" })), + ) + .await, + StatusCode::OK + ); + + // And must not afterwards. + let status = f + .request(Principal::TenantAdmin, "GET", "/api/v1/catalogs", None) + .await; + assert_eq!( + status, + StatusCode::UNAUTHORIZED, + "a revoked token kept working after logout (B0j)" + ); +} + +/// **B0k.** The OAuth code-exchange endpoint must be reachable without a +/// bearer token: it is the endpoint whose job is to issue the first one. +#[tokio::test] +#[serial] +async fn the_oauth_exchange_endpoint_is_reachable_unauthenticated() { + let _env = root_env(); + let app = app(); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/oauth/exchange") + .header("Content-Type", "application/json") + .body(Body::from(json!({ "code": "nonexistent" }).to_string())) + .unwrap(), + ) + .await + .unwrap(); + + assert_ne!( + response.status(), + StatusCode::UNAUTHORIZED, + "the OAuth code-exchange endpoint demanded the token it exists to \ + issue, making the login flow unreachable (B0k)" + ); +} + +/// Unauthenticated requests are refused across the board. +/// +/// The floor under everything above: if this regresses, the per-principal +/// assertions stop meaning anything. +#[tokio::test] +#[serial] +async fn protected_routes_reject_anonymous_callers() { + let _env = root_env(); + let app = app(); + + for (method, uri) in [ + ("GET", "/api/v1/catalogs"), + ("POST", "/api/v1/catalogs"), + ("GET", "/api/v1/warehouses"), + ("POST", "/api/v1/tokens"), + ("GET", "/api/v1/users"), + ("GET", "/api/v1/tenants"), + ("POST", "/api/v1/permissions"), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("Content-Type", "application/json") + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "{method} {uri} served an anonymous caller" + ); + } +} diff --git a/pangolin/pangolin_api/src/authz_utils.rs b/pangolin/pangolin_api/src/authz_utils.rs index 999f5fd..c2abfec 100644 --- a/pangolin/pangolin_api/src/authz_utils.rs +++ b/pangolin/pangolin_api/src/authz_utils.rs @@ -3,12 +3,26 @@ use pangolin_core::permission::{Action, Permission, PermissionScope}; use pangolin_core::user::UserRole; use uuid::Uuid; +/// Does a `Tenant`-scoped grant apply to a resource in `resource_tenant_id`? +/// +/// B0i: all three access checks below used a bare `PermissionScope::Tenant => +/// true`, never comparing the grant's own `tenant_id` against the resource's. +/// A tenant-wide grant issued in tenant A therefore satisfied access checks for +/// resources in tenant B. Nothing exploited it today only because callers +/// pre-scope their store queries to one tenant - but every path that can +/// surface cross-tenant rows (root impersonation, search, dashboards) leaked +/// through it, and the invariant was one refactor away from mattering. +fn tenant_grant_applies(perm: &Permission, resource_tenant_id: Uuid) -> bool { + perm.tenant_id == resource_tenant_id +} + /// Check if a user has access to a catalog based on their permissions /// /// Checks for Read or Discoverable actions on: /// - Exact catalog scope -/// - Tenant-wide scope +/// - Tenant-wide scope, when the grant belongs to the resource's tenant pub fn has_catalog_access( + resource_tenant_id: Uuid, catalog_id: Uuid, permissions: &[Permission], required_actions: &[Action], @@ -18,7 +32,8 @@ pub fn has_catalog_access( let scope_matches = matches!( &perm.scope, PermissionScope::Catalog { catalog_id: cid } if *cid == catalog_id - ) || matches!(&perm.scope, PermissionScope::Tenant); + ) || (matches!(&perm.scope, PermissionScope::Tenant) + && tenant_grant_applies(perm, resource_tenant_id)); // Check if permission has any of the required actions let has_action = required_actions @@ -34,8 +49,9 @@ pub fn has_catalog_access( /// Checks for Read or Discoverable actions on: /// - Exact namespace scope /// - Parent catalog scope -/// - Tenant-wide scope +/// - Tenant-wide scope, when the grant belongs to the resource's tenant pub fn has_namespace_access( + resource_tenant_id: Uuid, catalog_id: Uuid, namespace: &str, permissions: &[Permission], @@ -49,7 +65,7 @@ pub fn has_namespace_access( namespace: ns, } => *cid == catalog_id && ns == namespace, PermissionScope::Catalog { catalog_id: cid } => *cid == catalog_id, - PermissionScope::Tenant => true, + PermissionScope::Tenant => tenant_grant_applies(perm, resource_tenant_id), _ => false, }; @@ -68,8 +84,9 @@ pub fn has_namespace_access( /// - Exact asset scope /// - Parent namespace scope /// - Parent catalog scope -/// - Tenant-wide scope +/// - Tenant-wide scope, when the grant belongs to the resource's tenant pub fn has_asset_access( + resource_tenant_id: Uuid, catalog_id: Uuid, namespace: &str, asset_id: Uuid, @@ -89,7 +106,7 @@ pub fn has_asset_access( namespace: ns, } => *cid == catalog_id && ns == namespace, PermissionScope::Catalog { catalog_id: cid } => *cid == catalog_id, - PermissionScope::Tenant => true, + PermissionScope::Tenant => tenant_grant_applies(perm, resource_tenant_id), _ => false, }; @@ -107,6 +124,7 @@ pub fn has_asset_access( /// Returns only catalogs the user has Read or Discoverable access to. /// Root and TenantAdmin users bypass filtering. pub fn filter_catalogs( + resource_tenant_id: Uuid, catalogs: Vec, permissions: &[Permission], user_role: UserRole, @@ -120,7 +138,14 @@ pub fn filter_catalogs( catalogs .into_iter() - .filter(|catalog| has_catalog_access(catalog.id, permissions, &required_actions)) + .filter(|catalog| { + has_catalog_access( + resource_tenant_id, + catalog.id, + permissions, + &required_actions, + ) + }) .collect() } @@ -129,6 +154,7 @@ pub fn filter_catalogs( /// Returns only namespaces the user has Read or Discoverable access to. /// Root and TenantAdmin users bypass filtering. pub fn filter_namespaces( + resource_tenant_id: Uuid, namespaces: Vec<(Namespace, String)>, permissions: &[Permission], user_role: UserRole, @@ -147,7 +173,13 @@ pub fn filter_namespaces( // Get catalog ID from the map if let Some(&catalog_id) = catalog_id_map.get(catalog_name) { let namespace_str = namespace.name.join("."); - has_namespace_access(catalog_id, &namespace_str, permissions, &required_actions) + has_namespace_access( + resource_tenant_id, + catalog_id, + &namespace_str, + permissions, + &required_actions, + ) } else { false } @@ -160,6 +192,7 @@ pub fn filter_namespaces( /// Returns only assets the user has Read or Discoverable access to. /// Root and TenantAdmin users bypass filtering. pub fn filter_assets( + resource_tenant_id: Uuid, assets: Vec<( Asset, Option, @@ -196,6 +229,7 @@ pub fn filter_assets( if let Some(&catalog_id) = catalog_id_map.get(catalog_name) { let namespace_str = namespace.join("."); has_asset_access( + resource_tenant_id, catalog_id, &namespace_str, asset.id, @@ -215,23 +249,29 @@ mod tests { use chrono::Utc; use std::collections::HashSet; - #[test] - fn test_has_catalog_access_with_catalog_permission() { - let catalog_id = Uuid::new_v4(); + /// Build a permission with a single `Read` action. + fn read_permission(tenant_id: Uuid, scope: PermissionScope) -> Permission { let mut actions = HashSet::new(); actions.insert(Action::Read); - - let permission = Permission { + Permission { id: Uuid::new_v4(), user_id: Uuid::new_v4(), - tenant_id: Uuid::new_v4(), - scope: PermissionScope::Catalog { catalog_id }, + tenant_id, + scope, actions, granted_by: Uuid::new_v4(), granted_at: Utc::now(), - }; + } + } + + #[test] + fn test_has_catalog_access_with_catalog_permission() { + let tenant_id = Uuid::new_v4(); + let catalog_id = Uuid::new_v4(); + let permission = read_permission(tenant_id, PermissionScope::Catalog { catalog_id }); assert!(has_catalog_access( + tenant_id, catalog_id, &[permission], &[Action::Read] @@ -240,47 +280,70 @@ mod tests { #[test] fn test_has_catalog_access_with_tenant_permission() { + let tenant_id = Uuid::new_v4(); let catalog_id = Uuid::new_v4(); - let mut actions = HashSet::new(); - actions.insert(Action::Read); - - let permission = Permission { - id: Uuid::new_v4(), - user_id: Uuid::new_v4(), - tenant_id: Uuid::new_v4(), - scope: PermissionScope::Tenant, - actions, - granted_by: Uuid::new_v4(), - granted_at: Utc::now(), - }; + let permission = read_permission(tenant_id, PermissionScope::Tenant); assert!(has_catalog_access( + tenant_id, catalog_id, &[permission], &[Action::Read] )); } + /// Regression test for B0i: a `Tenant`-scoped grant issued in tenant A must + /// not satisfy access for a resource in tenant B. + #[test] + fn tenant_scoped_grant_does_not_cross_tenants() { + let tenant_a = Uuid::new_v4(); + let tenant_b = Uuid::new_v4(); + let catalog_id = Uuid::new_v4(); + let asset_id = Uuid::new_v4(); + let permission = read_permission(tenant_a, PermissionScope::Tenant); + let grants = [permission]; + + assert!(has_catalog_access( + tenant_a, + catalog_id, + &grants, + &[Action::Read] + )); + assert!( + !has_catalog_access(tenant_b, catalog_id, &grants, &[Action::Read]), + "a tenant-A grant must not authorize a tenant-B catalog" + ); + assert!( + !has_namespace_access(tenant_b, catalog_id, "sales", &grants, &[Action::Read]), + "a tenant-A grant must not authorize a tenant-B namespace" + ); + assert!( + !has_asset_access( + tenant_b, + catalog_id, + "sales", + asset_id, + &grants, + &[Action::Read] + ), + "a tenant-A grant must not authorize a tenant-B asset" + ); + } + #[test] fn test_has_catalog_access_without_permission() { + let tenant_id = Uuid::new_v4(); let catalog_id = Uuid::new_v4(); let other_catalog_id = Uuid::new_v4(); - let mut actions = HashSet::new(); - actions.insert(Action::Read); - - let permission = Permission { - id: Uuid::new_v4(), - user_id: Uuid::new_v4(), - tenant_id: Uuid::new_v4(), - scope: PermissionScope::Catalog { + let permission = read_permission( + tenant_id, + PermissionScope::Catalog { catalog_id: other_catalog_id, }, - actions, - granted_by: Uuid::new_v4(), - granted_at: Utc::now(), - }; + ); assert!(!has_catalog_access( + tenant_id, catalog_id, &[permission], &[Action::Read] @@ -299,7 +362,7 @@ mod tests { properties: std::collections::HashMap::new(), }]; - let filtered = filter_catalogs(catalogs.clone(), &[], UserRole::Root); + let filtered = filter_catalogs(Uuid::new_v4(), catalogs.clone(), &[], UserRole::Root); assert_eq!(filtered.len(), catalogs.len()); } @@ -316,20 +379,15 @@ mod tests { properties: std::collections::HashMap::new(), }]; - let mut actions = HashSet::new(); - actions.insert(Action::Read); - - let permission = Permission { - id: Uuid::new_v4(), - user_id: Uuid::new_v4(), - tenant_id: Uuid::new_v4(), - scope: PermissionScope::Catalog { catalog_id }, - actions, - granted_by: Uuid::new_v4(), - granted_at: Utc::now(), - }; + let tenant_id = Uuid::new_v4(); + let permission = read_permission(tenant_id, PermissionScope::Catalog { catalog_id }); - let filtered = filter_catalogs(catalogs.clone(), &[permission], UserRole::TenantUser); + let filtered = filter_catalogs( + tenant_id, + catalogs.clone(), + &[permission], + UserRole::TenantUser, + ); assert_eq!(filtered.len(), 1); } @@ -346,7 +404,7 @@ mod tests { properties: std::collections::HashMap::new(), }]; - let filtered = filter_catalogs(catalogs, &[], UserRole::TenantUser); + let filtered = filter_catalogs(Uuid::new_v4(), catalogs, &[], UserRole::TenantUser); assert_eq!(filtered.len(), 0); } } diff --git a/pangolin/pangolin_api/src/business_metadata_handlers.rs b/pangolin/pangolin_api/src/business_metadata_handlers.rs index 1b41c7e..f77272f 100644 --- a/pangolin/pangolin_api/src/business_metadata_handlers.rs +++ b/pangolin/pangolin_api/src/business_metadata_handlers.rs @@ -14,6 +14,7 @@ use utoipa::ToSchema; use uuid::Uuid; #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct AddMetadataRequest { pub description: Option, pub tags: Vec, @@ -26,6 +27,33 @@ pub struct MetadataResponse { pub metadata: BusinessMetadata, } +/// Resolve an asset's permission scope, for the handlers keyed only by asset id. +/// +/// Several business-metadata handlers took an asset id straight off the path +/// and acted on it with no authorization at all - the same class as B0e. An +/// asset id alone is not a scope, so the catalog and namespace have to be +/// resolved first; `get_asset_by_id` does that in one lookup. +async fn asset_scope( + store: &AppState, + tenant_id: uuid::Uuid, + asset_id: uuid::Uuid, +) -> Result, anyhow::Error> { + let Some((asset, catalog_name, namespace)) = store.get_asset_by_id(tenant_id, asset_id).await? + else { + return Ok(None); + }; + + let Some(catalog) = store.get_catalog(tenant_id, catalog_name).await? else { + return Ok(None); + }; + + Ok(Some(PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace.join("."), + asset_id: asset.id, + })) +} + #[utoipa::path( post, path = "/api/v1/assets/{asset_id}/metadata", @@ -171,10 +199,31 @@ pub async fn get_business_metadata( )] pub async fn delete_business_metadata( State(store): State, - Extension(_session): Extension, + Extension(session): Extension, Path(asset_id): Path, ) -> impl IntoResponse { - // Check permission logic + // The `// Check permission logic` that stood here was the whole of the + // authorization: any authenticated tenant member could delete any asset's + // business metadata - descriptions, tags, and the `discoverable` flag that + // governs who can see the asset at all. + let tenant_id = session.tenant_id.unwrap_or_default(); + let scope = match asset_scope(&store, tenant_id, asset_id).await { + Ok(Some(scope)) => scope, + Ok(None) => return (StatusCode::NOT_FOUND, "Asset not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_business_metadata: failed to resolve asset"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + match crate::authz::check_permission(&store, &session, &Action::Delete, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_business_metadata: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } match store.delete_business_metadata(asset_id).await { Ok(_) => StatusCode::NO_CONTENT.into_response(), @@ -186,12 +235,42 @@ pub async fn delete_business_metadata( } } +/// Query parameters for asset search. +/// +/// B35: `tags` was typed `Option>` and extracted with axum's +/// `Query`, which uses `serde_urlencoded` - and `serde_urlencoded` cannot +/// deserialize repeated keys (`?tags=a&tags=b`) into a `Vec`. The UI sent +/// exactly that, so every tag-filtered search returned `400 Bad Request`: +/// tag-filtered search was broken end to end and had presumably never worked. +/// +/// A comma-separated string is the fix that keeps working with a plain `Query` +/// extractor, and it is also what an operator would type by hand. Repeated +/// keys are still accepted by the deserializer below, so a client that sends +/// the old shape gets the last value rather than a 400. #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct SearchRequest { pub query: String, + /// Comma-separated tag names, e.g. `?tags=pii,finance`. + #[serde(default, deserialize_with = "deserialize_comma_separated")] pub tags: Option>, } +/// Deserialize `a,b,c` into `["a", "b", "c"]`, trimming and dropping blanks. +fn deserialize_comma_separated<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let raw = Option::::deserialize(deserializer)?; + Ok(raw.map(|value| { + value + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() + })) +} + #[utoipa::path( get, path = "/api/v1/assets/search", @@ -243,6 +322,7 @@ pub async fn search_assets( // Apply permission-based filtering let filtered_results = crate::authz_utils::filter_assets( + tenant_id, results, &permissions, session.role.clone(), @@ -262,6 +342,7 @@ pub async fn search_assets( let ns_str = namespace.join("."); let required_actions = vec![pangolin_core::permission::Action::Read]; crate::authz_utils::has_asset_access( + tenant_id, catalog_id, &ns_str, asset.id, diff --git a/pangolin/pangolin_api/src/cached_store.rs b/pangolin/pangolin_api/src/cached_store.rs index ff99fed..52a36e0 100644 --- a/pangolin/pangolin_api/src/cached_store.rs +++ b/pangolin/pangolin_api/src/cached_store.rs @@ -101,16 +101,25 @@ impl CatalogStore for CachedCatalogStore { } async fn delete_warehouse(&self, tenant_id: Uuid, name: String) -> Result<()> { - // Invalidate cache on delete + // B16m: delete first, *then* invalidate. Invalidating first opened a + // window in which a concurrent `get_warehouse` missed the cache, read the + // still-present row, and re-inserted it with a full TTL - so the deleted + // warehouse's cloud credentials kept being vended for up to the TTL after + // delete returned success. Deleting first means any racing read either + // sees the row (and the invalidate below clears it) or does not find it. + let key = (tenant_id, name.clone()); + let result = self.inner.delete_warehouse(tenant_id, name.clone()).await; + tracing::info!( "Cache INVALIDATE (Delete) for warehouse: {}/{}", tenant_id, name ); - self.warehouse_cache - .invalidate(&(tenant_id, name.clone())) - .await; - self.inner.delete_warehouse(tenant_id, name).await + // Invalidate on the error path too: the delete may have partially + // applied, and a stale credential entry is the worse failure. + self.warehouse_cache.invalidate(&key).await; + + result } // --- Passthrough Operations (Uncached) --- @@ -224,6 +233,18 @@ impl CatalogStore for CachedCatalogStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + self.inner + .replace_namespace_properties(tenant_id, catalog_name, namespace, properties) + .await + } + // Asset async fn create_asset( &self, @@ -476,6 +497,9 @@ impl CatalogStore for CachedCatalogStore { async fn write_file(&self, location: &str, content: Vec) -> Result<()> { self.inner.write_file(location, content).await } + async fn delete_file(&self, location: &str) -> Result<()> { + self.inner.delete_file(location).await + } // Maintenance Operations async fn expire_snapshots( diff --git a/pangolin/pangolin_api/src/cleanup_job.rs b/pangolin/pangolin_api/src/cleanup_job.rs index 478a122..584b2bb 100644 --- a/pangolin/pangolin_api/src/cleanup_job.rs +++ b/pangolin/pangolin_api/src/cleanup_job.rs @@ -1,31 +1,97 @@ +//! Periodic removal of expired revocation records. +//! +//! Two things were wrong here, and the first makes the second moot. +//! +//! **It was never started.** `start_token_cleanup_job` was defined, the module +//! was declared, and nothing anywhere called it — a grep across the workspace +//! found only the definition and its own test. So the `revoked_tokens` +//! collection grew for the life of the deployment: every logout, every +//! administrative revocation, kept forever. The revocation check reads that +//! table on every authenticated request (A-28/C-14). +//! +//! **It was uncoordinated.** Every replica would have run it on the same +//! schedule. `cleanup_expired_tokens` is a `DELETE ... WHERE expires_at < now`, +//! so N replicas doing it concurrently is *correct* — the operation is +//! idempotent and the second one simply deletes nothing. What it is not is +//! free: replicas started by the same rolling deploy tick within milliseconds +//! of each other, so the database takes N identical delete scans at the same +//! instant, forever, on a schedule nobody staggered. +//! +//! The fix for that is jitter rather than leader election. Leader election +//! needs a lock table, a lease, and a story for what happens when the leader +//! dies mid-sweep; all of that is complexity spent to serialise an operation +//! that is safe to run concurrently. Spreading the replicas across the interval +//! removes the stampede, which is the actual cost. + use pangolin_store::CatalogStore; use std::sync::Arc; use std::time::Duration; -use tokio::time::interval; -/// Background job that periodically cleans up expired revoked tokens +/// How often each replica sweeps. +pub const DEFAULT_INTERVAL: Duration = Duration::from_secs(3600); + +/// Pick a start delay somewhere inside the interval. +/// +/// Without this, replicas from one deploy sweep in lockstep forever. The +/// randomness only has to be uneven, not unguessable, so this uses the process +/// id and the wall clock rather than pulling in an RNG. +fn stagger(interval: Duration) -> Duration { + use std::time::{SystemTime, UNIX_EPOCH}; + + let seed = std::process::id() as u128 + ^ SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + + let millis = interval.as_millis().max(1); + Duration::from_millis((seed % millis) as u64) +} + +/// Sweep expired revocation records until cancelled. /// -/// This job runs every hour and removes tokens from the blacklist that have expired. -/// This prevents the blacklist from growing indefinitely. +/// Runs forever; spawn it. Errors are logged and the loop continues, because a +/// database blip should not permanently stop the sweep. pub async fn start_token_cleanup_job(store: Arc) { - let mut cleanup_interval = interval(Duration::from_secs(3600)); // Run every hour + run_with_interval(store, DEFAULT_INTERVAL).await +} - tracing::info!("Token cleanup job started (runs every hour)"); +pub async fn run_with_interval(store: Arc, interval: Duration) { + let delay = stagger(interval); + tracing::info!( + interval_secs = interval.as_secs(), + first_sweep_in_secs = delay.as_secs(), + "token cleanup job started" + ); + tokio::time::sleep(delay).await; + + let mut ticker = tokio::time::interval(interval); + // Without this, a sweep that overruns the interval makes tokio fire the + // missed ticks back to back to catch up - turning one slow sweep into a + // burst of them against a database that is evidently already struggling. + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { - cleanup_interval.tick().await; - - match store.cleanup_expired_tokens().await { - Ok(count) => { - if count > 0 { - tracing::info!("Token cleanup job: removed {} expired tokens", count); - } else { - tracing::debug!("Token cleanup job: no expired tokens to remove"); - } - } - Err(e) => { - tracing::error!("Token cleanup job failed: {}", e); - } + ticker.tick().await; + sweep_once(&store).await; + } +} + +/// One sweep. Separate so a test can drive it without waiting an hour. +pub async fn sweep_once(store: &Arc) { + match store.cleanup_expired_tokens().await { + Ok(0) => tracing::debug!("token cleanup: nothing expired"), + Ok(count) => { + tracing::info!( + removed = count, + "token cleanup: removed expired revocations" + ) + } + Err(e) => { + // Logged, not fatal. The next tick tries again; a transient + // database error should not stop the sweep for the life of the + // process. + tracing::error!(error = %e, "token cleanup failed"); } } } @@ -38,45 +104,89 @@ mod tests { use uuid::Uuid; #[tokio::test] - async fn test_cleanup_job_removes_expired_tokens() { + async fn a_sweep_removes_expired_revocations_and_keeps_live_ones() { let store = Arc::new(MemoryStore::new()) as Arc; - // Add some expired tokens - let expired_token1 = Uuid::new_v4(); - let expired_token2 = Uuid::new_v4(); + let expired = Uuid::new_v4(); let past = Utc::now() - ChronoDuration::hours(1); - store - .revoke_token(expired_token1, past, Some("Test 1".to_string())) + .revoke_token(expired, past, Some("expired".to_string())) .await .unwrap(); + + let live = Uuid::new_v4(); + let future = Utc::now() + ChronoDuration::hours(24); store - .revoke_token(expired_token2, past, Some("Test 2".to_string())) + .revoke_token(live, future, Some("still valid".to_string())) .await .unwrap(); - // Add a valid token - let valid_token = Uuid::new_v4(); - let future = Utc::now() + ChronoDuration::hours(24); + sweep_once(&store).await; + + assert!( + !store.is_token_revoked(expired).await.unwrap(), + "an expired revocation should have been swept" + ); + assert!( + store.is_token_revoked(live).await.unwrap(), + "a live revocation must survive the sweep, or logging out stops \ + working an hour later" + ); + } + + #[tokio::test] + async fn sweeping_twice_is_harmless() { + // Every replica runs this. Concurrency safety is the reason jitter is + // enough and leader election is not needed. + let store = Arc::new(MemoryStore::new()) as Arc; + let expired = Uuid::new_v4(); store - .revoke_token(valid_token, future, Some("Test 3".to_string())) + .revoke_token(expired, Utc::now() - ChronoDuration::hours(1), None) .await .unwrap(); - // Verify all are revoked - assert!(store.is_token_revoked(expired_token1).await.unwrap()); - assert!(store.is_token_revoked(expired_token2).await.unwrap()); - assert!(store.is_token_revoked(valid_token).await.unwrap()); + sweep_once(&store).await; + sweep_once(&store).await; + assert!(!store.is_token_revoked(expired).await.unwrap()); + } + + #[test] + fn the_stagger_lands_inside_the_interval() { + let interval = Duration::from_secs(3600); + for _ in 0..50 { + let delay = stagger(interval); + assert!( + delay < interval, + "a stagger of {delay:?} is not inside {interval:?}; the first \ + sweep would be skipped or doubled" + ); + } + } + + #[tokio::test] + async fn the_job_actually_sweeps_when_run() { + // The defect this module existed with: the function was never called, + // so none of the above mattered. This drives the real loop. + let store = Arc::new(MemoryStore::new()) as Arc; + let expired = Uuid::new_v4(); + store + .revoke_token(expired, Utc::now() - ChronoDuration::hours(1), None) + .await + .unwrap(); - // Run cleanup - let count = store.cleanup_expired_tokens().await.unwrap(); - assert_eq!(count, 2, "Should have cleaned up 2 expired tokens"); + let job_store = store.clone(); + let handle = tokio::spawn(async move { + run_with_interval(job_store, Duration::from_millis(50)).await; + }); - // Verify expired tokens are removed - assert!(!store.is_token_revoked(expired_token1).await.unwrap()); - assert!(!store.is_token_revoked(expired_token2).await.unwrap()); + // The stagger is bounded by the interval, so this is enough for at + // least one sweep. + tokio::time::sleep(Duration::from_millis(400)).await; + handle.abort(); - // Verify valid token still exists - assert!(store.is_token_revoked(valid_token).await.unwrap()); + assert!( + !store.is_token_revoked(expired).await.unwrap(), + "the running job did not sweep" + ); } } diff --git a/pangolin/pangolin_api/src/config.rs b/pangolin/pangolin_api/src/config.rs index 919bcbb..e8a3c28 100644 --- a/pangolin/pangolin_api/src/config.rs +++ b/pangolin/pangolin_api/src/config.rs @@ -102,6 +102,13 @@ pub struct AppConfig { pub oauth_redirect_allowlist: Vec, /// Whether API keys minted before the key-id format are still accepted. pub allow_legacy_api_keys: bool, + /// Whether JWTs carrying no `jti` are still accepted (off by default). + /// + /// `Claims.jti` is `Option` "for compatibility", and the middleware + /// used to skip the revocation check for such tokens - making them + /// unrevocable for their full lifetime (B0o). They are now rejected unless + /// an operator opts in for a migration window. + pub allow_tokens_without_jti: bool, /// Bind address and port. pub bind_address: String, pub port: u16, @@ -125,6 +132,19 @@ pub struct AppConfig { /// CORS origins. `None` means "allow any", which is only safe behind a /// trusted gateway and is no longer the default in production. pub cors_allowed_origins: Option>, + + /// Failed authentication attempts allowed per window, per source address + /// and separately per account. 0 disables throttling. C-5: the login + /// endpoint had no throttle of any kind and was brute-forceable. + pub auth_rate_limit: u32, + /// The window those attempts are counted over. + pub auth_rate_window: Duration, + /// Honour `X-Forwarded-For` when deriving the client address. + /// + /// Off by default, and that default matters: trusting the header when you + /// are *not* behind a proxy lets a caller set it per request and bypass the + /// per-address limit entirely. + pub trust_forwarded_for: bool, } static CONFIG: OnceLock = OnceLock::new(); @@ -213,7 +233,14 @@ impl AppConfig { let bind_address = env_opt("PANGOLIN_BIND_ADDRESS").unwrap_or_else(|| "0.0.0.0".to_string()); - if no_auth && !dev_mode && !is_loopback(&bind_address) { + // B0h: the guard used to read `no_auth && !dev_mode && !is_loopback(..)`. + // That `!dev_mode` term meant `PANGOLIN_NO_AUTH=true PANGOLIN_DEV_MODE=true` + // started happily on the default `0.0.0.0` bind and treated every + // anonymous request as `TenantAdmin` - and those two flags are routinely + // set together in compose and dev setups, so the escape hatch was the + // common case. Dev mode relaxes secret strength, never network exposure: + // if auth is off, the listener must be loopback, unconditionally. + if no_auth && !is_loopback(&bind_address) { return Err(ConfigError::NoAuthOnPublicBind(bind_address)); } @@ -250,6 +277,7 @@ impl AppConfig { frontend_url, oauth_redirect_allowlist, allow_legacy_api_keys: env_bool("PANGOLIN_ALLOW_LEGACY_API_KEYS"), + allow_tokens_without_jti: env_bool("PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI"), bind_address, port: env_parsed("PORT", 8080u16)?, log_format, @@ -257,6 +285,12 @@ impl AppConfig { body_limit_bytes: env_parsed("PANGOLIN_MAX_BODY_BYTES", 16 * 1024 * 1024)?, concurrency_limit: env_parsed("PANGOLIN_MAX_CONCURRENT_REQUESTS", 512)?, shutdown_grace: Duration::from_secs(env_parsed("PANGOLIN_SHUTDOWN_GRACE_SECS", 25)?), + auth_rate_limit: env_parsed("PANGOLIN_AUTH_RATE_LIMIT", 10u32)?, + auth_rate_window: Duration::from_secs(env_parsed( + "PANGOLIN_AUTH_RATE_WINDOW_SECS", + 60, + )?), + trust_forwarded_for: env_bool("PANGOLIN_TRUST_FORWARDED_FOR"), metrics_enabled: env_opt("PANGOLIN_METRICS_ENABLED") .map(|v| matches!(v.to_ascii_lowercase().as_str(), "true" | "1" | "yes")) .unwrap_or(true), @@ -342,6 +376,14 @@ pub fn allow_legacy_api_keys() -> bool { env_bool("PANGOLIN_ALLOW_LEGACY_API_KEYS") } +/// Whether JWTs with no `jti` are still honoured (off by default, see B0o). +pub fn allow_tokens_without_jti() -> bool { + if let Some(cfg) = CONFIG.get() { + return cfg.allow_tokens_without_jti; + } + env_bool("PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI") +} + /// Constant-time string comparison, for credential checks. /// /// `==` on `&str` short-circuits on the first differing byte and therefore diff --git a/pangolin/pangolin_api/src/credential_signers/azure_signer.rs b/pangolin/pangolin_api/src/credential_signers/azure_signer.rs index cdb7343..db712a8 100644 --- a/pangolin/pangolin_api/src/credential_signers/azure_signer.rs +++ b/pangolin/pangolin_api/src/credential_signers/azure_signer.rs @@ -49,13 +49,18 @@ impl AzureSasSigner { #[async_trait] impl CredentialSigner for AzureSasSigner { + // These are read only under the feature; without the allow, a default + // build warns on every one of them. Prefixing them with `_` instead - + // which is what was here - meant the cfg block referenced names that did + // not exist, so the feature could not compile at all. + #[allow(unused_variables)] async fn generate_credentials( &self, - _resource_path: &str, - _permissions: &[String], + resource_path: &str, + permissions: &[String], duration: Duration, ) -> Result { - let _expires_at = Utc::now() + duration; + let expires_at = Utc::now() + duration; // Check for account key first (works regardless of feature flags) if let Some(account_key) = &self.account_key { diff --git a/pangolin/pangolin_api/src/credential_signers/gcp_signer.rs b/pangolin/pangolin_api/src/credential_signers/gcp_signer.rs index a4fc8c2..4d84b6d 100644 --- a/pangolin/pangolin_api/src/credential_signers/gcp_signer.rs +++ b/pangolin/pangolin_api/src/credential_signers/gcp_signer.rs @@ -1,5 +1,9 @@ use super::{CredentialSigner, VendedCredentials}; use anyhow::Result; +// Only the feature path raises errors with the macro; importing it +// unconditionally warns in a default build. +#[cfg(feature = "gcp-oauth")] +use anyhow::anyhow; use async_trait::async_trait; use chrono::{Duration, Utc}; use std::collections::HashMap; @@ -30,10 +34,15 @@ impl GcpTokenSigner { #[async_trait] impl CredentialSigner for GcpTokenSigner { + // These are read only under the feature; without the allow, a default + // build warns on every one of them. Prefixing them with `_` instead - + // which is what was here - meant the cfg block referenced names that did + // not exist, so the feature could not compile at all. + #[allow(unused_variables)] async fn generate_credentials( &self, - _resource_path: &str, - _permissions: &[String], + resource_path: &str, + permissions: &[String], duration: Duration, ) -> Result { #[cfg(feature = "gcp-oauth")] diff --git a/pangolin/pangolin_api/src/credential_signers/s3_signer.rs b/pangolin/pangolin_api/src/credential_signers/s3_signer.rs index aaafe28..76fdd19 100644 --- a/pangolin/pangolin_api/src/credential_signers/s3_signer.rs +++ b/pangolin/pangolin_api/src/credential_signers/s3_signer.rs @@ -44,11 +44,14 @@ impl S3Signer { #[async_trait] impl CredentialSigner for S3Signer { + // `duration` is read only under `aws-sts`; the other two are unused either + // way. Without the allow, a default build warns on all three. + #[allow(unused_variables)] async fn generate_credentials( &self, _resource_path: &str, _permissions: &[String], - _duration: Duration, + duration: Duration, ) -> Result { #[cfg(feature = "aws-sts")] { @@ -102,9 +105,12 @@ impl CredentialSigner for S3Signer { config.insert("s3.endpoint".to_string(), endpoint.clone()); } - let expires_at = chrono::DateTime::parse_from_rfc3339(creds.expiration()) - .ok() - .map(|dt| dt.with_timezone(&Utc)); + // `expiration()` hands back an `aws_smithy_types::DateTime`, not + // an RFC3339 string, so this went through `parse_from_rfc3339` + // on a value that was never text. + let expiry = creds.expiration(); + let expires_at = + chrono::DateTime::from_timestamp(expiry.secs(), expiry.subsec_nanos()); tracing::info!("✅ Successfully assumed AWS role"); diff --git a/pangolin/pangolin_api/src/dashboard_handlers.rs b/pangolin/pangolin_api/src/dashboard_handlers.rs index 02f81b6..f43b4e1 100644 --- a/pangolin/pangolin_api/src/dashboard_handlers.rs +++ b/pangolin/pangolin_api/src/dashboard_handlers.rs @@ -132,6 +132,7 @@ pub async fn get_dashboard_stats( .await .map_err(ApiError::from)?; let accessible_catalogs = crate::authz_utils::filter_catalogs( + tenant_id, all_catalogs, &permissions, session.role.clone(), @@ -172,6 +173,7 @@ pub async fn get_dashboard_stats( .collect(); let filtered = crate::authz_utils::filter_namespaces( + tenant_id, namespace_tuples, &permissions, session.role.clone(), @@ -186,6 +188,7 @@ pub async fn get_dashboard_stats( let mut accessible_tables_count = 0; if let Ok(all_assets) = store.search_assets(tenant_id, "", None).await { let filtered_assets = crate::authz_utils::filter_assets( + tenant_id, all_assets, &permissions, session.role.clone(), diff --git a/pangolin/pangolin_api/src/federated_catalog_handlers.rs b/pangolin/pangolin_api/src/federated_catalog_handlers.rs index 6003fb1..1042cf8 100644 --- a/pangolin/pangolin_api/src/federated_catalog_handlers.rs +++ b/pangolin/pangolin_api/src/federated_catalog_handlers.rs @@ -18,6 +18,7 @@ type AppState = Arc; // Request/Response types #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateFederatedCatalogRequest { pub name: String, pub config: FederatedCatalogConfig, diff --git a/pangolin/pangolin_api/src/iceberg/commit.rs b/pangolin/pangolin_api/src/iceberg/commit.rs index 1ff21b3..9357ac2 100644 --- a/pangolin/pangolin_api/src/iceberg/commit.rs +++ b/pangolin/pangolin_api/src/iceberg/commit.rs @@ -207,14 +207,22 @@ pub fn check_requirements( } } - // Pangolin does not track partition-field id assignment separately - // from the specs themselves, so this cannot be verified. Refusing - // is the honest answer: the alternative is pretending a - // precondition held when it was never examined. - CommitRequirement::AssertLastAssignedPartitionId { .. } => { - return Err(CommitError::Unsupported { - operation: "assert-last-assigned-partition-id".into(), - }) + // Now checkable: `last-partition-id` is a real field on + // `TableMetadata` (B12). Before it existed this requirement had to + // be refused, because the alternative was pretending a precondition + // held when it had never been examined. + CommitRequirement::AssertLastAssignedPartitionId { + last_assigned_partition_id, + } => { + if metadata.last_partition_id != *last_assigned_partition_id { + return Err(CommitError::RequirementFailed { + requirement: "assert-last-assigned-partition-id".into(), + detail: format!( + "last-partition-id is {}, expected {last_assigned_partition_id}", + metadata.last_partition_id + ), + }); + } } CommitRequirement::Unknown => { @@ -236,6 +244,18 @@ pub fn apply_updates( updates: &[CommitUpdate], branch: &str, ) -> Result<(), CommitError> { + // B16c: `-1` in `set-current-schema` / `set-default-spec` / + // `set-default-sort-order` means "the one added *by this commit*". The old + // code resolved it against `metadata.schemas.last()` etc., which for an + // existing table is never empty - so a `-1` sent *without* a preceding + // `add-schema` silently repointed the table at whatever happened to be last + // in the persisted vector (arbitrary if the vector is not in creation + // order), instead of hitting the error arm the message claims. Tracking + // what this commit actually added makes the sentinel mean what it says. + let mut last_added_schema_id: Option = None; + let mut last_added_spec_id: Option = None; + let mut last_added_sort_order_id: Option = None; + for update in updates { match update { CommitUpdate::AssignUuid { uuid } => { @@ -266,17 +286,33 @@ pub fn apply_updates( serde_json::from_value(schema.clone()).map_err(|e| CommitError::Invalid { detail: format!("add-schema: {e}"), })?; - metadata.last_column_id = metadata - .last_column_id - .max(new_schema.fields.iter().map(|f| f.id).max().unwrap_or(0)); + // Reject a duplicate id rather than pushing a second schema the + // rest of the metadata cannot tell apart (B16c). + if metadata + .schemas + .iter() + .any(|s| s.schema_id == new_schema.schema_id) + { + return Err(CommitError::Invalid { + detail: format!( + "add-schema: schema id {} already exists", + new_schema.schema_id + ), + }); + } + // `max_field_id` walks nested fields; taking the max over the + // top-level `fields` alone understates `last-column-id` for any + // schema containing a struct, list or map. + metadata.last_column_id = metadata.last_column_id.max(new_schema.max_field_id()); + last_added_schema_id = Some(new_schema.schema_id); metadata.schemas.push(new_schema); } CommitUpdate::SetCurrentSchema { schema_id } => { // -1 means "the schema added by this same commit". if *schema_id == -1 { - match metadata.schemas.last() { - Some(last) => metadata.current_schema_id = last.schema_id, + match last_added_schema_id { + Some(id) => metadata.current_schema_id = id, None => { return Err(CommitError::Invalid { detail: "set-current-schema: -1 with no schema in this commit" @@ -299,7 +335,7 @@ pub fn apply_updates( serde_json::from_value(snapshot.clone()).map_err(|e| CommitError::Invalid { detail: format!("add-snapshot: {e}"), })?; - add_snapshot(metadata, snapshot_obj, branch); + add_snapshot(metadata, snapshot_obj, branch)?; } CommitUpdate::SetSnapshotRef { @@ -371,18 +407,26 @@ pub fn apply_updates( serde_json::from_value(spec.clone()).map_err(|e| CommitError::Invalid { detail: format!("add-spec: {e}"), })?; + if metadata + .partition_specs + .iter() + .any(|s| s.spec_id == new_spec.spec_id) + { + return Err(CommitError::Invalid { + detail: format!("add-spec: spec id {} already exists", new_spec.spec_id), + }); + } + last_added_spec_id = Some(new_spec.spec_id); metadata.partition_specs.push(new_spec); + // Keep the required `last-partition-id` true (B12). + metadata.recompute_last_partition_id(); } CommitUpdate::SetDefaultSpec { spec_id } => { let target = if *spec_id == -1 { - metadata - .partition_specs - .last() - .map(|s| s.spec_id) - .ok_or_else(|| CommitError::Invalid { - detail: "set-default-spec: -1 with no spec in this commit".into(), - })? + last_added_spec_id.ok_or_else(|| CommitError::Invalid { + detail: "set-default-spec: -1 with no spec in this commit".into(), + })? } else { *spec_id }; @@ -401,19 +445,28 @@ pub fn apply_updates( detail: format!("add-sort-order: {e}"), } })?; + if metadata + .sort_orders + .iter() + .any(|o| o.order_id == new_order.order_id) + { + return Err(CommitError::Invalid { + detail: format!( + "add-sort-order: sort order id {} already exists", + new_order.order_id + ), + }); + } + last_added_sort_order_id = Some(new_order.order_id); metadata.sort_orders.push(new_order); } CommitUpdate::SetDefaultSortOrder { sort_order_id } => { let target = if *sort_order_id == -1 { - metadata - .sort_orders - .last() - .map(|o| o.order_id) - .ok_or_else(|| CommitError::Invalid { - detail: "set-default-sort-order: -1 with no sort order in this commit" - .into(), - })? + last_added_sort_order_id.ok_or_else(|| CommitError::Invalid { + detail: "set-default-sort-order: -1 with no sort order in this commit" + .into(), + })? } else { *sort_order_id }; @@ -466,7 +519,11 @@ pub fn apply_updates( } /// Append a snapshot and move the branch to it. -fn add_snapshot(metadata: &mut TableMetadata, snapshot: Snapshot, branch: &str) { +fn add_snapshot( + metadata: &mut TableMetadata, + snapshot: Snapshot, + branch: &str, +) -> Result<(), CommitError> { let snapshot_id = snapshot.snapshot_id; let timestamp_ms = if snapshot.timestamp_ms > 0 { snapshot.timestamp_ms @@ -475,10 +532,21 @@ fn add_snapshot(metadata: &mut TableMetadata, snapshot: Snapshot, branch: &str) }; // The sequence number is a monotonic counter, *not* a snapshot ID (A-3). - // Honour the client's value when it advances the counter; otherwise assign - // the next one. - let next_sequence = metadata.last_sequence_number + 1; - let sequence_number = if snapshot.sequence_number > metadata.last_sequence_number { + // + // B15: the old rule was "honour any client value greater than the current + // counter". A client could therefore submit `i64::MAX`, after which the next + // commit computed `last_sequence_number + 1` and overflowed - a panic in + // debug builds, a wrap in release, and corrupt commit ordering either way. + // The counter is the server's to advance: a client value is accepted only if + // it is exactly the next one, and anything else is assigned rather than + // honoured. + let next_sequence = metadata + .last_sequence_number + .checked_add(1) + .ok_or_else(|| CommitError::Invalid { + detail: "add-snapshot: sequence number counter is exhausted".into(), + })?; + let sequence_number = if snapshot.sequence_number == next_sequence { snapshot.sequence_number } else { next_sequence @@ -494,15 +562,22 @@ fn add_snapshot(metadata: &mut TableMetadata, snapshot: Snapshot, branch: &str) .push(snapshot); metadata.last_sequence_number = sequence_number; metadata.last_updated_ms = timestamp_ms; - metadata.current_snapshot_id = Some(snapshot_id); - metadata - .snapshot_log - .get_or_insert_with(Vec::new) - .push(SnapshotLogEntry { - timestamp_ms, - snapshot_id, - }); + // B16: `current_snapshot_id` and the `snapshot_log` describe *main*. Setting + // them for any branch meant a `dev`-branch commit changed what `main` + // readers resolve if the metadata document was ever shared across branches + // or exported. Same for fabricating a `main` ref pointing at the branch's + // snapshot: that silently published unreviewed work to main. + if branch == MAIN_REF { + metadata.current_snapshot_id = Some(snapshot_id); + metadata + .snapshot_log + .get_or_insert_with(Vec::new) + .push(SnapshotLogEntry { + timestamp_ms, + snapshot_id, + }); + } let refs = metadata.refs.get_or_insert_with(HashMap::new); refs.insert( @@ -515,18 +590,8 @@ fn add_snapshot(metadata: &mut TableMetadata, snapshot: Snapshot, branch: &str) max_ref_age_ms: None, }, ); - if branch != MAIN_REF { - // Keep `main` consistent with `current_snapshot_id` for readers that - // predate ref tracking. - refs.entry(MAIN_REF.to_string()) - .or_insert(SnapshotReference { - snapshot_id, - ref_type: "branch".to_string(), - min_snapshots_to_keep: None, - max_snapshot_age_ms: None, - max_ref_age_ms: None, - }); - } + + Ok(()) } #[cfg(test)] @@ -545,6 +610,7 @@ mod tests { last_column_id: 1, current_schema_id: 0, schemas: vec![Schema { + type_: "struct".to_string(), schema_id: 0, identifier_field_ids: None, fields: vec![NestedField { @@ -557,6 +623,7 @@ mod tests { }], current_partition_spec_id: 0, partition_specs: vec![], + last_partition_id: 999, default_sort_order_id: 0, sort_orders: vec![], properties: None, @@ -921,8 +988,12 @@ mod tests { assert_eq!(snapshots[1].sequence_number, 2); } + /// B15: the counter is the server's. A client value is honoured only when it + /// is exactly the next one; anything else is replaced by the next value + /// rather than letting the client jump the counter to, say, `i64::MAX` and + /// overflow the *following* commit. #[test] - fn a_client_supplied_sequence_number_may_advance_the_counter() { + fn a_client_supplied_sequence_number_cannot_jump_the_counter() { let mut metadata = base_metadata(); let mut snap = snapshot_json(1, None); snap["sequence-number"] = serde_json::json!(5); @@ -932,25 +1003,100 @@ mod tests { MAIN_REF, ) .unwrap(); - assert_eq!(metadata.last_sequence_number, 5); + assert_eq!(metadata.last_sequence_number, 1); + + // The exact next value is accepted. + let mut snap = snapshot_json(2, None); + snap["sequence-number"] = serde_json::json!(2); + apply_updates( + &mut metadata, + &[CommitUpdate::AddSnapshot { snapshot: snap }], + MAIN_REF, + ) + .unwrap(); + assert_eq!(metadata.last_sequence_number, 2); + } + + /// B15 regression: `i64::MAX` used to be honoured verbatim, so the next + /// commit's `last_sequence_number + 1` overflowed. + #[test] + fn an_absurd_client_sequence_number_does_not_overflow_the_next_commit() { + let mut metadata = base_metadata(); + let mut snap = snapshot_json(1, None); + snap["sequence-number"] = serde_json::json!(i64::MAX); + apply_updates( + &mut metadata, + &[CommitUpdate::AddSnapshot { snapshot: snap }], + MAIN_REF, + ) + .unwrap(); + assert_eq!(metadata.last_sequence_number, 1); + + // The follow-up commit is ordinary arithmetic, not an overflow. + apply_updates( + &mut metadata, + &[CommitUpdate::AddSnapshot { + snapshot: snapshot_json(2, None), + }], + MAIN_REF, + ) + .unwrap(); + assert_eq!(metadata.last_sequence_number, 2); } #[test] - fn adding_a_snapshot_records_the_snapshot_log_and_moves_the_branch() { + fn adding_a_snapshot_on_main_records_the_snapshot_log_and_moves_the_branch() { let mut metadata = base_metadata(); apply_updates( &mut metadata, &[CommitUpdate::AddSnapshot { snapshot: snapshot_json(99, None), }], - "feature", + MAIN_REF, ) .unwrap(); assert_eq!(metadata.current_snapshot_id, Some(99)); - assert_eq!(ref_snapshot_id(&metadata, "feature"), Some(99)); + assert_eq!(ref_snapshot_id(&metadata, MAIN_REF), Some(99)); assert_eq!(metadata.snapshot_log.as_ref().unwrap().len(), 1); } + /// B16: a commit to a feature branch must move *only* that branch. It used + /// to set `current_snapshot_id` for any branch and fabricate a `main` ref + /// pointing at the branch's snapshot, so a `dev` commit changed what `main` + /// readers resolve. + #[test] + fn committing_to_a_feature_branch_leaves_main_alone() { + let mut metadata = base_metadata(); + let main_before = ref_snapshot_id(&metadata, MAIN_REF); + + apply_updates( + &mut metadata, + &[CommitUpdate::AddSnapshot { + snapshot: snapshot_json(99, None), + }], + "feature", + ) + .unwrap(); + + assert_eq!(ref_snapshot_id(&metadata, "feature"), Some(99)); + assert_eq!( + ref_snapshot_id(&metadata, MAIN_REF), + main_before, + "a feature-branch commit must not move main" + ); + assert_eq!( + metadata.current_snapshot_id, main_before, + "current-snapshot-id describes main, not the committed branch" + ); + assert!( + metadata + .snapshot_log + .as_ref() + .is_none_or(|log| log.is_empty()), + "the snapshot log describes main's history" + ); + } + #[test] fn metadata_round_trips_through_json_with_refs() { let mut metadata = base_metadata(); diff --git a/pangolin/pangolin_api/src/iceberg/error.rs b/pangolin/pangolin_api/src/iceberg/error.rs index d05168d..b7f3b18 100644 --- a/pangolin/pangolin_api/src/iceberg/error.rs +++ b/pangolin/pangolin_api/src/iceberg/error.rs @@ -8,7 +8,10 @@ //! //! Pangolin emitted a flat `{"error": ""}`, and most Iceberg handlers //! bypassed the error type entirely and returned bare `(StatusCode, &str)` -//! tuples with a plain-text body (A-6). Engines parse the envelope to tell a +//! tuples with a plain-text body (A-6, and still open as B16j at the August +//! audit despite this module reading as though it were resolved). Every return +//! in `iceberg/` now routes through the helpers below; there are zero bare +//! tuples left in that module. Engines parse the envelope to tell a //! `NoSuchTableException` from a `CommitFailedException`, which is what drives //! their retry logic, so a non-conforming body breaks retries rather than //! merely looking untidy. @@ -80,6 +83,47 @@ pub fn forbidden(detail: &str) -> Response { iceberg_error(StatusCode::FORBIDDEN, "ForbiddenException", detail) } +/// `404` for a view that does not exist. +pub fn no_such_view(identifier: &str) -> Response { + iceberg_error( + StatusCode::NOT_FOUND, + "NoSuchViewException", + &format!("View does not exist: {identifier}"), + ) +} + +/// `400` for a malformed or unusable request. +pub fn bad_request(detail: &str) -> Response { + iceberg_error(StatusCode::BAD_REQUEST, "BadRequestException", detail) +} + +/// `409` for a table that already exists. +pub fn table_already_exists(identifier: &str) -> Response { + iceberg_error( + StatusCode::CONFLICT, + "AlreadyExistsException", + &format!("Table already exists: {identifier}"), + ) +} + +/// `409` for a namespace that already exists. +pub fn namespace_already_exists(namespace: &str) -> Response { + iceberg_error( + StatusCode::CONFLICT, + "AlreadyExistsException", + &format!("Namespace already exists: {namespace}"), + ) +} + +/// `409` for a namespace that still has children. +pub fn namespace_not_empty(namespace: &str) -> Response { + iceberg_error( + StatusCode::CONFLICT, + "NamespaceNotEmptyException", + &format!("Namespace is not empty: {namespace}"), + ) +} + /// `500`, with the underlying cause logged rather than returned. pub fn internal(context: &str) -> Response { iceberg_error( diff --git a/pangolin/pangolin_api/src/iceberg/mod.rs b/pangolin/pangolin_api/src/iceberg/mod.rs index 1b93310..db57d9a 100644 --- a/pangolin/pangolin_api/src/iceberg/mod.rs +++ b/pangolin/pangolin_api/src/iceberg/mod.rs @@ -18,7 +18,10 @@ pub mod tables; pub mod types; // Re-export types for convenience -pub use error::iceberg_error; +pub use error::{ + bad_request, forbidden, iceberg_error, internal, namespace_already_exists, namespace_not_empty, + no_such_namespace, no_such_table, no_such_view, table_already_exists, +}; pub use types::*; pub type AppState = std::sync::Arc; diff --git a/pangolin/pangolin_api/src/iceberg/namespaces.rs b/pangolin/pangolin_api/src/iceberg/namespaces.rs index d2dc88f..770af8e 100644 --- a/pangolin/pangolin_api/src/iceberg/namespaces.rs +++ b/pangolin/pangolin_api/src/iceberg/namespaces.rs @@ -1,5 +1,8 @@ use super::types::*; -use super::{check_and_forward_if_federated, AppState}; +use super::{ + check_and_forward_if_federated, forbidden, internal, namespace_already_exists, + no_such_namespace, AppState, +}; use crate::auth::TenantId; use crate::authz::check_permission; use axum::{ @@ -37,7 +40,7 @@ pub async fn list_namespaces( Extension(session): Extension, Path(prefix): Path, Query(params): Query, - Query(pagination): Query, + Query(page): Query, ) -> impl IntoResponse { let tenant_id = tenant.0; let catalog_name = prefix.clone(); @@ -67,9 +70,10 @@ pub async fn list_namespaces( // Local catalog handling let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "list_namespaces: failed to load catalog"); + return internal("Failed to load catalog"); } }; @@ -79,31 +83,193 @@ pub async fn list_namespaces( }; match check_permission(&store, &session, &Action::List, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "list_namespaces: permission check failed"); + return internal("Permission check failed"); } } + let (offset, limit) = page.resolve(); + let pagination = PaginationParams { + limit: Some(limit as usize), + offset: Some(offset as usize), + }; + match store .list_namespaces(tenant_id, &catalog_name, params.parent, Some(pagination)) .await { Ok(namespaces) => { + let returned = namespaces.len(); let ns_list: Vec> = namespaces.into_iter().map(|n| n.name).collect(); ( StatusCode::OK, Json(ListNamespacesResponse { namespaces: ns_list, + next_page_token: next_page_token(returned, offset, limit), }), ) .into_response() } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "list_namespaces: failed to list namespaces"); + internal("Failed to list namespaces") + } + } +} + +/// Load a namespace's metadata (`loadNamespaceMetadata`). +/// +/// Part of completing the Iceberg REST surface: this endpoint was on the +/// README's "not implemented" list, so clients could create a namespace and set +/// its properties but never read them back. +#[utoipa::path( + get, + path = "/v1/{prefix}/namespaces/{namespace}", + tag = "Iceberg REST", + params( + ("prefix" = String, Path, description = "Catalog name"), + ("namespace" = String, Path, description = "Namespace name") + ), + responses( + (status = 200, description = "Namespace metadata", body = CreateNamespaceResponse), + (status = 403, description = "Forbidden"), + (status = 404, description = "Namespace not found"), + (status = 500, description = "Internal server error") + ), + security(("bearer_auth" = [])) +)] +pub async fn load_namespace_metadata( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path((prefix, namespace)): Path<(String, String)>, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = prefix; + + let path = format!("/namespaces/{}", namespace); + if let Some(response) = check_and_forward_if_federated( + &store, + tenant_id, + &catalog_name, + Method::GET, + &path, + None, + HeaderMap::new(), + ) + .await + { + return response; + } + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "load_namespace_metadata: failed to load catalog"); + return internal("Failed to load catalog"); + } + }; + + let (namespace_parts, _branch) = parse_namespace(&namespace); + + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden"), + Err(e) => { + tracing::error!(error = %e, "load_namespace_metadata: permission check failed"); + return internal("Permission check failed"); + } + } + + match store + .get_namespace(tenant_id, &catalog_name, namespace_parts.clone()) + .await + { + Ok(Some(ns)) => ( + StatusCode::OK, + Json(CreateNamespaceResponse { + namespace: ns.name, + properties: ns.properties, + }), + ) + .into_response(), + Ok(None) => no_such_namespace(&namespace_parts.join(".")), + Err(e) => { + tracing::error!(error = %e, "load_namespace_metadata: failed to load namespace"); + internal("Failed to load namespace") + } + } +} + +/// Check whether a namespace exists (`namespaceExists`). +/// +/// `HEAD` with an empty body, per the spec. Also on the README's +/// "not implemented" list. +#[utoipa::path( + head, + path = "/v1/{prefix}/namespaces/{namespace}", + tag = "Iceberg REST", + params( + ("prefix" = String, Path, description = "Catalog name"), + ("namespace" = String, Path, description = "Namespace name") + ), + responses( + (status = 204, description = "Namespace exists"), + (status = 403, description = "Forbidden"), + (status = 404, description = "Namespace not found") + ), + security(("bearer_auth" = [])) +)] +pub async fn namespace_exists( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path((prefix, namespace)): Path<(String, String)>, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = prefix; + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "namespace_exists: failed to load catalog"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + + let (namespace_parts, _branch) = parse_namespace(&namespace); + + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (), + Ok(false) => return StatusCode::FORBIDDEN.into_response(), + Err(e) => { + tracing::error!(error = %e, "namespace_exists: permission check failed"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + } + + match store + .get_namespace(tenant_id, &catalog_name, namespace_parts) + .await + { + Ok(Some(_)) => StatusCode::NO_CONTENT.into_response(), + Ok(None) => StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "namespace_exists: failed to load namespace"); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } } } @@ -140,12 +306,34 @@ pub async fn create_namespace( catalog_name ); + // B16k: federated forwarding was missing here (and on delete and the + // namespace tree) although `list_namespaces` had it. On a `Federated` + // catalog, creating a namespace built a *local shadow* and returned 200 + // while `GET` listed the remote - the two views diverged permanently, and + // delete reported success for a namespace still present upstream. + let path = "/namespaces".to_string(); + let body_bytes = serde_json::to_vec(&payload).ok().map(Bytes::from); + if let Some(response) = check_and_forward_if_federated( + &store, + tenant_id, + &catalog_name, + Method::POST, + &path, + body_bytes, + HeaderMap::new(), + ) + .await + { + return response; + } + // Resolve catalog ID let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "create_namespace: failed to load catalog"); + return internal("Failed to load catalog"); } }; @@ -155,13 +343,10 @@ pub async fn create_namespace( }; match check_permission(&store, &session, &Action::Create, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "create_namespace: permission check failed"); + return internal("Permission check failed"); } } @@ -170,6 +355,21 @@ pub async fn create_namespace( properties: payload.properties.unwrap_or_default(), }; + // Report a conflict rather than silently overwriting an existing namespace's + // properties, which is what the create path did on backends whose insert is + // an upsert. + match store + .get_namespace(tenant_id, &catalog_name, ns.name.clone()) + .await + { + Ok(Some(_)) => return namespace_already_exists(&ns.name.join(".")), + Ok(None) => {} + Err(e) => { + tracing::error!(error = %e, "create_namespace: existence check failed"); + return internal("Failed to check namespace"); + } + } + match store .create_namespace(tenant_id, &catalog_name, ns.clone()) .await @@ -200,7 +400,10 @@ pub async fn create_namespace( ) .into_response() } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_namespace: failed to create namespace"); + internal("Failed to create namespace") + } } } @@ -230,16 +433,33 @@ pub async fn delete_namespace( let tenant_id = tenant.0; let catalog_name = prefix; + // Federated forwarding (B16k) - see `create_namespace`. + let path = format!("/namespaces/{}", namespace); + if let Some(response) = check_and_forward_if_federated( + &store, + tenant_id, + &catalog_name, + Method::DELETE, + &path, + None, + HeaderMap::new(), + ) + .await + { + return response; + } + // Resolve catalog ID let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "delete_namespace: failed to load catalog"); + return internal("Failed to load catalog"); } }; - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let (namespace_parts, _branch) = parse_namespace(&namespace); // Check Permissions let scope = PermissionScope::Namespace { @@ -249,13 +469,10 @@ pub async fn delete_namespace( match check_permission(&store, &session, &Action::Delete, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "delete_namespace: permission check failed"); + return internal("Permission check failed"); } } @@ -282,11 +499,25 @@ pub async fn delete_namespace( StatusCode::NO_CONTENT.into_response() } - Err(_) => (StatusCode::NOT_FOUND, "Namespace not found").into_response(), + Err(e) => { + tracing::debug!(error = %e, "delete_namespace: namespace not deleted"); + no_such_namespace(&namespace_parts.join(".")) + } } } -/// Update namespace properties +/// Update a namespace's properties. +/// +/// Two fixes here: +/// +/// * **B0d** - the handler bound `Extension(_session)` (deliberately discarding +/// it) and never called `check_permission`, and never resolved the catalog at +/// all. Any tenant member could rewrite any namespace's properties, including +/// `location`, which later table creation derives paths from. +/// * **B16h** - `removals` were silently ignored. A request carrying removals +/// got `200 OK` with `removed: []` / `missing: []` while nothing was removed: +/// exactly the "silent success" failure class 0.6.0 set out to eliminate. The +/// three response lists are now reported honestly. #[utoipa::path( post, path = "/v1/{prefix}/namespaces/{namespace}/properties", @@ -307,7 +538,7 @@ pub async fn delete_namespace( pub async fn update_namespace_properties( State(store): State, Extension(tenant): Extension, - Extension(_session): Extension, + Extension(session): Extension, Path((prefix, namespace)): Path<(String, String)>, Json(payload): Json, ) -> impl IntoResponse { @@ -331,26 +562,44 @@ pub async fn update_namespace_properties( { return response; } - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); - - // For MVP, we only support updates. Removals are ignored or TODO. - if let Some(updates) = payload.updates { - match store - .update_namespace_properties(tenant_id, &catalog_name, namespace_parts, updates.clone()) - .await - { - Ok(_) => { - let response = UpdateNamespacePropertiesResponse { - updated: updates.keys().cloned().collect(), - removed: vec![], - missing: vec![], - }; - (StatusCode::OK, Json(response)).into_response() - } - Err(_) => (StatusCode::NOT_FOUND, "Namespace not found").into_response(), + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "update_namespace_properties: failed to load catalog"); + return internal("Failed to load catalog"); + } + }; + + let (namespace_parts, _branch) = parse_namespace(&namespace); + + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Write, &scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden"), + Err(e) => { + tracing::error!(error = %e, "update_namespace_properties: permission check failed"); + return internal("Permission check failed"); } - } else { - ( + } + + let updates = payload.updates.unwrap_or_default(); + let removals = payload.removals.unwrap_or_default(); + + // The spec rejects a key that appears in both lists rather than picking a + // winner. + if let Some(conflict) = removals.iter().find(|k| updates.contains_key(*k)) { + return super::bad_request(&format!( + "Property {conflict} appears in both updates and removals" + )); + } + + if updates.is_empty() && removals.is_empty() { + return ( StatusCode::OK, Json(UpdateNamespacePropertiesResponse { updated: vec![], @@ -358,7 +607,59 @@ pub async fn update_namespace_properties( missing: vec![], }), ) - .into_response() + .into_response(); + } + + // Read-modify-write: removals cannot be expressed by the merging store + // method, so the resulting map is computed here and written wholesale. + let existing = match store + .get_namespace(tenant_id, &catalog_name, namespace_parts.clone()) + .await + { + Ok(Some(ns)) => ns.properties, + Ok(None) => return no_such_namespace(&namespace_parts.join(".")), + Err(e) => { + tracing::error!(error = %e, "update_namespace_properties: failed to load namespace"); + return internal("Failed to load namespace"); + } + }; + + let mut properties = existing; + let mut removed = Vec::new(); + let mut missing = Vec::new(); + for key in &removals { + if properties.remove(key).is_some() { + removed.push(key.clone()); + } else { + missing.push(key.clone()); + } + } + + let updated: Vec = updates.keys().cloned().collect(); + properties.extend(updates); + + match store + .replace_namespace_properties( + tenant_id, + &catalog_name, + namespace_parts.clone(), + properties, + ) + .await + { + Ok(_) => ( + StatusCode::OK, + Json(UpdateNamespacePropertiesResponse { + updated, + removed, + missing, + }), + ) + .into_response(), + Err(e) => { + tracing::error!(error = %e, "update_namespace_properties: failed to write properties"); + no_such_namespace(&namespace_parts.join(".")) + } } } @@ -387,12 +688,29 @@ pub async fn list_namespaces_tree( let tenant_id = tenant.0; let catalog_name = prefix.clone(); + // Federated forwarding (B16k): without it the tree renders the local shadow + // while every other view shows the remote. + if let Some(response) = check_and_forward_if_federated( + &store, + tenant_id, + &catalog_name, + Method::GET, + "/namespaces", + None, + HeaderMap::new(), + ) + .await + { + return response; + } + // Resolve catalog ID let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "list_namespaces_tree: failed to load catalog"); + return internal("Failed to load catalog"); } }; @@ -402,13 +720,10 @@ pub async fn list_namespaces_tree( }; match check_permission(&store, &session, &Action::List, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "list_namespaces_tree: permission check failed"); + return internal("Permission check failed"); } } @@ -455,10 +770,9 @@ pub async fn list_namespaces_tree( ) .into_response() } - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to list namespaces: {}", e), - ) - .into_response(), + Err(e) => { + tracing::error!(error = %e, "list_namespaces_tree: failed to list namespaces"); + internal("Failed to list namespaces") + } } } diff --git a/pangolin/pangolin_api/src/iceberg/oauth.rs b/pangolin/pangolin_api/src/iceberg/oauth.rs index 9ff80c2..1998520 100644 --- a/pangolin/pangolin_api/src/iceberg/oauth.rs +++ b/pangolin/pangolin_api/src/iceberg/oauth.rs @@ -18,6 +18,7 @@ use crate::iceberg::AppState; use utoipa::ToSchema; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct OAuthTokenRequest { #[schema(example = "client_credentials")] grant_type: String, @@ -86,9 +87,15 @@ pub async fn handle_oauth_token( .map_err(ApiError::InternalError)? .ok_or_else(|| ApiError::unauthorized("Invalid client_id"))?; - // 4. Verify Active Status - if !service_user.active { - return Err(ApiError::unauthorized("Client is inactive")); + // 4. Verify Active Status *and* expiry. + // + // B0g: this checked only `active`, not `is_valid()` (= `active && + // !is_expired()`). The API-key path in `auth_middleware` uses `is_valid()` + // correctly, so an *expired* service user was refused there but could still + // exchange `client_credentials` here for a fresh 1-hour JWT - fully + // bypassing key expiry, and renewably. + if !service_user.is_valid() { + return Err(ApiError::unauthorized("Client is inactive or expired")); } // 5. Verify Secret (API Key) diff --git a/pangolin/pangolin_api/src/iceberg/tables.rs b/pangolin/pangolin_api/src/iceberg/tables.rs index 4c1bfdd..40f091b 100644 --- a/pangolin/pangolin_api/src/iceberg/tables.rs +++ b/pangolin/pangolin_api/src/iceberg/tables.rs @@ -1,5 +1,8 @@ use super::types::*; -use super::{check_and_forward_if_federated, commit, iceberg_error, AppState}; +use super::{ + bad_request, check_and_forward_if_federated, commit, forbidden, iceberg_error, internal, + no_such_namespace, no_such_table, table_already_exists, AppState, +}; use crate::auth::TenantId; use crate::authz::check_permission; use axum::{ @@ -11,7 +14,7 @@ use axum::{ use bytes::Bytes; use chrono::Utc; use pangolin_core::iceberg_metadata::{ - NestedField, PartitionSpec, Schema, SortOrder, TableMetadata, Type, + MetadataLogEntry, PartitionSpec, Schema, SortOrder, TableMetadata, }; use pangolin_core::model::{Asset, AssetType}; use pangolin_core::permission::{Action, PermissionScope}; @@ -21,6 +24,11 @@ use std::collections::HashMap; use std::sync::Arc; use uuid::Uuid; +/// How many previous metadata files to keep in `metadata-log` when the table +/// does not set `write.metadata.previous-versions-max`. Matches the Iceberg +/// default. +const DEFAULT_PREVIOUS_VERSIONS_MAX: usize = 100; + /// List tables in a namespace #[utoipa::path( get, @@ -43,7 +51,7 @@ pub async fn list_tables( Extension(tenant): Extension, Extension(session): Extension, Path((prefix, namespace)): Path<(String, String)>, - Query(pagination): Query, + Query(page): Query, ) -> impl IntoResponse { let tenant_id = tenant.0; let catalog_name = prefix.clone(); @@ -66,32 +74,34 @@ pub async fn list_tables( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "list_tables: failed to load catalog"); + return internal("Failed to load catalog"); } }; - let (ns_name, branch) = parse_table_identifier(&namespace); + let (ns_vec, branch) = parse_namespace(&namespace); // Check Permissions let scope = PermissionScope::Namespace { catalog_id: catalog.id, - namespace: ns_name.clone(), + namespace: ns_vec.join("."), }; match check_permission(&store, &session, &Action::List, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "list_tables: permission check failed"); + return internal("Permission check failed"); } } - let ns_vec = vec![ns_name]; + let (offset, limit) = page.resolve(); + let pagination = PaginationParams { + limit: Some(limit as usize), + offset: Some(offset as usize), + }; match store .list_assets( @@ -104,6 +114,11 @@ pub async fn list_tables( .await { Ok(assets) => { + // The page token has to be computed from the number of rows the + // store returned, before the asset-type filter narrows it - the + // store is what applied `limit`, so a full page of rows means there + // may be more even if none of them survive the filter. + let returned = assets.len(); let identifiers: Vec = assets .into_iter() .filter(|a| a.kind == AssetType::IcebergTable) @@ -112,13 +127,24 @@ pub async fn list_tables( name: a.name, }) .collect(); - (StatusCode::OK, Json(ListTablesResponse { identifiers })).into_response() + ( + StatusCode::OK, + Json(ListTablesResponse { + identifiers, + next_page_token: next_page_token(returned, offset, limit), + }), + ) + .into_response() + } + Err(e) => { + tracing::error!(error = %e, "list_tables: failed to list assets"); + internal("Failed to list tables") } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), } } #[derive(Debug, serde::Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] pub struct MaintenanceRequest { pub job_type: String, // "expire_snapshots" or "remove_orphan_files" pub retention_ms: Option, @@ -126,6 +152,13 @@ pub struct MaintenanceRequest { } /// Perform maintenance on a table +/// +/// Two bugs fixed here (B0f): +/// 1. the catalog from the path was discarded and the literal `"default"` was +/// passed to `expire_snapshots`/`remove_orphan_files`, so destructive +/// maintenance ran against the wrong catalog entirely; +/// 2. there was no session and no permission check, so any tenant member could +/// trigger snapshot expiry and orphan-file deletion on any table. #[utoipa::path( post, path = "/api/v1/catalogs/{prefix}/namespaces/{namespace}/tables/{table}/maintenance", @@ -139,6 +172,8 @@ pub struct MaintenanceRequest { responses( (status = 200, description = "Maintenance accepted", body = serde_json::Value), (status = 400, description = "Bad request"), + (status = 403, description = "Forbidden"), + (status = 404, description = "Catalog or table not found"), (status = 500, description = "Internal server error") ), security(("bearer_auth" = [])) @@ -146,24 +181,67 @@ pub struct MaintenanceRequest { pub async fn perform_maintenance( State(store): State>, Extension(tenant_id): Extension, - Path((_prefix, namespace, table)): Path<(String, String, String)>, + Extension(session): Extension, + Path((prefix, namespace, table)): Path<(String, String, String)>, Json(payload): Json, ) -> Result, StatusCode> { - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); - // Parse table@branch - let (table_name, branch_name) = if let Some((t, b)) = table.split_once('@') { - (t.to_string(), Some(b.to_string())) - } else { - (table.to_string(), None) + let tenant = tenant_id.0; + let catalog_name = prefix; + + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let (table_name, branch_from_table) = parse_table_identifier(&table); + let branch_name = branch_from_table.or(branch_from_ns); + + let catalog = match store.get_catalog(tenant, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return Err(StatusCode::NOT_FOUND), + Err(e) => { + tracing::error!(error = %e, "maintenance: failed to load catalog"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + }; + + let asset = match store + .get_asset( + tenant, + &catalog_name, + branch_name.clone(), + namespace_parts.clone(), + table_name.clone(), + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return Err(StatusCode::NOT_FOUND), + Err(e) => { + tracing::error!(error = %e, "maintenance: failed to load asset"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } }; + // Snapshot expiry and orphan-file removal both destroy data, so they need + // Delete, not merely Write. + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + asset_id: asset.id, + }; + match check_permission(&store, &session, &Action::Delete, &scope).await { + Ok(true) => (), + Ok(false) => return Err(StatusCode::FORBIDDEN), + Err(e) => { + tracing::error!(error = %e, "maintenance: permission check failed"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + match payload.job_type.as_str() { "expire_snapshots" => { let retention = payload.retention_ms.unwrap_or(86400000); // Default 1 day store .expire_snapshots( - tenant_id.0, - "default", + tenant, + &catalog_name, branch_name, namespace_parts, table_name, @@ -179,8 +257,8 @@ pub async fn perform_maintenance( let older_than = payload.older_than_ms.unwrap_or(86400000); // Default 1 day store .remove_orphan_files( - tenant_id.0, - "default", + tenant, + &catalog_name, branch_name, namespace_parts, table_name, @@ -256,16 +334,18 @@ pub async fn create_table( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "create_table: failed to load catalog"); + return internal("Failed to load catalog"); } }; let (tbl_name, branch_from_name) = parse_table_identifier(&payload.name); - let (ns_name, branch_from_ns) = parse_table_identifier(&namespace); + let (ns_vec, branch_from_ns) = parse_namespace(&namespace); let branch_from_query = params.get("branch").cloned(); let branch = branch_from_name.or(branch_from_ns).or(branch_from_query); + let ns_name = ns_vec.join("."); let scope = PermissionScope::Namespace { catalog_id: catalog.id, @@ -273,18 +353,13 @@ pub async fn create_table( }; match check_permission(&store, &session, &Action::Create, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "create_table: permission check failed"); + return internal("Permission check failed"); } } - let ns_vec = vec![ns_name.clone()]; - let table_uuid = Uuid::new_v4(); let location = if let Some(loc) = payload.location { loc @@ -328,66 +403,57 @@ pub async fn create_table( ) }; - let schema_fields = if let Some(schema_value) = &payload.schema { - if let Some(fields) = schema_value.get("fields").and_then(|f| f.as_array()) { - fields - .iter() - .filter_map(|field| { - let id = field.get("id")?.as_i64()? as i32; - let name = field.get("name")?.as_str()?.to_string(); - let required = false; - let field_type_str = field.get("type")?.as_str()?; - - let field_type = match field_type_str { - "int" | "integer" => Type::Primitive("long".to_string()), - "long" => Type::Primitive("long".to_string()), - "string" => Type::Primitive("string".to_string()), - "boolean" => Type::Primitive("boolean".to_string()), - "float" => Type::Primitive("float".to_string()), - "double" => Type::Primitive("double".to_string()), - "date" => Type::Primitive("date".to_string()), - "time" => Type::Primitive("time".to_string()), - "timestamp" => Type::Primitive("timestamp".to_string()), - "timestamptz" => Type::Primitive("timestamptz".to_string()), - "binary" => Type::Primitive("binary".to_string()), - "uuid" => Type::Primitive("uuid".to_string()), - _ => Type::Primitive(field_type_str.to_string()), - }; - - Some(NestedField { - id, - name, - required, - field_type, - doc: None, - }) - }) - .collect() - } else { - vec![] - } - } else { - vec![] + // B16f: the schema used to be hand-parsed field by field, which silently + // lost data three ways - `required` was hardcoded to `false` so every column + // became optional, `int` was widened to `long`, and `field.get("type")? + // .as_str()?` inside a `filter_map` returned `None` for any struct/list/map/ + // decimal/fixed column, so those columns were *dropped* and `last_column_id` + // was computed from the survivors. The table was created `200 OK` with a + // schema missing columns. + // + // Deserializing straight into the core `Schema` (as the commit path already + // does) keeps nullability and complex types, and a malformed schema is now a + // 400 rather than a quietly mangled table. + let schema = match &payload.schema { + Some(schema_value) => match serde_json::from_value::(schema_value.clone()) { + Ok(mut s) => { + s.schema_id = 0; + if s.identifier_field_ids.is_none() { + s.identifier_field_ids = Some(vec![]); + } + s + } + Err(e) => { + return bad_request(&format!("Invalid schema: {}", e)); + } + }, + None => Schema { + type_: Schema::STRUCT.to_string(), + schema_id: 0, + identifier_field_ids: Some(vec![]), + fields: vec![], + }, }; + let last_column_id = schema.max_field_id(); + let metadata = TableMetadata { format_version: 2, table_uuid, location: location.clone(), last_sequence_number: 0, last_updated_ms: Utc::now().timestamp_millis(), - last_column_id: schema_fields.iter().map(|f| f.id).max().unwrap_or(0), - schemas: vec![Schema { - schema_id: 0, - identifier_field_ids: Some(vec![]), - fields: schema_fields, - }], + last_column_id, + schemas: vec![schema], current_schema_id: 0, current_partition_spec_id: 0, partition_specs: vec![PartitionSpec { spec_id: 0, fields: vec![], }], + // Required by the v2 spec; an empty unpartitioned spec assigns nothing, + // so the highest assigned partition id is the "unpartitioned" sentinel. + last_partition_id: pangolin_core::iceberg_metadata::PARTITION_FIELD_ID_START - 1, default_sort_order_id: 0, sort_orders: vec![SortOrder { order_id: 0, @@ -401,7 +467,13 @@ pub async fn create_table( refs: None, }; - let metadata_json = serde_json::to_string(&metadata).unwrap(); + let metadata_json = match serde_json::to_string(&metadata) { + Ok(json) => json, + Err(e) => { + tracing::error!(error = %e, "create_table: failed to serialize metadata"); + return internal("Failed to serialize table metadata"); + } + }; let metadata_location = format!( "{}/metadata/00000-{}.metadata.json", location, @@ -423,23 +495,30 @@ pub async fn create_table( }, }; + // B16g: write the metadata file *first*, then register the asset. The old + // order registered the asset and only then wrote the file, so a failed write + // left a permanently broken table - registered, pointing at a file that does + // not exist, with `load_table` 500ing and `update_table` 404ing and no repair + // path. This is also the order the commit path already uses. + if let Err(e) = store + .write_file(&metadata_location, metadata_json.into_bytes()) + .await + { + tracing::error!("Failed to write metadata file: {}", e); + return internal("Failed to write metadata"); + } + match store - .create_asset(tenant_id, &catalog_name, branch, ns_vec, asset.clone()) + .create_asset( + tenant_id, + &catalog_name, + branch, + ns_vec.clone(), + asset.clone(), + ) .await { Ok(_) => { - if let Err(e) = store - .write_file(&metadata_location, metadata_json.into_bytes()) - .await - { - tracing::error!("Failed to write metadata file: {}", e); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to write metadata", - ) - .into_response(); - } - let _ = store .log_audit_event( tenant_id, @@ -493,7 +572,12 @@ pub async fn create_table( ( StatusCode::OK, Json(TableResponse::with_credentials( - Some(location.clone()), + // B16e: this returned `location` - the table *directory* - + // where the spec (and `load_table`, correctly) return the + // metadata *file*. A client that keeps the returned `Table` + // (PyIceberg does) ended up with a `metadata_location` it + // could neither read nor refresh from. + Some(metadata_location.clone()), metadata, credentials, Some(table_uuid), @@ -501,11 +585,173 @@ pub async fn create_table( ) .into_response() } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_table: failed to register asset"); + // The metadata file was written before registration (B16g); with the + // asset unregistered it is unreferenced, so clean it up rather than + // leaving an orphan behind. + if let Err(cleanup) = store.delete_file(&metadata_location).await { + tracing::warn!( + error = %cleanup, + location = %metadata_location, + "could not remove the orphaned metadata file after a failed create_asset" + ); + } + internal("Failed to create table") + } } } /// Load a table +/// The spec's `registerTable` request body. +#[derive(serde::Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RegisterTableRequest { + pub name: String, + #[serde(rename = "metadata-location")] + pub metadata_location: String, + /// Spec-optional; some clients send it, and `deny_unknown_fields` would + /// otherwise reject an otherwise valid request. + #[serde(default, rename = "overwrite")] + pub overwrite: Option, +} + +/// `POST /v1/{prefix}/namespaces/{namespace}/register` — the spec's +/// `registerTable`. +/// +/// A-5: this route did not exist. It is how an engine adopts a table whose +/// metadata already sits in object storage — a migration from another catalog, +/// a restore, or a table written directly by a job. Without it the only way to +/// get such a table into Pangolin was to recreate it and lose its history. +/// +/// Distinct from `create_table`: nothing is written to storage. The metadata +/// file already exists and this points the catalog at it. +pub async fn register_table( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path((prefix, namespace)): Path<(String, String)>, + Json(payload): Json, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = prefix; + + let (tbl_name, branch_from_name) = parse_table_identifier(&payload.name); + let (ns_vec, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name.or(branch_from_ns); + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "register_table: failed to load catalog"); + return internal("Failed to load catalog"); + } + }; + + // Namespace-scoped Create, the same check `create_table` makes. Registering + // is a create: it brings a new table into the namespace. + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: ns_vec.join("."), + }; + match check_permission(&store, &session, &Action::Create, &scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden"), + Err(e) => { + tracing::error!(error = %e, "register_table: permission check failed"); + return internal("Permission check failed"); + } + } + + // Refuse to shadow an existing table. The spec has `overwrite` for this and + // defaults it to false; silently replacing a table's metadata pointer would + // orphan its history. + let existing = store + .get_asset( + tenant_id, + &catalog_name, + branch.clone(), + ns_vec.clone(), + tbl_name.clone(), + ) + .await; + match existing { + Ok(Some(_)) if !payload.overwrite.unwrap_or(false) => { + return table_already_exists(&format!("{}.{}", ns_vec.join("."), tbl_name)); + } + Ok(_) => {} + Err(e) => { + tracing::error!(error = %e, "register_table: failed to check for an existing table"); + return internal("Failed to check for an existing table"); + } + } + + // The metadata file must actually be there. Registering a location that + // does not exist would leave a table that every subsequent `loadTable` + // fails on, and the failure would surface far from this request. + if let Err(e) = store.read_file(&payload.metadata_location).await { + tracing::warn!( + error = %e, + location = %payload.metadata_location, + "register_table: the metadata location could not be read" + ); + return ( + axum::http::StatusCode::BAD_REQUEST, + Json(serde_json::json!({ + "error": { + "message": format!( + "metadata-location {:?} could not be read: {e}", + payload.metadata_location + ), + "type": "BadRequestException", + "code": 400 + } + })), + ) + .into_response(); + } + + let mut properties = HashMap::new(); + properties.insert( + "metadata_location".to_string(), + payload.metadata_location.clone(), + ); + + let asset = pangolin_core::model::Asset { + id: uuid::Uuid::new_v4(), + name: tbl_name.clone(), + kind: pangolin_core::model::AssetType::IcebergTable, + location: payload.metadata_location.clone(), + properties, + }; + + if let Err(e) = store + .create_asset(tenant_id, &catalog_name, branch, ns_vec.clone(), asset) + .await + { + tracing::error!(error = %e, "register_table: failed to record the asset"); + return internal("Failed to register the table"); + } + + tracing::info!( + table = %tbl_name, + location = %payload.metadata_location, + "registered an existing table" + ); + + // The spec wants the loaded table back. Re-reading through `load_table`'s + // path would duplicate the credential-vending logic, so the caller is + // pointed at the metadata it just supplied. + ( + axum::http::StatusCode::OK, + Json(serde_json::json!({ + "metadata-location": payload.metadata_location, + })), + ) + .into_response() +} + #[utoipa::path( get, path = "/v1/{prefix}/namespaces/{namespace}/tables/{table}", @@ -560,18 +806,19 @@ pub async fn load_table( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "load_table: failed to load catalog"); + return internal("Failed to load catalog"); } }; + // B16a: shared namespace parsing, so a nested namespace resolves the same + // way here as it does on the commit path. let (tbl_name, branch_from_name) = parse_table_identifier(&table); - let (ns_name, branch_from_ns) = parse_table_identifier(&namespace); + let (ns_vec, branch_from_ns) = parse_namespace(&namespace); let branch = branch_from_name.or(branch_from_ns); - let ns_vec = vec![ns_name]; - let asset = match store .get_asset( tenant_id, @@ -583,9 +830,10 @@ pub async fn load_table( .await { Ok(Some(a)) => a, - Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_table(&format!("{}.{}", ns_vec.join("."), tbl_name)), + Err(e) => { + tracing::error!(error = %e, "load_table: failed to load asset"); + return internal("Failed to load table"); } }; @@ -597,13 +845,10 @@ pub async fn load_table( match check_permission(&store, &session, &Action::Read, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "load_table: permission check failed"); + return internal("Permission check failed"); } } @@ -612,12 +857,9 @@ pub async fn load_table( if let Some(location) = current_metadata_location { let metadata_bytes = match store.read_file(&location).await { Ok(bytes) => bytes, - Err(_) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to read metadata file", - ) - .into_response() + Err(e) => { + tracing::error!(error = %e, "load_table: failed to read metadata file"); + return internal("Failed to read metadata file"); } }; @@ -629,15 +871,13 @@ pub async fn load_table( .await { Ok(Ok(m)) => m, - Ok(Err(_)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to parse metadata", - ) - .into_response() + Ok(Err(e)) => { + tracing::error!(error = %e, "update_table: failed to parse metadata"); + return internal("Failed to parse metadata"); } - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Task join error").into_response() + Err(e) => { + tracing::error!(error = %e, "update_table: metadata parse task panicked"); + return internal("Failed to parse metadata"); } }; @@ -683,7 +923,7 @@ pub async fn load_table( ) .into_response() } else { - (StatusCode::NOT_FOUND, "Metadata location not found").into_response() + no_such_table(&format!("{}.{}", ns_vec.join("."), tbl_name)) } } @@ -736,15 +976,23 @@ pub async fn update_table( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "update_table: failed to load catalog"); + return internal("Failed to load catalog"); } }; + // B16a: this used to split the namespace on 0x1F while `create_table` and + // `load_table` went through `parse_table_identifier` (a *single*-element + // namespace). A table created in namespace `a\x1Fb` was registered under + // `["a\x1Fb"]` and looked up here under `["a", "b"]`, so every commit to a + // nested namespace 404'd and the CAS loop below never ran at all. let (table_name, branch_from_name) = parse_table_identifier(&table); - let branch = branch_from_name.unwrap_or("main".to_string()); - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .unwrap_or_else(|| "main".to_string()); let asset = match store .get_asset( @@ -757,9 +1005,10 @@ pub async fn update_table( .await { Ok(Some(a)) => a, - Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_table(&format!("{}.{}", namespace_parts.join("."), table_name)), + Err(e) => { + tracing::error!(error = %e, "update_table: failed to load asset"); + return internal("Failed to load table"); } }; @@ -771,13 +1020,10 @@ pub async fn update_table( match check_permission(&store, &session, &Action::Write, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "update_table: permission check failed"); + return internal("Permission check failed"); } } @@ -798,9 +1044,12 @@ pub async fn update_table( .await { Ok(Some(a)) => a, - Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => { + return no_such_table(&format!("{}.{}", namespace_parts.join("."), table_name)) + } + Err(e) => { + tracing::error!(error = %e, "update_table: failed to re-read asset"); + return internal("Failed to load table"); } }; @@ -809,16 +1058,13 @@ pub async fn update_table( let metadata_bytes = if let Some(loc) = ¤t_metadata_location { match store.read_file(loc).await { Ok(bytes) => bytes, - Err(_) => { - return (StatusCode::NOT_FOUND, "Failed to read metadata file").into_response() + Err(e) => { + tracing::error!(error = %e, "update_table: failed to read metadata file"); + return internal("Failed to read metadata file"); } } } else { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Table corrupted (no metadata)", - ) - .into_response(); + return internal("Table corrupted (no metadata location)"); }; // Parse metadata in a blocking task to avoid stalling the executor @@ -829,15 +1075,13 @@ pub async fn update_table( .await { Ok(Ok(m)) => m, - Ok(Err(_)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to parse metadata", - ) - .into_response() + Ok(Err(e)) => { + tracing::error!(error = %e, "load_table: failed to parse metadata"); + return internal("Failed to parse metadata"); } - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Task join error").into_response() + Err(e) => { + tracing::error!(error = %e, "load_table: metadata parse task panicked"); + return internal("Failed to parse metadata"); } }; @@ -854,6 +1098,36 @@ pub async fn update_table( return commit_error_response(e); } + // B16b: `last-updated-ms` was only ever assigned inside `add_snapshot`, + // so a commit of only `set-properties` / `add-schema` / `set-location` / + // `add-spec` / `set-snapshot-ref` / `remove-snapshots` published a new + // metadata file carrying an *unchanged* timestamp - and any consumer + // that orders or dedupes metadata by that field treated the two versions + // as identical. Every successful set of updates bumps it. + metadata.last_updated_ms = Utc::now().timestamp_millis(); + + // B13: record the metadata file this one supersedes. `metadata-log` was + // initialised to an empty vec at table creation and never appended to, + // so metadata time-travel and previous-version cleanup + // (`write.metadata.previous-versions-max`) had nothing to work with. + if let Some(previous) = ¤t_metadata_location { + let log = metadata.metadata_log.get_or_insert_with(Vec::new); + log.push(MetadataLogEntry { + timestamp_ms: metadata.last_updated_ms, + metadata_file: previous.clone(), + }); + let max_entries = metadata + .properties + .as_ref() + .and_then(|p| p.get("write.metadata.previous-versions-max")) + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_PREVIOUS_VERSIONS_MAX); + if log.len() > max_entries { + let excess = log.len() - max_entries; + log.drain(0..excess); + } + } + let new_metadata_location = format!( "{}/metadata/00000-{}.metadata.json", metadata.location, @@ -863,23 +1137,16 @@ pub async fn update_table( Ok(json) => json, Err(e) => { tracing::error!(error = %e, "could not serialise table metadata"); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to serialise table metadata", - ) - .into_response(); + return internal("Failed to serialise table metadata"); } }; - if let Err(_) = store + if store .write_file(&new_metadata_location, metadata_json.into_bytes()) .await + .is_err() { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to write new metadata", - ) - .into_response(); + return internal("Failed to write new metadata"); } match store @@ -938,6 +1205,19 @@ pub async fn update_table( // Re-read and re-check requirements on the next pass. tracing::debug!(error = %e, attempt = retries, "metadata CAS lost, retrying"); crate::metrics::inc(&crate::metrics::COMMIT_CAS_RETRIES); + // B16d: the metadata file was written *before* the CAS, so on a + // lost CAS it is unreferenced - and the old code just + // `continue`d, orphaning it. Under contention that leaked up to + // one file per retry, with up to five left behind on a final + // give-up, and an orphan is indistinguishable from live metadata + // from the outside so nothing could reap it later. + if let Err(cleanup) = store.delete_file(&new_metadata_location).await { + tracing::warn!( + error = %cleanup, + location = %new_metadata_location, + "could not remove the metadata file orphaned by a lost CAS" + ); + } retries += 1; continue; } @@ -974,6 +1254,13 @@ fn commit_error_response(error: commit::CommitError) -> axum::response::Response } /// Rename a table +/// +/// B0c: this handler bound `Extension(session)` but never called +/// `check_permission` - the only table handler that didn't. Any tenant member +/// could move any table into any namespace: an effective delete (the table +/// vanishes from where its readers look for it) and a way to smuggle a table +/// into a namespace where the caller *does* have read rights. It now needs +/// `Write` on the source table and `Create` on the destination namespace. #[utoipa::path( post, path = "/v1/{prefix}/tables/rename", @@ -984,6 +1271,7 @@ fn commit_error_response(error: commit::CommitError) -> axum::response::Response request_body = RenameTableRequest, responses( (status = 204, description = "Table renamed"), + (status = 403, description = "Forbidden"), (status = 404, description = "Source table not found"), (status = 500, description = "Internal server error") ), @@ -1022,6 +1310,85 @@ pub async fn rename_table( let dest_name = payload.destination.name; let branch = Some("main".to_string()); + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "rename_table: failed to load catalog"); + return internal("Failed to load catalog"); + } + }; + + let source_asset = match store + .get_asset( + tenant_id, + &catalog_name, + branch.clone(), + source_ns.clone(), + source_name.clone(), + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return no_such_table(&format!("{}.{}", source_ns.join("."), source_name)), + Err(e) => { + tracing::error!(error = %e, "rename_table: failed to load source asset"); + return internal("Failed to load source table"); + } + }; + + // Write on the source: renaming is a mutation of the table's identity, and + // from every reader's point of view it is a delete at the old path. + let source_scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: source_ns.join("."), + asset_id: source_asset.id, + }; + match check_permission(&store, &session, &Action::Write, &source_scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden: no write access to the source table"), + Err(e) => { + tracing::error!(error = %e, "rename_table: source permission check failed"); + return internal("Permission check failed"); + } + } + + // Create on the destination namespace: otherwise a caller with write on one + // table could plant it anywhere they can read. + let dest_scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: dest_ns.join("."), + }; + match check_permission(&store, &session, &Action::Create, &dest_scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden: no create access in the destination namespace"), + Err(e) => { + tracing::error!(error = %e, "rename_table: destination permission check failed"); + return internal("Permission check failed"); + } + } + + // The spec returns 409 rather than clobbering an existing destination. + match store + .get_asset( + tenant_id, + &catalog_name, + branch.clone(), + dest_ns.clone(), + dest_name.clone(), + ) + .await + { + Ok(Some(_)) => { + return table_already_exists(&format!("{}.{}", dest_ns.join("."), dest_name)) + } + Ok(None) => {} + Err(e) => { + tracing::error!(error = %e, "rename_table: destination existence check failed"); + return internal("Failed to check the destination table"); + } + } + match store .rename_asset( tenant_id, @@ -1044,7 +1411,7 @@ pub async fn rename_table( session.username.clone(), pangolin_core::audit::AuditAction::RenameTable, pangolin_core::audit::ResourceType::Table, - None, // Cannot determine asset ID easily without lookup + Some(source_asset.id), format!( "{}/{}.{} -> {}.{}", catalog_name, @@ -1059,7 +1426,10 @@ pub async fn rename_table( StatusCode::NO_CONTENT.into_response() } - Err(_) => (StatusCode::NOT_FOUND, "Table not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "rename_table: rename failed"); + no_such_table(&format!("{}.{}", source_ns.join("."), source_name)) + } } } @@ -1107,15 +1477,18 @@ pub async fn delete_table( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "delete_table: failed to load catalog"); + return internal("Failed to load catalog"); } }; let (table_name, branch_from_name) = parse_table_identifier(&table); - let branch = branch_from_name.or(Some("main".to_string())); - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .or(Some("main".to_string())); let asset = match store .get_asset( @@ -1128,9 +1501,10 @@ pub async fn delete_table( .await { Ok(Some(a)) => a, - Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_table(&format!("{}.{}", namespace_parts.join("."), table_name)), + Err(e) => { + tracing::error!(error = %e, "delete_table: failed to load asset"); + return internal("Failed to load table"); } }; @@ -1142,13 +1516,10 @@ pub async fn delete_table( match check_permission(&store, &session, &Action::Delete, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "delete_table: permission check failed"); + return internal("Permission check failed"); } } @@ -1172,7 +1543,7 @@ pub async fn delete_table( session.username.clone(), pangolin_core::audit::AuditAction::DropTable, pangolin_core::audit::ResourceType::Table, - None, + Some(asset.id), format!("{}/{}/{}", catalog_name, namespace, table), ), ) @@ -1180,7 +1551,10 @@ pub async fn delete_table( StatusCode::NO_CONTENT.into_response() } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_table: delete failed"); + internal("Failed to delete table") + } } } @@ -1203,6 +1577,7 @@ pub async fn delete_table( pub async fn table_exists( State(store): State, Extension(tenant): Extension, + Extension(session): Extension, Path((prefix, namespace, table)): Path<(String, String, String)>, ) -> impl IntoResponse { let tenant_id = tenant.0; @@ -1223,26 +1598,52 @@ pub async fn table_exists( return response; } - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); - let (table_name, branch_name) = if let Some((t, b)) = table.split_once('@') { - (t.to_string(), Some(b.to_string())) - } else { - (table.to_string(), None) + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let (table_name, branch_from_table) = parse_table_identifier(&table); + let branch_name = branch_from_table.or(branch_from_ns); + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "table_exists: failed to load catalog"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } }; - match store + let asset = match store .get_asset( tenant_id, &catalog_name, branch_name, - namespace_parts, + namespace_parts.clone(), table_name, ) .await { - Ok(Some(_)) => StatusCode::OK.into_response(), - Ok(None) => StatusCode::NOT_FOUND.into_response(), - Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + Ok(Some(a)) => a, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "table_exists: failed to load asset"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + + // Existence is information. Without a check this endpoint is an oracle that + // reports whether a table the caller cannot read exists - every sibling + // handler gates on Read, so this one does too. + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + asset_id: asset.id, + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => StatusCode::OK.into_response(), + Ok(false) => StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "table_exists: permission check failed"); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } } } diff --git a/pangolin/pangolin_api/src/iceberg/types.rs b/pangolin/pangolin_api/src/iceberg/types.rs index d792784..fba94c8 100644 --- a/pangolin/pangolin_api/src/iceberg/types.rs +++ b/pangolin/pangolin_api/src/iceberg/types.rs @@ -12,6 +12,9 @@ pub struct CatalogConfig { #[derive(Serialize, ToSchema)] pub struct ListNamespacesResponse { pub namespaces: Vec>, + /// Continuation token; absent on the final page (B16i). + #[serde(rename = "next-page-token", skip_serializing_if = "Option::is_none")] + pub next_page_token: Option, } #[derive(Deserialize, IntoParams)] @@ -32,6 +35,7 @@ pub struct ListNamespacesTreeResponse { } #[derive(Serialize, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateNamespaceRequest { pub namespace: Vec, pub properties: Option>, @@ -44,6 +48,7 @@ pub struct CreateNamespaceResponse { } #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateTableRequest { pub name: String, pub location: Option, @@ -107,6 +112,13 @@ impl TableResponse { #[derive(Serialize, Deserialize, ToSchema)] pub struct ListTablesResponse { pub identifiers: Vec, + /// Continuation token; absent on the final page (B16i). + #[serde( + rename = "next-page-token", + skip_serializing_if = "Option::is_none", + default + )] + pub next_page_token: Option, } #[derive(Serialize, Deserialize, ToSchema)] @@ -124,6 +136,7 @@ pub struct PartitionField { } #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CommitTableRequest { pub identifier: Option, pub requirements: Vec, @@ -258,13 +271,117 @@ pub fn parse_table_identifier(identifier: &str) -> (String, Option) { } } +/// The unit separator the Iceberg REST spec uses to encode a multi-level +/// namespace inside a single path segment. +pub const NAMESPACE_SEPARATOR: char = '\u{1F}'; + +/// Parse a namespace path segment into its levels plus an optional branch. +/// +/// This is the single parser for namespace path segments (B16a). Handlers used +/// to disagree: `list_tables`/`create_table`/`load_table` went through +/// [`parse_table_identifier`], which yields a *single-element* namespace, while +/// `update_table`/`delete_table`/`table_exists` split on `0x1F` and yielded the +/// real multi-element path. So a table created in namespace `a\x1Fb` was +/// registered under `["a\x1Fb"]` but looked up under `["a", "b"]` on commit - +/// a guaranteed `404 Table not found`, with the CAS loop never running. +/// +/// The `@branch` suffix is stripped first so a `ns@branch` form works +/// everywhere, not just on the handlers that happened to call +/// `parse_table_identifier`. +pub fn parse_namespace(namespace: &str) -> (Vec, Option) { + let (path, branch) = match namespace.split_once('@') { + Some((path, branch)) if !branch.is_empty() => (path, Some(branch.to_string())), + _ => (namespace, None), + }; + + let levels: Vec = path + .split(NAMESPACE_SEPARATOR) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(); + + (levels, branch) +} + +/// Spec-shaped pagination query parameters. +/// +/// The Iceberg REST spec paginates with `pageToken`/`pageSize`; Pangolin only +/// understood `limit`/`offset`, so a spec client's paging parameters were +/// silently ignored and it had no way to detect a truncated listing (B16i). +/// Both spellings are accepted, with the spec ones taking precedence. +#[derive(Deserialize, IntoParams, Default)] +pub struct IcebergPageParams { + #[serde(rename = "pageToken")] + pub page_token: Option, + #[serde(rename = "pageSize")] + pub page_size: Option, + pub limit: Option, + pub offset: Option, +} + +/// Default page size when a client asks for pagination without naming one. +pub const DEFAULT_PAGE_SIZE: u32 = 100; + +impl IcebergPageParams { + /// Resolve to `(offset, limit)`. + /// + /// The page token is an opaque encoding of the offset, per the spec's + /// "clients must treat the token as opaque" rule; the encoding here is just + /// a prefixed decimal so it stays debuggable, and an unparseable token + /// degrades to offset 0 rather than erroring the listing. + pub fn resolve(&self) -> (u32, u32) { + let limit = self + .page_size + .or(self.limit) + .filter(|l| *l > 0) + .unwrap_or(DEFAULT_PAGE_SIZE); + + let offset = self + .page_token + .as_deref() + .and_then(decode_page_token) + .or(self.offset) + .unwrap_or(0); + + (offset, limit) + } +} + +/// Encode an offset as an opaque continuation token. +pub fn encode_page_token(offset: u32) -> String { + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; + URL_SAFE_NO_PAD.encode(format!("o:{}", offset)) +} + +/// Decode a continuation token produced by [`encode_page_token`]. +pub fn decode_page_token(token: &str) -> Option { + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; + let decoded = URL_SAFE_NO_PAD.decode(token).ok()?; + let decoded = String::from_utf8(decoded).ok()?; + decoded.strip_prefix("o:")?.parse().ok() +} + +/// Compute the `next-page-token` for a listing. +/// +/// Returns `None` when the page came back short, which is how a client knows it +/// has reached the end. +pub fn next_page_token(returned: usize, offset: u32, limit: u32) -> Option { + if returned as u32 == limit { + Some(encode_page_token(offset + limit)) + } else { + None + } +} + #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct RenameTableRequest { pub source: TableIdentifier, pub destination: TableIdentifier, } #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateNamespacePropertiesRequest { pub removals: Option>, pub updates: Option>, diff --git a/pangolin/pangolin_api/src/lib.rs b/pangolin/pangolin_api/src/lib.rs index e44e056..a464060 100644 --- a/pangolin/pangolin_api/src/lib.rs +++ b/pangolin/pangolin_api/src/lib.rs @@ -24,6 +24,9 @@ pub mod audit_handlers; pub mod audit_tests; pub mod auth_middleware; pub mod authz; +/// Permission matrix: who is allowed to call what (roadmap improvement #0). +#[cfg(test)] +pub mod authz_matrix_tests; pub mod authz_utils; // Permission filtering utilities pub mod business_metadata_handlers; pub mod config; @@ -36,7 +39,9 @@ pub mod metrics; pub mod oauth_handlers; pub mod oauth_state; pub mod observability; +pub mod oidc; pub mod public_paths; +pub mod rate_limit; /// Shared test fixtures. /// /// Compiled only for tests: these helpers used to ship inside the release @@ -78,6 +83,9 @@ pub fn app_with_config( concurrency_limit: config.concurrency_limit, metrics_enabled: config.metrics_enabled, cors_allowed_origins: config.cors_allowed_origins.clone(), + auth_rate_limit: config.auth_rate_limit, + auth_rate_window: config.auth_rate_window, + trust_forwarded_for: config.trust_forwarded_for, }, ) } @@ -95,6 +103,12 @@ pub struct RouterOptions { pub metrics_enabled: bool, /// Explicit CORS origins. `None` allows any origin. pub cors_allowed_origins: Option>, + /// Failed authentication attempts allowed per window. 0 disables. + pub auth_rate_limit: u32, + /// The window those attempts are counted over. + pub auth_rate_window: std::time::Duration, + /// Honour `X-Forwarded-For` when identifying the client. + pub trust_forwarded_for: bool, } impl Default for RouterOptions { @@ -105,6 +119,15 @@ impl Default for RouterOptions { concurrency_limit: 512, metrics_enabled: true, cors_allowed_origins: None, + // Off in the library default, on in `app_with_config`. + // + // `RouterOptions::default()` is what the in-process tests build + // with, and they drive the login endpoint far harder than any real + // client. A production default belongs where production config is + // read - `PANGOLIN_AUTH_RATE_LIMIT`, which defaults to 10 a minute. + auth_rate_limit: 0, + auth_rate_window: std::time::Duration::from_secs(60), + trust_forwarded_for: false, } } } @@ -115,6 +138,14 @@ pub fn app_with_options( ) -> Router { let cors = build_cors(options.cors_allowed_origins.as_deref()); + // Shared with the login handler, which applies the per-account half of the + // limit; the middleware below only sees the source address. + let auth_limiter = std::sync::Arc::new(rate_limit::RateLimiter::new( + options.auth_rate_limit, + options.auth_rate_window, + )); + rate_limit::set_auth_limiter(auth_limiter.clone()); + let metrics_route = if options.metrics_enabled { Router::new().route("/metrics", get(metrics::metrics_handler)) } else { @@ -151,8 +182,22 @@ pub fn app_with_options( get(iceberg::namespaces::list_namespaces).post(iceberg::namespaces::create_namespace), ) .route( + // `loadNamespaceMetadata` and `namespaceExists` were on the README's + // "not implemented" list: a client could create a namespace and set + // properties but never read them back, and had no cheap existence + // probe. "/v1/:prefix/namespaces/:namespace", - delete(iceberg::namespaces::delete_namespace), + get(iceberg::namespaces::load_namespace_metadata) + .head(iceberg::namespaces::namespace_exists) + .delete(iceberg::namespaces::delete_namespace), + ) + .route( + // A-5: `registerTable`. How an engine adopts a table whose metadata + // already exists in storage - a migration from another catalog, a + // restore, or a table written directly by a job. Without it the + // only way in was to recreate the table and lose its history. + "/v1/:prefix/namespaces/:namespace/register", + post(iceberg::tables::register_table), ) .route( "/v1/:prefix/namespaces/:namespace/properties", @@ -192,7 +237,9 @@ pub fn app_with_options( ) .route( "/v1/:prefix/v1/namespaces/:namespace", - delete(iceberg::namespaces::delete_namespace), + get(iceberg::namespaces::load_namespace_metadata) + .head(iceberg::namespaces::namespace_exists) + .delete(iceberg::namespaces::delete_namespace), ) .route( "/v1/:prefix/v1/namespaces/:namespace/properties", @@ -252,7 +299,12 @@ pub fn app_with_options( "/api/v1/branches/:name/rebase", post(pangolin_handlers::rebase_branch), ) // Rebase endpoint - .route("/api/v1/branches/:name", get(pangolin_handlers::get_branch)) + .route( + // B33: the UI's branch-delete control has been 404ing because only + // GET was registered here. + "/api/v1/branches/:name", + get(pangolin_handlers::get_branch).delete(pangolin_handlers::delete_branch), + ) .route( "/api/v1/branches/:name/commits", get(pangolin_handlers::list_commits), @@ -373,11 +425,19 @@ pub fn app_with_options( // Asset Management (Views) .route( "/v1/:prefix/namespaces/:namespace/views", - post(asset_handlers::create_view), + // A-5: `listViews` was missing, so an engine could create a view and + // load one it already knew the name of, but never discover what + // existed. `SHOW VIEWS` calls this. + post(asset_handlers::create_view).get(asset_handlers::list_views), ) .route( "/v1/:prefix/namespaces/:namespace/views/:view", - get(asset_handlers::get_view), + // A-5: `dropView` and `viewExists` were missing too. Without the + // former a view created through the Iceberg API could never be + // removed through it. + get(asset_handlers::get_view) + .head(asset_handlers::view_exists) + .delete(asset_handlers::drop_view), ) // Signing APIs .route( @@ -517,6 +577,13 @@ pub fn app_with_options( "/api/v1/oauth/exchange", post(oauth_handlers::oauth_exchange), ) + .route( + // B33: the login page needs to know which providers are configured + // *before* anyone is authenticated; without this it rendered all + // four buttons unconditionally, several of them dead. + "/api/v1/oauth/providers", + get(oauth_handlers::list_oauth_providers), + ) // Business Metadata (by asset id) .route( "/api/v1/business-metadata/:asset_id", @@ -555,18 +622,47 @@ pub fn app_with_options( }, auth_middleware::auth_middleware, )) + // C-5: throttle the credential endpoints. Outside the auth middleware, + // so a refused attempt costs a cache lookup rather than a bcrypt round - + // the point is to make guessing cheap for us and expensive for the + // attacker, and verifying the password first would invert that. + .layer(axum::middleware::from_fn_with_state( + rate_limit::RateLimitState { + limiter: auth_limiter.clone(), + trust_forwarded: options.trust_forwarded_for, + }, + rate_limit::throttle_auth, + )) // Resource safety. There were previously no limits of any kind (A-20): // a single large POST could be buffered without bound and a slow // backend request had no deadline. + // + // Layer order matters, and `.layer()` applies *outermost last*, so this + // list reads inside-out: body limit is innermost, then the concurrency + // limiter, then the timeout, then load shedding. + // + // B16l: the timeout used to sit *inside* the concurrency limiter, so a + // request queued for one of the permits had no deadline at all - the + // 30s clock only started once it was admitted. Under sustained overload + // the queue grew without bound and clients saw latencies far past + // `PANGOLIN_REQUEST_TIMEOUT_SECS`. With the timeout outside, the + // deadline covers queueing time, which is what a client's timeout budget + // actually cares about. .layer(DefaultBodyLimit::max(options.body_limit_bytes)) - .layer(tower_http::timeout::TimeoutLayer::new( - options.request_timeout, - )) // GlobalConcurrencyLimitLayer rather than ConcurrencyLimitLayer: the // latter's service is not `Clone`, which axum requires. .layer(tower::limit::GlobalConcurrencyLimitLayer::new( options.concurrency_limit, )) + .layer(tower_http::timeout::TimeoutLayer::new( + options.request_timeout, + )) + // B0m: token issuance could be driven to panic by a request-controlled + // `expires_in_hours`, and with no catch-panic layer the panic tore down + // the whole connection task rather than failing one request. The + // arithmetic is now total, but a panic anywhere else should still be a + // 500 for one caller rather than a dropped connection for several. + .layer(tower_http::catch_panic::CatchPanicLayer::new()) // Request IDs, access logging and metrics. `tower-http` was already // built with the `trace` feature but `TraceLayer` was never applied. .layer(axum::middleware::from_fn(observability::track_request)) diff --git a/pangolin/pangolin_api/src/main.rs b/pangolin/pangolin_api/src/main.rs index 18d2d1a..bcd0351 100644 --- a/pangolin/pangolin_api/src/main.rs +++ b/pangolin/pangolin_api/src/main.rs @@ -1,6 +1,6 @@ use pangolin_api::auth_middleware::{create_session, generate_token}; use pangolin_api::config::{AppConfig, LogFormat}; -use pangolin_api::{app_with_config, health}; +use pangolin_api::{app_with_config, cleanup_job, health}; use pangolin_core::model::Tenant; use pangolin_core::user::{User, UserRole}; use pangolin_store::{CatalogStore, MemoryStore, MongoStore, PostgresStore, SqliteStore}; @@ -47,6 +47,17 @@ async fn main() { administrator. For evaluation only." ); } + // C-11. Said out loud rather than left to the documentation: warehouse + // credentials are only encrypted at rest when a key is configured, and a + // silent no-op is precisely the failure mode this audit keeps finding. + if !pangolin_store::secrets::is_enabled() { + tracing::warn!( + "PANGOLIN_ENCRYPTION_KEY is not set, so warehouse cloud credentials are stored in \ + the catalog database in plaintext. Anything that can read one row - a backup, a \ + replica, a snapshot - holds every tenant's cloud keys. Generate a key with \ + `openssl rand -base64 32`; see docs/operations/encryption.md for the migration." + ); + } let store = match build_store().await { Ok(s) => s, @@ -91,6 +102,7 @@ async fn main() { let shutdown_grace = config.shutdown_grace; let store_for_health: Arc = store.clone(); + let store_for_cleanup: Arc = store.clone(); let app = app_with_config(store, config.clone()); if config.install().is_err() { @@ -98,6 +110,17 @@ async fn main() { } health::set_store(store_for_health); + // Start the revocation sweep. It was dead code: `start_token_cleanup_job` + // existed, the module was declared, and nothing called it - so + // `revoked_tokens` grew for the life of the deployment, and the revocation + // check reads that table on every authenticated request. + // + // Every replica runs it. The sweep is a `DELETE ... WHERE expires_at < now` + // and therefore idempotent, so concurrent runs are correct; the job + // staggers its own start so replicas from one deploy do not sweep in + // lockstep forever. + tokio::spawn(cleanup_job::start_token_cleanup_job(store_for_cleanup)); + let listener = match tokio::net::TcpListener::bind(addr).await { Ok(l) => l, Err(e) => { @@ -112,13 +135,35 @@ async fn main() { // Graceful shutdown. Without this, a Kubernetes rolling update severs every // in-flight request; a SIGTERM between writing a table's metadata file and // the compare-and-swap that publishes it leaks an orphaned metadata file. - let serve = axum::serve(listener, app).with_graceful_shutdown(shutdown_signal(shutdown_grace)); - - if let Err(e) = serve.await { - tracing::error!("server error: {e}"); - std::process::exit(1); + // `into_make_service_with_connect_info` rather than the plain service: the + // authentication throttle keys on the peer address, and without this the + // `ConnectInfo` extractor finds nothing and every attempt shares one bucket. + let serve = axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(shutdown_signal(shutdown_grace)); + + // B16n: `shutdown_grace` used to be logged and nothing else. There was no + // bound on the drain at all, so `with_graceful_shutdown` waited for + // in-flight connections *indefinitely* - one hung upstream call blocked + // SIGTERM past the k8s termination grace period and into a SIGKILL, which + // is exactly the mid-commit kill this whole path exists to avoid. + // + // The first attempt at that bound was `tokio::time::timeout(shutdown_grace, + // serve)`, which is wrong in the worst possible way: `serve` is the whole + // server, not the drain, so it bounded the *lifetime of the process*. The + // server exited cleanly `shutdown_grace` seconds after startup - 25 by + // default - having received no signal at all. Every container would have + // crash-looped. The deadline has to start when the signal arrives, so it + // lives in `shutdown_signal` now, armed only once a signal is seen. + match serve.await { + Ok(()) => tracing::info!("shutdown complete"), + Err(e) => { + tracing::error!("server error: {e}"); + std::process::exit(1); + } } - tracing::info!("shutdown complete"); } /// Probe this instance's readiness endpoint. Returns a process exit code. @@ -367,4 +412,30 @@ async fn shutdown_signal(grace: std::time::Duration) { // while in-flight requests finish. health::mark_draining(); tracing::info!(grace_secs = grace.as_secs(), "draining in-flight requests"); + + // Give the load balancer a moment to observe the failing readiness probe + // before the listener stops accepting. Without this pause the LB can still + // be routing new connections at the instant we stop accepting them, which + // shows up to clients as connection resets during every rolling update. + // Capped so it can never consume the whole grace budget. + let deregistration_pause = std::cmp::min(grace / 4, std::time::Duration::from_secs(5)); + tokio::time::sleep(deregistration_pause).await; + + // Arm the hard deadline *here*, once a signal has actually been seen, + // rather than around the whole server. `with_graceful_shutdown` waits for + // in-flight connections with no bound of its own, so a single hung request + // would otherwise hold the process past Kubernetes' termination grace + // period and earn a SIGKILL mid-commit - the exact failure this path + // exists to prevent. + // + // Detached so returning from here still lets axum begin its drain; the + // task only wins if the drain overruns. + tokio::spawn(async move { + tokio::time::sleep(grace).await; + tracing::warn!( + grace_secs = grace.as_secs(), + "drain did not finish within the shutdown grace period; exiting anyway" + ); + std::process::exit(0); + }); } diff --git a/pangolin/pangolin_api/src/merge_handlers.rs b/pangolin/pangolin_api/src/merge_handlers.rs index 5ea9ddf..e563b47 100644 --- a/pangolin/pangolin_api/src/merge_handlers.rs +++ b/pangolin/pangolin_api/src/merge_handlers.rs @@ -17,6 +17,7 @@ type AppState = Arc; // Request/Response types #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct ResolveConflictRequest { pub strategy: ResolutionStrategy, pub resolved_value: Option, diff --git a/pangolin/pangolin_api/src/metrics.rs b/pangolin/pangolin_api/src/metrics.rs index 2804b2a..42c5287 100644 --- a/pangolin/pangolin_api/src/metrics.rs +++ b/pangolin/pangolin_api/src/metrics.rs @@ -65,6 +65,15 @@ counters! { pub AUDIT_WRITE_FAILURES => "pangolin_audit_write_failures_total", "Audit records that could not be persisted."; pub WAREHOUSE_CACHE_HITS => "pangolin_warehouse_cache_hits_total", "Warehouse lookups served from the in-process cache."; pub WAREHOUSE_CACHE_MISSES => "pangolin_warehouse_cache_misses_total", "Warehouse lookups that reached the store."; + pub AUTH_THROTTLED => "pangolin_auth_throttled_total", "Authentication attempts refused by the rate limiter."; +} + +/// One authentication attempt refused by the rate limiter. +/// +/// Worth alerting on: a sustained non-zero rate is either an attack or a +/// misconfigured client, and both want a human. +pub fn record_auth_throttled() { + inc(&AUTH_THROTTLED); } /// Increment a counter by one. diff --git a/pangolin/pangolin_api/src/oauth_handlers.rs b/pangolin/pangolin_api/src/oauth_handlers.rs index a3bf06e..b4be963 100644 --- a/pangolin/pangolin_api/src/oauth_handlers.rs +++ b/pangolin/pangolin_api/src/oauth_handlers.rs @@ -25,6 +25,50 @@ pub struct OAuthUserInfo { pub sub: String, pub email: String, pub name: Option, + /// Whether the provider asserts the address has been verified. + /// + /// Absent on providers that do not report it - notably GitHub's + /// `/user` endpoint - which is why an absent value is treated as + /// *unverified* (B0l). + #[serde(default)] + pub email_verified: Option, +} + +impl OAuthUserInfo { + fn email_is_verified(&self) -> bool { + self.email_verified.unwrap_or(false) + } +} + +/// Domains whose *verified* addresses may link to a pre-existing local account. +/// +/// Empty by default: with no allowlist configured, email never links an account, +/// and only a `(provider, subject)` match does. +fn email_link_domain_allowlist() -> Vec { + std::env::var("PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS") + .ok() + .map(|v| { + v.split(',') + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty()) + .collect() + }) + .unwrap_or_default() +} + +/// May `email` be used to adopt an existing local account? +fn email_may_link(user_info: &OAuthUserInfo, allowlist: &[String]) -> bool { + if !user_info.email_is_verified() { + return false; + } + let Some(domain) = user_info + .email + .rsplit_once('@') + .map(|(_, d)| d.to_ascii_lowercase()) + else { + return false; + }; + allowlist.contains(&domain) } #[derive(Deserialize, ToSchema)] @@ -75,7 +119,77 @@ pub async fn oauth_authorize( // Signed, single-use state (A-9). let state = crate::oauth_state::issue(&crate::config::jwt_secret(), &provider, redirect_index); - let auth_url = build_auth_url(&config, &state); + let Some(state_nonce) = crate::oauth_state::peek_nonce(&crate::config::jwt_secret(), &state) + else { + tracing::error!("could not read back the nonce from a state we just issued"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + }; + + // C-2/C-3: PKCE and an OIDC nonce, for providers that are OIDC. + // + // The verifier is kept server-side, never in `state`; see `PendingLogin`. + // A provider with no issuer (GitHub) gets neither, because it has no + // id_token to bind a nonce into and its token endpoint would reject the + // extra parameters. + let issuer = crate::oidc::issuer_for(&provider); + let mut oidc_params: Vec<(String, String)> = Vec::new(); + + if let Some(issuer) = &issuer { + let discovery = match crate::oidc::discover(issuer).await { + Ok(d) => Some(d), + Err(e) => { + // Discovery is best-effort at authorize time. Failing the login + // because a metadata document is briefly unreachable would be a + // worse outcome than proceeding without PKCE - but if the + // operator has demanded OIDC, proceeding is not acceptable. + if crate::oidc::require_oidc() { + tracing::error!(error = %e, provider = %provider, "OIDC discovery failed and PANGOLIN_OIDC_REQUIRE is set"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "OIDC discovery failed; cannot start a login while \ + PANGOLIN_OIDC_REQUIRE is set", + ) + .into_response(); + } + tracing::warn!(error = %e, provider = %provider, "OIDC discovery failed; continuing without PKCE"); + None + } + }; + + let (pkce, nonce) = match (crate::oidc::generate_pkce(), crate::oidc::generate_nonce()) { + (Ok(p), Ok(n)) => (p, n), + _ => { + tracing::error!("could not generate PKCE or nonce material"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error") + .into_response(); + } + }; + + if discovery + .as_ref() + .map(|d| d.supports_s256_pkce()) + .unwrap_or(false) + { + oidc_params.push(("code_challenge".to_string(), pkce.challenge.clone())); + oidc_params.push(("code_challenge_method".to_string(), "S256".to_string())); + } + oidc_params.push(("nonce".to_string(), nonce.clone())); + + crate::oauth_state::remember_login(&state_nonce, pkce.verifier, nonce); + } else if crate::oidc::require_oidc() { + tracing::warn!( + provider = %provider, + "refused a non-OIDC provider because PANGOLIN_OIDC_REQUIRE is set" + ); + return ( + StatusCode::BAD_REQUEST, + "this provider does not support OpenID Connect, and \ + PANGOLIN_OIDC_REQUIRE is set", + ) + .into_response(); + } + + let auth_url = build_auth_url(&config, &state, &oidc_params); Redirect::to(&auth_url).into_response() } @@ -146,9 +260,31 @@ pub async fn oauth_callback( } }; - // 1. Exchange code for access token - let access_token = match exchange_code_for_token(&config, &callback.code).await { - Ok(token) => token, + // 0b. Recover the PKCE verifier and OIDC nonce for this login. Absent for a + // non-OIDC provider, and absent if the state was issued before these + // existed - both are handled below rather than assumed. + let pending = crate::oauth_state::take_login(&state_payload.nonce); + let issuer = crate::oidc::issuer_for(&provider); + + if issuer.is_none() && crate::oidc::require_oidc() { + tracing::warn!(provider = %provider, "refused a non-OIDC callback because PANGOLIN_OIDC_REQUIRE is set"); + return ( + StatusCode::BAD_REQUEST, + "this provider does not support OpenID Connect, and \ + PANGOLIN_OIDC_REQUIRE is set", + ) + .into_response(); + } + + // 1. Exchange code for tokens, presenting the PKCE verifier. + let tokens = match exchange_code_for_token( + &config, + &callback.code, + pending.as_ref().map(|p| p.pkce_verifier.as_str()), + ) + .await + { + Ok(t) => t, Err(e) => { return ( StatusCode::BAD_REQUEST, @@ -157,19 +293,62 @@ pub async fn oauth_callback( .into_response() } }; + let access_token = tokens.access_token.clone(); - // 2. Fetch user info from provider - let user_info = match fetch_user_info(&config, &access_token).await { - Ok(info) => info, + // 2. Establish who the user is. + // + // For an OIDC provider the answer comes from the *signed* id_token, not + // from the userinfo endpoint. That is the substantive difference between + // this and what came before: previously identity was whatever an HTTP + // response said, and any holder of any access token for this client could + // have elicited it. Now it is an assertion the provider signed, bound to + // this login by a nonce and to this application by `aud`. + let mut user_info = match validate_oidc_identity( + &provider, + issuer.as_deref(), + pending.as_ref(), + tokens.id_token.as_deref(), + &config.client_id, + ) + .await + { + Ok(Some(claims)) => claims, + Ok(None) => { + // Not an OIDC provider. Fall back to userinfo, which is all such a + // provider offers. + match fetch_user_info(&config, &access_token).await { + Ok(info) => info, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + format!("Failed to fetch user info: {}", e), + ) + .into_response() + } + } + } Err(e) => { + tracing::warn!(error = %e, provider = %provider, "id_token validation failed"); return ( - StatusCode::BAD_REQUEST, - format!("Failed to fetch user info: {}", e), + StatusCode::UNAUTHORIZED, + format!("OpenID Connect validation failed: {e}"), ) - .into_response() + .into_response(); } }; + // An OIDC id_token need not carry `email`; the userinfo endpoint fills the + // gap. The *subject* still comes from the signed token - only the + // display-level attributes are topped up here. + if user_info.email.is_empty() { + if let Ok(extra) = fetch_user_info(&config, &access_token).await { + user_info.email = extra.email; + if user_info.name.is_none() { + user_info.name = extra.name; + } + } + } + // 3. Map provider string to Enum let provider_enum = match provider.as_str() { "google" => OAuthProvider::Google, @@ -180,23 +359,32 @@ pub async fn oauth_callback( }; // 4. Find or Create User - // We assume email is unique and can be used to link accounts or create new ones - // For MVP, we'll create a new user if not found by email, or maybe by oauth_subject - // Ideally look up by (provider, subject) - - // Since CatalogStore doesn't expose `get_user_by_oauth` yet, we'll use `get_user_by_username` as a fallback or iterate - // But `CatalogStore` trait needs a method for this efficiently. - // For now, let's list users and filter (inefficient but works for MemoryStore) - // Or better, let's just stick to email for now if unique. - - // Let's implement a rudimentary lookup + // + // B0l: the match used to include `|| u.email == user_info.email`, with no + // `email_verified` check and no provider binding. Anyone who could set a + // matching address on *any* configured provider - GitHub happily reports + // unverified addresses - logged in as that Pangolin user, including the + // seeded `TenantAdmin`. Identity is `(provider, subject)`; an address is + // only allowed to adopt a pre-existing account when the provider says it is + // verified *and* its domain is one the operator listed. let all_users = store.list_users(None, None).await.unwrap_or_default(); + let allowlist = email_link_domain_allowlist(); + let may_link_by_email = email_may_link(&user_info, &allowlist); + let existing_user = all_users.into_iter().find(|u| { - (u.oauth_provider == Some(provider_enum.clone()) - && u.oauth_subject == Some(user_info.sub.clone())) - || u.email == user_info.email + let subject_match = u.oauth_provider == Some(provider_enum.clone()) + && u.oauth_subject == Some(user_info.sub.clone()); + let email_match = may_link_by_email && u.email == user_info.email; + subject_match || email_match }); + if !may_link_by_email && !allowlist.is_empty() && !user_info.email_is_verified() { + tracing::warn!( + provider = %provider, + "OAuth provider reported an unverified email; not linking by address" + ); + } + let user = match existing_user { Some(u) => { // Update last login or details if needed @@ -311,6 +499,7 @@ pub async fn oauth_callback( /// Request body for exchanging a one-time OAuth code for a session token. #[derive(Debug, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct OAuthExchangeRequest { pub code: String, } @@ -354,6 +543,93 @@ pub async fn oauth_exchange(Json(req): Json) -> Response { } } +/// The providers this deployment has actually configured. +/// +/// B33: the UI called `GET /api/v1/oauth/providers`, which did not exist, so it +/// 404'd - which is why the login page fell back to rendering all four provider +/// buttons unconditionally, including ones the operator had never configured. +/// Clicking those went nowhere. +/// +/// This endpoint is public (see `public_paths`): the login page has to render +/// before anyone is authenticated. It reveals only which provider *names* are +/// enabled - never a client id, and never a secret. +#[derive(serde::Serialize, utoipa::ToSchema)] +pub struct OAuthProvidersResponse { + pub providers: Vec, +} + +#[utoipa::path( + get, + path = "/api/v1/oauth/providers", + tag = "Authentication", + responses((status = 200, description = "Configured OAuth providers", body = OAuthProvidersResponse)) +)] +pub async fn list_oauth_providers() -> impl IntoResponse { + let providers = ["google", "microsoft", "github", "okta"] + .into_iter() + .filter(|p| get_oauth_config(p).is_some()) + .map(|p| p.to_string()) + .collect(); + + (StatusCode::OK, Json(OAuthProvidersResponse { providers })) +} + +/// Establish identity from a validated `id_token`, when the provider is OIDC. +/// +/// Returns `Ok(None)` for a provider that is not an OIDC provider, which is the +/// signal to fall back to the userinfo endpoint. Every other absence is an +/// error: if a provider *is* OIDC and no id_token arrived, or the login has no +/// remembered nonce, something is wrong and proceeding would mean skipping the +/// validation while appearing to have done it. +async fn validate_oidc_identity( + provider: &str, + issuer: Option<&str>, + pending: Option<&crate::oauth_state::PendingLogin>, + id_token: Option<&str>, + client_id: &str, +) -> Result, String> { + let Some(issuer) = issuer else { + return Ok(None); // not an OIDC provider + }; + + let Some(id_token) = id_token else { + return Err(format!( + "{provider} is configured as an OpenID Connect provider but returned \ + no id_token. Check that `openid` is among the requested scopes." + )); + }; + + let Some(pending) = pending else { + return Err( + "this login has no remembered nonce, so the id_token cannot be bound \ + to it. Start the login again." + .to_string(), + ); + }; + + let discovery = crate::oidc::discover(issuer) + .await + .map_err(|e| format!("OIDC discovery failed: {e}"))?; + + let claims = + crate::oidc::validate_id_token(id_token, &discovery, client_id, &pending.oidc_nonce) + .await + .map_err(|e| e.to_string())?; + + tracing::info!( + provider = %provider, + subject = %claims.sub, + "authenticated a user from a validated id_token" + ); + + Ok(Some(OAuthUserInfo { + sub: claims.sub, + email: claims.email.unwrap_or_default(), + name: claims.name, + email_verified: claims.email_verified, + })) +} + /// Get OAuth configuration for provider fn get_oauth_config(provider: &str) -> Option { // TODO: Load from environment variables or config file @@ -399,31 +675,62 @@ fn get_oauth_config(provider: &str) -> Option { } /// Build the provider authorization URL around an already-signed `state`. -fn build_auth_url(config: &OAuthConfig, state: &str) -> String { +/// +/// `extra` carries the OIDC parameters - `code_challenge`, +/// `code_challenge_method`, `nonce` - which are absent for a provider that is +/// not an OIDC provider. +fn build_auth_url(config: &OAuthConfig, state: &str, extra: &[(String, String)]) -> String { let scopes = config.scopes.join(" "); - format!( + let mut url = format!( "{}?client_id={}&redirect_uri={}&response_type=code&scope={}&state={}", config.get_auth_url(), urlencoding::encode(&config.client_id), urlencoding::encode(&config.redirect_uri), urlencoding::encode(&scopes), urlencoding::encode(state) - ) + ); + + for (key, value) in extra { + url.push('&'); + url.push_str(&urlencoding::encode(key)); + url.push('='); + url.push_str(&urlencoding::encode(value)); + } + + url } /// Exchange authorization code for access token -async fn exchange_code_for_token(config: &OAuthConfig, code: &str) -> Result { +/// What the provider returned from the token endpoint. +pub struct TokenExchange { + pub access_token: String, + /// Present only for OIDC providers. GitHub returns none. + pub id_token: Option, +} + +async fn exchange_code_for_token( + config: &OAuthConfig, + code: &str, + pkce_verifier: Option<&str>, +) -> Result { let client = reqwest::Client::new(); - let params = [ - ("client_id", &config.client_id), - ("client_secret", &config.client_secret), - ("code", &code.to_string()), - ("redirect_uri", &config.redirect_uri), - ("grant_type", &"authorization_code".to_string()), + let mut params: Vec<(&str, String)> = vec![ + ("client_id", config.client_id.clone()), + ("client_secret", config.client_secret.clone()), + ("code", code.to_string()), + ("redirect_uri", config.redirect_uri.clone()), + ("grant_type", "authorization_code".to_string()), ]; + // The other half of PKCE. The provider recomputes SHA-256 of this and + // compares it to the challenge sent at authorize time; a code stolen + // without the verifier cannot be redeemed. + if let Some(verifier) = pkce_verifier { + params.push(("code_verifier", verifier.to_string())); + } + let response = client .post(config.get_token_url()) .form(¶ms) @@ -440,6 +747,8 @@ async fn exchange_code_for_token(config: &OAuthConfig, code: &str) -> Result, // we might handle refresh_token later } @@ -448,7 +757,10 @@ async fn exchange_code_for_token(config: &OAuthConfig, code: &str) -> Result String { encode_signed(secret, &payload) } +/// What an in-flight OIDC login needs to remember between authorize and +/// callback. +/// +/// **This is deliberately server-side and not carried in `state`.** The PKCE +/// verifier is a secret: its entire purpose is that an attacker holding the +/// authorization code cannot redeem it. `state` travels to the provider and back +/// through the user's browser in the same URL as the code, so anyone positioned +/// to steal the code - a referrer header, a proxy log, shell history on a shared +/// machine - would also hold the verifier. Putting it there would make PKCE +/// decorative in exactly the situation it exists for. +/// +/// In-process, like the nonce store beside it, which is why OAuth needs session +/// affinity across replicas. See docs/operations/running-multiple-replicas.md. +#[derive(Debug, Clone)] +pub struct PendingLogin { + pub pkce_verifier: String, + pub oidc_nonce: String, + expires_at: u64, +} + +fn pending_logins() -> &'static Mutex> { + static STORE: OnceLock>> = OnceLock::new(); + STORE.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Remember the PKCE verifier and OIDC nonce for a login, keyed by its state +/// nonce. +pub fn remember_login(state_nonce: &str, pkce_verifier: String, oidc_nonce: String) { + let expires_at = now_secs() + STATE_TTL.as_secs(); + if let Ok(mut guard) = pending_logins().lock() { + // Opportunistic sweep: without it an abandoned authorize - a user who + // closes the tab at the provider - leaks an entry forever. + let now = now_secs(); + guard.retain(|_, pending| pending.expires_at > now); + guard.insert( + state_nonce.to_string(), + PendingLogin { + pkce_verifier, + oidc_nonce, + expires_at, + }, + ); + } +} + +/// Take the remembered login. Single-use, like the state it is keyed by. +pub fn take_login(state_nonce: &str) -> Option { + let mut guard = pending_logins().lock().ok()?; + let pending = guard.remove(state_nonce)?; + if pending.expires_at <= now_secs() { + return None; + } + Some(pending) +} + +/// Read the nonce out of a `state` this server just issued, without consuming +/// it. +/// +/// `issue` returns the encoded string, and the caller needs the nonce inside it +/// as the key for the pending-login store. Returning it from `issue` would be +/// tidier, but that signature is used in several places and a second return +/// value would be silently ignored at most of them. +/// +/// This still verifies the signature: reading an unverified payload, even one we +/// believe we just wrote, is the habit that makes the next reader assume it is +/// safe elsewhere. +pub fn peek_nonce(secret: &str, state: &str) -> Option { + let (encoded, signature) = state.rsplit_once('.')?; + if sign(secret, encoded.as_bytes()) != signature { + return None; + } + let body = B64.decode(encoded).ok()?; + let payload: StatePayload = serde_json::from_slice(&body).ok()?; + Some(payload.nonce) +} + fn encode_signed(secret: &str, payload: &StatePayload) -> String { let body = serde_json::to_vec(payload).expect("StatePayload is always serializable"); let encoded = B64.encode(&body); diff --git a/pangolin/pangolin_api/src/oidc.rs b/pangolin/pangolin_api/src/oidc.rs new file mode 100644 index 0000000..56fdc32 --- /dev/null +++ b/pangolin/pangolin_api/src/oidc.rs @@ -0,0 +1,685 @@ +//! OpenID Connect: discovery, JWKS, and `id_token` validation. +//! +//! C-2/C-3. What the OAuth flow did before this was *authorization*, not +//! authentication. It exchanged a code for an access token, called the +//! provider's userinfo endpoint, and believed whatever came back. That is +//! sufficient only if the access token cannot have come from anywhere else, +//! which is exactly what OIDC exists to establish. +//! +//! Three properties this adds, and what each one prevents: +//! +//! * **PKCE (RFC 7636).** The authorization code is bound to a secret the +//! client generated and never transmitted. An attacker who intercepts the +//! code - from a browser log, a referrer header, a shared machine - cannot +//! redeem it without the verifier. +//! * **`id_token` signature validation.** The provider signs an assertion about +//! who the user is. Verifying it against the provider's published keys means +//! identity comes from something the provider signed, not from an HTTP +//! response that any holder of *some* access token could have elicited. +//! * **Claim validation** - `iss`, `aud`, `exp`, `nonce`. Without `aud`, a token +//! minted for a *different* application at the same provider is accepted here; +//! this is the classic confused-deputy in OAuth logins. Without `nonce`, an +//! `id_token` captured from one login can be replayed into another. +//! +//! ## Not every provider is an OIDC provider +//! +//! GitHub's OAuth is not OIDC: there is no `id_token` and no JWKS. Pretending +//! otherwise would mean either failing GitHub logins or silently skipping +//! validation while claiming to do it. Instead a provider declares whether it is +//! OIDC-capable, and `PANGOLIN_OIDC_REQUIRE` decides whether a non-OIDC provider +//! may be used at all. An operator who needs every login to be OIDC-validated +//! can have that; one who needs GitHub can have that; nobody gets it by accident. + +use anyhow::{anyhow, Context, Result}; +use jsonwebtoken::jwk::{Jwk, JwkSet}; +use jsonwebtoken::{Algorithm, DecodingKey, Validation}; +use serde::Deserialize; +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +/// How long a discovery document is trusted before refetching. +const DISCOVERY_TTL: Duration = Duration::from_secs(3600); +/// How long a JWKS is trusted before refetching on a routine lookup. +const JWKS_TTL: Duration = Duration::from_secs(3600); +/// Minimum gap between forced JWKS refetches triggered by an unknown `kid`. +/// +/// Providers rotate keys, and the correct response to an unknown `kid` is to +/// refetch. Doing that unconditionally turns a stream of garbage tokens into a +/// denial-of-service against the provider's JWKS endpoint - and against our own +/// latency, since every such request would block on an outbound fetch. +const JWKS_REFETCH_COOLDOWN: Duration = Duration::from_secs(60); + +/// The subset of an OIDC discovery document this needs. +#[derive(Debug, Clone, Deserialize)] +pub struct Discovery { + pub issuer: String, + pub authorization_endpoint: String, + pub token_endpoint: String, + pub jwks_uri: String, + #[serde(default)] + pub userinfo_endpoint: Option, + /// Advertised PKCE methods. Absent means the provider does not say, which + /// is not the same as "does not support" - many providers support S256 and + /// omit the field. + #[serde(default)] + pub code_challenge_methods_supported: Option>, +} + +impl Discovery { + /// Whether the provider advertises S256 PKCE. + /// + /// `None` is treated as "probably yes": PKCE with a provider that ignores it + /// is harmless - the extra parameters are simply not used - whereas + /// withholding PKCE from a provider that supports it but does not advertise + /// it loses a real protection. + pub fn supports_s256_pkce(&self) -> bool { + match &self.code_challenge_methods_supported { + Some(methods) => methods.iter().any(|m| m == "S256"), + None => true, + } + } +} + +struct Cached { + value: T, + fetched_at: Instant, +} + +#[derive(Default)] +struct Caches { + discovery: HashMap>, + jwks: HashMap>, + last_forced_refetch: HashMap, +} + +fn caches() -> &'static Mutex { + static CACHES: OnceLock> = OnceLock::new(); + CACHES.get_or_init(|| Mutex::new(Caches::default())) +} + +/// Discard every cached document. For tests, and for an operator-triggered +/// reload after changing provider configuration. +pub fn clear_caches() { + if let Ok(mut guard) = caches().lock() { + guard.discovery.clear(); + guard.jwks.clear(); + guard.last_forced_refetch.clear(); + } +} + +fn http_client() -> Result { + reqwest::Client::builder() + // A provider that hangs must not hang the login. Without a timeout the + // request inherits the server's global request deadline, which is + // longer than a user will wait and longer than a healthy provider should take. + .timeout(Duration::from_secs(10)) + .build() + .context("could not build an HTTP client for OIDC discovery") +} + +/// Fetch (or reuse) a provider's discovery document. +/// +/// `issuer_url` is the issuer, not the full `.well-known` path; the suffix is +/// appended here so configuration cannot drift between providers. +pub async fn discover(issuer_url: &str) -> Result { + let key = issuer_url.to_string(); + + if let Ok(guard) = caches().lock() { + if let Some(entry) = guard.discovery.get(&key) { + if entry.fetched_at.elapsed() < DISCOVERY_TTL { + return Ok(entry.value.clone()); + } + } + } + + let url = format!( + "{}/.well-known/openid-configuration", + issuer_url.trim_end_matches('/') + ); + let response = http_client()? + .get(&url) + .send() + .await + .with_context(|| format!("could not reach the OIDC discovery document at {url}"))?; + + if !response.status().is_success() { + return Err(anyhow!( + "OIDC discovery at {url} returned HTTP {}", + response.status() + )); + } + + let discovery: Discovery = response + .json() + .await + .with_context(|| format!("the OIDC discovery document at {url} is not valid"))?; + + // The issuer in the document is authoritative and must match where we asked. + // A mismatch means either a misconfiguration or a provider impersonating + // another, and it is the check that makes `iss` validation meaningful later. + if discovery.issuer.trim_end_matches('/') != issuer_url.trim_end_matches('/') { + return Err(anyhow!( + "OIDC discovery mismatch: asked {issuer_url}, document declares issuer {}", + discovery.issuer + )); + } + + if let Ok(mut guard) = caches().lock() { + guard.discovery.insert( + key, + Cached { + value: discovery.clone(), + fetched_at: Instant::now(), + }, + ); + } + + Ok(discovery) +} + +async fn fetch_jwks(jwks_uri: &str) -> Result { + let response = http_client()? + .get(jwks_uri) + .send() + .await + .with_context(|| format!("could not reach the JWKS at {jwks_uri}"))?; + + if !response.status().is_success() { + return Err(anyhow!( + "JWKS at {jwks_uri} returned HTTP {}", + response.status() + )); + } + + response + .json::() + .await + .with_context(|| format!("the JWKS at {jwks_uri} is not valid")) +} + +/// The signing key for `kid`, refetching once if it is not already known. +/// +/// Key rotation is the reason for the refetch: a provider publishes a new key +/// and starts signing with it, and a cache that only expires on a timer rejects +/// every login until the timer fires. The cooldown stops a stream of tokens +/// carrying unknown `kid`s from turning that recovery into a hammer. +async fn signing_key(jwks_uri: &str, kid: &str) -> Result { + let cached = caches().lock().ok().and_then(|guard| { + guard + .jwks + .get(jwks_uri) + .filter(|entry| entry.fetched_at.elapsed() < JWKS_TTL) + .map(|entry| entry.value.clone()) + }); + + if let Some(set) = cached { + if let Some(key) = set.find(kid) { + return Ok(key.clone()); + } + // Known set, unknown kid: either rotation or rubbish. Rate-limit the + // distinction. + let may_refetch = caches() + .lock() + .ok() + .map(|mut guard| { + let allowed = guard + .last_forced_refetch + .get(jwks_uri) + .map(|at| at.elapsed() >= JWKS_REFETCH_COOLDOWN) + .unwrap_or(true); + if allowed { + guard + .last_forced_refetch + .insert(jwks_uri.to_string(), Instant::now()); + } + allowed + }) + .unwrap_or(false); + + if !may_refetch { + return Err(anyhow!( + "no signing key {kid} in the cached JWKS, and a refetch was \ + attempted too recently. If the provider has just rotated keys \ + this resolves within a minute." + )); + } + } + + let set = fetch_jwks(jwks_uri).await?; + let key = set + .find(kid) + .cloned() + .ok_or_else(|| anyhow!("the provider's JWKS has no signing key with kid {kid}"))?; + + if let Ok(mut guard) = caches().lock() { + guard.jwks.insert( + jwks_uri.to_string(), + Cached { + value: set, + fetched_at: Instant::now(), + }, + ); + } + + Ok(key) +} + +/// The claims this cares about. Providers send many more. +#[derive(Debug, Clone, Deserialize)] +pub struct IdTokenClaims { + pub sub: String, + #[serde(default)] + pub email: Option, + #[serde(default)] + pub email_verified: Option, + #[serde(default)] + pub name: Option, + #[serde(default)] + pub nonce: Option, + pub iss: String, + pub exp: usize, +} + +/// Verify an `id_token` and return its claims. +/// +/// Every check here has a specific attack behind it: +/// +/// * **signature** against the provider's published key - otherwise the token is +/// just a base64 string the caller wrote; +/// * **`iss`** must match the discovery document - otherwise a token from any +/// issuer is accepted; +/// * **`aud`** must contain our `client_id` - otherwise a token minted for a +/// *different* application at the same provider logs its holder in here, +/// which is the confused-deputy problem OAuth logins are famous for; +/// * **`exp`** with a small leeway for clock skew; +/// * **`nonce`** must equal the one bound into this login - otherwise an +/// `id_token` observed in one flow can be replayed into another. +pub async fn validate_id_token( + id_token: &str, + discovery: &Discovery, + client_id: &str, + expected_nonce: &str, +) -> Result { + let header = + jsonwebtoken::decode_header(id_token).context("the id_token is not a well-formed JWT")?; + + let kid = header + .kid + .ok_or_else(|| anyhow!("the id_token has no `kid`, so its signing key cannot be found"))?; + + // `alg` comes from the token, which the attacker controls, so it is used + // only to look up how to verify - never to decide *whether* to. `none` and + // the HMAC family are rejected outright: accepting HS256 here would let + // anyone who knows the (public) signing key material forge a token. + if !matches!( + header.alg, + Algorithm::RS256 + | Algorithm::RS384 + | Algorithm::RS512 + | Algorithm::ES256 + | Algorithm::ES384 + | Algorithm::PS256 + | Algorithm::PS384 + | Algorithm::PS512 + ) { + return Err(anyhow!( + "the id_token is signed with {:?}, which is not an asymmetric \ + algorithm. Only provider-signed tokens are acceptable.", + header.alg + )); + } + + let jwk = signing_key(&discovery.jwks_uri, &kid).await?; + let key = DecodingKey::from_jwk(&jwk) + .context("the provider's JWKS entry could not be used as a decoding key")?; + + let mut validation = Validation::new(header.alg); + validation.set_issuer(&[discovery.issuer.as_str()]); + validation.set_audience(&[client_id]); + validation.validate_exp = true; + // A minute of tolerance. Providers and this server rarely agree to the + // second, and a token rejected for being one second early is a login + // failure nobody can diagnose. + validation.leeway = 60; + + // The underlying error kind is carried through rather than flattened to + // "failed validation". Which check rejected the token is the difference + // between a misconfigured `client_id` and an actual confused-deputy attempt, + // and an operator reading a log needs to be able to tell them apart. It is + // also what lets the tests assert that a *specific* check fired, instead of + // passing because some unrelated check happened to reject the token. + let data = jsonwebtoken::decode::(id_token, &key, &validation).map_err(|e| { + let detail = match e.kind() { + jsonwebtoken::errors::ErrorKind::InvalidAudience => { + "audience mismatch: this id_token was minted for a different \ + application at the same provider" + } + jsonwebtoken::errors::ErrorKind::InvalidIssuer => { + "issuer mismatch: this id_token did not come from the configured \ + provider" + } + jsonwebtoken::errors::ErrorKind::ExpiredSignature => "the id_token has expired", + jsonwebtoken::errors::ErrorKind::InvalidSignature => { + "signature mismatch: the id_token was not signed by the provider's key" + } + _ => "the id_token failed validation", + }; + anyhow!("{detail} ({e})") + })?; + + // `nonce` is not something `jsonwebtoken` knows about, so it is checked + // here. Constant-time comparison is unnecessary - the nonce is not a secret + // an attacker is trying to guess byte by byte - but an exact match is. + match &data.claims.nonce { + Some(nonce) if nonce == expected_nonce => {} + Some(_) => { + return Err(anyhow!( + "the id_token's nonce does not match this login. This is what a \ + replayed token looks like." + )) + } + None => { + return Err(anyhow!( + "the id_token carries no nonce, so it cannot be bound to this \ + login and may be a replay from another" + )) + } + } + + Ok(data.claims) +} + +/// PKCE parameters for one authorization request. +pub struct Pkce { + pub verifier: String, + pub challenge: String, +} + +/// Generate an S256 PKCE pair. +/// +/// The verifier is 32 random bytes, base64url-encoded - comfortably inside RFC +/// 7636's 43-128 character range and drawn from the system CSPRNG, because a +/// guessable verifier provides no protection at all. +pub fn generate_pkce() -> Result { + use base64::Engine as _; + use ring::rand::{SecureRandom, SystemRandom}; + use sha2::{Digest, Sha256}; + + let mut bytes = [0u8; 32]; + SystemRandom::new() + .fill(&mut bytes) + .map_err(|_| anyhow!("could not draw PKCE randomness from the system RNG"))?; + + let verifier = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes); + let digest = Sha256::digest(verifier.as_bytes()); + let challenge = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(digest); + + Ok(Pkce { + verifier, + challenge, + }) +} + +/// An opaque, URL-safe random string, for the OIDC `nonce`. +pub fn generate_nonce() -> Result { + use base64::Engine as _; + use ring::rand::{SecureRandom, SystemRandom}; + + let mut bytes = [0u8; 24]; + SystemRandom::new() + .fill(&mut bytes) + .map_err(|_| anyhow!("could not draw nonce randomness from the system RNG"))?; + Ok(base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes)) +} + +/// The issuer URL for a provider, if it is OIDC-capable. +/// +/// GitHub is deliberately absent: its OAuth is not OIDC, there is no `id_token` +/// and no JWKS. Returning `None` is how the callback knows to fall back to the +/// userinfo endpoint - and how `require_oidc()` knows to refuse it when the +/// operator has demanded OIDC everywhere. +pub fn issuer_for(provider: &str) -> Option { + // An explicit override wins, so a self-hosted or non-standard deployment + // (Okta, Keycloak, Auth0, an internal IdP) is configurable without a code + // change. + let explicit = std::env::var(format!("PANGOLIN_{}_ISSUER", provider.to_ascii_uppercase())) + .ok() + .filter(|v| !v.trim().is_empty()); + if explicit.is_some() { + return explicit; + } + + match provider { + "google" => Some("https://accounts.google.com".to_string()), + "microsoft" => std::env::var("PANGOLIN_MICROSOFT_TENANT_ID") + .ok() + .map(|tenant| format!("https://login.microsoftonline.com/{tenant}/v2.0")), + "okta" => std::env::var("PANGOLIN_OKTA_DOMAIN") + .ok() + .map(|domain| format!("https://{domain}")), + // GitHub OAuth is not OIDC. + _ => None, + } +} + +/// Whether every login must be OIDC-validated. +/// +/// Off by default: turning it on without warning would break a working GitHub +/// deployment on upgrade. On, a provider with no issuer is refused rather than +/// quietly downgraded, which is the property an operator turning this on is +/// asking for. +pub fn require_oidc() -> bool { + std::env::var("PANGOLIN_OIDC_REQUIRE") + .map(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes")) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pkce_pairs_are_random_and_well_formed() { + let a = generate_pkce().unwrap(); + let b = generate_pkce().unwrap(); + + assert_ne!( + a.verifier, b.verifier, + "a repeated verifier would make PKCE decorative" + ); + // RFC 7636 section 4.1. + assert!( + (43..=128).contains(&a.verifier.len()), + "verifier length {} is outside RFC 7636's 43-128", + a.verifier.len() + ); + assert!( + !a.verifier.contains('+') && !a.verifier.contains('/') && !a.verifier.contains('='), + "the verifier must be base64url without padding: {}", + a.verifier + ); + assert_ne!( + a.challenge, a.verifier, + "S256 means the challenge is the hash, not the verifier itself - \ + sending the verifier as the challenge would defeat the whole point" + ); + } + + #[test] + fn the_challenge_is_the_sha256_of_the_verifier() { + use base64::Engine as _; + use sha2::{Digest, Sha256}; + + let pkce = generate_pkce().unwrap(); + let expected = base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(Sha256::digest(pkce.verifier.as_bytes())); + assert_eq!( + pkce.challenge, expected, + "the provider recomputes this; a mismatch fails every login" + ); + } + + #[test] + fn nonces_are_unique() { + let a = generate_nonce().unwrap(); + let b = generate_nonce().unwrap(); + assert_ne!(a, b); + assert!(a.len() >= 32, "a short nonce is a guessable nonce: {a}"); + } + + #[test] + fn github_is_not_treated_as_an_oidc_provider() { + // GitHub OAuth issues no id_token. Claiming otherwise would mean either + // failing every GitHub login or skipping validation while reporting it + // as done. + assert!(issuer_for("github").is_none()); + } + + #[test] + fn google_has_a_known_issuer() { + assert_eq!( + issuer_for("google").as_deref(), + Some("https://accounts.google.com") + ); + } + + #[test] + fn an_explicit_issuer_overrides_the_default() { + // Self-hosted and non-standard deployments must be configurable without + // a code change. + std::env::set_var("PANGOLIN_GOOGLE_ISSUER", "https://idp.internal/realms/x"); + let issuer = issuer_for("google"); + std::env::remove_var("PANGOLIN_GOOGLE_ISSUER"); + assert_eq!(issuer.as_deref(), Some("https://idp.internal/realms/x")); + } + + #[test] + fn pkce_is_offered_when_the_provider_is_silent() { + // Omitting the field is common and does not mean unsupported. Sending + // PKCE to a provider that ignores it costs nothing; withholding it from + // one that supports it loses a real protection. + let discovery = Discovery { + issuer: "https://i".into(), + authorization_endpoint: "https://i/a".into(), + token_endpoint: "https://i/t".into(), + jwks_uri: "https://i/j".into(), + userinfo_endpoint: None, + code_challenge_methods_supported: None, + }; + assert!(discovery.supports_s256_pkce()); + } + + #[test] + fn plain_pkce_alone_is_not_treated_as_s256() { + let discovery = Discovery { + issuer: "https://i".into(), + authorization_endpoint: "https://i/a".into(), + token_endpoint: "https://i/t".into(), + jwks_uri: "https://i/j".into(), + userinfo_endpoint: None, + code_challenge_methods_supported: Some(vec!["plain".into()]), + }; + assert!( + !discovery.supports_s256_pkce(), + "`plain` offers no protection against an intercepted challenge; it \ + must not be mistaken for S256" + ); + } + + #[tokio::test] + async fn an_id_token_signed_with_hmac_is_refused() { + // The `alg` header is attacker-controlled. Accepting HS256 would let + // anyone who knows the provider's *public* key forge a token, because + // for HMAC the verification key and the signing key are the same. + use jsonwebtoken::{encode, EncodingKey, Header}; + + #[derive(serde::Serialize)] + struct Claims { + sub: String, + iss: String, + aud: String, + exp: usize, + nonce: String, + } + + let mut header = Header::new(Algorithm::HS256); + header.kid = Some("attacker".into()); + let token = encode( + &header, + &Claims { + sub: "victim".into(), + iss: "https://issuer".into(), + aud: "client".into(), + exp: 9_999_999_999, + nonce: "n".into(), + }, + &EncodingKey::from_secret(b"public-key-material"), + ) + .unwrap(); + + let discovery = Discovery { + issuer: "https://issuer".into(), + authorization_endpoint: "https://issuer/a".into(), + token_endpoint: "https://issuer/t".into(), + jwks_uri: "https://issuer/jwks".into(), + userinfo_endpoint: None, + code_challenge_methods_supported: None, + }; + + let err = validate_id_token(&token, &discovery, "client", "n") + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("not an asymmetric"), + "an HMAC-signed id_token must be refused before any key lookup: {err}" + ); + } + + #[tokio::test] + async fn an_id_token_without_a_kid_is_refused() { + use jsonwebtoken::{encode, EncodingKey, Header}; + + #[derive(serde::Serialize)] + struct Claims { + sub: String, + exp: usize, + } + + let token = encode( + &Header::new(Algorithm::HS256), + &Claims { + sub: "x".into(), + exp: 9_999_999_999, + }, + &EncodingKey::from_secret(b"k"), + ) + .unwrap(); + + let discovery = Discovery { + issuer: "https://issuer".into(), + authorization_endpoint: "https://issuer/a".into(), + token_endpoint: "https://issuer/t".into(), + jwks_uri: "https://issuer/jwks".into(), + userinfo_endpoint: None, + code_challenge_methods_supported: None, + }; + + let err = validate_id_token(&token, &discovery, "client", "n") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("kid"), "got: {err}"); + } + + #[test] + fn oidc_is_not_required_by_default() { + std::env::remove_var("PANGOLIN_OIDC_REQUIRE"); + assert!( + !require_oidc(), + "defaulting this on would break a working GitHub deployment on \ + upgrade, with no warning" + ); + } +} diff --git a/pangolin/pangolin_api/src/optimization_handlers.rs b/pangolin/pangolin_api/src/optimization_handlers.rs index 63358fd..c46fd9b 100644 --- a/pangolin/pangolin_api/src/optimization_handlers.rs +++ b/pangolin/pangolin_api/src/optimization_handlers.rs @@ -106,8 +106,13 @@ pub async fn search_assets_by_name( catalogs.iter().map(|c| (c.name.clone(), c.id)).collect(); // Apply permission-based filtering - let filtered_assets = - crate::authz_utils::filter_assets(assets, &permissions, session.role.clone(), &catalog_map); + let filtered_assets = crate::authz_utils::filter_assets( + tenant_id, + assets, + &permissions, + session.role.clone(), + &catalog_map, + ); // Filter by catalog if specified let mut all_results = Vec::new(); @@ -128,6 +133,7 @@ pub async fn search_assets_by_name( let namespace_str = namespace.join("."); let required_actions = vec![pangolin_core::permission::Action::Read]; crate::authz_utils::has_asset_access( + tenant_id, catalog_id, &namespace_str, asset.id, @@ -171,6 +177,7 @@ pub async fn search_assets_by_name( } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct BulkDeleteAssetsRequest { /// List of asset UUIDs to delete (maximum 100) pub asset_ids: Vec, @@ -261,6 +268,7 @@ pub async fn bulk_delete_assets( } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct ValidateNamesRequest { /// Resource type: "catalog" or "warehouse" pub resource_type: String, @@ -401,8 +409,12 @@ pub async fn unified_search( .search_catalogs(tenant_id, &query.q) .await .map_err(ApiError::from)?; - let filtered_catalogs = - crate::authz_utils::filter_catalogs(catalogs, &permissions, session.role.clone()); + let filtered_catalogs = crate::authz_utils::filter_catalogs( + tenant_id, + catalogs, + &permissions, + session.role.clone(), + ); for c in filtered_catalogs { results.push(UnifiedSearchResult { id: Some(c.id.to_string()), @@ -430,6 +442,7 @@ pub async fn unified_search( .collect(); let filtered_namespaces = crate::authz_utils::filter_namespaces( + tenant_id, namespaces, &permissions, session.role.clone(), @@ -452,6 +465,7 @@ pub async fn unified_search( .await .map_err(ApiError::from)?; let filtered_assets = crate::authz_utils::filter_assets( + tenant_id, assets, &permissions, session.role.clone(), @@ -481,6 +495,7 @@ pub async fn unified_search( session.role, pangolin_core::user::UserRole::Root | pangolin_core::user::UserRole::TenantAdmin ) || crate::authz_utils::has_catalog_access( + tenant_id, catalog_id, &permissions, &[pangolin_core::permission::Action::Read], diff --git a/pangolin/pangolin_api/src/pangolin_handlers.rs b/pangolin/pangolin_api/src/pangolin_handlers.rs index 1cd126c..893c05b 100644 --- a/pangolin/pangolin_api/src/pangolin_handlers.rs +++ b/pangolin/pangolin_api/src/pangolin_handlers.rs @@ -19,6 +19,7 @@ use uuid::Uuid; pub type AppState = Arc; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateBranchRequest { name: String, branch_type: Option, // "ingest" or "experimental", defaults to experimental @@ -36,6 +37,7 @@ pub struct ListBranchParams { } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct MergeBranchRequest { pub source_branch: String, pub target_branch: String, @@ -164,79 +166,89 @@ pub async fn create_branch( name: payload.name.clone(), head_commit_id: None, branch_type: b_type.clone(), - assets: vec![], // Start empty, will be populated + assets: vec![], // Populated by the copy below }; - // Create branch first - if let Err(e) = store.create_branch(tenant_id, catalog_name, branch).await { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to create branch: {}", e), + // A-24. This used to be `create_branch`, then either a loop of + // `create_asset` or a `copy_assets_bulk`, as independent statements. Two + // things were wrong with that, and the second is worse than the first: + // + // 1. A failure partway through left a branch that existed holding an + // arbitrary subset of its assets, with no rollback and no repair tool. + // 2. The bulk-copy error was *logged and discarded* - `Err(e) => + // tracing::error!(...)` - and the handler then returned 200. The caller + // was told the branch was ready when it was empty. The per-asset loop + // did the same thing more quietly, with `if let Ok(_)` around each + // create and `continue` on a malformed name. + // + // The store now does both in one transaction where the backend can. Where + // it cannot, it says so and the fallback below is taken deliberately rather + // than by accident - and either way a failure is reported to the caller. + let copy_result = store + .create_branch_with_assets( + tenant_id, + catalog_name, + branch.clone(), + from_branch, + payload.assets.clone(), ) - .into_response(); - } + .await; - if let Some(assets_to_copy) = &payload.assets { - tracing::info!( - "Explicit asset list provided: {} assets", - assets_to_copy.len() - ); - // For explicit asset list, we still iterate for now as copy_assets_bulk doesn't take a list - // Optimization: We could add a filtered bulk copy in the future - for asset_name in assets_to_copy { - let parts: Vec<&str> = asset_name.split('.').collect(); - if parts.len() < 2 { - continue; // Skip invalid format - } - let table_name = parts.last().unwrap().to_string(); - let namespace_parts = parts[0..parts.len() - 1] - .iter() - .map(|s| s.to_string()) - .collect::>(); - - // Get asset from source branch - if let Ok(Some(asset)) = store - .get_asset( - tenant_id, - catalog_name, - Some(from_branch.to_string()), - namespace_parts.clone(), - table_name.clone(), + match copy_result { + Ok(count) => { + tracing::info!( + branch = %payload.name, + assets = count, + "created branch atomically" + ); + } + Err(e) if e.to_string().contains("not supported by this store") => { + // Backend without transaction support for this operation. Do it + // sequentially, but say so, and still fail loudly. + tracing::warn!( + branch = %payload.name, + "this backend cannot create a branch and copy its assets atomically; \ + a failure partway through will leave the branch incomplete" + ); + + if let Err(e) = store.create_branch(tenant_id, catalog_name, branch).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to create branch: {e}"), ) + .into_response(); + } + + match store + .copy_assets_bulk(tenant_id, catalog_name, from_branch, &payload.name, None) .await { - // Create asset in new branch with NEW ID to avoid unique constraint violation - let mut new_asset = asset.clone(); - new_asset.id = uuid::Uuid::new_v4(); - if let Ok(_) = store - .create_asset( - tenant_id, - catalog_name, - Some(payload.name.clone()), - namespace_parts, - new_asset, + Ok(count) => tracing::info!( + branch = %payload.name, + assets = count, + "copied assets into the new branch" + ), + Err(e) => { + // Previously logged and ignored. The branch exists and is + // incomplete; the caller has to know. + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!( + "Branch {:?} was created but copying its assets failed: {e}. \ + The branch is incomplete; delete it and retry.", + payload.name + ), ) - .await - { - // branch_assets.push(asset_name.clone()); // Unnecessary as create_asset updates branch + .into_response(); } } } - } else { - tracing::info!("No explicit assets. Auto-propagating from {}", from_branch); - - // Use optimized bulk copy - // Note: This works because create_branch (above) initialized the branch, and copy_assets_bulk appends to it (or updates it) - match store - .copy_assets_bulk(tenant_id, catalog_name, from_branch, &payload.name, None) - .await - { - Ok(count) => tracing::info!( - "Bulk copied {} assets to new branch {}", - count, - payload.name - ), - Err(e) => tracing::error!("Failed to bulk copy assets: {}", e), + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to create branch: {e}"), + ) + .into_response(); } } @@ -304,6 +316,99 @@ pub async fn get_branch( } } +/// Delete a branch. +/// +/// B33: the UI has had a "delete branch" control calling +/// `DELETE /api/v1/branches/{catalog}/{name}` since before this audit, but the +/// router only ever registered `GET` on `/api/v1/branches/:name` - so branch +/// deletion from the UI always 404'd. The store has supported it all along; the +/// route simply did not exist. Added here with the catalog as a query parameter, +/// matching `get_branch` rather than inventing a third path shape. +#[utoipa::path( + delete, + path = "/api/v1/branches/{name}", + tag = "Branches", + params( + ("name" = String, Path, description = "Branch name"), + ("catalog" = String, Query, description = "Catalog the branch belongs to") + ), + responses( + (status = 204, description = "Branch deleted"), + (status = 400, description = "The main branch cannot be deleted"), + (status = 403, description = "Forbidden"), + (status = 404, description = "Branch or catalog not found"), + (status = 500, description = "Internal server error") + ), + security(("bearer_auth" = [])) +)] +pub async fn delete_branch( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path(name): Path, + Query(params): Query, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = ¶ms.catalog; + + // Deleting `main` would strand every asset on it with no branch to reach + // them through. + if name == "main" { + return (StatusCode::BAD_REQUEST, "The main branch cannot be deleted").into_response(); + } + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_branch: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + let scope = PermissionScope::Catalog { + catalog_id: catalog.id, + }; + match crate::authz::check_permission(&store, &session, &Action::Delete, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_branch: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } + + match store + .delete_branch(tenant_id, catalog_name, name.clone()) + .await + { + Ok(_) => { + if let Err(e) = store + .log_audit_event( + tenant_id, + pangolin_core::audit::AuditLogEntry::success( + tenant_id, + Some(session.user_id), + session.username.clone(), + pangolin_core::audit::AuditAction::DeleteBranch, + pangolin_core::audit::ResourceType::Branch, + None, + format!("{}/{}", catalog_name, name), + ), + ) + .await + { + tracing::error!(error = %e, "failed to write the branch-delete audit record"); + } + StatusCode::NO_CONTENT.into_response() + } + Err(e) => { + tracing::debug!(error = %e, "delete_branch: branch not deleted"); + (StatusCode::NOT_FOUND, "Branch not found").into_response() + } + } +} + #[utoipa::path( post, path = "/api/v1/branches/merge", @@ -524,6 +629,7 @@ pub async fn list_commits( } #[derive(Deserialize)] +#[serde(deny_unknown_fields)] pub struct CreateTagRequest { name: String, catalog: Option, @@ -659,7 +765,7 @@ pub async fn delete_tag( pub async fn rebase_branch( State(store): State, Extension(tenant): Extension, - Extension(_session): Extension, + Extension(session): Extension, Path(branch_name): Path, Json(payload): Json, // We need catalog_name from body ) -> impl IntoResponse { @@ -669,12 +775,33 @@ pub async fn rebase_branch( // Source: main // Target: branch_name - // Check permissions - // TODO: Granular permissions? For now assume Write on Catalog - // Catalog is now required let catalog_name = &payload.catalog; + // The `// TODO: Granular permissions? For now assume Write on Catalog` that + // stood here *was* the authorization: any tenant member could rebase any + // branch, which rewrites that branch's contents. Same class as B0c-B0f. + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "rebase_branch: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + let scope = PermissionScope::Catalog { + catalog_id: catalog.id, + }; + match crate::authz::check_permission(&store, &session, &Action::Write, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "rebase_branch: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } + match store .merge_branch(tenant_id, catalog_name, "main".to_string(), branch_name) .await @@ -690,6 +817,7 @@ pub async fn rebase_branch( // Catalog Management #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateCatalogRequest { name: String, catalog_type: Option, @@ -700,6 +828,7 @@ pub struct CreateCatalogRequest { } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateCatalogRequest { warehouse_name: Option, storage_location: Option, @@ -751,7 +880,16 @@ pub async fn list_catalogs( let tenant_id = tenant.0; tracing::info!("list_catalogs called with tenant_id: {}", tenant_id); - match store.list_catalogs(tenant_id, Some(pagination)).await { + // B42: the store used to paginate first and `filter_catalogs` removed the + // unauthorized rows afterwards, so a `TenantUser` got variable-size pages - + // including *empty pages while more authorized data existed*. Any client + // that stops on an empty page (the normal idiom) silently missed data. + // + // Filtering has to happen before slicing. The permitted set is not + // expressible as a store predicate today, so the rows are fetched unpaged + // and the page is cut after filtering; the page window is applied below. + let requested = pagination; + match store.list_catalogs(tenant_id, None).await { Ok(catalogs) => { // Use authz_utils for consistent permission filtering let permissions = if matches!(session.role, UserRole::TenantUser) { @@ -769,8 +907,12 @@ pub async fn list_catalogs( Vec::new() // Root/TenantAdmin bypass filtering }; - let filtered_catalogs = - crate::authz_utils::filter_catalogs(catalogs, &permissions, session.role.clone()); + let filtered_catalogs = crate::authz_utils::filter_catalogs( + tenant_id, + catalogs, + &permissions, + session.role.clone(), + ); tracing::info!( "list_catalogs returning {} catalogs for user {}", @@ -778,8 +920,16 @@ pub async fn list_catalogs( session.user_id ); - let resp: Vec = - filtered_catalogs.into_iter().map(|c| c.into()).collect(); + // Now slice the *authorized* set, so a page is empty only when the + // caller has actually reached the end. + let offset = requested.offset.unwrap_or(0); + let limit = requested.limit.unwrap_or(usize::MAX); + let resp: Vec = filtered_catalogs + .into_iter() + .skip(offset) + .take(limit) + .map(|c| c.into()) + .collect(); (StatusCode::OK, Json(resp)).into_response() } Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), diff --git a/pangolin/pangolin_api/src/permission_handlers.rs b/pangolin/pangolin_api/src/permission_handlers.rs index c570d88..3bd9401 100644 --- a/pangolin/pangolin_api/src/permission_handlers.rs +++ b/pangolin/pangolin_api/src/permission_handlers.rs @@ -16,6 +16,7 @@ use uuid::Uuid; /// Request to create a new role #[derive(Debug, Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct CreateRoleRequest { pub name: String, pub description: Option, @@ -25,6 +26,7 @@ pub struct CreateRoleRequest { /// Request to assign a role to a user #[derive(Debug, Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct AssignRoleRequest { pub role_id: Uuid, } @@ -32,6 +34,7 @@ pub struct AssignRoleRequest { /// Request to grant a permission #[derive(Debug, Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct GrantPermissionRequest { pub user_id: Uuid, pub scope: PermissionScope, diff --git a/pangolin/pangolin_api/src/public_paths.rs b/pangolin/pangolin_api/src/public_paths.rs index ef128e3..c987141 100644 --- a/pangolin/pangolin_api/src/public_paths.rs +++ b/pangolin/pangolin_api/src/public_paths.rs @@ -53,6 +53,21 @@ pub fn is_public_path(path: &str) -> bool { ["oauth", "authorize", _provider] => true, ["oauth", "callback", _provider] => true, + // Redeeming the one-time code the OAuth callback hands back. + // + // B0k: this was missing, so the middleware demanded a bearer token on + // the very endpoint whose job is to obtain the first one. The 0.6.0 + // callback -> one-time code -> POST exchange flow was therefore + // unreachable in production: the browser landed with a `?code=...` it + // could never redeem. The code itself is single-use and short-lived, + // which is what makes this endpoint safe to expose unauthenticated. + ["api", "v1", "oauth", "exchange"] => true, + + // Which OAuth providers are configured. The login page needs this + // before anyone is authenticated (see B33); it reveals only provider + // names, never secrets. + ["api", "v1", "oauth", "providers"] => true, + _ => false, } } @@ -87,6 +102,18 @@ mod tests { } } + /// Regression test for B0k: without this the OAuth login flow cannot + /// complete, because the code-exchange endpoint demanded the token it + /// exists to issue. + #[test] + fn oauth_exchange_and_providers_are_public() { + assert!(is_public_path("/api/v1/oauth/exchange")); + assert!(is_public_path("/api/v1/oauth/providers")); + // ...but nothing deeper under the same prefix. + assert!(!is_public_path("/api/v1/oauth/exchange/steal")); + assert!(!is_public_path("/api/v1/oauth/tokens")); + } + /// Regression test for A-11: a resource named `config` must not bypass auth. #[test] fn resources_named_config_are_not_public() { diff --git a/pangolin/pangolin_api/src/rate_limit.rs b/pangolin/pangolin_api/src/rate_limit.rs new file mode 100644 index 0000000..4e02d52 --- /dev/null +++ b/pangolin/pangolin_api/src/rate_limit.rs @@ -0,0 +1,311 @@ +//! Throttling for the authentication endpoints. +//! +//! C-5. Before this, the login endpoint was brute-forceable: there were global +//! concurrency and body limits and a request timeout, but nothing that made the +//! thousandth password guess from one address any more expensive than the +//! first. Bcrypt makes each attempt slow, which raises the cost of a large +//! campaign but does nothing against a targeted guess at one weak password, and +//! it makes the endpoint an efficient way to burn the server's CPU. +//! +//! Two keys, deliberately: +//! +//! * **by source address**: bounds one attacker hammering many accounts; +//! * **by account**: bounds many sources hammering one account, which is what +//! a credential-stuffing list looks like and which a per-IP limit alone +//! cannot see. +//! +//! A fixed window rather than a token bucket. It is coarser, but the failure +//! mode of a fixed window under attack is that it lets through at most twice +//! the configured rate across a window boundary, whereas the failure mode of a +//! badly tuned bucket is a burst allowance that hands an attacker exactly the +//! sustained-guess budget the limit exists to remove. +//! +//! ## What this is not +//! +//! In-process, so the limit is **per replica**: with N replicas the effective +//! limit is N times the configured one. That is a real weakness and it is +//! stated in the operations documentation rather than hidden. A shared limiter +//! needs Redis or equivalent, which this project does not otherwise require; +//! adding a mandatory external dependency to the login path is a bigger +//! decision than this fix. + +use axum::body::Body; +use axum::extract::{ConnectInfo, State}; +use axum::http::{HeaderMap, Request, StatusCode}; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; +use axum::Json; +use moka::future::Cache; +use serde_json::json; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::time::Duration; + +/// The paths this applies to. +/// +/// Only the endpoints that accept a credential. Rate-limiting the whole API +/// would be a different feature with different tuning, and applying an +/// auth-shaped limit to ordinary catalog traffic would break normal use. +const THROTTLED_PATHS: &[&str] = &[ + "/api/v1/users/login", + "/api/v1/tokens", + "/api/v1/auth/oauth/callback", +]; + +pub fn is_throttled_path(path: &str) -> bool { + THROTTLED_PATHS.contains(&path) +} + +#[derive(Clone)] +pub struct RateLimiter { + counters: Cache, + limit: u32, + window: Duration, +} + +impl RateLimiter { + pub fn new(limit: u32, window: Duration) -> Self { + Self { + // Entries expire a window after they are created, which is what + // makes this a fixed window: the count for a key lives exactly as + // long as the window it belongs to. + counters: Cache::builder() + .time_to_live(window) + .max_capacity(100_000) + .build(), + limit, + window, + } + } + + /// Record an attempt against `key`. `Err(retry_after)` when over the limit. + pub async fn check(&self, key: &str) -> Result<(), Duration> { + if self.limit == 0 { + return Ok(()); // disabled + } + let current = self.counters.get(key).await.unwrap_or(0); + if current >= self.limit { + return Err(self.window); + } + self.counters.insert(key.to_string(), current + 1).await; + Ok(()) + } + + /// Forget a key. Called after a *successful* authentication so that a user + /// who mistypes a password several times and then gets it right is not + /// still carrying the failures. + pub async fn clear(&self, key: &str) { + self.counters.invalidate(key).await; + } + + pub fn limit(&self) -> u32 { + self.limit + } +} + +/// Resolve the client address. +/// +/// `X-Forwarded-For` is only honoured when the operator has said they run +/// behind a proxy. Trusting it unconditionally would make the limit trivially +/// bypassable - an attacker sets the header to a fresh value per request and +/// every attempt looks like a new client, which is worse than no limit at all +/// because it looks like protection. +pub fn client_key(headers: &HeaderMap, peer: Option, trust_forwarded: bool) -> String { + if trust_forwarded { + if let Some(forwarded) = headers.get("x-forwarded-for").and_then(|v| v.to_str().ok()) { + // Left-most entry is the originating client as recorded by the + // first proxy. + if let Some(first) = forwarded.split(',').next() { + let candidate = first.trim(); + if let Ok(ip) = candidate.parse::() { + return ip.to_string(); + } + } + } + } + peer.map(|s| s.ip().to_string()) + .unwrap_or_else(|| "unknown".to_string()) +} + +fn too_many(retry_after: Duration) -> Response { + ( + StatusCode::TOO_MANY_REQUESTS, + [("retry-after", retry_after.as_secs().to_string())], + Json(json!({ + "error": { + "message": "too many authentication attempts; try again later", + "type": "TooManyRequests", + "code": 429 + } + })), + ) + .into_response() +} + +/// The limiter the login handler uses for its per-account half. +/// +/// A `RwLock` rather than a `OnceLock` so each `app_with_options` installs its +/// own: several tests build an app in the same process, and a limiter shared +/// across them would make one test's login attempts throttle another's. +static AUTH_LIMITER: std::sync::RwLock>> = std::sync::RwLock::new(None); + +pub fn set_auth_limiter(limiter: Arc) { + if let Ok(mut guard) = AUTH_LIMITER.write() { + *guard = Some(limiter); + } +} + +pub fn auth_limiter() -> Option> { + AUTH_LIMITER.read().ok().and_then(|g| g.clone()) +} + +/// Throttle key for one account. Tenant-scoped, because the same username can +/// exist in two tenants and they are different accounts. +pub fn account_key(username: &str, tenant: Option) -> String { + match tenant { + Some(t) => format!("acct:{t}:{username}"), + None => format!("acct:-:{username}"), + } +} + +#[derive(Clone)] +pub struct RateLimitState { + pub limiter: Arc, + pub trust_forwarded: bool, +} + +/// Per-source-address throttling for the authentication endpoints. +/// +/// The per-account half lives in the login handler, which is the only place +/// that knows which account is being attempted - reading the body here would +/// mean buffering and re-attaching it for every request on these routes. +pub async fn throttle_auth( + State(state): State, + // Optional: `ConnectInfo` only exists when the app is served through + // `into_make_service_with_connect_info`, which `main` does but an + // in-process test calling the router directly does not. A required + // extractor here would turn every such request into a 500. + peer: Option>, + request: Request, + next: Next, +) -> Response { + if !is_throttled_path(request.uri().path()) { + return next.run(request).await; + } + + let key = format!( + "ip:{}", + client_key( + request.headers(), + peer.map(|ConnectInfo(addr)| addr), + state.trust_forwarded + ) + ); + + if let Err(retry_after) = state.limiter.check(&key).await { + tracing::warn!( + path = request.uri().path(), + "authentication attempts throttled for a source address" + ); + crate::metrics::record_auth_throttled(); + return too_many(retry_after); + } + + next.run(request).await +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::HeaderValue; + + #[tokio::test] + async fn allows_up_to_the_limit_then_refuses() { + let limiter = RateLimiter::new(3, Duration::from_secs(60)); + for i in 0..3 { + assert!(limiter.check("k").await.is_ok(), "attempt {i} should pass"); + } + assert!( + limiter.check("k").await.is_err(), + "the fourth attempt must be refused" + ); + } + + #[tokio::test] + async fn keys_are_independent() { + let limiter = RateLimiter::new(1, Duration::from_secs(60)); + assert!(limiter.check("a").await.is_ok()); + assert!(limiter.check("a").await.is_err()); + assert!( + limiter.check("b").await.is_ok(), + "one key's exhaustion must not affect another" + ); + } + + #[tokio::test] + async fn a_successful_login_clears_the_count() { + let limiter = RateLimiter::new(2, Duration::from_secs(60)); + assert!(limiter.check("u").await.is_ok()); + assert!(limiter.check("u").await.is_ok()); + assert!(limiter.check("u").await.is_err()); + + limiter.clear("u").await; + assert!( + limiter.check("u").await.is_ok(), + "clearing after a success must reset the window" + ); + } + + #[tokio::test] + async fn the_window_expires() { + let limiter = RateLimiter::new(1, Duration::from_millis(120)); + assert!(limiter.check("t").await.is_ok()); + assert!(limiter.check("t").await.is_err()); + tokio::time::sleep(Duration::from_millis(250)).await; + // moka expires lazily; a get after the TTL sees nothing. + limiter.counters.run_pending_tasks().await; + assert!( + limiter.check("t").await.is_ok(), + "the counter must not outlive its window" + ); + } + + #[tokio::test] + async fn a_zero_limit_disables_throttling() { + let limiter = RateLimiter::new(0, Duration::from_secs(60)); + for _ in 0..100 { + assert!(limiter.check("k").await.is_ok()); + } + } + + #[test] + fn forwarded_for_is_ignored_unless_the_operator_trusts_it() { + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", HeaderValue::from_static("203.0.113.9")); + let peer: SocketAddr = "198.51.100.1:5000".parse().unwrap(); + + assert_eq!( + client_key(&headers, Some(peer), false), + "198.51.100.1", + "an untrusted X-Forwarded-For must not select the key, or the limit \ + is bypassable by setting a header" + ); + assert_eq!(client_key(&headers, Some(peer), true), "203.0.113.9"); + } + + #[test] + fn a_malformed_forwarded_for_falls_back_to_the_peer() { + let mut headers = HeaderMap::new(); + headers.insert("x-forwarded-for", HeaderValue::from_static("not-an-ip")); + let peer: SocketAddr = "198.51.100.1:5000".parse().unwrap(); + assert_eq!(client_key(&headers, Some(peer), true), "198.51.100.1"); + } + + #[test] + fn only_the_credential_endpoints_are_throttled() { + assert!(is_throttled_path("/api/v1/users/login")); + assert!(is_throttled_path("/api/v1/tokens")); + assert!(!is_throttled_path("/api/v1/catalogs")); + assert!(!is_throttled_path("/health/ready")); + } +} diff --git a/pangolin/pangolin_api/src/service_user_handlers.rs b/pangolin/pangolin_api/src/service_user_handlers.rs index 22beb26..90868d4 100644 --- a/pangolin/pangolin_api/src/service_user_handlers.rs +++ b/pangolin/pangolin_api/src/service_user_handlers.rs @@ -18,6 +18,7 @@ type AppState = Arc; // Request/Response types #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateServiceUserRequest { pub name: String, pub description: Option, @@ -26,6 +27,7 @@ pub struct CreateServiceUserRequest { } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateServiceUserRequest { pub name: Option, pub description: Option, diff --git a/pangolin/pangolin_api/src/signing_handlers.rs b/pangolin/pangolin_api/src/signing_handlers.rs index f0c2f7b..70010f8 100644 --- a/pangolin/pangolin_api/src/signing_handlers.rs +++ b/pangolin/pangolin_api/src/signing_handlers.rs @@ -1,5 +1,6 @@ use crate::auth::TenantId; -use crate::iceberg::AppState; +use crate::authz::check_permission; +use crate::iceberg::{parse_namespace, parse_table_identifier, AppState}; use axum::Extension; use axum::{ extract::{Path, Query, State}, @@ -7,6 +8,8 @@ use axum::{ response::IntoResponse, Json, }; +use pangolin_core::permission::{Action, PermissionScope}; +use pangolin_core::user::UserSession; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use utoipa::{IntoParams, ToSchema}; @@ -180,6 +183,17 @@ pub async fn get_gcp_token(_service_account_key_json: &str) -> Result Result Result, Extension(tenant): Extension, + Extension(session): Extension, Path((catalog_name, namespace, table)): Path<(String, String, String)>, ) -> impl IntoResponse { let tenant_id = tenant.0; @@ -218,7 +233,55 @@ pub async fn get_table_credentials( Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), }; - // 2. Check if catalog has a warehouse + // 2. Resolve the asset the caller is asking for. Vending credentials for a + // table that does not exist is not a meaningful operation, and resolving it + // is what makes an asset-scoped permission check possible at all. + let (ns_levels, branch) = parse_namespace(&namespace); + let (table_name, branch_from_table) = parse_table_identifier(&table); + let branch = branch_from_table.or(branch); + + let asset = match store + .get_asset( + tenant_id, + &catalog_name, + branch, + ns_levels.clone(), + table_name.clone(), + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), + Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), + }; + + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: ns_levels.join("."), + asset_id: asset.id, + }; + + // 3. Read is the floor: without it, no credentials at all. + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Permission check failed: {}", e), + ) + .into_response() + } + } + + // Write is vended only when actually held, so a read-only principal gets + // read-only cloud credentials. + let can_write = matches!( + check_permission(&store, &session, &Action::Write, &scope).await, + Ok(true) + ); + + // 4. Check if catalog has a warehouse let warehouse_name = match catalog.warehouse_name { Some(name) => name, None => { @@ -231,16 +294,23 @@ pub async fn get_table_credentials( } }; - // 3. Get warehouse configuration + // 5. Get warehouse configuration let warehouse = match store.get_warehouse(tenant_id, warehouse_name).await { Ok(Some(wh)) => wh, Ok(None) => return (StatusCode::NOT_FOUND, "Warehouse not found").into_response(), Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), }; - // 4. Vend credentials using the credential signer infrastructure - let resource_path = format!("{}/{}", namespace, table); - let permissions = vec!["read".to_string(), "write".to_string()]; + // 6. Vend credentials using the credential signer infrastructure + let resource_path = if ns_levels.is_empty() { + table_name.clone() + } else { + format!("{}/{}", ns_levels.join("/"), table_name) + }; + let mut permissions = vec!["read".to_string()]; + if can_write { + permissions.push("write".to_string()); + } match crate::credential_vending::vend_credentials_for_warehouse( &warehouse, diff --git a/pangolin/pangolin_api/src/tenant_handlers.rs b/pangolin/pangolin_api/src/tenant_handlers.rs index d16738b..63304c1 100644 --- a/pangolin/pangolin_api/src/tenant_handlers.rs +++ b/pangolin/pangolin_api/src/tenant_handlers.rs @@ -15,12 +15,14 @@ use utoipa::ToSchema; use uuid::Uuid; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateTenantRequest { name: String, properties: Option>, } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateTenantRequest { name: Option, properties: Option>, diff --git a/pangolin/pangolin_api/src/token_handlers.rs b/pangolin/pangolin_api/src/token_handlers.rs index 83db728..1110159 100644 --- a/pangolin/pangolin_api/src/token_handlers.rs +++ b/pangolin/pangolin_api/src/token_handlers.rs @@ -15,6 +15,7 @@ use utoipa::ToSchema; use uuid::Uuid; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct GenerateTokenRequest { pub tenant_id: String, pub username: Option, @@ -22,6 +23,38 @@ pub struct GenerateTokenRequest { pub expires_in_hours: Option, } +/// Upper bound on a caller-requested token lifetime. +/// +/// `expires_in_hours` is attacker-controlled and used to be fed straight into +/// `chrono::Duration::hours`, which *panics* on a large enough value - before +/// the `checked_add_signed().unwrap()` below it could even run (B0m). There is +/// no `CatchPanicLayer` under the router, so that panic aborted the connection +/// task. Clamping first makes the arithmetic total. +const MAX_TOKEN_LIFETIME_HOURS: u64 = 24 * 365; + +/// Rank roles so a caller cannot mint a token more privileged than itself. +fn role_rank(role: &UserRole) -> u8 { + match role { + UserRole::Root => 3, + UserRole::TenantAdmin => 2, + UserRole::TenantUser => 1, + } +} + +/// Parse a role name from a token request. +/// +/// Accepts both the Debug-ish spellings the old code matched and the kebab-case +/// serde names, but unknown values are now an error rather than a silent +/// downgrade to `TenantUser`. +fn parse_role(name: &str) -> Option { + match name { + "Root" | "root" => Some(UserRole::Root), + "Admin" | "admin" | "TenantAdmin" | "tenant-admin" => Some(UserRole::TenantAdmin), + "TenantUser" | "tenant-user" | "user" => Some(UserRole::TenantUser), + _ => None, + } +} + #[derive(Serialize, ToSchema)] pub struct GenerateTokenResponse { pub token: String, @@ -29,8 +62,21 @@ pub struct GenerateTokenResponse { pub tenant_id: String, } -/// Generate a JWT token for a tenant -/// This endpoint allows generating tokens for testing and development +/// Generate a JWT token for a tenant. +/// +/// Authorization (B0a): this handler used to take no session at all, so any +/// authenticated principal - including the lowest-privilege `TenantUser` or any +/// service-user API key - could POST `{"tenant_id": "", "roles":["Root"]}` +/// and receive a signed `Root` token for an arbitrary tenant. That is a total +/// privilege escalation, since `check_permission` short-circuits for `Root`. +/// +/// The rules now are: +/// * `Root` may mint anything. +/// * `TenantAdmin` may mint only for its own tenant, and only a role at or +/// below its own. +/// * everyone else is refused. +/// A role supplied in the body is never trusted for a non-`Root` caller beyond +/// those bounds. #[utoipa::path( post, path = "/api/v1/tokens", @@ -39,60 +85,107 @@ pub struct GenerateTokenResponse { responses( (status = 200, description = "Token generated", body = GenerateTokenResponse), (status = 400, description = "Bad request"), + (status = 403, description = "Forbidden"), (status = 500, description = "Internal server error") - ) + ), + security(("bearer_auth" = [])) )] pub async fn generate_token( State(store): State, + Extension(session): Extension, Json(payload): Json, ) -> impl IntoResponse { // Validate tenant_id is a valid UUID - let _tenant_uuid = match Uuid::parse_str(&payload.tenant_id) { + let tenant_uuid = match Uuid::parse_str(&payload.tenant_id) { Ok(uuid) => uuid, Err(_) => return (StatusCode::BAD_REQUEST, "Invalid tenant_id format").into_response(), }; + let caller_is_root = session.role == UserRole::Root; + if !caller_is_root { + if session.role != UserRole::TenantAdmin { + return ( + StatusCode::FORBIDDEN, + "Root or tenant-admin access required to mint tokens", + ) + .into_response(); + } + if session.tenant_id != Some(tenant_uuid) { + return ( + StatusCode::FORBIDDEN, + "Cannot mint a token for another tenant", + ) + .into_response(); + } + } + let secret = crate::config::jwt_secret(); - let expires_in = payload.expires_in_hours.unwrap_or(24); + let expires_in = payload + .expires_in_hours + .unwrap_or(24) + .min(MAX_TOKEN_LIFETIME_HOURS); let now = chrono::Utc::now(); - let exp = now + let Some(exp) = now .checked_add_signed(chrono::Duration::hours(expires_in as i64)) - .unwrap() - .timestamp(); + .map(|t| t.timestamp()) + else { + return (StatusCode::BAD_REQUEST, "expires_in_hours out of range").into_response(); + }; let username = payload.username.unwrap_or_else(|| "api-user".to_string()); - // Map role strings to UserRole - // Default to lookup user role or TenantUser if not specified - let role = if let Some(roles) = &payload.roles { - if let Some(first_role) = roles.first() { - match first_role.as_str() { - "Root" | "root" => UserRole::Root, - "Admin" | "admin" | "TenantAdmin" | "tenant-admin" => UserRole::TenantAdmin, - _ => UserRole::TenantUser, - } - } else { - UserRole::TenantUser + // Map role strings to UserRole. An unknown name is now a 400 rather than a + // silent downgrade, so a typo cannot quietly hand out the wrong role. + let requested_role = if let Some(roles) = &payload.roles { + match roles.first() { + Some(first_role) => match parse_role(first_role) { + Some(r) => Some(r), + None => { + return ( + StatusCode::BAD_REQUEST, + format!("Unknown role: {}", first_role), + ) + .into_response() + } + }, + None => None, } } else { - // Try to lookup user - if let Ok(Some(user)) = store.get_user_by_username(&username).await { - tracing::info!( - "generate_token: Found user '{}' with role {:?} ({})", - username, - user.role, - user.id - ); - user.role - } else { - tracing::warn!( - "generate_token: User '{}' not found, defaulting to TenantUser", - username - ); - UserRole::TenantUser + None + }; + + let role = match requested_role { + Some(r) => r, + None => { + // Try to look up the user's own role. + if let Ok(Some(user)) = store.get_user_by_username(&username).await { + tracing::info!( + "generate_token: Found user '{}' with role {:?} ({})", + username, + user.role, + user.id + ); + user.role + } else { + tracing::warn!( + "generate_token: User '{}' not found, defaulting to TenantUser", + username + ); + UserRole::TenantUser + } } }; + // A non-root caller can never mint above its own rank, whatever the body or + // the looked-up user says. + if !caller_is_root && role_rank(&role) > role_rank(&session.role) { + return ( + StatusCode::FORBIDDEN, + "Cannot mint a token more privileged than the caller", + ) + .into_response(); + } + // sub MUST be a UUID for to_session() to work // If user exists, use their ID. Else generate one. let user_id = if let Ok(Some(user)) = store.get_user_by_username(&username).await { @@ -121,10 +214,7 @@ pub async fn generate_token( // Store token info for listing let token_info = TokenInfo { id: token_id, - tenant_id: match Uuid::parse_str(&payload.tenant_id) { - Ok(u) => u, - Err(_) => Uuid::default(), // Should be validated above - }, + tenant_id: tenant_uuid, user_id, username: username.clone(), expires_at: chrono::DateTime::from_timestamp(exp, 0).unwrap_or_default(), @@ -141,7 +231,7 @@ pub async fn generate_token( let response = GenerateTokenResponse { token, expires_at: chrono::DateTime::from_timestamp(exp, 0) - .unwrap() + .unwrap_or_default() .to_rfc3339(), tenant_id: payload.tenant_id, }; @@ -164,6 +254,7 @@ use pangolin_store::CatalogStore; use std::sync::Arc; #[derive(Debug, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct RevokeTokenRequest { pub reason: Option, } @@ -190,11 +281,22 @@ pub async fn revoke_current_token( Extension(session): Extension, Json(payload): Json, ) -> impl IntoResponse { - // Generate an expiration time (tokens typically expire in 24 hours) - let expires_at = Utc::now() + Duration::hours(24); - - // Use the user_id as the token_id for revocation - let token_id = session.user_id; + // Revoke until the token would have expired anyway; the blacklist entry + // only has to outlive the token. + let expires_at = session.expires_at.max(Utc::now()); + + // B0j: revoke the token's own `jti`. This used to revoke `session.user_id`, + // which no token ever carries as its `jti`, so the middleware's revocation + // check (keyed by `jti`) never matched: logout returned 200 and the token + // stayed valid for its full lifetime. + let Some(token_id) = session.token_id else { + // API-key and root-basic-auth sessions have no revocable JWT. + return ( + StatusCode::BAD_REQUEST, + "This session is not backed by a revocable token", + ) + .into_response(); + }; match store .revoke_token(token_id, expires_at, payload.reason) @@ -474,10 +576,16 @@ pub async fn rotate_token( let secret = crate::config::jwt_secret(); let now = chrono::Utc::now(); let expires_in = 24; // Default rotation to 24h - let exp = now + let Some(exp) = now .checked_add_signed(chrono::Duration::hours(expires_in)) - .unwrap() - .timestamp(); + .map(|t| t.timestamp()) + else { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to compute token expiry", + ) + .into_response(); + }; let token_id = Uuid::new_v4(); let tenant_id_str = session.tenant_id.map(|t| t.to_string()).unwrap_or_default(); @@ -514,21 +622,31 @@ pub async fn rotate_token( tracing::warn!("Failed to store rotated token info: {}", e); } - // 3. Revoke old token (if we knew its ID - session doesn't carry jti currently in UserSession struct?) - // Wait, UserSession struct usually just has user info. - // Current `auth_middleware` decodes claims but might not pass `jti` to `UserSession`. - // Let's check `UserSession`. If it doesn't have token ID, we can't revoke the *specific* old token easily. - // We can revoke ALL other tokens for this user? No, that's too aggressive. - // If we can't revoke the old one, "Rotation" is just "Get New Token". - // Ideally UserSession should have `token_id`. - // I'll skip revocation of old token for now if I lack the ID, but assume the client will discard it. - // Implementation: Just return new token. - // Update: We can update UserSession to include token_id (jti) later. + // 3. Revoke the old token. `UserSession` now carries the presenting + // token's `jti` (B0j), so rotation is a real rotation rather than + // "issue a second valid token and hope the client forgets the first". + if let Some(old_token_id) = session.token_id { + if let Err(e) = store + .revoke_token( + old_token_id, + session.expires_at.max(Utc::now()), + Some("Rotated".to_string()), + ) + .await + { + tracing::error!(error = %e, "failed to revoke the rotated-out token"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "Issued a new token but could not revoke the old one", + ) + .into_response(); + } + } let response = GenerateTokenResponse { token, expires_at: chrono::DateTime::from_timestamp(exp, 0) - .unwrap() + .unwrap_or_default() .to_rfc3339(), tenant_id: tenant_id_str, }; diff --git a/pangolin/pangolin_api/src/user_handlers.rs b/pangolin/pangolin_api/src/user_handlers.rs index 72b17f4..dbe4999 100644 --- a/pangolin/pangolin_api/src/user_handlers.rs +++ b/pangolin/pangolin_api/src/user_handlers.rs @@ -13,6 +13,7 @@ use uuid::Uuid; /// Request to create a new user #[derive(Debug, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateUserRequest { pub username: String, pub email: String, @@ -24,6 +25,7 @@ pub struct CreateUserRequest { /// Request to update a user #[derive(Debug, Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct UpdateUserRequest { pub email: Option, pub password: Option, @@ -33,6 +35,7 @@ pub struct UpdateUserRequest { /// User login request #[derive(Debug, Deserialize, Serialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct LoginRequest { pub username: String, pub password: String, @@ -439,6 +442,31 @@ pub async fn login( State(store): State>, Json(req): Json, ) -> Response { + // C-5, the per-account half of the throttle. The middleware bounds attempts + // per source address; this bounds them per account, which is the shape a + // credential-stuffing list has - many addresses, one target - and which a + // per-address limit cannot see. + // + // Checked before any password verification so a refused attempt costs a + // cache lookup rather than a bcrypt round. + let throttle_key = crate::rate_limit::account_key(&req.username, req.tenant_id); + let limiter = crate::rate_limit::auth_limiter(); + if let Some(limiter) = &limiter { + if limiter.check(&throttle_key).await.is_err() { + tracing::warn!( + username = %req.username, + "login attempts throttled for an account" + ); + crate::metrics::record_auth_throttled(); + return ( + StatusCode::TOO_MANY_REQUESTS, + [("retry-after", "60")], + "too many authentication attempts; try again later", + ) + .into_response(); + } + } + // 1. Check for Root User via Environment Variables (only if tenant_id is null) // This takes precedence over DB users to ensure Root is always accessible even if a tenant user shadows the name // Root basic credentials are only honoured when the operator has configured @@ -497,6 +525,9 @@ pub async fn login( expires_at: (chrono::Utc::now() + chrono::Duration::seconds(86400)).to_rfc3339(), }; + if let Some(limiter) = &limiter { + limiter.clear(&throttle_key).await; + } return (StatusCode::OK, Json(response)).into_response(); } } @@ -590,6 +621,12 @@ pub async fn login( expires_at: (chrono::Utc::now() + chrono::Duration::seconds(86400)).to_rfc3339(), }; + // The DB-user success path. Same reason as the root path above: a user who + // mistypes twice and then gets it right should not still be near the limit. + if let Some(limiter) = &limiter { + limiter.clear(&throttle_key).await; + } + (StatusCode::OK, Json(response)).into_response() } diff --git a/pangolin/pangolin_api/src/verification_tests.rs b/pangolin/pangolin_api/src/verification_tests.rs index 737d5cd..8ea5656 100644 --- a/pangolin/pangolin_api/src/verification_tests.rs +++ b/pangolin/pangolin_api/src/verification_tests.rs @@ -172,7 +172,10 @@ async fn test_verified_flow_regression() { .body(Body::from( json!({ "name": "test-tenant", - "organization": "TestOrg" + // `CreateTenantRequest` has no `organization` field. + // With `deny_unknown_fields` this is now a 422 rather + // than a silently dropped value (improvement #0). + "properties": { "organization": "TestOrg" } }) .to_string(), )) @@ -254,7 +257,10 @@ async fn test_verified_flow_regression() { json!({ "name": "test-catalog", "warehouse_name": "success-warehouse", - "type": "pangolin" + // `CreateCatalogRequest` takes `catalog_type`, not + // `type` - the old spelling was silently dropped and + // the catalog defaulted to Local anyway. + "catalog_type": "Local" }) .to_string(), )) @@ -304,7 +310,7 @@ async fn test_iceberg_namespace_creation() { .header("Authorization", format!("Bearer {}", root_token)) .header("Content-Type", "application/json") .body(Body::from( - json!({"name":"t1", "organization":"o1"}).to_string(), + json!({"name":"t1", "properties": {"organization":"o1"}}).to_string(), )) .unwrap(), ) @@ -339,7 +345,7 @@ async fn test_iceberg_namespace_creation() { .header("Authorization", format!("Bearer {}", ta_token)) .header("Content-Type", "application/json") .body(Body::from( - json!({"name":"cat","warehouse_name":"wh","type":"pangolin"}).to_string(), + json!({"name":"cat","warehouse_name":"wh","catalog_type":"Local"}).to_string(), )) .unwrap(), ) diff --git a/pangolin/pangolin_api/src/warehouse_handlers.rs b/pangolin/pangolin_api/src/warehouse_handlers.rs index 05e6e5c..597f498 100644 --- a/pangolin/pangolin_api/src/warehouse_handlers.rs +++ b/pangolin/pangolin_api/src/warehouse_handlers.rs @@ -18,6 +18,7 @@ pub struct GetCredentialsParams { } #[derive(Serialize, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateWarehouseRequest { pub name: String, pub use_sts: Option, // If true, use STS credential vending; if false, pass through static creds @@ -26,6 +27,7 @@ pub struct CreateWarehouseRequest { } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateWarehouseRequest { name: Option, use_sts: Option, diff --git a/pangolin/pangolin_api/tests/auth_rate_limit_tests.rs b/pangolin/pangolin_api/tests/auth_rate_limit_tests.rs new file mode 100644 index 0000000..b28b899 --- /dev/null +++ b/pangolin/pangolin_api/tests/auth_rate_limit_tests.rs @@ -0,0 +1,292 @@ +//! The login endpoint must actually refuse a brute-force attempt. +//! +//! C-5. `rate_limit`'s unit tests prove the counter behaves; they say nothing +//! about whether it is wired to anything. These drive the real router, because +//! the interesting failure mode is a limiter that exists and is never +//! consulted. That is how this endpoint came to be brute-forceable with a +//! limiter crate already in the dependency tree for something else. + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use pangolin_api::tests_common::EnvGuard; +use pangolin_api::{app_with_options, RouterOptions}; +use pangolin_store::memory::MemoryStore; +use serde_json::json; +use serial_test::serial; +use std::sync::Arc; +use std::time::Duration; +use tower::util::ServiceExt; + +/// A router with throttling on. `RouterOptions::default()` deliberately leaves +/// it off so the rest of the suite is unaffected, so this opts in explicitly. +fn throttled_app(limit: u32, window: Duration) -> axum::Router { + let store = Arc::new(MemoryStore::new()); + app_with_options( + store, + RouterOptions { + auth_rate_limit: limit, + auth_rate_window: window, + ..Default::default() + }, + ) +} + +fn login_request(username: &str, password: &str) -> Request { + Request::builder() + .method("POST") + .uri("/api/v1/users/login") + .header("content-type", "application/json") + .body(Body::from( + json!({ "username": username, "password": password }).to_string(), + )) + .unwrap() +} + +#[tokio::test] +#[serial] +async fn repeated_bad_passwords_are_eventually_refused() { + let _user = EnvGuard::new("PANGOLIN_ROOT_USER", "admin"); + let _pass = EnvGuard::new( + "PANGOLIN_ROOT_PASSWORD", + "a-real-password-not-a-placeholder", + ); + let app = throttled_app(3, Duration::from_secs(60)); + + let mut statuses = Vec::new(); + for _ in 0..6 { + let response = app + .clone() + .oneshot(login_request("admin", "wrong-password")) + .await + .unwrap(); + statuses.push(response.status()); + } + + assert!( + statuses.contains(&StatusCode::TOO_MANY_REQUESTS), + "six guesses against a limit of three must be throttled; got {statuses:?}" + ); + assert_eq!( + statuses[0], + StatusCode::UNAUTHORIZED, + "the first guess should be answered normally, not throttled" + ); + assert_eq!( + statuses[5], + StatusCode::TOO_MANY_REQUESTS, + "by the sixth the limiter must be refusing; got {:?}", + statuses[5] + ); +} + +#[tokio::test] +#[serial] +async fn a_throttled_response_says_when_to_retry() { + let _user = EnvGuard::new("PANGOLIN_ROOT_USER", "admin"); + let _pass = EnvGuard::new( + "PANGOLIN_ROOT_PASSWORD", + "a-real-password-not-a-placeholder", + ); + let app = throttled_app(1, Duration::from_secs(60)); + + let mut last = None; + for _ in 0..4 { + last = Some( + app.clone() + .oneshot(login_request("admin", "nope")) + .await + .unwrap(), + ); + } + + let response = last.unwrap(); + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + assert!( + response.headers().contains_key("retry-after"), + "a 429 without Retry-After leaves a well-behaved client guessing" + ); +} + +#[tokio::test] +#[serial] +async fn throttling_is_per_account_across_many_source_addresses() { + let _user = EnvGuard::new("PANGOLIN_ROOT_USER", "admin"); + let _pass = EnvGuard::new( + "PANGOLIN_ROOT_PASSWORD", + "a-real-password-not-a-placeholder", + ); + + // Every request arrives from a different address, so the per-address half + // never trips and anything refused here can only have been refused by the + // per-account key. This is the credential-stuffing shape - one target, a + // list of proxies - and it is precisely what a per-address limit alone + // cannot see. + let store = Arc::new(MemoryStore::new()); + let app = app_with_options( + store, + RouterOptions { + auth_rate_limit: 4, + auth_rate_window: Duration::from_secs(60), + trust_forwarded_for: true, + ..Default::default() + }, + ); + + let from = |ip: &str, user: &str| { + Request::builder() + .method("POST") + .uri("/api/v1/users/login") + .header("content-type", "application/json") + .header("x-forwarded-for", ip) + .body(Body::from( + json!({ "username": user, "password": "guess" }).to_string(), + )) + .unwrap() + }; + + for i in 0..4 { + let r = app + .clone() + .oneshot(from(&format!("203.0.113.{i}"), "victim")) + .await + .unwrap(); + assert_ne!( + r.status(), + StatusCode::TOO_MANY_REQUESTS, + "attempt {i} came from a fresh address and should not be throttled yet" + ); + } + + let refused = app + .clone() + .oneshot(from("203.0.113.99", "victim")) + .await + .unwrap(); + assert_eq!( + refused.status(), + StatusCode::TOO_MANY_REQUESTS, + "a fifth guess at the same account, from an address that has never been \ + seen, must still be refused" + ); + + // A different account from another fresh address must be unaffected. + let other = app + .clone() + .oneshot(from("198.51.100.7", "someone-else")) + .await + .unwrap(); + assert_ne!( + other.status(), + StatusCode::TOO_MANY_REQUESTS, + "throttling one account must not lock out every other account" + ); +} + +#[tokio::test] +#[serial] +async fn throttling_is_per_source_address() { + let _user = EnvGuard::new("PANGOLIN_ROOT_USER", "admin"); + let _pass = EnvGuard::new( + "PANGOLIN_ROOT_PASSWORD", + "a-real-password-not-a-placeholder", + ); + + // The mirror image: one address, a different account every time, so the + // per-account half never trips and only the per-address key can refuse. + let store = Arc::new(MemoryStore::new()); + let app = app_with_options( + store, + RouterOptions { + auth_rate_limit: 3, + auth_rate_window: Duration::from_secs(60), + trust_forwarded_for: true, + ..Default::default() + }, + ); + + let from_one_address = |user: &str| { + Request::builder() + .method("POST") + .uri("/api/v1/users/login") + .header("content-type", "application/json") + .header("x-forwarded-for", "203.0.113.5") + .body(Body::from( + json!({ "username": user, "password": "guess" }).to_string(), + )) + .unwrap() + }; + + for i in 0..3 { + let r = app + .clone() + .oneshot(from_one_address(&format!("user{i}"))) + .await + .unwrap(); + assert_ne!(r.status(), StatusCode::TOO_MANY_REQUESTS); + } + + let refused = app + .clone() + .oneshot(from_one_address("yet-another-user")) + .await + .unwrap(); + assert_eq!( + refused.status(), + StatusCode::TOO_MANY_REQUESTS, + "one address spraying many accounts must be throttled on the address" + ); +} + +#[tokio::test] +#[serial] +async fn ordinary_endpoints_are_not_throttled() { + let app = throttled_app(1, Duration::from_secs(60)); + + // Well past the limit, on a path that is not a credential endpoint. + for i in 0..10 { + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/health/ready") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_ne!( + response.status(), + StatusCode::TOO_MANY_REQUESTS, + "request {i} to /health/ready was throttled; an auth-shaped limit \ + must not apply to ordinary traffic" + ); + } +} + +#[tokio::test] +#[serial] +async fn throttling_is_off_unless_configured() { + let _user = EnvGuard::new("PANGOLIN_ROOT_USER", "admin"); + let _pass = EnvGuard::new( + "PANGOLIN_ROOT_PASSWORD", + "a-real-password-not-a-placeholder", + ); + // `RouterOptions::default()` has the limit at 0. That is what the rest of + // the suite builds with, so this pins the default rather than leaving the + // suite's behaviour to depend on it implicitly. + let store = Arc::new(MemoryStore::new()); + let app = app_with_options(store, RouterOptions::default()); + + for i in 0..25 { + let response = app + .clone() + .oneshot(login_request("admin", "wrong")) + .await + .unwrap(); + assert_ne!( + response.status(), + StatusCode::TOO_MANY_REQUESTS, + "attempt {i} was throttled with the limit disabled" + ); + } +} diff --git a/pangolin/pangolin_api/tests/business_metadata_test.rs b/pangolin/pangolin_api/tests/business_metadata_test.rs index c65fed4..8e785fd 100644 --- a/pangolin/pangolin_api/tests/business_metadata_test.rs +++ b/pangolin/pangolin_api/tests/business_metadata_test.rs @@ -124,6 +124,7 @@ async fn test_business_metadata_flow() { role: UserRole::Root, issued_at: chrono::Utc::now(), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + token_id: None, }; let token = pangolin_api::auth_middleware::generate_token(session, secret).unwrap(); diff --git a/pangolin/pangolin_api/tests/credential_vending_integration.rs b/pangolin/pangolin_api/tests/credential_vending_integration.rs index 2b06b02..4129f69 100644 --- a/pangolin/pangolin_api/tests/credential_vending_integration.rs +++ b/pangolin/pangolin_api/tests/credential_vending_integration.rs @@ -294,6 +294,6 @@ async fn test_permission_scoping() { assert!(read_write.config.contains_key("gcp-oauth-token")); // Both should succeed but in real implementation would have different scopes - assert!(read_only.config.get("gcp-oauth-token").is_some()); - assert!(read_write.config.get("gcp-oauth-token").is_some()); + assert!(read_only.config.contains_key("gcp-oauth-token")); + assert!(read_write.config.contains_key("gcp-oauth-token")); } diff --git a/pangolin/pangolin_api/tests/credential_vending_tests.rs b/pangolin/pangolin_api/tests/credential_vending_tests.rs index fc8240c..7a80bad 100644 --- a/pangolin/pangolin_api/tests/credential_vending_tests.rs +++ b/pangolin/pangolin_api/tests/credential_vending_tests.rs @@ -233,6 +233,7 @@ fn create_test_metadata() -> TableMetadata { last_updated_ms: 1234567890, last_column_id: 1, schemas: vec![Schema { + type_: "struct".to_string(), schema_id: 0, fields: vec![], identifier_field_ids: None, @@ -242,6 +243,7 @@ fn create_test_metadata() -> TableMetadata { spec_id: 0, fields: vec![], }], + last_partition_id: 999, properties: Some(HashMap::new()), current_snapshot_id: None, snapshots: None, diff --git a/pangolin/pangolin_api/tests/fixtures/oidc_test_key.pem b/pangolin/pangolin_api/tests/fixtures/oidc_test_key.pem new file mode 100644 index 0000000..397b5a7 --- /dev/null +++ b/pangolin/pangolin_api/tests/fixtures/oidc_test_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCRnegbM4IziGCu +okQjkCogSOcCP5XbMubX4KqKWt815QeroQJqcRgw19GkAbfMFp3dTNg6xo83W1eT +xRxNavMb90VlQ4gl4/hSUH59MylCTCvBxauWVGxsYiKv3BV7nLrAILRgwut57jQ9 +F4zFSXo/U8Q7ycm5Muhfopj7rQZ/0gYKvNHKlRCrieKsODENjXXgiQGX8viTfVop +9X4gm82zi2ps88HD6Agn2WigjKTUKus0d4y7qcS1T0aWlKS5Cr22D6XtcTUuTBHu +Dty1TIBY4KU1p77UaP8Z1T0ooGpDR0DT76a2N4RKPPUfuLujH/vVZdf97GZWn6v2 +86RnTpH5AgMBAAECggEAPxWRwHcIKRskB4P00RNbrLDoo2i4m0XqDAwg1d+YY9jd +AdW3HjXwrPhSsiKsZZZGveDUrGHpGapQvovePwveVndBcSVYjlG8+qw0oDjR3w7w +xfnE5FOexTWeoWFPTUYRAbspaeIH8cLojq7tJWs3ssevc9dvtPQBB62BaUcEUcsb +j0/TMpUYep0+DGixOmuGs0ekYnGFWgc4L2bRvdDvSRGdkhc3Ula4HSQA2T3f3OIb ++kg2VyY0mNkp79AuaRtz0DJ1OAULWPoZ5nf6s73FujgltIbanSg2OxjKn1NOaumZ ++WPrC5hjdVTnZ5OiAplWgw+UvKgdHtp8Km3FG2LZIQKBgQC2G5wY1wqvsq7lsHfT +6/dUD7A3B5hvDjseTfKpKtavhMp8t2Rd7nlFxVo/QQ54f4kHq+qrozCmJ3KtzYZK +Jb2XFcaEQ3d6Jqu2LnrNbhHMLBuj5k6YXEOvTr3aleI2m0NtJJ73eKp3hK9xfgV3 +FwgsTkLyI3AcElTFiuFvN5qX1wKBgQDMs83Rq9ZzlcPmUZndt10jDYaQC5NhPQMr +bO9q9HKNP9dNUvu4k3LZScpFK0ez6m9Q2PDZmXpiS6nvMR/4ZAmXDG256hTSfaiz +KxyYzWR0oDOV7xTrkpfUDi8Qs4KRcfy2u4ZPvaDSpy546mIIHaCQlpcIp83tur0P +NILiORSqrwKBgCl/+z+x6c6GYtMXNweFc7Slapvv0C4myRQr4Uvp5kjTcy2ewXGi +geKDigB2O+z762djJzR8GP8UaE7xUlQda6o9nSLRGS4uF92JlBSyq72io53jQy/3 +frk5sYxZsdaN5Xy+5rcwuXBJPY4YkPhFuGgYYas6pjbPeqAV+S/WNRW7AoGAMSUB +xtLQe5N7IrJ4lEhPbfjzU2XUDkZNmoFewXjKf6rSD9haYqfTrOMQUqbfYgxoiSps +OHGmdi7gL3LF0CaEPuVW+ol+UkKk287/4Jd/BJjpiZeKmDvrg5ecKRBPyodpOp2u +0zodgDVu7MyomHY0dEITJJrUz7xDY4Eh4xn9IjUCgYBBL6eIys12SHJLn5lOeKSo +3bjbHrzhEE4d9mxhzIjpO1yuqQ98SJYKrpSc1nOpY2CVBj/WKlAnvQHSxQ5/DYgT +qktMdwYHjijVsmJvJcR64H9QB4omeA+WIlxW+LG572lyhWQmWv6BKUbprjJ+jMV5 +cVV7fUk9RNCouvfY/BpfTA== +-----END PRIVATE KEY----- diff --git a/pangolin/pangolin_api/tests/fixtures/oidc_test_modulus.txt b/pangolin/pangolin_api/tests/fixtures/oidc_test_modulus.txt new file mode 100644 index 0000000..3e4b720 --- /dev/null +++ b/pangolin/pangolin_api/tests/fixtures/oidc_test_modulus.txt @@ -0,0 +1 @@ +kZ3oGzOCM4hgrqJEI5AqIEjnAj-V2zLm1-CqilrfNeUHq6ECanEYMNfRpAG3zBad3UzYOsaPN1tXk8UcTWrzG_dFZUOIJeP4UlB-fTMpQkwrwcWrllRsbGIir9wVe5y6wCC0YMLree40PReMxUl6P1PEO8nJuTLoX6KY-60Gf9IGCrzRypUQq4nirDgxDY114IkBl_L4k31aKfV-IJvNs4tqbPPBw-gIJ9looIyk1CrrNHeMu6nEtU9GlpSkuQq9tg-l7XE1LkwR7g7ctUyAWOClNae-1Gj_GdU9KKBqQ0dA0--mtjeESjz1H7i7ox_71WXX_exmVp-r9vOkZ06R-Q diff --git a/pangolin/pangolin_api/tests/full_system_test.rs b/pangolin/pangolin_api/tests/full_system_test.rs index d3cc45d..7f9e69b 100644 --- a/pangolin/pangolin_api/tests/full_system_test.rs +++ b/pangolin/pangolin_api/tests/full_system_test.rs @@ -24,7 +24,9 @@ async fn test_full_system_flow() { let app = app(store.clone()); // 2. Create Tenant with Initial Admin - let tenant_id = Uuid::new_v4(); + // + // The client-chosen id is gone: `CreateTenantRequest` never accepted one, + // so this only ever looked like it was being used. let admin_username = "system_admin"; let admin_password = "secure_password"; @@ -35,11 +37,11 @@ async fn test_full_system_flow() { .header("Authorization", "Basic YWRtaW46cGFzc3dvcmQ=") // root auth .body(Body::from( json!({ + // Only `name` and `properties` exist on + // `CreateTenantRequest`; the rest were silently discarded, so + // the tenant id below never matched what the server assigned. "name": "E2ETenant", - "id": tenant_id.to_string(), // In memory store usually ignores ID in payload but useful if supported - "properties": {}, - "admin_username": admin_username, - "admin_password": admin_password + "properties": {} }) .to_string(), )) @@ -103,7 +105,33 @@ async fn test_full_system_flow() { println!("Logged in as Tenant Admin. Token length: {}", token.len()); - // 4. Create Catalog + // 4. Create the warehouse the catalog references. + // + // This step did not exist. The catalog payload below named a warehouse + // called "default" under a *kebab-case* key the server does not read, so it + // was dropped, `warehouse_name` arrived as `None`, and the existence check + // was skipped entirely - the test passed while asserting nothing about + // warehouses. With the key corrected the check runs, and it needs a + // warehouse to find. + let create_warehouse_req = Request::builder() + .method("POST") + .uri("/api/v1/warehouses") + .header("Authorization", &auth_header) + .header("Content-Type", "application/json") + .body(Body::from( + json!({ + "name": "default", + "use_sts": false, + "storage_config": { "type": "filesystem", "root": "/tmp/pangolin-e2e" } + }) + .to_string(), + )) + .unwrap(); + + let resp = app.clone().oneshot(create_warehouse_req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::CREATED); + + // 5. Create Catalog // Note: We need to use the Tenant ID header for tenant-scoped operations let create_catalog_req = Request::builder() .method("POST") @@ -112,10 +140,17 @@ async fn test_full_system_flow() { .header("Content-Type", "application/json") .body(Body::from( json!({ + // Three phantom fields lived here: `type` (the field is + // `catalog_type`) and kebab-case `warehouse-name` / + // `storage-location` (the struct has no rename_all, so they are + // snake_case). All three were silently dropped, so this test + // asserted 201 for a catalog created with no warehouse and no + // storage location - the exact class of drift + // `deny_unknown_fields` now catches. "name": "data_catalog", - "type": "nessie", - "warehouse-name": "default", - "storage-location": "s3://bucket/data", + "catalog_type": "Local", + "warehouse_name": "default", + "storage_location": "s3://bucket/data", "properties": {} }) .to_string(), diff --git a/pangolin/pangolin_api/tests/iceberg_handlers_test.rs b/pangolin/pangolin_api/tests/iceberg_handlers_test.rs index ed59401..09623b2 100644 --- a/pangolin/pangolin_api/tests/iceberg_handlers_test.rs +++ b/pangolin/pangolin_api/tests/iceberg_handlers_test.rs @@ -28,6 +28,7 @@ fn test_add_snapshot_deserializes_full_object() { current_schema_id: 0, current_partition_spec_id: 0, partition_specs: vec![], + last_partition_id: 999, default_sort_order_id: 0, sort_orders: vec![], properties: None, @@ -77,6 +78,7 @@ fn test_add_snapshot_handles_multiple_snapshots() { current_schema_id: 0, current_partition_spec_id: 0, partition_specs: vec![], + last_partition_id: 999, default_sort_order_id: 0, sort_orders: vec![], properties: None, @@ -140,6 +142,7 @@ fn test_table_response_includes_config() { current_schema_id: 0, current_partition_spec_id: 0, partition_specs: vec![], + last_partition_id: 999, default_sort_order_id: 0, sort_orders: vec![], properties: None, diff --git a/pangolin/pangolin_api/tests/iceberg_missing_endpoints_tests.rs b/pangolin/pangolin_api/tests/iceberg_missing_endpoints_tests.rs new file mode 100644 index 0000000..4d44687 --- /dev/null +++ b/pangolin/pangolin_api/tests/iceberg_missing_endpoints_tests.rs @@ -0,0 +1,378 @@ +//! The Iceberg REST operations that had no route at all (A-5). +//! +//! `listViews`, `viewExists`, `dropView` and `registerTable` returned `404` for +//! the *route*, not for the resource — indistinguishable, from a client's point +//! of view, from a catalog that simply had no such view. Spark's `SHOW VIEWS`, +//! `DROP VIEW` and any migration from another catalog all need these. +//! +//! `commitTransaction` is deliberately still absent; see the note at the bottom +//! of this file. + +use axum::body::Body; +use axum::http::{header, Request, StatusCode}; +use pangolin_api::app; +use pangolin_api::tests_common::EnvGuard; +use pangolin_core::model::{Catalog, CatalogType, Namespace, Tenant}; +use pangolin_store::memory::MemoryStore; +use serial_test::serial; +use std::collections::HashMap; +use std::sync::Arc; +use tower::ServiceExt; +use uuid::Uuid; + +async fn setup() -> ( + Arc, + EnvGuard, +) { + let guard = EnvGuard::new("PANGOLIN_NO_AUTH", "true"); + let store = Arc::new(MemoryStore::new()) as Arc; + let tenant_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(); + + store + .create_tenant(Tenant { + id: tenant_id, + name: "t".to_string(), + properties: HashMap::new(), + }) + .await + .unwrap(); + + store + .create_catalog( + tenant_id, + Catalog { + id: Uuid::new_v4(), + name: "cat".to_string(), + catalog_type: CatalogType::Local, + warehouse_name: None, + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }, + ) + .await + .unwrap(); + + store + .create_namespace( + tenant_id, + "cat", + Namespace { + name: vec!["sales".to_string()], + properties: HashMap::new(), + }, + ) + .await + .unwrap(); + + (store, guard) +} + +fn json_request(method: &str, uri: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(uri) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body.to_string())) + .unwrap() +} + +async fn create_view(app: &axum::Router, name: &str) -> StatusCode { + app.clone() + .oneshot(json_request( + "POST", + "/v1/cat/namespaces/sales/views", + // `CreateViewRequest` is {name, sql, dialect?, properties?} and is + // `deny_unknown_fields`, so a spec-shaped `schema` is rejected. + serde_json::json!({ + "name": name, + "sql": "SELECT 1" + }), + )) + .await + .unwrap() + .status() +} + +#[tokio::test] +#[serial] +async fn list_views_returns_what_was_created() { + let (store, _guard) = setup().await; + let app = app(store); + + assert!( + create_view(&app, "daily_totals").await.is_success(), + "the view should be creatable" + ); + + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/v1/cat/namespaces/sales/views") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::OK, + "listViews had no route at all; a client could not discover any view" + ); + + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + + let identifiers = body + .get("identifiers") + .and_then(|v| v.as_array()) + .expect("the spec's response is {\"identifiers\": [...]}"); + assert!( + identifiers + .iter() + .any(|i| i.get("name").and_then(|n| n.as_str()) == Some("daily_totals")), + "the created view is missing from the listing: {body}" + ); +} + +#[tokio::test] +#[serial] +async fn view_exists_answers_without_a_body() { + let (store, _guard) = setup().await; + let app = app(store); + assert!(create_view(&app, "v1").await.is_success()); + + let present = app + .clone() + .oneshot( + Request::builder() + .method("HEAD") + .uri("/v1/cat/namespaces/sales/views/v1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(present.status(), StatusCode::NO_CONTENT); + + let absent = app + .clone() + .oneshot( + Request::builder() + .method("HEAD") + .uri("/v1/cat/namespaces/sales/views/nope") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(absent.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +#[serial] +async fn drop_view_removes_it() { + let (store, _guard) = setup().await; + let app = app(store); + assert!(create_view(&app, "doomed").await.is_success()); + + let dropped = app + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri("/v1/cat/namespaces/sales/views/doomed") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + dropped.status(), + StatusCode::NO_CONTENT, + "dropView had no route; a view created through the Iceberg API could \ + never be removed through it" + ); + + let after = app + .clone() + .oneshot( + Request::builder() + .method("HEAD") + .uri("/v1/cat/namespaces/sales/views/doomed") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + after.status(), + StatusCode::NOT_FOUND, + "the view survived its drop" + ); +} + +/// Dropping a *table* through the view endpoint must not work. +/// +/// Views and tables are both assets; without an explicit kind check, a caller +/// permitted to drop views could remove a table by addressing it as one. +#[tokio::test] +#[serial] +async fn drop_view_refuses_a_table() { + let (store, _guard) = setup().await; + + let tenant_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(); + store + .create_asset( + tenant_id, + "cat", + Some("main".to_string()), + vec!["sales".to_string()], + pangolin_core::model::Asset { + id: Uuid::new_v4(), + name: "real_table".to_string(), + kind: pangolin_core::model::AssetType::IcebergTable, + location: "s3://bucket/real_table".to_string(), + properties: HashMap::new(), + }, + ) + .await + .unwrap(); + + let app = app(store.clone()); + let response = app + .oneshot( + Request::builder() + .method("DELETE") + .uri("/v1/cat/namespaces/sales/views/real_table") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "a table must not be droppable through the view endpoint" + ); + + assert!( + store + .get_asset( + tenant_id, + "cat", + Some("main".to_string()), + vec!["sales".to_string()], + "real_table".to_string() + ) + .await + .unwrap() + .is_some(), + "the table was deleted through the view endpoint" + ); +} + +/// `registerTable` must reject a metadata location it cannot read. +/// +/// Registering a location that is not there would leave a table whose every +/// subsequent `loadTable` fails, with the failure surfacing far from the +/// request that caused it. +#[tokio::test] +#[serial] +async fn register_table_rejects_an_unreadable_location() { + let (store, _guard) = setup().await; + let app = app(store); + + let response = app + .oneshot(json_request( + "POST", + "/v1/cat/namespaces/sales/register", + serde_json::json!({ + "name": "adopted", + "metadata-location": "s3://nowhere/does-not-exist.metadata.json" + }), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "an unreadable metadata location must be refused at registration time" + ); +} + +/// The route exists at all — which is the thing A-5 was about. +/// +/// Before this, `POST .../register` was a routing 404, indistinguishable to a +/// client from a catalog that had rejected the request. +#[tokio::test] +#[serial] +async fn register_table_is_routed() { + let (store, _guard) = setup().await; + let app = app(store); + + let response = app + .oneshot(json_request( + "POST", + "/v1/cat/namespaces/sales/register", + serde_json::json!({ + "name": "adopted", + "metadata-location": "s3://nowhere/x.json" + }), + )) + .await + .unwrap(); + + assert_ne!( + response.status(), + StatusCode::METHOD_NOT_ALLOWED, + "the register route is not wired" + ); + // A 400 from the handler is a real answer; a 404 here would mean no route. + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +/// `commitTransaction` is still absent, on purpose. +/// +/// The spec's `POST /v1/{prefix}/transactions/commit` is a *multi-table atomic* +/// commit: either every table's metadata pointer moves or none does. Pangolin's +/// commit path does compare-and-swap per table through the store trait, and +/// there is no cross-table transaction behind it. +/// +/// Routing it and committing the tables one at a time would be worse than +/// leaving it unrouted. An engine that sees the endpoint will rely on the +/// atomicity the spec promises, and a partial failure would leave half a +/// multi-table change applied with no way to tell. A 404 makes the client fall +/// back to per-table commits, which is what actually happens today and is +/// honest about it. +/// +/// This test pins that decision so it is a choice rather than an oversight. +#[tokio::test] +#[serial] +async fn commit_transaction_is_deliberately_absent() { + let (store, _guard) = setup().await; + let app = app(store); + + let response = app + .oneshot(json_request( + "POST", + "/v1/cat/transactions/commit", + serde_json::json!({ "table-changes": [] }), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "commitTransaction must stay unrouted until the store can commit \ + several tables atomically. If this test fails because someone added \ + the route, check that it is genuinely atomic across tables." + ); +} diff --git a/pangolin/pangolin_api/tests/merge_tests.rs b/pangolin/pangolin_api/tests/merge_tests.rs index 374ed1e..1d76c56 100644 --- a/pangolin/pangolin_api/tests/merge_tests.rs +++ b/pangolin/pangolin_api/tests/merge_tests.rs @@ -33,10 +33,10 @@ async fn test_merge_branch_flow() { .header("Authorization", "Basic YWRtaW46cGFzc3dvcmQ=") // admin:password .body(Body::from( json!({ - "name": "MergeTestTenant", - "id": tenant_id.to_string(), - "admin_username": "merge_admin", - "admin_password": "password" + // `CreateTenantRequest` takes only `name` and `properties`. + // `id`, `admin_username` and `admin_password` were all silently + // discarded; `deny_unknown_fields` now names them. + "name": "MergeTestTenant" }) .to_string(), )) diff --git a/pangolin/pangolin_api/tests/oidc_validation_tests.rs b/pangolin/pangolin_api/tests/oidc_validation_tests.rs new file mode 100644 index 0000000..1edc870 --- /dev/null +++ b/pangolin/pangolin_api/tests/oidc_validation_tests.rs @@ -0,0 +1,340 @@ +//! `id_token` validation against a real OIDC provider. +//! +//! C-2/C-3. These stand up a fake provider with `wiremock` — a real discovery +//! document, a real JWKS, and tokens signed with a real 2048-bit RSA key — and +//! drive the actual validation path. Nothing here is mocked at the crypto +//! layer, because the properties under test *are* the crypto: a test that +//! stubbed out signature checking would pass against code that skipped them. +//! +//! Each test names the attack it prevents. An OIDC implementation that accepts +//! the tokens below is not an OIDC implementation; it is a decoder. + +use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; +use pangolin_api::oidc; +use serde::Serialize; +use serde_json::json; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const KEY_PEM: &str = include_str!("fixtures/oidc_test_key.pem"); +const MODULUS: &str = include_str!("fixtures/oidc_test_modulus.txt"); +const KID: &str = "test-key-1"; + +#[derive(Serialize)] +struct Claims { + sub: String, + iss: String, + aud: String, + exp: usize, + iat: usize, + #[serde(skip_serializing_if = "Option::is_none")] + nonce: Option, + #[serde(skip_serializing_if = "Option::is_none")] + email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + email_verified: Option, +} + +fn now() -> usize { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() as usize +} + +fn claims(issuer: &str, audience: &str, nonce: Option<&str>) -> Claims { + Claims { + sub: "user-subject-123".to_string(), + iss: issuer.to_string(), + aud: audience.to_string(), + exp: now() + 600, + iat: now(), + nonce: nonce.map(|n| n.to_string()), + email: Some("someone@example.com".to_string()), + email_verified: Some(true), + } +} + +fn sign(claims: &Claims, kid: &str) -> String { + let mut header = Header::new(Algorithm::RS256); + header.kid = Some(kid.to_string()); + encode( + &header, + claims, + &EncodingKey::from_rsa_pem(KEY_PEM.as_bytes()).expect("the fixture key is valid RSA PEM"), + ) + .expect("signing the test token") +} + +/// A provider serving discovery and a JWKS containing our test key. +async fn provider() -> MockServer { + let server = MockServer::start().await; + let issuer = server.uri(); + + Mock::given(method("GET")) + .and(path("/.well-known/openid-configuration")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "issuer": issuer, + "authorization_endpoint": format!("{issuer}/authorize"), + "token_endpoint": format!("{issuer}/token"), + "jwks_uri": format!("{issuer}/jwks"), + "code_challenge_methods_supported": ["S256"], + }))) + .mount(&server) + .await; + + Mock::given(method("GET")) + .and(path("/jwks")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "keys": [{ + "kty": "RSA", + "use": "sig", + "alg": "RS256", + "kid": KID, + "n": MODULUS.trim(), + "e": "AQAB", + }] + }))) + .mount(&server) + .await; + + server +} + +#[tokio::test] +async fn a_properly_signed_token_is_accepted() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + + let discovery = oidc::discover(&issuer).await.expect("discovery"); + assert!(discovery.supports_s256_pkce()); + + let token = sign(&claims(&issuer, "my-client", Some("the-nonce")), KID); + let validated = oidc::validate_id_token(&token, &discovery, "my-client", "the-nonce") + .await + .expect("a correctly signed and scoped token should validate"); + + assert_eq!(validated.sub, "user-subject-123"); + assert_eq!(validated.email.as_deref(), Some("someone@example.com")); + assert_eq!(validated.email_verified, Some(true)); +} + +/// A token minted for a *different* application at the same provider. +/// +/// This is the confused-deputy problem that makes `aud` validation +/// non-negotiable: without it, any site the user logs into with the same +/// provider can take the id_token it received and present it here. +#[tokio::test] +async fn a_token_for_another_audience_is_refused() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + let discovery = oidc::discover(&issuer).await.unwrap(); + + let token = sign(&claims(&issuer, "some-other-app", Some("n")), KID); + let err = oidc::validate_id_token(&token, &discovery, "my-client", "n") + .await + .expect_err("a token for another audience must be refused") + .to_string(); + + assert!( + err.contains("audience"), + "the rejection must be specifically about the audience - otherwise this \ + test would pass on any unrelated failure and prove nothing about \ + confused-deputy protection. Got: {err}" + ); +} + +/// A token from a different issuer, correctly signed by *that* issuer's key. +#[tokio::test] +async fn a_token_from_another_issuer_is_refused() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + let discovery = oidc::discover(&issuer).await.unwrap(); + + let token = sign( + &claims("https://attacker.example", "my-client", Some("n")), + KID, + ); + let err = oidc::validate_id_token(&token, &discovery, "my-client", "n") + .await + .expect_err("a token from another issuer must be refused") + .to_string(); + + assert!( + err.contains("issuer"), + "the rejection must be specifically about the issuer: {err}" + ); +} + +/// An id_token captured from one login and replayed into another. +#[tokio::test] +async fn a_replayed_token_is_refused() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + let discovery = oidc::discover(&issuer).await.unwrap(); + + // Signed for a login whose nonce was "first-login". + let token = sign(&claims(&issuer, "my-client", Some("first-login")), KID); + + // Presented to a login that issued "second-login". + let err = oidc::validate_id_token(&token, &discovery, "my-client", "second-login") + .await + .expect_err("a token bound to another login must be refused") + .to_string(); + + assert!( + err.contains("nonce"), + "the error should name the nonce mismatch: {err}" + ); +} + +/// A token with no `nonce` at all cannot be bound to a login. +#[tokio::test] +async fn a_token_without_a_nonce_is_refused() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + let discovery = oidc::discover(&issuer).await.unwrap(); + + let token = sign(&claims(&issuer, "my-client", None), KID); + let err = oidc::validate_id_token(&token, &discovery, "my-client", "expected") + .await + .expect_err("a token without a nonce must be refused") + .to_string(); + + assert!(err.contains("nonce"), "got: {err}"); +} + +#[tokio::test] +async fn an_expired_token_is_refused() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + let discovery = oidc::discover(&issuer).await.unwrap(); + + let mut expired = claims(&issuer, "my-client", Some("n")); + // Well past the 60s leeway. + expired.exp = now() - 3600; + let token = sign(&expired, KID); + + let err = oidc::validate_id_token(&token, &discovery, "my-client", "n") + .await + .expect_err("an expired token must be refused") + .to_string(); + + assert!( + err.contains("expired"), + "the rejection must be specifically about expiry: {err}" + ); +} + +/// A token whose `kid` is not in the provider's JWKS. +/// +/// Also exercises the rate-limited refetch: an unknown `kid` triggers one +/// refetch, and a flood of them must not hammer the provider. +#[tokio::test] +async fn a_token_signed_with_an_unknown_key_is_refused() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + let discovery = oidc::discover(&issuer).await.unwrap(); + + let token = sign(&claims(&issuer, "my-client", Some("n")), "some-other-kid"); + let err = oidc::validate_id_token(&token, &discovery, "my-client", "n") + .await + .expect_err("a token signed with an unpublished key must be refused") + .to_string(); + + assert!( + err.contains("kid") || err.contains("signing key"), + "got: {err}" + ); +} + +/// A discovery document whose `issuer` disagrees with where it was fetched. +/// +/// Without this check, `iss` validation is circular: the attacker supplies the +/// document *and* the token, so both agree. +#[tokio::test] +async fn discovery_with_a_mismatched_issuer_is_refused() { + oidc::clear_caches(); + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/.well-known/openid-configuration")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "issuer": "https://accounts.google.com", + "authorization_endpoint": "https://accounts.google.com/authorize", + "token_endpoint": "https://accounts.google.com/token", + "jwks_uri": "https://accounts.google.com/jwks", + }))) + .mount(&server) + .await; + + let err = oidc::discover(&server.uri()) + .await + .expect_err("a document claiming to be another issuer must be refused") + .to_string(); + + assert!(err.contains("mismatch"), "got: {err}"); +} + +/// The discovery document is cached, so a login does not fetch it every time. +#[tokio::test] +async fn discovery_is_cached() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + + oidc::discover(&issuer).await.unwrap(); + oidc::discover(&issuer).await.unwrap(); + oidc::discover(&issuer).await.unwrap(); + + let discovery_requests = server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path().contains("openid-configuration")) + .count(); + + assert_eq!( + discovery_requests, 1, + "discovery should be fetched once and cached; fetching per login adds a \ + provider round trip to every authentication" + ); +} + +/// The JWKS is cached across validations. +#[tokio::test] +async fn the_jwks_is_cached_across_validations() { + oidc::clear_caches(); + let server = provider().await; + let issuer = server.uri(); + let discovery = oidc::discover(&issuer).await.unwrap(); + + for _ in 0..5 { + let token = sign(&claims(&issuer, "my-client", Some("n")), KID); + oidc::validate_id_token(&token, &discovery, "my-client", "n") + .await + .unwrap(); + } + + let jwks_requests = server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path().contains("jwks")) + .count(); + + assert_eq!( + jwks_requests, 1, + "the JWKS should be fetched once; fetching per validation puts an \ + outbound request on every login" + ); +} diff --git a/pangolin/pangolin_api/tests/signing_handlers_test.rs b/pangolin/pangolin_api/tests/signing_handlers_test.rs index 246507a..a838f0d 100644 --- a/pangolin/pangolin_api/tests/signing_handlers_test.rs +++ b/pangolin/pangolin_api/tests/signing_handlers_test.rs @@ -338,6 +338,7 @@ fn test_table_response_includes_credentials() { schemas: vec![], current_schema_id: 0, partition_specs: vec![], + last_partition_id: 999, current_partition_spec_id: 0, properties: None, current_snapshot_id: None, @@ -394,6 +395,7 @@ fn test_table_response_without_credentials() { schemas: vec![], current_schema_id: 0, partition_specs: vec![], + last_partition_id: 999, current_partition_spec_id: 0, properties: None, current_snapshot_id: None, diff --git a/pangolin/pangolin_cli_admin/Cargo.toml b/pangolin/pangolin_cli_admin/Cargo.toml index 396535b..ad6507b 100644 --- a/pangolin/pangolin_cli_admin/Cargo.toml +++ b/pangolin/pangolin_cli_admin/Cargo.toml @@ -15,7 +15,7 @@ categories = ["database"] workspace = true [dependencies] -pangolin_cli_common = { path = "../pangolin_cli_common", version = "0.6.0" } +pangolin_cli_common = { path = "../pangolin_cli_common", version = "0.7.0" } clap = { version = "4.4", features = ["derive", "env"] } tokio = { version = "1.0", features = ["full"] } rustyline = "14.0" diff --git a/pangolin/pangolin_cli_admin/src/handlers/catalogs.rs b/pangolin/pangolin_cli_admin/src/handlers/catalogs.rs index 7245886..12a2bdc 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/catalogs.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/catalogs.rs @@ -47,10 +47,16 @@ pub async fn handle_create_catalog( )); } + // B_cli1: this sent `warehouse` and `type`. `CreateCatalogRequest` has + // neither - it takes `warehouse_name` and `catalog_type` - and serde + // ignored both unknown fields, so `warehouse_name` arrived as `None` (which + // *skips the warehouse existence check*) and `catalog_type` fell back to + // `Local`. The required `--warehouse` flag was thrown away and the CLI + // printed success for a catalog with no warehouse attached. let body = serde_json::json!({ "name": name, - "warehouse": warehouse, - "type": "pangea" // defaulting to internal type + "warehouse_name": warehouse, + "catalog_type": "Local" }); let res = client.post("/api/v1/catalogs", &body).await?; diff --git a/pangolin/pangolin_cli_admin/src/handlers/federated.rs b/pangolin/pangolin_cli_admin/src/handlers/federated.rs index 960bd8a..f9ccf8f 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/federated.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/federated.rs @@ -9,7 +9,7 @@ pub async fn handle_sync_federated_catalog( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/catalogs/{}/sync", name), + &format!("/api/v1/federated-catalogs/{}/sync", name), &serde_json::json!({}), ) .await?; @@ -25,7 +25,7 @@ pub async fn handle_get_federated_catalog_stats( name: String, ) -> Result<(), CliError> { let res = client - .get(&format!("/api/v1/catalogs/{}/stats", name)) + .get(&format!("/api/v1/federated-catalogs/{}/stats", name)) .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!( @@ -60,14 +60,20 @@ pub async fn handle_create_federated_catalog( } } + // B_cli3: this POSTed to `/api/v1/catalogs` with a `type` field that + // `CreateCatalogRequest` does not have, so serde dropped it and the command + // created an ordinary *Local* catalog while reporting a federated one. The + // dedicated endpoint takes the config the federated proxy actually reads. + props.insert( + "storage_location".to_string(), + serde_json::Value::String(storage_location.clone()), + ); let body = serde_json::json!({ "name": name, - "type": "federated", - "storage_location": storage_location, - "properties": props + "config": { "properties": props } }); - let res = client.post("/api/v1/catalogs", &body).await?; + let res = client.post("/api/v1/federated-catalogs", &body).await?; if !res.status().is_success() { let s = res.status(); let t = res.text().await.unwrap_or_default(); @@ -100,10 +106,13 @@ pub async fn handle_list_federated_catalogs( .await .map_err(|e| CliError::ApiError(e.to_string()))?; - // Filter client side just in case API doesn't support query param perfectly yet or to be safe + // B_cli3: this filtered on `i["type"] == "federated"`. The response key is + // `catalog_type` and its value is `"Federated"` - so the predicate never + // matched and `list-federated-catalogs` always printed an empty table, even + // with federated catalogs present. let fed_cats: Vec> = items .iter() - .filter(|i| i["type"].as_str() == Some("federated")) + .filter(|i| i["catalog_type"].as_str() == Some("Federated")) .map(|i| { vec![ i["name"].as_str().unwrap_or("").to_string(), @@ -139,7 +148,7 @@ pub async fn handle_test_federated_catalog( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/catalogs/{}/test", name), + &format!("/api/v1/federated-catalogs/{}/test", name), &serde_json::json!({}), ) .await?; diff --git a/pangolin/pangolin_cli_admin/src/handlers/governance.rs b/pangolin/pangolin_cli_admin/src/handlers/governance.rs index 240a05e..0efd70b 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/governance.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/governance.rs @@ -132,7 +132,12 @@ async fn resolve_scope( .await .map_err(|e| CliError::ApiError(e.to_string()))?; if let Some(c) = catalogs.iter().find(|c| c["name"].as_str() == Some(path)) { - let id = c["id"].as_str().unwrap().to_string(); + // B_cli5: `.unwrap()` here panicked the whole CLI whenever the + // server omitted `id` - a crash rather than an error message. + let id = c["id"] + .as_str() + .ok_or_else(|| CliError::ApiError(format!("Catalog '{}' has no id", path)))? + .to_string(); Ok(serde_json::json!({ "type": "catalog", "catalog_id": id @@ -160,7 +165,12 @@ async fn resolve_scope( .iter() .find(|c| c["name"].as_str() == Some(cat_name)) { - let id = c["id"].as_str().unwrap().to_string(); + // B_cli5: `.unwrap()` here panicked the whole CLI whenever the + // server omitted `id` - a crash rather than an error message. + let id = c["id"] + .as_str() + .ok_or_else(|| CliError::ApiError(format!("Catalog '{}' has no id", path)))? + .to_string(); Ok(serde_json::json!({ "type": "namespace", "catalog_id": id, @@ -185,7 +195,10 @@ async fn resolve_scope( .iter() .find(|c| c["name"].as_str() == Some(cat_name)) { - c["id"].as_str().unwrap().to_string() + c["id"] + .as_str() + .ok_or_else(|| CliError::ApiError("Catalog has no id".to_string()))? + .to_string() } else { return Err(CliError::ApiError(format!( "Catalog '{}' not found", @@ -283,11 +296,46 @@ pub async fn handle_revoke_permission( action: String, resource: String, ) -> Result<(), CliError> { - let url = format!( - "/api/v1/permissions?role={}&action={}&resource={}", - role, action, resource - ); - let res = client.delete(&url).await?; + // B_cli5: this issued `DELETE /api/v1/permissions?role=..&action=..&resource=..`. + // The only delete route is `DELETE /api/v1/permissions/{id}` - there is no + // filter form - so the request 405'd and no permission was ever revoked. + // The grant has to be located first. + let res = client.get("/api/v1/permissions").await?; + if !res.status().is_success() { + return Err(CliError::ApiError(format!("Error: {}", res.status()))); + } + let permissions: Vec = res + .json() + .await + .map_err(|e| CliError::ApiError(e.to_string()))?; + + let target = permissions.iter().find(|p| { + let actions_match = p["actions"] + .as_array() + .map(|a| a.iter().any(|v| v.as_str() == Some(action.as_str()))) + .unwrap_or(false); + let scope_matches = p["scope"]["type"].as_str() == Some(resource.as_str()) + || p["scope"]["catalog_id"].as_str() == Some(resource.as_str()) + || p["scope"]["namespace"].as_str() == Some(resource.as_str()) + || p["scope"]["asset_id"].as_str() == Some(resource.as_str()) + || p["scope"]["tag_name"].as_str() == Some(resource.as_str()); + actions_match && scope_matches + }); + + let Some(permission) = target else { + return Err(CliError::ApiError(format!( + "No permission found granting '{}' on '{}' (role filter: {})", + action, resource, role + ))); + }; + + let id = permission["id"] + .as_str() + .ok_or_else(|| CliError::ApiError("Permission has no id".to_string()))?; + + let res = client + .delete(&format!("/api/v1/permissions/{}", id)) + .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!("Error: {}", res.status()))); } @@ -464,11 +512,17 @@ pub async fn handle_request_access( // So permission arg is NOT in main.rs invocation for RequestAccess command. // I should remove `permission` arg from here. ) -> Result<(), CliError> { - let payload = serde_json::json!({ - "asset_id": asset_id, - "reason": reason - }); - let res = client.post("/api/v1/access-requests", &payload).await?; + // B_cli5: this POSTed to `/api/v1/access-requests`, which is registered for + // GET only - creating a request lives under the asset + // (`POST /api/v1/assets/{id}/access-requests`), and the asset id belongs in + // the path rather than the body. + let payload = serde_json::json!({ "reason": reason }); + let res = client + .post( + &format!("/api/v1/assets/{}/access-requests", asset_id), + &payload, + ) + .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!("Failed: {}", res.status()))); } diff --git a/pangolin/pangolin_cli_admin/src/handlers/merge.rs b/pangolin/pangolin_cli_admin/src/handlers/merge.rs index 4ee93d5..5b3de51 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/merge.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/merge.rs @@ -18,7 +18,15 @@ pub async fn handle_list_merge_operations( query.push_str(&pag); } - let res = client.get(&format!("/api/v1/merges?{}", query)).await?; + // B_cli2: every merge command targeted `/api/v1/merges/...`, a prefix + // the router has never registered - all six 404'd. Listing is scoped by + // catalog in the path, not by a query parameter. + let res = client + .get(&format!( + "/api/v1/catalogs/{}/merge-operations?{}", + catalog, query + )) + .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!("Error: {}", res.status()))); } @@ -58,7 +66,9 @@ pub async fn handle_get_merge_operation( client: &PangolinClient, id: String, ) -> Result<(), CliError> { - let res = client.get(&format!("/api/v1/merges/{}", id)).await?; + let res = client + .get(&format!("/api/v1/merge-operations/{}", id)) + .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!( "Failed to get merge operation: {}", @@ -97,9 +107,9 @@ pub async fn handle_list_merge_conflicts( ) -> Result<(), CliError> { let q = pangolin_cli_common::utils::pagination_query(limit, offset); let path = if q.is_empty() { - format!("/api/v1/merges/{}/conflicts", id) + format!("/api/v1/merge-operations/{}/conflicts", id) } else { - format!("/api/v1/merges/{}/conflicts?{}", id, q) + format!("/api/v1/merge-operations/{}/conflicts?{}", id, q) }; let res = client.get(&path).await?; if !res.status().is_success() { @@ -176,7 +186,7 @@ pub async fn handle_resolve_merge_conflict( let res = client .post( - &format!("/api/v1/merges/conflicts/{}/resolve", conflict_id), + &format!("/api/v1/conflicts/{}/resolve", conflict_id), &payload, ) .await?; @@ -198,7 +208,7 @@ pub async fn handle_complete_merge_operation( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/merges/{}/complete", id), + &format!("/api/v1/merge-operations/{}/complete", id), &serde_json::json!({}), ) .await?; @@ -220,7 +230,7 @@ pub async fn handle_abort_merge_operation( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/merges/{}/abort", id), + &format!("/api/v1/merge-operations/{}/abort", id), &serde_json::json!({}), ) .await?; diff --git a/pangolin/pangolin_cli_admin/src/handlers/tokens.rs b/pangolin/pangolin_cli_admin/src/handlers/tokens.rs index e524e7e..83a42ac 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/tokens.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/tokens.rs @@ -5,7 +5,10 @@ use serde_json::Value; pub async fn handle_revoke_token(client: &PangolinClient) -> Result<(), CliError> { let res = client - .post("/api/v1/tokens/revoke", &serde_json::json!({})) + // B_cli4: `/api/v1/tokens/revoke` does not exist; the routes are under + // `/api/v1/auth/`. Both revocation commands 404'd, so the documented + // logout path never worked from the admin CLI. + .post("/api/v1/auth/revoke", &serde_json::json!({})) .await?; if !res.status().is_success() { @@ -32,7 +35,7 @@ pub async fn handle_revoke_token_by_id( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/tokens/revoke/{}", id), + &format!("/api/v1/auth/revoke/{}", id), &serde_json::json!({}), ) .await?; diff --git a/pangolin/pangolin_cli_admin/src/handlers/users.rs b/pangolin/pangolin_cli_admin/src/handlers/users.rs index 67ec2d3..e0e0036 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/users.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/users.rs @@ -5,9 +5,29 @@ use pangolin_cli_common::utils::print_table; use serde_json::Value; pub async fn handle_delete_user(client: &PangolinClient, username: String) -> Result<(), CliError> { - let res = client - .delete(&format!("/api/v1/users/{}", username)) - .await?; + // B_cli5: this interpolated the *username* into a route whose handler takes + // `Path`, so every `delete-user` returned 400 before reaching any + // logic. The CLI takes a username because that is what an operator knows, + // so resolve it to an id first. + let listing = client.get("/api/v1/users").await?; + if !listing.status().is_success() { + return Err(CliError::ApiError(format!( + "Failed to look up user: {}", + listing.status() + ))); + } + let users: Vec = listing + .json() + .await + .map_err(|e| CliError::ApiError(e.to_string()))?; + + let user_id = users + .iter() + .find(|u| u["username"].as_str() == Some(username.as_str())) + .and_then(|u| u["id"].as_str()) + .ok_or_else(|| CliError::ApiError(format!("User '{}' not found", username)))?; + + let res = client.delete(&format!("/api/v1/users/{}", user_id)).await?; if !res.status().is_success() { let status = res.status(); let t = res.text().await.unwrap_or_default(); @@ -96,8 +116,15 @@ pub async fn handle_update_user( ) -> Result<(), CliError> { let mut payload = serde_json::json!({}); - if let Some(u) = username { - payload["username"] = serde_json::Value::String(u); + // B_cli6: `--username` was accepted and written into the payload, but + // `UpdateUserRequest` has no `username` field - the server cannot rename a + // user - so serde dropped it and the CLI reported success for a rename that + // never happened. Saying so is better than pretending. + if username.is_some() { + return Err(CliError::ApiError( + "Renaming a user is not supported by the server; --username is not applied." + .to_string(), + )); } if let Some(e) = email { diff --git a/pangolin/pangolin_cli_admin/src/main.rs b/pangolin/pangolin_cli_admin/src/main.rs index 610da61..a1b1a90 100644 --- a/pangolin/pangolin_cli_admin/src/main.rs +++ b/pangolin/pangolin_cli_admin/src/main.rs @@ -33,7 +33,20 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); // Load saved config - let config_manager = ConfigManager::new(args.profile.as_deref()).unwrap(); + // B_cli7: this was `.unwrap()`, so with neither $HOME nor $XDG_CONFIG_HOME + // set - a container, a systemd unit, a CI runner - the admin CLI panicked + // with a backtrace instead of saying what was wrong. The user CLI already + // used `?` here. + let config_manager = match ConfigManager::new(args.profile.as_deref()) { + Ok(m) => m, + Err(e) => { + eprintln!( + "Could not locate a config directory: {e}\n\ + Set $HOME or $XDG_CONFIG_HOME, or pass --url and --token." + ); + std::process::exit(1); + } + }; let mut config = config_manager.load().unwrap_or_default(); // Override URL if provided via Args or Env @@ -307,6 +320,10 @@ async fn main() -> anyhow::Result<()> { handlers::merge::handle_list_merge_conflicts(&client, merge_id, limit, offset) .await? } + // B_cli6: `--merge-id` is accepted and discarded. The resolve + // route is keyed on the conflict alone (`/api/v1/conflicts/{id}/resolve`), + // so the flag is genuinely redundant rather than lost - noted here + // so the discard is deliberate rather than a silent drop. AdminCommand::ResolveConflict { merge_id: _, conflict_id, @@ -429,7 +446,17 @@ async fn main() -> anyhow::Result<()> { AdminCommand::ListNamespaceTree { catalog } => { handlers::explorer::handle_namespace_tree(&client, catalog).await? } - _ => println!("Command not available in non-interactive mode."), + // B_cli6/B_cli7: this printed a note and fell through to + // `return Ok(())`, so a command the non-interactive path cannot run + // exited 0 - indistinguishable from success to any script or CI job. + // (`assign-role` and `revoke-user-role` land here.) + other => { + eprintln!( + "Command {:?} is only available in interactive mode.", + std::mem::discriminant(&other) + ); + std::process::exit(2); + } } return Ok(()); } diff --git a/pangolin/pangolin_cli_common/src/client.rs b/pangolin/pangolin_cli_common/src/client.rs index c85629d..c369f44 100644 --- a/pangolin/pangolin_cli_common/src/client.rs +++ b/pangolin/pangolin_cli_common/src/client.rs @@ -9,12 +9,25 @@ pub struct PangolinClient { pub config: CliConfig, } +/// Default per-request timeout. +/// +/// B_cli7: `Client::new()` has no timeout, so a hung or unreachable server left +/// the CLI blocked indefinitely with no output and no way out but Ctrl-C. +const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); +const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + impl PangolinClient { pub fn new(config: CliConfig) -> Self { - Self { - client: Client::new(), - config, - } + let client = Client::builder() + .timeout(REQUEST_TIMEOUT) + .connect_timeout(CONNECT_TIMEOUT) + .build() + // The builder only fails on a TLS backend problem, which would make + // every request fail anyway; falling back keeps `new` infallible for + // its callers. + .unwrap_or_else(|_| Client::new()); + + Self { client, config } } pub fn update_config(&mut self, config: CliConfig) { diff --git a/pangolin/pangolin_cli_common/src/config.rs b/pangolin/pangolin_cli_common/src/config.rs index 7fddd86..5646876 100644 --- a/pangolin/pangolin_cli_common/src/config.rs +++ b/pangolin/pangolin_cli_common/src/config.rs @@ -60,8 +60,22 @@ impl ConfigManager { .map_err(|e| CliError::ConfigError(e.to_string()))?; if let Some(parent) = self.config_path.parent() { fs::create_dir_all(parent)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = fs::set_permissions(parent, fs::Permissions::from_mode(0o700)); + } } fs::write(&self.config_path, content)?; + + // B_cli7: this file holds the auth token and was written at the process + // umask - typically 0644 - so any local account could read it. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&self.config_path, fs::Permissions::from_mode(0o600))?; + } + Ok(()) } } diff --git a/pangolin/pangolin_cli_user/Cargo.toml b/pangolin/pangolin_cli_user/Cargo.toml index d6c38e2..fa97ed4 100644 --- a/pangolin/pangolin_cli_user/Cargo.toml +++ b/pangolin/pangolin_cli_user/Cargo.toml @@ -15,7 +15,7 @@ categories = ["database"] workspace = true [dependencies] -pangolin_cli_common = { path = "../pangolin_cli_common", version = "0.6.0" } +pangolin_cli_common = { path = "../pangolin_cli_common", version = "0.7.0" } clap = { version = "4.4", features = ["derive", "env"] } tokio = { version = "1.0", features = ["full"] } rustyline = "14.0" diff --git a/pangolin/pangolin_cli_user/src/handlers.rs b/pangolin/pangolin_cli_user/src/handlers.rs index 5ec6403..66cef4f 100644 --- a/pangolin/pangolin_cli_user/src/handlers.rs +++ b/pangolin/pangolin_cli_user/src/handlers.rs @@ -77,11 +77,60 @@ pub async fn handle_list_catalogs( Ok(()) } -pub async fn handle_search(_client: &PangolinClient, query: String) -> Result<(), CliError> { - // Placeholder - Search API not fully implemented yet in handlers.rs of common? - // We will just mock it for now or hit the real endpoint if available - println!("Searching for '{}'...", query); - println!("(Search functionality pending backend index implementation)"); +/// Search assets. +/// +/// B_cli8: this printed "(Search functionality pending backend index +/// implementation)" and returned. Two search endpoints have existed the whole +/// time; the placeholder simply told users the feature was missing. +pub async fn handle_search(client: &PangolinClient, query: String) -> Result<(), CliError> { + let res = client + .get(&format!( + "/api/v1/assets/search?query={}", + // Percent-encode the term so a query containing & or = does not + // split into extra parameters. + query + .chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(), + other => other + .to_string() + .bytes() + .map(|b| format!("%{:02X}", b)) + .collect::(), + }) + .collect::() + )) + .await?; + + if !res.status().is_success() { + let status = res.status(); + let text = res.text().await.unwrap_or_default(); + return Err(CliError::ApiError(format!("{} - {}", status, text))); + } + + let results: Vec = res + .json() + .await + .map_err(|e| CliError::ApiError(e.to_string()))?; + + if results.is_empty() { + println!("No assets matched '{}'.", query); + return Ok(()); + } + + let rows: Vec> = results + .iter() + .map(|r| { + vec![ + r["name"].as_str().unwrap_or("-").to_string(), + r["kind"].as_str().unwrap_or("-").to_string(), + r["catalog"].as_str().unwrap_or("-").to_string(), + r["namespace"].as_str().unwrap_or("-").to_string(), + ] + }) + .collect(); + + print_table(vec!["Name", "Type", "Catalog", "Namespace"], rows); Ok(()) } @@ -91,11 +140,11 @@ pub async fn handle_generate_code( table: String, ) -> Result<(), CliError> { let url = &client.config.base_url; - let token = client - .config - .auth_token - .as_deref() - .unwrap_or(""); + // B_cli8: this interpolated the *live* JWT into the generated snippet, which + // is explicitly copy-paste output - it went into scrollback, into whatever + // the user pasted it into, and often into a committed file. A placeholder is + // what a code sample should carry; the reader substitutes their own. + let token = ""; let parts: Vec<&str> = table.split('.').collect(); let (catalog, namespace, table_name) = if parts.len() == 3 { @@ -277,10 +326,14 @@ pub async fn handle_merge_branch( source: String, target: String, ) -> Result<(), CliError> { + // B_cli8: this sent `source`/`target`. `MergeBranchRequest` takes + // `source_branch`/`target_branch`, so the request 422'd - and with + // `deny_unknown_fields` on the server it now says which field is wrong + // rather than reporting a missing one. let body = serde_json::json!({ "catalog": catalog, - "source": source, - "target": target + "source_branch": source, + "target_branch": target }); let res = client.post("/api/v1/branches/merge", &body).await?; if !res.status().is_success() { @@ -381,12 +434,33 @@ pub async fn handle_list_requests( Ok(()) } +/// Request access to an asset. +/// +/// B_cli8: the whole body was `Ok(())`. The command accepted its arguments, +/// contacted nothing, and returned success - so a user could "request access", +/// see no error, and wait indefinitely for a review of a request that was never +/// created. pub async fn handle_request_access( - _client: &PangolinClient, - _resource: String, + client: &PangolinClient, + resource: String, _role: String, - _reason: String, + reason: String, ) -> Result<(), CliError> { + let body = serde_json::json!({ "reason": reason }); + let res = client + .post( + &format!("/api/v1/assets/{}/access-requests", resource), + &body, + ) + .await?; + + if !res.status().is_success() { + let status = res.status(); + let text = res.text().await.unwrap_or_default(); + return Err(CliError::ApiError(format!("{} - {}", status, text))); + } + + println!("✅ Access request submitted for asset {}.", resource); Ok(()) } @@ -396,13 +470,23 @@ pub async fn handle_get_token( description: String, expires_in: u32, ) -> Result<(), CliError> { + // B_cli8: this sent `description`, which `GenerateTokenRequest` does not + // have (it was silently dropped), and passed `client.config.tenant_id` + // straight through - which is `None` when no tenant is configured, so the + // server received `"tenant_id": null` and rejected it with an unhelpful + // parse error. `tenant_id` is required, so say so up front. + let _ = description; + + let Some(tenant_id) = client.config.tenant_id.clone() else { + return Err(CliError::ApiError( + "No tenant selected. Set one with `pangolin config set-tenant `.".to_string(), + )); + }; + let body = serde_json::json!({ - "description": description, - "expires_in_hours": expires_in * 24, // Endpoint expects hours? Wait, previous code said expires_in_days but endpoint is hours? - // token_handlers.rs uses `payload.expires_in_hours.unwrap_or(24)`. - // The CLI arg is `expires_in` (u32). The payload key in `handle_get_token` was `expires_in_days`. - // Let's check GenerateTokenRequest struct in `token_handlers.rs`. - "tenant_id": client.config.tenant_id, + // The server's field is `expires_in_hours`; the CLI flag is a day count. + "expires_in_hours": u64::from(expires_in) * 24, + "tenant_id": tenant_id, "username": client.config.username }); diff --git a/pangolin/pangolin_core/src/audit.rs b/pangolin/pangolin_core/src/audit.rs index 2091388..1f3ce09 100644 --- a/pangolin/pangolin_core/src/audit.rs +++ b/pangolin/pangolin_core/src/audit.rs @@ -206,6 +206,55 @@ pub enum AuditResult { Failure, } +/// Serialize an audit enum to the spelling that goes into storage. +/// +/// B22: the SQLite backend persisted `format!("{:?}", action)` - the Debug +/// spelling, `"CreateBranch"` - and then read it back by lowercasing to +/// `"createbranch"` and deserializing against serde's snake_case +/// (`"create_branch"`). Nothing matched, and the result was +/// `.unwrap_or(AuditAction::CreateCatalog)`, so nearly every multi-word action +/// in the SQLite audit trail was recorded as `CreateCatalog`. The audit log is +/// the one artefact that has to be right after an incident. +/// +/// Both directions now go through serde, so the write and read spellings cannot +/// drift apart again. +pub fn audit_enum_to_stored(value: &T) -> String { + serde_json::to_value(value) + .ok() + .and_then(|v| v.as_str().map(str::to_owned)) + .unwrap_or_else(|| format!("{:?}", value)) +} + +/// Parse an audit enum from its stored spelling. +/// +/// Accepts the canonical serde name and, for rows written before B22 was fixed, +/// the legacy Debug name. An unrecognised value is an error rather than a silent +/// substitution. +pub fn audit_enum_from_stored(stored: &str) -> Result +where + T: serde::de::DeserializeOwned, +{ + // Canonical snake_case. + if let Ok(v) = serde_json::from_value::(serde_json::Value::String(stored.to_string())) { + return Ok(v); + } + + // Legacy Debug spelling: "CreateBranch" -> "create_branch". + let mut snake = String::with_capacity(stored.len() + 4); + for (i, ch) in stored.chars().enumerate() { + if ch.is_ascii_uppercase() { + if i != 0 { + snake.push('_'); + } + snake.push(ch.to_ascii_lowercase()); + } else { + snake.push(ch); + } + } + serde_json::from_value::(serde_json::Value::String(snake)) + .map_err(|_| format!("unknown audit enum value {stored:?}")) +} + /// Filter for querying audit logs #[derive(Debug, Clone, Deserialize)] #[cfg_attr(feature = "utoipa", derive(ToSchema))] diff --git a/pangolin/pangolin_core/src/iceberg_metadata.rs b/pangolin/pangolin_core/src/iceberg_metadata.rs index f925e58..51003fb 100644 --- a/pangolin/pangolin_core/src/iceberg_metadata.rs +++ b/pangolin/pangolin_core/src/iceberg_metadata.rs @@ -3,6 +3,10 @@ use std::collections::HashMap; use utoipa::ToSchema; use uuid::Uuid; +/// Partition field ids are assigned from 1000 upward by the Iceberg spec, so an +/// unpartitioned table's highest *assigned* partition id is 999. +pub const PARTITION_FIELD_ID_START: i32 = 1000; + #[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] #[serde(rename_all = "kebab-case")] pub struct TableMetadata { @@ -14,8 +18,25 @@ pub struct TableMetadata { pub last_column_id: i32, pub current_schema_id: i32, pub schemas: Vec, + /// The default partition spec id. + /// + /// The spec field is `default-spec-id`. Under the struct's kebab-case rule + /// this serialized as `current-partition-spec-id` (B11), which no + /// spec-conformant reader looks for - metadata Pangolin wrote could not be + /// read as v2 metadata by an external engine reading the file directly, and + /// a conformant engine's metadata could not round-trip in. The alias keeps + /// already-written Pangolin files parseable. + #[serde(rename = "default-spec-id", alias = "current-partition-spec-id")] pub current_partition_spec_id: i32, pub partition_specs: Vec, + /// Highest assigned partition field id. + /// + /// Required by the v2 spec and missing entirely before (B12); Java-based + /// readers reject metadata without it. Defaulted on read so files Pangolin + /// wrote earlier still parse, and recomputed from `partition_specs` by + /// [`TableMetadata::recompute_last_partition_id`]. + #[serde(default = "default_last_partition_id")] + pub last_partition_id: i32, pub default_sort_order_id: i32, pub sort_orders: Vec, pub properties: Option>, @@ -32,6 +53,28 @@ pub struct TableMetadata { pub refs: Option>, } +fn default_last_partition_id() -> i32 { + PARTITION_FIELD_ID_START - 1 +} + +impl TableMetadata { + /// Recompute `last_partition_id` from the partition specs. + /// + /// Called after any change to `partition_specs` so the field stays true; + /// the spec defines it as the highest partition field id ever assigned, so + /// it only ever moves up. + pub fn recompute_last_partition_id(&mut self) { + let highest = self + .partition_specs + .iter() + .flat_map(|spec| spec.fields.iter()) + .map(|f| f.field_id) + .max() + .unwrap_or(PARTITION_FIELD_ID_START - 1); + self.last_partition_id = self.last_partition_id.max(highest); + } +} + /// A named branch or tag pointing at a snapshot. #[derive(Debug, Serialize, Deserialize, Clone, ToSchema, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] @@ -51,11 +94,79 @@ pub struct SnapshotReference { #[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] #[serde(rename_all = "kebab-case")] pub struct Schema { + /// Always `"struct"`. + /// + /// Spec schemas *are* struct types and conformant writers emit this; it was + /// missing entirely (B14), so strict readers rejected the schema object. + #[serde(rename = "type", default = "struct_type_name")] + pub type_: String, + /// Defaulted because a `createTable` request body may legitimately omit it - + /// the server assigns the id. Without the default, deserializing an incoming + /// schema (which `create_table` now does instead of hand-parsing it, B16f) + /// would reject a spec-legal request. + #[serde(default)] pub schema_id: i32, + /// Omitted when absent rather than written as an explicit `null` (B14) - + /// some strict parsers reject `"identifier-field-ids": null`. + #[serde(default, skip_serializing_if = "Option::is_none")] pub identifier_field_ids: Option>, pub fields: Vec, } +fn struct_type_name() -> String { + Schema::STRUCT.to_string() +} + +impl Schema { + /// The only legal value of a schema's `type` field. + pub const STRUCT: &'static str = "struct"; + + /// Highest field id in this schema, including nested fields. + /// + /// `last-column-id` must cover nested ids too; computing it from the + /// top-level fields alone (as `create_table` used to) understates it for any + /// schema containing a struct, list or map. + pub fn max_field_id(&self) -> i32 { + fn walk(t: &Type, acc: &mut i32) { + match t { + Type::Primitive(_) => {} + Type::Struct { fields, .. } => { + for f in fields { + *acc = (*acc).max(f.id); + walk(&f.field_type, acc); + } + } + Type::List { + element_id, + element, + .. + } => { + *acc = (*acc).max(*element_id); + walk(element, acc); + } + Type::Map { + key_id, + key, + value_id, + value, + .. + } => { + *acc = (*acc).max(*key_id).max(*value_id); + walk(key, acc); + walk(value, acc); + } + } + } + + let mut max = 0; + for field in &self.fields { + max = max.max(field.id); + walk(&field.field_type, &mut max); + } + max + } +} + #[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] #[serde(rename_all = "kebab-case")] pub struct NestedField { @@ -64,6 +175,8 @@ pub struct NestedField { pub required: bool, #[serde(rename = "type")] pub field_type: Type, + /// Omitted when absent rather than serialized as `"doc": null` (B14). + #[serde(default, skip_serializing_if = "Option::is_none")] pub doc: Option, } diff --git a/pangolin/pangolin_core/src/model.rs b/pangolin/pangolin_core/src/model.rs index 0b94f89..ca18774 100644 --- a/pangolin/pangolin_core/src/model.rs +++ b/pangolin/pangolin_core/src/model.rs @@ -111,6 +111,73 @@ pub enum AssetType { Other, } +impl AssetType { + /// The canonical persisted spelling, e.g. `"DELTA_TABLE"`. + /// + /// B7: all three persistent backends wrote `format!("{:?}", asset.kind)` - + /// the *Debug* spelling - and read it back through a match that recognised + /// only `IcebergTable` and `View`, defaulting everything else to + /// `IcebergTable`. A `DeltaTable`, `MlModel`, `Lance` or any of the other + /// 13 variants round-tripped as an Iceberg table, silently defeating the + /// headline "tracks any lakehouse asset type" feature. Going through serde + /// means the enum's own rename policy is the single source of truth, and + /// adding a variant cannot reintroduce the drift. + pub fn as_stored_str(&self) -> String { + serde_json::to_value(self) + .ok() + .and_then(|v| v.as_str().map(str::to_owned)) + // The enum is a plain unit-variant enum, so serialization cannot + // fail; the Debug spelling is a belt-and-braces fallback only. + .unwrap_or_else(|| format!("{:?}", self)) + } + + /// Parse a persisted asset type. + /// + /// Accepts the canonical serde spelling *and* the legacy Debug spelling, so + /// rows written before this fix keep loading. An unrecognised value is an + /// error rather than a silent downgrade - a wrong asset type is worse than a + /// loud failure, because it misroutes every reader downstream. + pub fn from_stored_str(value: &str) -> Result { + if let Ok(parsed) = + serde_json::from_value::(serde_json::Value::String(value.to_string())) + { + return Ok(parsed); + } + + // Legacy Debug spellings, e.g. "IcebergTable". + for candidate in Self::all() { + if format!("{:?}", candidate) == value { + return Ok(candidate); + } + } + + Err(format!("unknown asset type {value:?}")) + } + + /// Every variant, for round-trip tests and legacy parsing. + pub fn all() -> Vec { + vec![ + Self::IcebergTable, + Self::DeltaTable, + Self::HudiTable, + Self::ParquetTable, + Self::CsvTable, + Self::JsonTable, + Self::View, + Self::MlModel, + Self::ApachePaimon, + Self::Vortex, + Self::Lance, + Self::Nimble, + Self::Directory, + Self::VideoFile, + Self::ImageFile, + Self::DbConnString, + Self::Other, + ] + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Asset { pub id: Uuid, diff --git a/pangolin/pangolin_core/src/user.rs b/pangolin/pangolin_core/src/user.rs index 3c6eaac..2a88a5b 100644 --- a/pangolin/pangolin_core/src/user.rs +++ b/pangolin/pangolin_core/src/user.rs @@ -56,6 +56,14 @@ pub struct UserSession { pub role: UserRole, pub issued_at: DateTime, pub expires_at: DateTime, + /// The `jti` of the bearer token that produced this session, when there was + /// one. Revocation is keyed by `jti`, so logout has to revoke *this* id - + /// revoking `user_id` (as it once did) blacklists an id no token ever + /// carries, and the token keeps working until it expires naturally. + /// + /// `None` for sessions with no underlying JWT (API keys, root basic auth). + #[serde(default)] + pub token_id: Option, } /// Service user for API key authentication diff --git a/pangolin/pangolin_store/Cargo.toml b/pangolin/pangolin_store/Cargo.toml index f8b001b..305f0cd 100644 --- a/pangolin/pangolin_store/Cargo.toml +++ b/pangolin/pangolin_store/Cargo.toml @@ -15,7 +15,7 @@ categories = ["database"] workspace = true [dependencies] -pangolin_core = { path = "../pangolin_core", version = "0.6.0", features = ["sqlx"] } +pangolin_core = { path = "../pangolin_core", version = "0.7.0", features = ["sqlx"] } object_store = { workspace = true } async-trait = { workspace = true } dashmap = { workspace = true } @@ -30,9 +30,14 @@ bytes = { workspace = true } tracing = { workspace = true } chrono = { workspace = true } aws-sdk-s3 = { version = "1.0", features = ["behavior-version-latest"] } -aws-config = { version = "1.0", features = ["behavior-version-latest"] } +aws-config = { version = "1.10", features = ["behavior-version-latest"] } aws-credential-types = "1.0" sqlx = { version = "0.8.6", features = ["runtime-tokio-rustls", "postgres", "sqlite", "uuid", "chrono", "json"] } +# C-11, warehouse credential encryption at rest. `ring` is already in the tree +# via jsonwebtoken, object_store and rustls, so this declares a dependency that +# was already being compiled rather than adding one. +ring = "0.17" +base64 = "0.22" mongodb = "3.4.1" bson = { version = "2.14", features = ["uuid-1", "chrono-0_4"] } aws-sdk-sts = { version = "1.0", features = ["behavior-version-latest"] } @@ -59,3 +64,6 @@ test-support = [] [dev-dependencies] pangolin_store = { path = ".", features = ["test-support"] } +# The few tests that exercise PANGOLIN_ENCRYPTION_KEY handling mutate a +# process-global, so they must not run concurrently with each other. +serial_test = "2.0" diff --git a/pangolin/pangolin_store/migrations/20260810000000_add_business_metadata.sql b/pangolin/pangolin_store/migrations/20260810000000_add_business_metadata.sql new file mode 100644 index 0000000..cf2382e --- /dev/null +++ b/pangolin/pangolin_store/migrations/20260810000000_add_business_metadata.sql @@ -0,0 +1,41 @@ +-- Business metadata for assets: descriptions, tags, discoverability. +-- +-- This table was never created on Postgres. `search_assets` has always run +-- `LEFT JOIN business_metadata m ON a.id = m.asset_id`, so *every* asset search +-- on the Postgres backend failed outright with +-- `relation "business_metadata" does not exist` - not an empty result, a hard +-- SQL error. The three CRUD methods were not implemented either, so the trait's +-- "Operation not supported by this store" default answered them. +-- +-- Found by running the cross-backend parity suite against a live Postgres for +-- the first time. SQLite, Mongo and the memory backend all had it. +-- +-- Column types follow the Postgres conventions already in this schema: UUID for +-- ids, JSONB for structured values, TIMESTAMPTZ for times - rather than the +-- TEXT/INTEGER encoding SQLite uses. + +CREATE TABLE IF NOT EXISTS business_metadata ( + id UUID PRIMARY KEY, + asset_id UUID NOT NULL UNIQUE, + description TEXT, + tags JSONB NOT NULL DEFAULT '[]'::jsonb, + properties JSONB NOT NULL DEFAULT '{}'::jsonb, + discoverable BOOLEAN NOT NULL DEFAULT FALSE, + created_by UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_by UUID NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT fk_business_metadata_asset + FOREIGN KEY (asset_id) REFERENCES assets(id) ON DELETE CASCADE +); + +-- `search_assets` joins on asset_id and filters on description; the unique +-- constraint above already indexes asset_id. +CREATE INDEX IF NOT EXISTS idx_business_metadata_discoverable + ON business_metadata(discoverable) + WHERE discoverable; + +-- Tag filtering uses the containment operator (`tags @> $n::jsonb`), which GIN +-- serves directly. +CREATE INDEX IF NOT EXISTS idx_business_metadata_tags + ON business_metadata USING GIN (tags); diff --git a/pangolin/pangolin_store/sql/sqlite_schema.sql b/pangolin/pangolin_store/sql/sqlite_schema.sql index f3d7ab5..46bf7fb 100644 --- a/pangolin/pangolin_store/sql/sqlite_schema.sql +++ b/pangolin/pangolin_store/sql/sqlite_schema.sql @@ -113,14 +113,29 @@ CREATE TABLE IF NOT EXISTS commits ( CREATE INDEX IF NOT EXISTS idx_commits_tenant ON commits(tenant_id); -- Audit Logs +-- The columns here had drifted out of step with the code that writes them. +-- The table declared the original (actor, resource, details) shape while +-- `sqlite/audit_logs.rs` inserted the full AuditLogEntry - user_id, username, +-- resource_type, resource_id, resource_name, ip_address, user_agent, result, +-- error_message, metadata. Every `log_audit_event` on SQLite therefore failed +-- at runtime with "table audit_logs has no column named user_id", so the +-- backend recorded *no* audit trail at all. Found by the cross-backend parity +-- suite, which is the first thing to exercise audit logging on SQLite. CREATE TABLE IF NOT EXISTS audit_logs ( id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL, - timestamp INTEGER NOT NULL, - actor TEXT NOT NULL, + user_id TEXT, + username TEXT NOT NULL, action TEXT NOT NULL, - resource TEXT NOT NULL, - details TEXT, -- JSON + resource_type TEXT NOT NULL, + resource_id TEXT, + resource_name TEXT NOT NULL, + timestamp INTEGER NOT NULL, + ip_address TEXT, + user_agent TEXT, + result TEXT NOT NULL, + error_message TEXT, + metadata TEXT, -- JSON FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_audit_logs_tenant_ts ON audit_logs(tenant_id, timestamp DESC); diff --git a/pangolin/pangolin_store/src/azure_signer.rs b/pangolin/pangolin_store/src/azure_signer.rs index cdc1c38..f2c97bc 100644 --- a/pangolin/pangolin_store/src/azure_signer.rs +++ b/pangolin/pangolin_store/src/azure_signer.rs @@ -36,12 +36,16 @@ impl AzureSigner { pub fn parse_azure_path(path: &str) -> Result<(String, String)> { // Parse az://container/blob/path OR abfs://container@account/path OR abfss://container@account/path - let (scheme, rest) = if path.starts_with("az://") { - ("az", &path[5..]) - } else if path.starts_with("abfs://") { - ("abfs", &path[7..]) - } else if path.starts_with("abfss://") { - ("abfss", &path[8..]) + // `strip_prefix` rather than `starts_with` plus a hand-counted `&path[n..]`, + // where the index and the prefix have to be kept in step by hand. The three + // prefixes are mutually exclusive - `abfss://` does not start with + // `abfs://` - so the order of the arms does not matter. + let (scheme, rest) = if let Some(rest) = path.strip_prefix("az://") { + ("az", rest) + } else if let Some(rest) = path.strip_prefix("abfss://") { + ("abfss", rest) + } else if let Some(rest) = path.strip_prefix("abfs://") { + ("abfs", rest) } else { return Err(anyhow::anyhow!("Invalid Azure path: {}", path)); }; diff --git a/pangolin/pangolin_store/src/file_delete.rs b/pangolin/pangolin_store/src/file_delete.rs new file mode 100644 index 0000000..a2a59a0 --- /dev/null +++ b/pangolin/pangolin_store/src/file_delete.rs @@ -0,0 +1,64 @@ +//! Best-effort deletion of a single object at a warehouse location. +//! +//! Added for the metadata-orphan problem (B16d/B16g): the Iceberg commit loop +//! writes a full metadata file *before* attempting the compare-and-swap that +//! publishes it. On a lost CAS it retried and wrote a fresh file, abandoning the +//! previous one, and up to five orphans were left behind on a final give-up. +//! Orphaned metadata files are indistinguishable from live ones from the +//! outside, so they cannot be reaped later by inspection alone - they have to be +//! removed at the moment the writer knows they are unreferenced. +//! +//! Every backend routes here so the four of them cannot drift apart, which is +//! the failure mode most of the storage-layer audit findings share. + +use anyhow::Result; +use object_store::ObjectStore; +use std::collections::HashMap; + +/// Delete `location`, resolving credentials from `storage_config` when the +/// location points at object storage. +/// +/// A missing object is *not* an error: callers use this to clean up after a +/// failure, and the object may never have been written. +pub async fn delete_location( + storage_config: Option<&HashMap>, + location: &str, +) -> Result<()> { + if let Some(rest) = location.strip_prefix("file://") { + return match tokio::fs::remove_file(rest).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(anyhow::anyhow!("Failed to delete {}: {}", rest, e)), + }; + } + + let is_object_store = location.starts_with("s3://") + || location.starts_with("az://") + || location.starts_with("abfs://") + || location.starts_with("gs://"); + + if !is_object_store { + // A bare local path. + return match tokio::fs::remove_file(location).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(anyhow::anyhow!("Failed to delete {}: {}", location, e)), + }; + } + + let empty = HashMap::new(); + let config = storage_config.unwrap_or(&empty); + let store = crate::object_store_factory::create_object_store(config, location)?; + + let key = location + .split_once("://") + .and_then(|(_, rest)| rest.split_once('/')) + .map(|(_, key)| key) + .unwrap_or(location); + + match store.delete(&object_store::path::Path::from(key)).await { + Ok(()) => Ok(()), + Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(e) => Err(anyhow::anyhow!("Failed to delete {}: {}", location, e)), + } +} diff --git a/pangolin/pangolin_store/src/lib.rs b/pangolin/pangolin_store/src/lib.rs index f645eba..2092673 100644 --- a/pangolin/pangolin_store/src/lib.rs +++ b/pangolin/pangolin_store/src/lib.rs @@ -1,9 +1,11 @@ pub mod aws_utils; pub mod azure_signer; +pub mod file_delete; pub mod gcp_signer; pub mod memory; pub mod mongo; pub mod postgres; +pub mod secrets; pub mod signer; pub mod sqlite; /// Helpers for locating the databases backend integration tests need. @@ -28,6 +30,7 @@ pub use sqlite::SqliteStore; pub mod metadata_cache; pub mod object_store_cache; pub mod object_store_factory; +pub mod search; pub use metadata_cache::MetadataCache; pub use object_store_cache::ObjectStoreCache; pub use signer::SignerImpl; @@ -118,6 +121,7 @@ pub trait CatalogStore: Send + Sync + Signer { catalog_name: &str, namespace: Vec, ) -> Result<()>; + /// Merge `properties` into the namespace's existing properties. async fn update_namespace_properties( &self, tenant_id: Uuid, @@ -125,6 +129,19 @@ pub trait CatalogStore: Send + Sync + Signer { namespace: Vec, properties: std::collections::HashMap, ) -> Result<()>; + /// Replace the namespace's properties with `properties`. + /// + /// The merge-only method above cannot express a removal, which is why the + /// Iceberg `updateProperties` handler silently dropped every `removals` + /// entry while reporting success (B16h). Errors when the namespace does not + /// exist, so "updated nothing" and "no such namespace" are distinguishable. + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()>; // Asset Operations async fn create_asset( @@ -293,6 +310,34 @@ pub trait CatalogStore: Send + Sync + Signer { async fn create_commit(&self, tenant_id: Uuid, commit: Commit) -> Result<()>; async fn get_commit(&self, tenant_id: Uuid, commit_id: Uuid) -> Result>; + /// Create a branch and populate it from another, atomically. + /// + /// The remainder of A-24. Creating a branch by copy used to be + /// `create_branch` followed by `copy_assets_bulk` (or a loop of + /// `create_asset`) as independent statements, so a failure partway through + /// left a branch that existed and was missing an arbitrary subset of its + /// assets, with no rollback and no repair tool. Worse, the handler logged + /// the copy failure and returned `200`, so the caller was told the branch + /// was ready. + /// + /// `assets` selects what to copy: `None` takes everything on `src_branch`, + /// `Some(names)` takes only those, in `namespace.table` form. Returns the + /// number copied. + /// + /// Backends that cannot do this atomically must say so rather than + /// pretending: the default is an error, and the caller falls back to the + /// non-atomic path explicitly. + async fn create_branch_with_assets( + &self, + _tenant_id: Uuid, + _catalog_name: &str, + _branch: pangolin_core::model::Branch, + _src_branch: &str, + _assets: Option>, + ) -> Result { + Err(anyhow::anyhow!("Operation not supported by this store")) + } + /// Bulk copy assets from one branch to another /// Returns the number of assets copied async fn copy_assets_bulk( @@ -340,6 +385,13 @@ pub trait CatalogStore: Send + Sync + Signer { // Generic File IO (for metadata files) async fn read_file(&self, location: &str) -> Result>; async fn write_file(&self, location: &str, content: Vec) -> Result<()>; + /// Delete a single file, resolving warehouse credentials from the location. + /// + /// Needed so the commit path can reclaim the metadata file it wrote before + /// losing a compare-and-swap (B16d), and so `create_table` can clean up + /// after a failed registration (B16g). Deleting something that is not there + /// succeeds. + async fn delete_file(&self, location: &str) -> Result<()>; // Maintenance Operations async fn expire_snapshots( diff --git a/pangolin/pangolin_store/src/memory.rs.bak b/pangolin/pangolin_store/src/memory.rs.bak deleted file mode 100644 index 8ad28a6..0000000 --- a/pangolin/pangolin_store/src/memory.rs.bak +++ /dev/null @@ -1,1831 +0,0 @@ -use crate::CatalogStore; -use crate::signer::{Signer, Credentials}; -use async_trait::async_trait; -use dashmap::DashMap; -use chrono::Utc; -use pangolin_core::model::{ - Catalog, CatalogType, Namespace, Warehouse, Asset, Commit, Branch, Tag, BranchType, Tenant, - VendingStrategy, SystemSettings, SyncStats -}; -use pangolin_core::user::User; -use pangolin_core::permission::{Role, UserRole, Permission}; -use pangolin_core::audit::AuditLogEntry; -use uuid::Uuid; -use anyhow::Result; -use std::sync::Arc; -use pangolin_core::business_metadata::{BusinessMetadata, AccessRequest}; - -use tracing; - - -#[derive(Clone)] -pub struct MemoryStore { - tenants: Arc>, - warehouses: Arc>, // Key: (TenantId, WarehouseName) - catalogs: Arc>, // Key: (TenantId, CatalogName) - namespaces: Arc>, // Key: (TenantId, CatalogName, NamespaceString) - // Key: (TenantId, CatalogName, BranchName, NamespaceString, AssetName) - assets: Arc>, - branches: Arc>, // Key: (TenantId, CatalogName, BranchName) - tags: Arc>, // Key: (TenantId, CatalogName, TagName) - commits: Arc>, // Key: (TenantId, CommitId) - files: Arc>>, // Key: Location - audit_events: Arc>>, // Changed to DashMap for consistency, key tenant_id - // New fields - users: Arc>, - roles: Arc>, - signer: crate::signer::SignerImpl, - user_roles: Arc>, - permissions: Arc>, - business_metadata: Arc>, - access_requests: Arc>, - service_users: Arc>, - merge_operations: Arc>, - merge_conflicts: Arc>, - // Optimization: Direct lookup for assets by ID - // Key: AssetID, Value: (CatalogName, Namespace, Branch, AssetName) - assets_by_id: Arc, Option, String)>>, - // Token revocation - revoked_tokens: Arc>, - // Active tokens (for listing - in real DB this would be querying sessions/tokens table) - // We only store TokenInfo here. The actual validation is stateless JWT + Revocation Check. - // But to "List Tokens", we need to store them. - active_tokens: Arc>, - // System Settings: Tenant -> Settings - system_settings: Arc>, - // Federated Stats: (TenantId, CatalogName) -> SyncStats - federated_stats: Arc>, - // Performance optimizations - object_store_cache: crate::ObjectStoreCache, - metadata_cache: crate::MetadataCache, -} - -impl MemoryStore { - pub fn new() -> Self { - Self { - tenants: Arc::new(DashMap::new()), - warehouses: Arc::new(DashMap::new()), - catalogs: Arc::new(DashMap::new()), - namespaces: Arc::new(DashMap::new()), - assets: Arc::new(DashMap::new()), - branches: Arc::new(DashMap::new()), - tags: Arc::new(DashMap::new()), - commits: Arc::new(DashMap::new()), - files: Arc::new(DashMap::new()), - audit_events: Arc::new(DashMap::new()), - users: Arc::new(DashMap::new()), - roles: Arc::new(DashMap::new()), - signer: crate::signer::SignerImpl::new("memory_key".to_string()), - user_roles: Arc::new(DashMap::new()), - permissions: Arc::new(DashMap::new()), - business_metadata: Arc::new(DashMap::new()), - access_requests: Arc::new(DashMap::new()), - service_users: Arc::new(DashMap::new()), - merge_operations: Arc::new(DashMap::new()), - merge_conflicts: Arc::new(DashMap::new()), - assets_by_id: Arc::new(DashMap::new()), - revoked_tokens: Arc::new(DashMap::new()), - active_tokens: Arc::new(DashMap::new()), - system_settings: Arc::new(DashMap::new()), - federated_stats: Arc::new(DashMap::new()), - object_store_cache: crate::ObjectStoreCache::new(), - metadata_cache: crate::MetadataCache::default(), - } - } -} - - -#[async_trait] -impl CatalogStore for MemoryStore { - async fn create_tenant(&self, tenant: Tenant) -> Result<()> { - self.tenants.insert(tenant.id, tenant); - Ok(()) - } - - async fn get_tenant(&self, tenant_id: Uuid) -> Result> { - if let Some(t) = self.tenants.get(&tenant_id) { - Ok(Some(t.value().clone())) - } else { - Ok(None) - } - } - - async fn list_tenants(&self) -> Result> { - let tenants = self.tenants.iter().map(|t| t.value().clone()).collect(); - Ok(tenants) - } - - async fn update_tenant(&self, tenant_id: Uuid, updates: pangolin_core::model::TenantUpdate) -> Result { - if let Some(mut tenant) = self.tenants.get_mut(&tenant_id) { - if let Some(name) = updates.name { - tenant.name = name; - } - if let Some(properties) = updates.properties { - tenant.properties.extend(properties); - } - Ok(tenant.clone()) - } else { - Err(anyhow::anyhow!("Tenant not found")) - } - } - - async fn delete_tenant(&self, tenant_id: Uuid) -> Result<()> { - if self.tenants.remove(&tenant_id).is_some() { - // TODO: Cascade delete warehouses and catalogs - Ok(()) - } else { - Err(anyhow::anyhow!("Tenant not found")) - } - } - - async fn create_warehouse(&self, tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { - let key = (tenant_id, warehouse.name.clone()); - self.warehouses.insert(key, warehouse); - Ok(()) - } - - async fn get_warehouse(&self, tenant_id: Uuid, name: String) -> Result> { - let key = (tenant_id, name); - if let Some(w) = self.warehouses.get(&key) { - Ok(Some(w.value().clone())) - } else { - Ok(None) - } - } - - async fn list_warehouses(&self, tenant_id: Uuid) -> Result> { - let warehouses = self.warehouses.iter() - .filter(|r| r.key().0 == tenant_id) - .map(|r| r.value().clone()) - .collect(); - Ok(warehouses) - } - - async fn update_warehouse(&self, tenant_id: Uuid, name: String, updates: pangolin_core::model::WarehouseUpdate) -> Result { - let key = (tenant_id, name.clone()); - if let Some(mut warehouse) = self.warehouses.get_mut(&key) { - if let Some(new_name) = updates.name { - // If name is changing, we need to remove old key and insert with new key - let mut w = warehouse.clone(); - w.name = new_name.clone(); - drop(warehouse); // Release the mutable reference - self.warehouses.remove(&key); - let new_key = (tenant_id, new_name); - self.warehouses.insert(new_key, w.clone()); - return Ok(w); - } - if let Some(config) = updates.storage_config { - warehouse.storage_config.extend(config); - } - if let Some(use_sts) = updates.use_sts { - warehouse.use_sts = use_sts; - } - Ok(warehouse.clone()) - } else { - Err(anyhow::anyhow!("Warehouse '{}' not found", name)) - } - } - - async fn delete_warehouse(&self, tenant_id: Uuid, name: String) -> Result<()> { - let key = (tenant_id, name.clone()); - - if self.warehouses.remove(&key).is_some() { - Ok(()) - } else { - Err(anyhow::anyhow!("Warehouse '{}' not found", name)) - } - } - - async fn create_catalog(&self, tenant_id: Uuid, catalog: Catalog) -> Result<()> { - let key = (tenant_id, catalog.name.clone()); - self.catalogs.insert(key, catalog); - Ok(()) - } - - async fn get_catalog(&self, tenant_id: Uuid, name: String) -> Result> { - let key = (tenant_id, name); - if let Some(c) = self.catalogs.get(&key) { - Ok(Some(c.value().clone())) - } else { - Ok(None) - } - } - - async fn list_catalogs(&self, tenant_id: Uuid) -> Result> { - let mut catalogs = Vec::new(); - for entry in self.catalogs.iter() { - let (tid, _) = entry.key(); - if *tid == tenant_id { - catalogs.push(entry.value().clone()); - } - } - Ok(catalogs) - } - - async fn update_catalog(&self, tenant_id: Uuid, name: String, updates: pangolin_core::model::CatalogUpdate) -> Result { - let key = (tenant_id, name.clone()); - if let Some(mut catalog) = self.catalogs.get_mut(&key) { - if let Some(warehouse_name) = updates.warehouse_name { - catalog.warehouse_name = Some(warehouse_name); - } - if let Some(storage_location) = updates.storage_location { - catalog.storage_location = Some(storage_location); - } - if let Some(properties) = updates.properties { - catalog.properties.extend(properties); - } - Ok(catalog.clone()) - } else { - Err(anyhow::anyhow!("Catalog '{}' not found", name)) - } - } - - async fn delete_catalog(&self, tenant_id: Uuid, name: String) -> Result<()> { - let key = (tenant_id, name.clone()); - if self.catalogs.remove(&key).is_some() { - // Cascade delete: Remove all associated resources - // Note: In a real database, this would be handled by foreign keys. - // In MemoryStore, we must manually iterate and remove. - - // Remove Namespaces - self.namespaces.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - - // Remove Assets - self.assets.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - - // Remove Branches - self.branches.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - - // Remove Tags - self.tags.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - - // Clean up assets_by_id index - // This is expensive O(N) since we have to scan the whole index - // But deletion is rare. - self.assets_by_id.retain(|_, v| v.0 != name); - - Ok(()) - } else { - Err(anyhow::anyhow!("Catalog not found")) - } - } - - async fn create_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Namespace) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), namespace.to_string()); - self.namespaces.insert(key, namespace); - Ok(()) - } - - async fn list_namespaces(&self, tenant_id: Uuid, catalog_name: &str, parent: Option) -> Result> { - let parent_prefix = parent.unwrap_or_default(); - tracing::info!("DEBUG_MEM: list_namespaces tid={} cat={} parent='{}'", tenant_id, catalog_name, parent_prefix); - let mut namespaces = Vec::new(); - for entry in self.namespaces.iter() { - let (tid, cat, ns_str) = entry.key(); - tracing::info!("DEBUG_MEM: Checking entry tid={} cat={} ns={}", tid, cat, ns_str); - if *tid == tenant_id && cat == catalog_name && (parent_prefix.is_empty() || ns_str.starts_with(&parent_prefix)) { - namespaces.push(entry.value().clone()); - } - } - tracing::info!("DEBUG_MEM: Found {} namespaces", namespaces.len()); - Ok(namespaces) - } - - async fn get_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec) -> Result> { - let key = (tenant_id, catalog_name.to_string(), namespace.join(".")); - if let Some(n) = self.namespaces.get(&key) { - Ok(Some(n.value().clone())) - } else { - Ok(None) - } - } - - async fn delete_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec) -> Result<()> { - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), ns_str); - if self.namespaces.remove(&key).is_some() { - Ok(()) - } else { - Err(anyhow::anyhow!("Namespace not found")) - } - } - - async fn update_namespace_properties(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec, properties: std::collections::HashMap) -> Result<()> { - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), ns_str); - - if let Some(mut ns) = self.namespaces.get_mut(&key) { - ns.properties.extend(properties); - Ok(()) - } else { - Err(anyhow::anyhow!("Namespace not found")) - } - } - - async fn create_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, asset: Asset) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - - // 1. Insert Asset - let asset_full_name = format!("{}.{}", namespace.join("."), asset.name); - let key = (tenant_id, catalog_name.to_string(), branch_name.clone(), namespace.join("\x1F"), asset.name.clone()); - self.assets.insert(key, asset.clone()); - - // 2. Update optimized lookups - self.assets_by_id.insert(asset.id, (catalog_name.to_string(), namespace.clone(), Some(branch_name.clone()), asset.name.clone())); - - // 2. Ensure Branch Exists and Update Asset List - let mut branch_obj = self.get_branch(tenant_id, catalog_name, branch_name.clone()).await? - .unwrap_or_else(|| { - Branch { - name: branch_name.clone(), - head_commit_id: None, - branch_type: BranchType::Experimental, - assets: vec![], - } - }); - - if !branch_obj.assets.contains(&asset_full_name) { - branch_obj.assets.push(asset_full_name); - self.create_branch(tenant_id, catalog_name, branch_obj).await?; - } - - Ok(()) - } - - async fn get_asset_by_id(&self, tenant_id: Uuid, asset_id: Uuid) -> Result)>> { - if let Some(entry) = self.assets_by_id.get(&asset_id) { - let (catalog_name, namespace, branch, name) = entry.value().clone(); - // Verify tenant ownership (implicit via proper key lookup) purely for safety - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let key = (tenant_id, catalog_name.clone(), branch_name, namespace.join("\x1F"), name); - - if let Some(asset) = self.assets.get(&key) { - return Ok(Some((asset.value().clone(), catalog_name, namespace))); - } - } - Ok(None) - } - - async fn get_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let key = (tenant_id, catalog_name.to_string(), branch_name, namespace.join("\x1F"), name); - if let Some(a) = self.assets.get(&key) { - Ok(Some(a.value().clone())) - } else { - Ok(None) - } - } - - async fn list_assets(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let ns_str = namespace.join("\x1F"); - let mut assets = Vec::new(); - for entry in self.assets.iter() { - let (tid, cat, b_name, ns, _) = entry.key(); - if *tid == tenant_id && cat == catalog_name && *b_name == branch_name && *ns == ns_str { - assets.push(entry.value().clone()); - } - } - Ok(assets) - } - - async fn delete_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), branch_name, ns_str, name); - if let Some((_, asset)) = self.assets.remove(&key) { - self.assets_by_id.remove(&asset.id); - } - Ok(()) - } - - async fn rename_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, source_namespace: Vec, source_name: String, dest_namespace: Vec, dest_name: String) -> Result<()> { - let branch_val = branch.unwrap_or_else(|| "main".to_string()); - let src_ns_str = source_namespace.join("\x1F"); - let src_key = (tenant_id, catalog_name.to_string(), branch_val.clone(), src_ns_str, source_name); - let dest_key = (tenant_id, catalog_name.to_string(), branch_val.clone(), dest_namespace.join("\x1F"), dest_name.clone()); - - if let Some((_, mut asset)) = self.assets.remove(&src_key) { - asset.name = dest_name; - // Update index - self.assets_by_id.insert(asset.id, (catalog_name.to_string(), dest_namespace, Some(branch_val), asset.name.clone())); - - self.assets.insert(dest_key, asset); - Ok(()) - } else { - Err(anyhow::anyhow!("Asset not found")) - } - } - - async fn count_namespaces(&self, tenant_id: Uuid) -> Result { - // Efficient counting for MemoryStore - // We iterate over the DashMap, but it's much faster than constructing full Namespace objects - let count = self.namespaces.iter() - .filter(|entry| entry.key().0 == tenant_id) - .count(); - Ok(count) - } - - async fn count_assets(&self, tenant_id: Uuid) -> Result { - // Efficient counting for MemoryStore - let count = self.assets.iter() - .filter(|entry| entry.key().0 == tenant_id) - .count(); - Ok(count) - } - - async fn create_branch(&self, tenant_id: Uuid, catalog_name: &str, branch: Branch) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), branch.name.clone()); - self.branches.insert(key, branch); - Ok(()) - } - - async fn get_branch(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result> { - let key = (tenant_id, catalog_name.to_string(), name); - if let Some(b) = self.branches.get(&key) { - Ok(Some(b.value().clone())) - } else { - Ok(None) - } - } - - async fn list_branches(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let mut branches = Vec::new(); - for entry in self.branches.iter() { - let (tid, cat, _) = entry.key(); - if *tid == tenant_id && cat == catalog_name { - branches.push(entry.value().clone()); - } - } - Ok(branches) - } - - async fn delete_branch(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), name.clone()); - if self.branches.remove(&key).is_some() { - // Also remove assets associated with this branch - self.assets.retain(|k, _| !(k.0 == tenant_id && k.1 == catalog_name && k.2 == name)); - Ok(()) - } else { - Err(anyhow::anyhow!("Branch '{}' not found", name)) - } - } - - async fn merge_branch(&self, tenant_id: Uuid, catalog_name: &str, source_branch_name: String, target_branch_name: String) -> Result<()> { - // 1. Get Source Branch - let source_branch = self.get_branch(tenant_id, catalog_name, source_branch_name.clone()).await? - .ok_or_else(|| anyhow::anyhow!("Source branch not found"))?; - - // 2. Get Target Branch - let mut target_branch = self.get_branch(tenant_id, catalog_name, target_branch_name.clone()).await? - .ok_or_else(|| anyhow::anyhow!("Target branch not found"))?; - - // 3. Iterate assets tracked by source branch - for asset_str in &source_branch.assets { - let parts: Vec<&str> = asset_str.split('.').collect(); - if parts.len() < 2 { continue; } - - let asset_name = parts.last().unwrap().to_string(); - let namespace_parts: Vec = parts[0..parts.len()-1].iter().map(|s| s.to_string()).collect(); - - // Get asset from source - if let Some(asset) = self.get_asset(tenant_id, catalog_name, Some(source_branch_name.clone()), namespace_parts.clone(), asset_name).await? { - // Write to target - self.create_asset(tenant_id, catalog_name, Some(target_branch_name.clone()), namespace_parts.clone(), asset).await?; - - // Ensure branch exists - let mut branch = self.get_branch(tenant_id, catalog_name, target_branch_name.clone()).await? - .unwrap_or_else(|| { - tracing::info!("MemoryStore: Branch {} not found, creating new struct", target_branch_name); - Branch { - name: target_branch_name.clone(), - head_commit_id: None, - branch_type: BranchType::Experimental, - assets: vec![], - }}); - - let full_asset_name = asset_str.to_string(); - if !branch.assets.contains(&full_asset_name) { - tracing::info!("MemoryStore: Adding asset {} to branch {}", full_asset_name, target_branch_name); - branch.assets.push(full_asset_name.clone()); - self.create_branch(tenant_id, catalog_name, branch).await?; - } else { - tracing::info!("MemoryStore: Asset {} already in branch {}", full_asset_name, target_branch_name); - } - } - } - - // 4. Update Target Branch asset list - // This block is now redundant because assets are added to the target branch within the loop. - // Keeping it commented out or removing it depends on desired behavior. - // For now, let's assume the in-loop update is sufficient. - // for asset_name in source_branch.assets { - // if !target_branch.assets.contains(&asset_name) { - // target_branch.assets.push(asset_name); - // } - // } - - // The target_branch variable might not be fully up-to-date if `create_branch` was called inside the loop. - // Re-fetch or ensure `create_branch` updates the existing one. - // Given `create_branch` inserts, it effectively overwrites if key exists. - // So, the loop's `create_branch` calls would update the branch. - // This final `create_branch` call might be redundant or intended to ensure the final state. - // Let's remove the redundant update of target_branch.assets and the final create_branch call - // if the loop already handles it. - // Based on the instruction, the new code is inserted *inside* the `if let Some(asset) = ...` block. - // The original `// 4. Update Target Branch asset list` and `self.create_branch(tenant_id, catalog_name, target_branch).await?;` - // are still present in the original code. The instruction does not remove them. - // So, I will keep them as is, even if they might be logically redundant after the change. - - // 4. Update Target Branch asset list - for asset_name in source_branch.assets { - if !target_branch.assets.contains(&asset_name) { - target_branch.assets.push(asset_name); - } - } - - self.create_branch(tenant_id, catalog_name, target_branch).await?; - - Ok(()) - } - - // Tag Operations - async fn create_tag(&self, tenant_id: Uuid, catalog_name: &str, tag: Tag) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), tag.name.clone()); - self.tags.insert(key, tag); - Ok(()) - } - - async fn get_tag(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result> { - let key = (tenant_id, catalog_name.to_string(), name); - if let Some(tag) = self.tags.get(&key) { - Ok(Some(tag.value().clone())) - } else { - Ok(None) - } - } - - async fn list_tags(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let mut tags = Vec::new(); - for r in self.tags.iter() { - let (tid, cname, _) = r.key(); - if *tid == tenant_id && cname == catalog_name { - tags.push(r.value().clone()); - } - } - Ok(tags) - } - - async fn delete_tag(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), name); - self.tags.remove(&key); - Ok(()) - } - - async fn create_commit(&self, tenant_id: Uuid, commit: Commit) -> Result<()> { - let key = (tenant_id, commit.id); - self.commits.insert(key, commit); - Ok(()) - } - async fn get_commit(&self, tenant_id: Uuid, commit_id: Uuid) -> Result> { - let key = (tenant_id, commit_id); - if let Some(c) = self.commits.get(&key) { - Ok(Some(c.value().clone())) - } else { - Ok(None) - } - } - - async fn get_metadata_location(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let key = (tenant_id, catalog_name.to_string(), branch_name, namespace.join("\x1F"), table); - - if let Some(asset) = self.assets.get(&key) { - let loc = asset.properties.get("metadata_location").cloned().unwrap_or(asset.location.clone()); - Ok(Some(loc)) - } else { - Ok(None) - } - } - - async fn update_metadata_location(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String, expected_location: Option, new_location: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let key = (tenant_id, catalog_name.to_string(), branch_name, namespace.join("\x1F"), table); - - if let Some(mut asset) = self.assets.get_mut(&key) { - let current_loc = asset.properties.get("metadata_location").cloned().unwrap_or(asset.location.clone()); - - // CAS Check - if let Some(expected) = expected_location { - if current_loc != expected { - return Err(anyhow::anyhow!("CAS failure: expected {} but found {}", expected, current_loc)); - } - } - - asset.location = new_location.clone(); - asset.properties.insert("metadata_location".to_string(), new_location); - Ok(()) - } else { - Err(anyhow::anyhow!("Table not found")) - } - } - - - async fn read_file(&self, location: &str) -> Result> { - // Use metadata cache for metadata.json files - if location.ends_with("metadata.json") || location.ends_with(".metadata.json") { - return self.metadata_cache.get_or_fetch(location, || async { - self.read_file_uncached(location).await - }).await; - } - - // Non-metadata files bypass cache - self.read_file_uncached(location).await - } - - async fn write_file(&self, location: &str, content: Vec) -> Result<()> { - // Invalidate metadata cache on write - if location.ends_with("metadata.json") || location.ends_with(".metadata.json") { - self.metadata_cache.invalidate(location).await; - } - - // Dual write: Memory + Object Store - self.files.insert(location.to_string(), content.clone()); - - if let Some(warehouse) = self.get_warehouse_for_location(location) { - if location.starts_with("s3://") || location.starts_with("az://") || location.starts_with("gs://") { - // Use object store cache - let cache_key = self.get_object_store_cache_key(&warehouse.storage_config, location); - let store = self.object_store_cache.get_or_insert(cache_key, || { - Arc::new(crate::object_store_factory::create_object_store(&warehouse.storage_config, location) - .expect("Failed to create object store")) - }); - - let path = self.extract_object_store_path(location); - store.put(&path, content.into()).await?; - } - } - - Ok(()) - } - - async fn expire_snapshots(&self, _tenant_id: Uuid, _catalog_name: &str, _branch: Option, _namespace: Vec, _table: String, _retention_ms: i64) -> Result<()> { - tracing::info!("MemoryStore: Expiring snapshots (placeholder)"); - Ok(()) - } - - async fn remove_orphan_files(&self, _tenant_id: Uuid, _catalog_name: &str, _branch: Option, _namespace: Vec, _table: String, _older_than_ms: i64) -> Result<()> { - tracing::info!("MemoryStore: Removing orphan files (placeholder)"); - Ok(()) - } - - // Audit Operations - async fn log_audit_event(&self, tenant_id: Uuid, event: pangolin_core::audit::AuditLogEntry) -> Result<()> { - // Log to tracing - tracing::info!("AUDIT: {:?}", event); - // Store in map - self.audit_events.entry(tenant_id).or_insert_with(Vec::new).push(event); - Ok(()) - } - - async fn list_audit_events(&self, tenant_id: Uuid, filter: Option) -> Result> { - if let Some(events) = self.audit_events.get(&tenant_id) { - let mut filtered = events.clone(); - - // Apply filters if provided - if let Some(f) = filter { - filtered.retain(|event| { - // Filter by user_id - if let Some(user_id) = f.user_id { - if event.user_id != Some(user_id) { - return false; - } - } - - // Filter by action - if let Some(ref action) = f.action { - if &event.action != action { - return false; - } - } - - // Filter by resource_type - if let Some(ref resource_type) = f.resource_type { - if &event.resource_type != resource_type { - return false; - } - } - - // Filter by resource_id - if let Some(resource_id) = f.resource_id { - if event.resource_id != Some(resource_id) { - return false; - } - } - - // Filter by start_time - if let Some(start_time) = f.start_time { - if event.timestamp < start_time { - return false; - } - } - - // Filter by end_time - if let Some(end_time) = f.end_time { - if event.timestamp > end_time { - return false; - } - } - - // Filter by result - if let Some(ref result) = f.result { - if &event.result != result { - return false; - } - } - - true - }); - - // Apply pagination - let offset = f.offset.unwrap_or(0); - let limit = f.limit.unwrap_or(100); - - filtered = filtered.into_iter() - .skip(offset) - .take(limit) - .collect(); - } - - Ok(filtered) - } else { - Ok(vec![]) - } - } - - async fn get_audit_event(&self, tenant_id: Uuid, event_id: Uuid) -> Result> { - if let Some(events) = self.audit_events.get(&tenant_id) { - Ok(events.iter().find(|e| e.id == event_id).cloned()) - } else { - Ok(None) - } - } - - async fn count_audit_events(&self, tenant_id: Uuid, filter: Option) -> Result { - if let Some(events) = self.audit_events.get(&tenant_id) { - if let Some(f) = filter { - let count = events.iter().filter(|event| { - // Same filtering logic as list_audit_events - if let Some(user_id) = f.user_id { - if event.user_id != Some(user_id) { - return false; - } - } - if let Some(ref action) = f.action { - if &event.action != action { - return false; - } - } - if let Some(ref resource_type) = f.resource_type { - if &event.resource_type != resource_type { - return false; - } - } - if let Some(resource_id) = f.resource_id { - if event.resource_id != Some(resource_id) { - return false; - } - } - if let Some(start_time) = f.start_time { - if event.timestamp < start_time { - return false; - } - } - if let Some(end_time) = f.end_time { - if event.timestamp > end_time { - return false; - } - } - if let Some(ref result) = f.result { - if &event.result != result { - return false; - } - } - true - }).count(); - Ok(count) - } else { - Ok(events.len()) - } - } else { - Ok(0) - } - } - - // User Operations - async fn create_user(&self, user: User) -> Result<()> { - self.users.insert(user.id, user); - Ok(()) - } - - async fn get_user(&self, user_id: Uuid) -> Result> { - if let Some(user) = self.users.get(&user_id) { - Ok(Some(user.value().clone())) - } else { - Ok(None) - } - } - - async fn get_user_by_username(&self, username: &str) -> Result> { - // Linear search for now, could add index - for entry in self.users.iter() { - if entry.value().username == username { - return Ok(Some(entry.value().clone())); - } - } - Ok(None) - } - - async fn list_users(&self, tenant_id: Option) -> Result> { - let users = self.users.iter() - .filter(|entry| { - match tenant_id { - Some(tid) => entry.value().tenant_id == Some(tid), - None => true // Root listing or all users - } - }) - .map(|entry| entry.value().clone()) - .collect(); - Ok(users) - } - - async fn update_user(&self, user: User) -> Result<()> { - if self.users.contains_key(&user.id) { - self.users.insert(user.id, user); - Ok(()) - } else { - Err(anyhow::anyhow!("User not found")) - } - } - - async fn delete_user(&self, user_id: Uuid) -> Result<()> { - if self.users.remove(&user_id).is_some() { - Ok(()) - } else { - Err(anyhow::anyhow!("User not found")) - } - } - // Role Operations - async fn create_role(&self, role: pangolin_core::permission::Role) -> Result<()> { - self.roles.insert(role.id, role); - Ok(()) - } - - async fn get_role(&self, role_id: Uuid) -> Result> { - Ok(self.roles.get(&role_id).map(|r| r.value().clone())) - } - - async fn list_roles(&self, tenant_id: Uuid) -> Result> { - Ok(self.roles.iter() - .filter(|r| r.value().tenant_id == tenant_id) - .map(|r| r.value().clone()) - .collect()) - } - - async fn assign_role(&self, user_role: UserRole) -> Result<()> { - let key = (user_role.user_id, user_role.role_id); - self.user_roles.insert(key, user_role); - Ok(()) - } - - async fn revoke_role(&self, user_id: Uuid, role_id: Uuid) -> Result<()> { - let key = (user_id, role_id); - self.user_roles.remove(&key); - Ok(()) - } - - async fn get_user_roles(&self, user_id: Uuid) -> Result> { - Ok(self.user_roles.iter() - .filter(|r| r.key().0 == user_id) - .map(|r| r.value().clone()) - .collect()) - } - - async fn delete_role(&self, role_id: Uuid) -> Result<()> { - self.roles.remove(&role_id); - Ok(()) - } - - - - async fn update_role(&self, role: Role) -> Result<()> { - // Just overwrite - self.roles.insert(role.id, role); - Ok(()) - } - - async fn create_permission(&self, permission: Permission) -> Result<()> { - self.permissions.insert(permission.id, permission); - Ok(()) - } - - async fn revoke_permission(&self, permission_id: Uuid) -> Result<()> { - self.permissions.remove(&permission_id); - Ok(()) - } - - async fn list_user_permissions(&self, user_id: Uuid) -> Result> { - let mut permissions: Vec = self.permissions.iter() - .filter(|p| p.value().user_id == user_id) - .map(|p| p.value().clone()) - .collect(); - - // Add permissions from roles - let user_roles = self.get_user_roles(user_id).await?; - for user_role in user_roles { - if let Some(role_entry) = self.roles.get(&user_role.role_id) { - let role = role_entry.value(); - for grant in &role.permissions { - // Synthesize a Permission object from the Role's PermissionGrant - let synthesized_perm = Permission { - id: Uuid::new_v4(), // Temporary ID for the aggregated result - user_id, - scope: grant.scope.clone(), - actions: grant.actions.clone(), - granted_by: role.created_by, - granted_at: role.created_at, - }; - permissions.push(synthesized_perm); - } - } - } - - Ok(permissions) - } - - async fn list_permissions(&self, tenant_id: Uuid) -> Result> { - let mut permissions = Vec::new(); - for entry in self.permissions.iter() { - let perm = entry.value(); - // Look up user to check tenant - if let Some(user_entry) = self.users.get(&perm.user_id) { - if user_entry.value().tenant_id == Some(tenant_id) { - permissions.push(perm.clone()); - } - } - } - Ok(permissions) - } - - async fn upsert_business_metadata(&self, metadata: pangolin_core::business_metadata::BusinessMetadata) -> Result<()> { - self.business_metadata.insert(metadata.asset_id, metadata); - Ok(()) - } - - async fn get_business_metadata(&self, asset_id: Uuid) -> Result> { - Ok(self.business_metadata.get(&asset_id).map(|m| m.value().clone())) - } - - async fn delete_business_metadata(&self, asset_id: Uuid) -> Result<()> { - self.business_metadata.remove(&asset_id); - Ok(()) - } - - async fn search_assets(&self, tenant_id: Uuid, query: &str, tags: Option>) -> Result, String, Vec)>> { - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - - // Iterate through all assets for this tenant - for entry in self.assets.iter() { - let key = entry.key(); // (tenant_id, catalog, branch, namespace_str, name) - if key.0 != tenant_id { - continue; - } - - let asset = entry.value().clone(); - let metadata = self.business_metadata.get(&asset.id).map(|m| m.value().clone()); - - // Check if asset matches search criteria (Name OR Description) - let name_matches = asset.name.to_lowercase().contains(&query_lower); - - let description_matches = if let Some(ref meta) = metadata { - if let Some(ref desc) = meta.description { - desc.to_lowercase().contains(&query_lower) - } else { - false - } - } else { - false - }; - - let tags_match = if let Some(ref search_tags) = tags { - if let Some(ref meta) = metadata { - search_tags.iter().any(|tag| meta.tags.contains(tag)) - } else { - false - } - } else { - true // No tag filter - }; - - if (name_matches || description_matches) && tags_match { - // key.1 is catalog_name, key.3 is namespace_str - let catalog_name = key.1.clone(); - let namespace = key.3.split('\x1F').map(String::from).collect(); - results.push((asset, metadata, catalog_name, namespace)); - } - } - - Ok(results) - } - - async fn search_catalogs(&self, tenant_id: Uuid, query: &str) -> Result> { - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - for entry in self.catalogs.iter() { - let (tid, _name) = entry.key(); - if *tid == tenant_id && entry.value().name.to_lowercase().contains(&query_lower) { - results.push(entry.value().clone()); - } - } - Ok(results) - } - - async fn search_namespaces(&self, tenant_id: Uuid, query: &str) -> Result> { - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - for entry in self.namespaces.iter() { - let (tid, catalog_name, _ns_str) = entry.key(); - if *tid == tenant_id { - let ns = entry.value(); - if ns.to_string().to_lowercase().contains(&query_lower) { - results.push((ns.clone(), catalog_name.clone())); - } - } - } - Ok(results) - } - - async fn search_branches(&self, tenant_id: Uuid, query: &str) -> Result> { - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - for entry in self.branches.iter() { - let (tid, catalog_name, _branch_name) = entry.key(); - if *tid == tenant_id { - let branch = entry.value(); - if branch.name.to_lowercase().contains(&query_lower) { - results.push((branch.clone(), catalog_name.clone())); - } - } - } - Ok(results) - } - - // Access Request Operations - async fn create_access_request(&self, request: AccessRequest) -> Result<()> { - self.access_requests.insert(request.id, request); - Ok(()) - } - - async fn get_access_request(&self, id: Uuid) -> Result> { - Ok(self.access_requests.get(&id).map(|r| r.value().clone())) - } - - async fn list_access_requests(&self, tenant_id: Uuid) -> Result> { - let mut requests = Vec::new(); - // Efficient scan filtering by tenant_id directly - for req in self.access_requests.iter() { - if req.value().tenant_id == tenant_id { - requests.push(req.value().clone()); - } - } - Ok(requests) - } - - async fn update_access_request(&self, request: AccessRequest) -> Result<()> { - self.access_requests.insert(request.id, request); - Ok(()) - } - - // Service User Operations - async fn create_service_user(&self, service_user: pangolin_core::user::ServiceUser) -> Result<()> { - self.service_users.insert(service_user.id, service_user); - Ok(()) - } - - async fn get_service_user(&self, id: Uuid) -> Result> { - Ok(self.service_users.get(&id).map(|r| r.value().clone())) - } - - async fn get_service_user_by_api_key_hash(&self, api_key_hash: &str) -> Result> { - // Linear search through all service users to find matching hash - for entry in self.service_users.iter() { - if entry.value().api_key_hash == api_key_hash { - return Ok(Some(entry.value().clone())); - } - } - Ok(None) - } - - async fn list_service_users(&self, tenant_id: Uuid) -> Result> { - Ok(self.service_users - .iter() - .filter(|entry| entry.value().tenant_id == tenant_id) - .map(|entry| entry.value().clone()) - .collect()) - } - - async fn update_service_user( - &self, - id: Uuid, - name: Option, - description: Option, - active: Option, - ) -> Result<()> { - if let Some(mut service_user) = self.service_users.get_mut(&id) { - if let Some(n) = name { - service_user.name = n; - } - if let Some(d) = description { - service_user.description = Some(d); - } - if let Some(a) = active { - service_user.active = a; - } - Ok(()) - } else { - Err(anyhow::anyhow!("Service user not found")) - } - } - - async fn delete_service_user(&self, id: Uuid) -> Result<()> { - self.service_users.remove(&id); - Ok(()) - } - - async fn update_service_user_last_used(&self, id: Uuid, timestamp: chrono::DateTime) -> Result<()> { - if let Some(mut service_user) = self.service_users.get_mut(&id) { - service_user.last_used = Some(timestamp); - Ok(()) - } else { - Err(anyhow::anyhow!("Service user not found")) - } - } - - // Merge Operation Methods - async fn create_merge_operation(&self, operation: pangolin_core::model::MergeOperation) -> Result<()> { - self.merge_operations.insert(operation.id, operation); - Ok(()) - } - - async fn get_merge_operation(&self, operation_id: Uuid) -> Result> { - Ok(self.merge_operations.get(&operation_id).map(|r| r.value().clone())) - } - - async fn list_merge_operations(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - Ok(self.merge_operations - .iter() - .filter(|r| r.value().tenant_id == tenant_id && r.value().catalog_name == catalog_name) - .map(|r| r.value().clone()) - .collect()) - } - - async fn update_merge_operation_status(&self, operation_id: Uuid, status: pangolin_core::model::MergeStatus) -> Result<()> { - if let Some(mut operation) = self.merge_operations.get_mut(&operation_id) { - operation.status = status; - Ok(()) - } else { - Err(anyhow::anyhow!("Merge operation not found")) - } - } - - async fn complete_merge_operation(&self, operation_id: Uuid, result_commit_id: Uuid) -> Result<()> { - if let Some(mut operation) = self.merge_operations.get_mut(&operation_id) { - operation.status = pangolin_core::model::MergeStatus::Completed; - operation.result_commit_id = Some(result_commit_id); - operation.completed_at = Some(chrono::Utc::now()); - Ok(()) - } else { - Err(anyhow::anyhow!("Merge operation not found")) - } - } - - async fn abort_merge_operation(&self, operation_id: Uuid) -> Result<()> { - if let Some(mut operation) = self.merge_operations.get_mut(&operation_id) { - operation.status = pangolin_core::model::MergeStatus::Aborted; - operation.completed_at = Some(chrono::Utc::now()); - Ok(()) - } else { - Err(anyhow::anyhow!("Merge operation not found")) - } - } - - // Merge Conflict Methods - async fn create_merge_conflict(&self, conflict: pangolin_core::model::MergeConflict) -> Result<()> { - self.merge_conflicts.insert(conflict.id, conflict); - Ok(()) - } - - async fn get_merge_conflict(&self, conflict_id: Uuid) -> Result> { - Ok(self.merge_conflicts.get(&conflict_id).map(|r| r.value().clone())) - } - - async fn list_merge_conflicts(&self, operation_id: Uuid) -> Result> { - Ok(self.merge_conflicts - .iter() - .filter(|r| r.value().merge_operation_id == operation_id) - .map(|r| r.value().clone()) - .collect()) - } - - async fn resolve_merge_conflict(&self, conflict_id: Uuid, resolution: pangolin_core::model::ConflictResolution) -> Result<()> { - if let Some(mut conflict) = self.merge_conflicts.get_mut(&conflict_id) { - conflict.resolution = Some(resolution); - Ok(()) - } else { - Err(anyhow::anyhow!("Merge conflict not found")) - } - } - - async fn add_conflict_to_operation(&self, operation_id: Uuid, conflict_id: Uuid) -> Result<()> { - if let Some(mut operation) = self.merge_operations.get_mut(&operation_id) { - if !operation.conflicts.contains(&conflict_id) { - operation.conflicts.push(conflict_id); - } - Ok(()) - } else { - Err(anyhow::anyhow!("Merge operation not found")) - } - } - - // Token Revocation Operations - async fn revoke_token(&self, token_id: Uuid, expires_at: chrono::DateTime, reason: Option) -> Result<()> { - let revoked = pangolin_core::token::RevokedToken::new(token_id, expires_at, reason); - self.revoked_tokens.insert(token_id, revoked); - Ok(()) - } - - async fn is_token_revoked(&self, token_id: Uuid) -> Result { - Ok(self.revoked_tokens.contains_key(&token_id)) - } - - async fn cleanup_expired_tokens(&self) -> Result { - let now = chrono::Utc::now(); - let to_remove: Vec = self.revoked_tokens - .iter() - .filter(|entry| entry.value().is_expired()) - .map(|entry| *entry.key()) - .collect(); - - let count = to_remove.len(); - for token_id in to_remove { - self.revoked_tokens.remove(&token_id); - } - Ok(count) - } - - // Token Operations - async fn list_active_tokens(&self, tenant_id: Uuid, user_id: Uuid) -> Result> { - let mut tokens = Vec::new(); - // Return tokens that match user and are not revoked/expired - for entry in self.active_tokens.iter() { - let token = entry.value(); - if token.tenant_id == tenant_id && token.user_id == user_id { - // Check revocation - if !self.revoked_tokens.contains_key(&token.id) && token.expires_at > Utc::now() { - tokens.push(token.clone()); - } - } - } - Ok(tokens) - } - - async fn store_token(&self, token: pangolin_core::token::TokenInfo) -> Result<()> { - self.active_tokens.insert(token.id, token); - Ok(()) - } - - // System Configuration - async fn get_system_settings(&self, tenant_id: Uuid) -> Result { - if let Some(s) = self.system_settings.get(&tenant_id) { - Ok(s.value().clone()) - } else { - // Return default if not set - Ok(SystemSettings::default()) - } - } - - async fn update_system_settings(&self, tenant_id: Uuid, settings: SystemSettings) -> Result { - self.system_settings.insert(tenant_id, settings.clone()); - Ok(settings) - } - - // Federated Catalog Operations - async fn sync_federated_catalog(&self, tenant_id: Uuid, catalog_name: &str) -> Result<()> { - let stats = SyncStats { - last_synced_at: Some(Utc::now()), - sync_status: "Success".to_string(), - tables_synced: 42, - namespaces_synced: 5, - error_message: None, - }; - self.federated_stats.insert((tenant_id, catalog_name.to_string()), stats); - Ok(()) - } - - async fn get_federated_catalog_stats(&self, tenant_id: Uuid, catalog_name: &str) -> Result { - if let Some(stats) = self.federated_stats.get(&(tenant_id, catalog_name.to_string())) { - Ok(stats.value().clone()) - } else { - // Return empty stats if never synced - Ok(SyncStats { - last_synced_at: None, - sync_status: "Not Synced".to_string(), - tables_synced: 0, - namespaces_synced: 0, - error_message: None, - }) - } - } -} - - - -impl MemoryStore { - pub fn get_warehouse_for_location(&self, location: &str) -> Option { - let warehouses_map: Vec = self.warehouses - .iter() - .map(|entry| entry.value().clone()) - .collect(); - - println!("DEBUG_MEM: get_warehouse_for_location checking {} warehouses for location: {}", warehouses_map.len(), location); - for w in &warehouses_map { - println!("DEBUG_MEM: Warehouse {} config keys: {:?}", w.name, w.storage_config.keys()); - } - - for warehouse in warehouses_map { - if let Some(bucket) = warehouse.storage_config.get("s3.bucket").or_else(|| warehouse.storage_config.get("bucket")) { - if location.contains(bucket) { - return Some(warehouse); - } - } - if let Some(container) = warehouse.storage_config.get("azure.container") { - if location.contains(container) { - return Some(warehouse); - } - } - if let Some(bucket) = warehouse.storage_config.get("gcp.bucket") { - if location.contains(bucket) { - return Some(warehouse); - } - } - } - None - } - - // Helper methods for performance optimizations - fn get_object_store_cache_key(&self, config: &std::collections::HashMap, location: &str) -> String { - let endpoint = config.get("s3.endpoint").or_else(|| config.get("endpoint")).or_else(|| config.get("azure.endpoint")).or_else(|| config.get("gcp.endpoint")).map(|s| s.as_str()).unwrap_or(""); - let bucket = self.extract_bucket_from_location(location); - let access_key = config.get("s3.access-key-id").or_else(|| config.get("access_key_id")).or_else(|| config.get("azure.account-name")).or_else(|| config.get("gcp.service-account")).map(|s| s.as_str()).unwrap_or(""); - let region = config.get("s3.region").or_else(|| config.get("region")).map(|s| s.as_str()).unwrap_or("us-east-1"); - - crate::ObjectStoreCache::cache_key(endpoint, &bucket, access_key, region) - } - - fn extract_bucket_from_location(&self, location: &str) -> String { - if let Some(rest) = location.strip_prefix("s3://").or_else(|| location.strip_prefix("az://")).or_else(|| location.strip_prefix("gs://")) { - if let Some((bucket, _)) = rest.split_once('/') { - return bucket.to_string(); - } - return rest.to_string(); - } - "default".to_string() - } - - fn extract_object_store_path(&self, location: &str) -> object_store::path::Path { - if let Some(rest) = location.strip_prefix("s3://").or_else(|| location.strip_prefix("az://")).or_else(|| location.strip_prefix("gs://")) { - if let Some((_, key)) = rest.split_once('/') { - return object_store::path::Path::from(key); - } - return object_store::path::Path::from(rest); - } - object_store::path::Path::from(location) - } - - async fn read_file_uncached(&self, location: &str) -> Result> { - // Try to read from object store first if configured - if let Some(warehouse) = self.get_warehouse_for_location(location) { - // Basic heuristic to skip memory-only locations if any - if location.starts_with("s3://") || location.starts_with("az://") || location.starts_with("gs://") { - // Use object store cache - let cache_key = self.get_object_store_cache_key(&warehouse.storage_config, location); - let store = self.object_store_cache.get_or_insert(cache_key, || { - Arc::new(crate::object_store_factory::create_object_store(&warehouse.storage_config, location) - .expect("Failed to create object store")) - }); - - let path = self.extract_object_store_path(location); - - match store.get(&path).await { - Ok(result) => return Ok(result.bytes().await?.to_vec()), - Err(e) => { - tracing::warn!("Failed to read from object store for {}, falling back to memory: {}", location, e); - } - } - } - } - - if let Some(data) = self.files.get(location) { - Ok(data.value().clone()) - } else { - Err(anyhow::anyhow!("File not found: {}", location)) - } - } -} - -#[async_trait] -impl Signer for MemoryStore { - async fn get_table_credentials(&self, location: &str) -> Result { - // 1. Find the warehouse that owns this location - // Iterate over all warehouses - let warehouses_map: Vec = self.warehouses - .iter() - .map(|entry| entry.value().clone()) - .collect(); - - // Simple prefix match. In real world, we might want more robust matching. - let mut target_warehouse = None; - - for warehouse in warehouses_map { - // Check AWS S3 - if let Some(bucket) = warehouse.storage_config.get("s3.bucket") { - if location.contains(bucket) { - target_warehouse = Some(warehouse); - break; - } - } - // Check Azure - if let Some(container) = warehouse.storage_config.get("azure.container") { - if location.contains(container) { - target_warehouse = Some(warehouse); - break; - } - } - // Check GCP - if let Some(bucket) = warehouse.storage_config.get("gcp.bucket") { - if location.contains(bucket) { - target_warehouse = Some(warehouse); - break; - } - } - } - - let warehouse = target_warehouse.ok_or_else(|| anyhow::anyhow!("No warehouse found for location: {}", location))?; - - // 2. Check Vending Strategy - match &warehouse.vending_strategy { - Some(VendingStrategy::AwsSts { role_arn: _, external_id: _ }) => { - Err(anyhow::anyhow!("AWS STS vending not implemented yet via VendingStrategy in MemoryStore")) - } - Some(VendingStrategy::AwsStatic { access_key_id, secret_access_key }) => { - Ok(Credentials::Aws { - access_key_id: access_key_id.clone(), - secret_access_key: secret_access_key.clone(), - session_token: None, - expiration: None, - }) - } - Some(VendingStrategy::AzureSas { account_name, account_key }) => { - #[cfg(feature = "azure")] - { - let signer = crate::azure_signer::AzureSigner::new(account_name.clone(), account_key.clone()); - let sas_token = signer.generate_sas_token(location).await?; - Ok(Credentials::Azure { - sas_token, - account_name: account_name.clone(), - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - }) - } - #[cfg(not(feature = "azure"))] - Err(anyhow::anyhow!("Azure vending requires 'azure' feature")) - } - Some(VendingStrategy::GcpDownscoped { service_account_email, private_key }) => { - #[cfg(feature = "gcp")] - { - let signer = crate::gcp_signer::GcpSigner::new(service_account_email.clone(), private_key.clone()); - let access_token = signer.generate_downscoped_token(location).await?; - Ok(Credentials::Gcp { - access_token, - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - }) - } - #[cfg(not(feature = "gcp"))] - Err(anyhow::anyhow!("GCP vending requires 'gcp' feature")) - } - Some(VendingStrategy::None) => Err(anyhow::anyhow!("Vending disabled")), - None => { - // Backward compatibility logic - let access_key = warehouse.storage_config.get("s3.access-key-id") - .ok_or_else(|| anyhow::anyhow!("Missing s3.access-key-id"))?; - let secret_key = warehouse.storage_config.get("s3.secret-access-key") - .ok_or_else(|| anyhow::anyhow!("Missing s3.secret-access-key"))?; - - if warehouse.use_sts { - // Existing STS Logic restored for backward compatibility - // MemoryStore doesn't usually make external calls, but to parity other stores: - let region = warehouse.storage_config.get("s3.region") - .map(|s| s.as_str()) - .unwrap_or("us-east-1"); - - let endpoint = warehouse.storage_config.get("s3.endpoint") - .map(|s| s.as_str()); - - let creds = aws_credential_types::Credentials::new( - access_key.to_string(), - secret_key.to_string(), - None, - None, - "legacy_provider" - ); - - let config_loader = aws_config::from_env() - .region(aws_config::Region::new(region.to_string())) - .credentials_provider(creds); - - let config = if let Some(ep) = endpoint { - config_loader.endpoint_url(ep).load().await - } else { - config_loader.load().await - }; - - let client = aws_sdk_sts::Client::new(&config); - - // For testing purposes, if we are in a test env without AWS creds, - // this client.get_session_token().send().await will likely fail. - // This failure is what we expect in the regression test "execution attempt". - - let role_arn = warehouse.storage_config.get("s3.role-arn").map(|s| s.as_str()); - - if let Some(arn) = role_arn { - let resp = client.assume_role() - .role_arn(arn) - .role_session_name("pangolin-memory-legacy") - .send() - .await - .map_err(|e| anyhow::anyhow!("STS AssumeRole failed: {}", e))?; - - let c = resp.credentials.ok_or_else(|| anyhow::anyhow!("No credentials in AssumeRole response"))?; - Ok(Credentials::Aws { - access_key_id: c.access_key_id, - secret_access_key: c.secret_access_key, - session_token: Some(c.session_token), - expiration: chrono::DateTime::from_timestamp(c.expiration.secs(), c.expiration.subsec_nanos()), - // Note: MemoryStore doesn't return Credentials struct in same way? - // Actually MemoryStore::get_table_credentials returns Credentials struct. - }) - } else { - let resp = client.get_session_token() - .send() - .await - .map_err(|e| anyhow::anyhow!("STS GetSessionToken failed: {}", e))?; - - let c = resp.credentials.ok_or_else(|| anyhow::anyhow!("No credentials in GetSessionToken response"))?; - Ok(Credentials::Aws { - access_key_id: c.access_key_id, - secret_access_key: c.secret_access_key, - session_token: Some(c.session_token), - expiration: chrono::DateTime::from_timestamp(c.expiration.secs(), c.expiration.subsec_nanos()), - }) - } - } else { - Ok(Credentials::Aws { - access_key_id: access_key.clone(), - secret_access_key: secret_key.clone(), - session_token: None, - expiration: None, - }) - } - } - } - } - - async fn presign_get(&self, _location: &str) -> Result { - // Stub: Presigning requires keeping an ObjectStore client around or rebuilding it. - // For this task, we focus on table credentials. - // Stub: Presigning requires keeping an ObjectStore client around or rebuilding it. - // For this task, we focus on table credentials. - Err(anyhow::anyhow!("MemoryStore does not support presigning yet")) - } - - -} - - -#[cfg(test)] -mod tests { - use super::*; - use pangolin_core::user::User; - use pangolin_core::permission::Permission; - use pangolin_core::model::{Tenant, Warehouse, AssetType}; - use std::collections::HashMap; - use chrono::Utc; - - #[tokio::test] - async fn test_tenant_operations() { - let store = MemoryStore::new(); - let tenant_id = Uuid::new_v4(); - let tenant = Tenant { - id: tenant_id, - name: "test_tenant".to_string(), - properties: HashMap::new(), - }; - - store.create_tenant(tenant.clone()).await.unwrap(); - let fetched = store.get_tenant(tenant_id).await.unwrap(); - assert!(fetched.is_some()); - assert_eq!(fetched.unwrap().name, "test_tenant"); - - let list = store.list_tenants().await.unwrap(); - assert_eq!(list.len(), 1); - } - - #[tokio::test] - async fn test_warehouse_operations() { - let store = MemoryStore::new(); - let tenant_id = Uuid::new_v4(); - let warehouse = Warehouse { - id: Uuid::new_v4(), - name: "main_warehouse".to_string(), - tenant_id, - storage_config: HashMap::new(), - use_sts: false, - vending_strategy: None, - }; - - store.create_warehouse(tenant_id, warehouse.clone()).await.unwrap(); - let fetched = store.get_warehouse(tenant_id, "main_warehouse".to_string()).await.unwrap(); - assert!(fetched.is_some()); - - let list = store.list_warehouses(tenant_id).await.unwrap(); - assert_eq!(list.len(), 1); - } - - #[tokio::test] - async fn test_asset_operations() { - let store = MemoryStore::new(); - let tenant_id = Uuid::new_v4(); - let catalog = "default"; - let namespace = vec!["ns1".to_string()]; - - let asset = Asset { - id: Uuid::new_v4(), - name: "tbl1".to_string(), - kind: AssetType::IcebergTable, - location: "s3://loc".to_string(), - properties: HashMap::new(), - }; - - store.create_asset(tenant_id, catalog, None, namespace.clone(), asset.clone()).await.unwrap(); - - let fetched = store.get_asset(tenant_id, catalog, None, namespace.clone(), "tbl1".to_string()).await.unwrap(); - assert!(fetched.is_some()); - - // Rename - store.rename_asset(tenant_id, catalog, None, namespace.clone(), "tbl1".to_string(), namespace.clone(), "tbl2".to_string()).await.unwrap(); - let old = store.get_asset(tenant_id, catalog, None, namespace.clone(), "tbl1".to_string()).await.unwrap(); - assert!(old.is_none()); - let new = store.get_asset(tenant_id, catalog, None, namespace.clone(), "tbl2".to_string()).await.unwrap(); - assert!(new.is_some()); - - store.delete_asset(tenant_id, catalog, None, namespace.clone(), "tbl2".to_string()).await.unwrap(); - let deleted = store.get_asset(tenant_id, catalog, None, namespace.clone(), "tbl2".to_string()).await.unwrap(); - assert!(deleted.is_none()); - } - - #[tokio::test] - async fn test_asset_update_consistency() { - let store = MemoryStore::new(); - crate::tests::test_asset_update_consistency(&store).await; - } - - #[tokio::test] - async fn test_list_permissions_filtering() { - use pangolin_core::permission::{Action, PermissionScope}; - use std::collections::HashSet; - - let store = MemoryStore::new(); - let tenant1 = Uuid::new_v4(); - let tenant2 = Uuid::new_v4(); - - // Create users for tenants - let user1 = Uuid::new_v4(); - let user2 = Uuid::new_v4(); - - let u1 = User { - id: user1, - username: "user1".to_string(), - email: "user1@example.com".to_string(), - password_hash: Some("hash".to_string()), - role: pangolin_core::user::UserRole::TenantUser, - tenant_id: Some(tenant1), - created_at: Utc::now(), - updated_at: Utc::now(), - last_login: None, - active: true, - oauth_provider: None, - oauth_subject: None, - }; - store.create_user(u1).await.unwrap(); - - let u2 = User { - id: user2, - username: "user2".to_string(), - email: "user2@example.com".to_string(), - password_hash: Some("hash".to_string()), - role: pangolin_core::user::UserRole::TenantUser, - tenant_id: Some(tenant2), - created_at: Utc::now(), - updated_at: Utc::now(), - last_login: None, - active: true, - oauth_provider: None, - oauth_subject: None, - }; - store.create_user(u2).await.unwrap(); - - // Grant permissions - let p1 = Permission::new( - user1, - PermissionScope::Catalog { catalog_id: Uuid::new_v4() }, - HashSet::from([Action::Read]), - Uuid::new_v4(), - ); - store.create_permission(p1.clone()).await.unwrap(); - - let p2 = Permission::new( - user2, - PermissionScope::Catalog { catalog_id: Uuid::new_v4() }, - HashSet::from([Action::Write]), - Uuid::new_v4(), - ); - store.create_permission(p2.clone()).await.unwrap(); - - // Test tenant filtering - let perms_t1 = store.list_permissions(tenant1).await.unwrap(); - assert_eq!(perms_t1.len(), 1); - assert_eq!(perms_t1[0].user_id, user1); - - let perms_t2 = store.list_permissions(tenant2).await.unwrap(); - assert_eq!(perms_t2.len(), 1); - assert_eq!(perms_t2[0].user_id, user2); - - // Test user filtering - let perms_u1 = store.list_user_permissions(user1).await.unwrap(); - assert_eq!(perms_u1.len(), 1); - assert_eq!(perms_u1[0].id, p1.id); - } - - #[tokio::test] - async fn test_list_user_permissions_aggregation() { - use pangolin_core::permission::{Action, PermissionScope, Role, UserRole}; - use std::collections::HashSet; - - let store = MemoryStore::new(); - let tenant_id = Uuid::new_v4(); - let user_id = Uuid::new_v4(); - let admin_id = Uuid::new_v4(); - - // 1. Create a Role with permissions - let mut role = Role::new("test-role".to_string(), None, tenant_id, admin_id); - let role_scope = PermissionScope::Catalog { catalog_id: Uuid::new_v4() }; - let mut role_actions = HashSet::new(); - role_actions.insert(Action::Read); - role.add_permission(role_scope.clone(), role_actions); - store.create_role(role.clone()).await.unwrap(); - - // 2. Assign role to user - let user_role = UserRole::new(user_id, role.id, admin_id); - store.assign_role(user_role).await.unwrap(); - - // 3. Create a direct permission - let direct_scope = PermissionScope::Catalog { catalog_id: Uuid::new_v4() }; - let mut direct_actions = HashSet::new(); - direct_actions.insert(Action::Write); - let direct_perm = Permission::new(user_id, direct_scope.clone(), direct_actions, admin_id); - store.create_permission(direct_perm.clone()).await.unwrap(); - - // 4. List user permissions and verify aggregation - let aggregated_perms = store.list_user_permissions(user_id).await.unwrap(); - - assert_eq!(aggregated_perms.len(), 2, "Should have 2 permissions (1 direct, 1 from role)"); - - let has_direct = aggregated_perms.iter().any(|p| p.scope == direct_scope && p.actions.contains(&Action::Write)); - let has_role_based = aggregated_perms.iter().any(|p| p.scope == role_scope && p.actions.contains(&Action::Read)); - - assert!(has_direct, "Aggregated permissions should include direct permission"); - assert!(has_role_based, "Aggregated permissions should include role-based permission"); - } -} - - diff --git a/pangolin/pangolin_store/src/memory/access_requests.rs b/pangolin/pangolin_store/src/memory/access_requests.rs index 788180f..c7b3514 100644 --- a/pangolin/pangolin_store/src/memory/access_requests.rs +++ b/pangolin/pangolin_store/src/memory/access_requests.rs @@ -44,13 +44,7 @@ impl MemoryStore { .filter(|req| req.value().tenant_id == tenant_id) .map(|req| req.value().clone()); - let requests = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let requests = crate::memory::main::paginate_sorted(iter, pagination, |r| r.id); Ok(requests) } } diff --git a/pangolin/pangolin_store/src/memory/assets.rs b/pangolin/pangolin_store/src/memory/assets.rs index bd0b006..621a186 100644 --- a/pangolin/pangolin_store/src/memory/assets.rs +++ b/pangolin/pangolin_store/src/memory/assets.rs @@ -29,6 +29,7 @@ impl MemoryStore { self.assets_by_id.insert( asset.id, ( + tenant_id, catalog_name.to_string(), namespace.clone(), Some(branch_name.clone()), @@ -83,8 +84,12 @@ impl MemoryStore { asset_id: Uuid, ) -> Result)>> { if let Some(entry) = self.assets_by_id.get(&asset_id) { - let (catalog_name, namespace, branch, name) = entry.value().clone(); - // Verify tenant ownership (implicit via proper key lookup) purely for safety + let (owner_tenant, catalog_name, namespace, branch, name) = entry.value().clone(); + // The index is global across tenants, so ownership is checked here + // rather than relying on the composite key lookup below to miss. + if owner_tenant != tenant_id { + return Ok(None); + } let branch_name = branch.unwrap_or_else(|| "main".to_string()); let key = ( tenant_id, @@ -120,13 +125,8 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let assets: Vec = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let assets: Vec = + crate::memory::main::paginate_sorted(iter, pagination, |a| a.name.clone()); Ok(assets) } pub(crate) async fn delete_asset_internal( @@ -184,6 +184,7 @@ impl MemoryStore { self.assets_by_id.insert( asset.id, ( + tenant_id, catalog_name.to_string(), dest_namespace, Some(branch_val), @@ -245,15 +246,17 @@ impl MemoryStore { false }; - let tags_match = if let Some(ref search_tags) = tags { - if let Some(ref meta) = metadata { - search_tags.iter().any(|tag| meta.tags.contains(tag)) - } else { - false - } - } else { - true - }; + // B28: this was an ANY-match, while Postgres (`@>`) and Mongo + // (`$all`) required all requested tags - and an *empty* tag list + // returned nothing here but everything on the other three. + // `crate::search::tags_match` is the single definition of the + // chosen ALL-match semantic, with an empty list meaning "no tag + // filter". + let owned_tags = metadata + .as_ref() + .map(|m| m.tags.clone()) + .unwrap_or_default(); + let tags_match = crate::search::tags_match(&owned_tags, tags.as_deref()); if (name_matches || description_matches) && tags_match { let namespace: Vec = @@ -316,6 +319,7 @@ impl MemoryStore { self.assets_by_id.insert( asset.id, ( + tenant_id, catalog_name.to_string(), ns.clone(), Some(dest_branch.to_string()), diff --git a/pangolin/pangolin_store/src/memory/audit.rs b/pangolin/pangolin_store/src/memory/audit.rs index f80d834..f26d385 100644 --- a/pangolin/pangolin_store/src/memory/audit.rs +++ b/pangolin/pangolin_store/src/memory/audit.rs @@ -2,6 +2,9 @@ use super::MemoryStore; use anyhow::Result; use uuid::Uuid; +/// Default cap on a listing, matching the SQL backends' `LIMIT 100`. +const DEFAULT_AUDIT_LIMIT: usize = 100; + impl MemoryStore { pub(crate) async fn log_audit_event_internal( &self, @@ -19,6 +22,14 @@ impl MemoryStore { tenant_id: Uuid, filter: Option, ) -> Result> { + // Read the pagination window out before the filter is consumed below. + let pagination = filter.as_ref().map(|f| { + ( + f.offset.unwrap_or(0), + f.limit.unwrap_or(DEFAULT_AUDIT_LIMIT), + ) + }); + if let Some(events) = self.audit_events.get(&tenant_id) { let mut filtered = events.clone(); @@ -76,13 +87,22 @@ impl MemoryStore { true }); + } - // Apply pagination - let offset = f.offset.unwrap_or(0); - let limit = f.limit.unwrap_or(100); + // B29: two divergences from the SQL backends, both fixed here. + // + // 1. Events were returned in *insertion* order (oldest first) while + // every SQL backend uses `ORDER BY timestamp DESC`. A caller + // asking for "the last 100 events" got the *first* 100. + // 2. Pagination lived inside the `if let Some(filter)` block, so a + // filterless listing returned the tenant's entire audit history + // while the SQL backends capped it at 100. On a busy tenant that + // is an unbounded allocation driven by an unauthenticated-shaped + // call pattern. + filtered.sort_by(|a, b| b.timestamp.cmp(&a.timestamp)); - filtered = filtered.into_iter().skip(offset).take(limit).collect(); - } + let (offset, limit) = pagination.unwrap_or((0, DEFAULT_AUDIT_LIMIT)); + let filtered = filtered.into_iter().skip(offset).take(limit).collect(); Ok(filtered) } else { diff --git a/pangolin/pangolin_store/src/memory/branches.rs b/pangolin/pangolin_store/src/memory/branches.rs index ef8cc15..4171a85 100644 --- a/pangolin/pangolin_store/src/memory/branches.rs +++ b/pangolin/pangolin_store/src/memory/branches.rs @@ -62,13 +62,8 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let branches: Vec = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let branches: Vec = + crate::memory::main::paginate_sorted(iter, pagination, |b| b.name.clone()); Ok(branches) } pub(crate) async fn delete_branch_internal( @@ -101,7 +96,8 @@ impl MemoryStore { source_branch_name: String, target_branch_name: String, ) -> Result<()> { - self.get_branch_internal(tenant_id, catalog_name, source_branch_name.clone()) + let source_branch = self + .get_branch_internal(tenant_id, catalog_name, source_branch_name.clone()) .await? .ok_or_else(|| anyhow::anyhow!("Source branch '{}' not found", source_branch_name))?; @@ -126,21 +122,34 @@ impl MemoryStore { let namespace_parts: Vec = namespace_key.split('\x1F').map(|s| s.to_string()).collect(); + // B25: the copy used to keep `asset.id`. `create_asset_internal` + // writes `assets_by_id[asset.id]`, so the shared id was repointed at + // the *target*-branch copy and every subsequent `get_asset_by_id` + // for the source branch's asset resolved to the wrong branch. A + // merged copy is a distinct row and needs a distinct identity. + let mut copied = asset.clone(); + copied.id = Uuid::new_v4(); + self.create_asset_internal( tenant_id, catalog_name, Some(target_branch_name.clone()), namespace_parts.clone(), - asset.clone(), + copied.clone(), ) .await?; - let qualified = format!("{}.{}", namespace_parts.join("."), asset.name); + let qualified = format!("{}.{}", namespace_parts.join("."), copied.name); if !target_branch.assets.contains(&qualified) { target_branch.assets.push(qualified); } } + // B25: the target's head was never advanced, unlike all three other + // backends, so after a merge the target branch still pointed at its + // pre-merge commit. + target_branch.head_commit_id = source_branch.head_commit_id; + self.create_branch_internal(tenant_id, catalog_name, target_branch) .await?; diff --git a/pangolin/pangolin_store/src/memory/catalogs.rs b/pangolin/pangolin_store/src/memory/catalogs.rs index f91401c..d29163e 100644 --- a/pangolin/pangolin_store/src/memory/catalogs.rs +++ b/pangolin/pangolin_store/src/memory/catalogs.rs @@ -118,10 +118,16 @@ impl MemoryStore { // Remove Tags self.tags.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - // Clean up assets_by_id index - // This is expensive O(N) since we have to scan the whole index - // But deletion is rare. - self.assets_by_id.retain(|_, v| v.0 != name); + // Clean up assets_by_id index. + // + // B6: this filtered on the catalog *name* alone, so deleting + // tenant A's `sales` catalog also evicted tenant B's `sales` assets + // from the by-id index - `get_asset_by_id` then returned `None` for + // rows that were still perfectly present. The index value now + // carries the tenant, so the cascade matches on both. + // O(N) over the index, but catalog deletion is rare. + self.assets_by_id + .retain(|_, v| !(v.0 == tenant_id && v.1 == name)); Ok(()) } else { diff --git a/pangolin/pangolin_store/src/memory/io.rs b/pangolin/pangolin_store/src/memory/io.rs index f5b7daa..6e9bd59 100644 --- a/pangolin/pangolin_store/src/memory/io.rs +++ b/pangolin/pangolin_store/src/memory/io.rs @@ -52,21 +52,38 @@ impl MemoryStore { ); if let Some(mut asset) = self.assets.get_mut(&key) { + // Resolved exactly as `get_metadata_location_internal` resolves it - + // property first, then the asset's own location - so the CAS + // compares against the value a reader would have seen. This mirrors + // SQLite, where the fallback is the `metadata_location` column. let current_loc = asset .properties .get("metadata_location") .cloned() - .unwrap_or(asset.location.clone()); + .or_else(|| { + if asset.location.is_empty() { + None + } else { + Some(asset.location.clone()) + } + }); - // CAS Check - if let Some(expected) = expected_location { - if current_loc != expected { - return Err(anyhow::anyhow!( - "CAS failure: expected {} but found {}", - expected, - current_loc - )); - } + // CAS check. + // + // B26: this used to be `if let Some(expected) = expected_location`, + // which skipped the check entirely when the caller passed `None`. + // But `None` is not "don't check" - it is the *create-path* + // assertion "there must be no metadata location yet". Skipping it + // meant a create-table race that Postgres and SQLite correctly + // rejected silently succeeded in dev and in every memory-backed + // test, which is precisely where such a race would have been caught. + // The unconditional comparison matches the SQLite form. + if current_loc != expected_location { + return Err(anyhow::anyhow!( + "CAS failure: expected {:?} but found {:?}", + expected_location, + current_loc + )); } asset.location = new_location.clone(); diff --git a/pangolin/pangolin_store/src/memory/main.rs b/pangolin/pangolin_store/src/memory/main.rs index 39f9a94..a816faf 100644 --- a/pangolin/pangolin_store/src/memory/main.rs +++ b/pangolin/pangolin_store/src/memory/main.rs @@ -31,7 +31,15 @@ pub struct MemoryStore { pub(crate) service_users: Arc>, pub(crate) merge_operations: Arc>, pub(crate) merge_conflicts: Arc>, - pub(crate) assets_by_id: Arc, Option, String)>>, + /// Asset id -> (tenant, catalog, namespace, branch, name). + /// + /// B6: the tenant used to be absent from the value, so `delete_catalog`'s + /// index cleanup could only filter on the catalog *name* - and tenant A + /// deleting a catalog called `sales` broke `get_asset_by_id` for tenant B's + /// unrelated catalog of the same name. Catalog names are per-tenant, so the + /// index key has to be too. + pub(crate) assets_by_id: + Arc, Option, String)>>, pub(crate) revoked_tokens: Arc>, pub(crate) active_tokens: Arc>, pub(crate) system_settings: Arc>, @@ -79,3 +87,36 @@ impl MemoryStore { } } } + +/// Deterministically page an in-memory listing. +/// +/// B27: every memory-backend listing paged straight over DashMap iteration +/// order, which is a hash order that varies run to run and shifts as entries are +/// inserted or removed. Two consecutive pages could therefore repeat a row or +/// skip one entirely - the same defect the SQL backends had from `LIMIT/OFFSET` +/// with no `ORDER BY`, and it makes the "two pages cover the set exactly once" +/// property untestable. +/// +/// Sorting by a stable key before slicing gives the memory backend the same +/// observable ordering the SQL backends now get from `ORDER BY`. +pub(crate) fn paginate_sorted( + items: impl Iterator, + pagination: Option, + key: F, +) -> Vec +where + F: Fn(&T) -> K, + K: Ord, +{ + let mut all: Vec = items.collect(); + all.sort_by_key(&key); + + match pagination { + Some(p) => all + .into_iter() + .skip(p.offset.unwrap_or(0)) + .take(p.limit.unwrap_or(usize::MAX)) + .collect(), + None => all, + } +} diff --git a/pangolin/pangolin_store/src/memory/mod.rs b/pangolin/pangolin_store/src/memory/mod.rs index 7a95761..4d31a96 100644 --- a/pangolin/pangolin_store/src/memory/mod.rs +++ b/pangolin/pangolin_store/src/memory/mod.rs @@ -171,6 +171,17 @@ impl CatalogStore for MemoryStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + self.replace_namespace_properties_internal(tenant_id, catalog_name, namespace, properties) + .await + } + async fn count_namespaces(&self, tenant_id: Uuid) -> Result { self.count_namespaces_internal(tenant_id).await } @@ -413,6 +424,15 @@ impl CatalogStore for MemoryStore { self.write_file_internal(location, content).await } + async fn delete_file(&self, location: &str) -> Result<()> { + self.metadata_cache.invalidate(location).await; + self.files.remove(location); + let storage_config = self + .get_warehouse_for_location(location) + .map(|w| w.storage_config); + crate::file_delete::delete_location(storage_config.as_ref(), location).await + } + async fn expire_snapshots( &self, tenant_id: Uuid, diff --git a/pangolin/pangolin_store/src/memory/namespaces.rs b/pangolin/pangolin_store/src/memory/namespaces.rs index 643eef9..c195719 100644 --- a/pangolin/pangolin_store/src/memory/namespaces.rs +++ b/pangolin/pangolin_store/src/memory/namespaces.rs @@ -3,6 +3,17 @@ use anyhow::Result; use pangolin_core::model::*; use uuid::Uuid; +/// The single encoding of a namespace path into a map key. +/// +/// B17: create/get keyed with `join(".")` (via `Namespace::to_string`) while +/// delete/update keyed with `join("\x1F")`. The two never met, so a multi-level +/// namespace could never be deleted or updated on the memory backend - both +/// returned "Namespace not found" - while the SQL backends succeeded. One helper +/// used everywhere makes that class of divergence impossible. +fn ns_key(tenant_id: Uuid, catalog_name: &str, namespace: &[String]) -> (Uuid, String, String) { + (tenant_id, catalog_name.to_string(), namespace.join(".")) +} + impl MemoryStore { pub(crate) async fn create_namespace_internal( &self, @@ -10,7 +21,7 @@ impl MemoryStore { catalog_name: &str, namespace: Namespace, ) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), namespace.to_string()); + let key = ns_key(tenant_id, catalog_name, &namespace.name); self.namespaces.insert(key, namespace); Ok(()) } @@ -40,13 +51,8 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let namespaces: Vec = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let namespaces: Vec = + crate::memory::main::paginate_sorted(iter, pagination, |n| n.name.clone()); tracing::info!("DEBUG_MEM: Found {} namespaces", namespaces.len()); Ok(namespaces) @@ -57,7 +63,7 @@ impl MemoryStore { catalog_name: &str, namespace: Vec, ) -> Result> { - let key = (tenant_id, catalog_name.to_string(), namespace.join(".")); + let key = ns_key(tenant_id, catalog_name, &namespace); if let Some(n) = self.namespaces.get(&key) { Ok(Some(n.value().clone())) } else { @@ -70,8 +76,7 @@ impl MemoryStore { catalog_name: &str, namespace: Vec, ) -> Result<()> { - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), ns_str); + let key = ns_key(tenant_id, catalog_name, &namespace); if self.namespaces.remove(&key).is_some() { Ok(()) } else { @@ -85,8 +90,7 @@ impl MemoryStore { namespace: Vec, properties: std::collections::HashMap, ) -> Result<()> { - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), ns_str); + let key = ns_key(tenant_id, catalog_name, &namespace); if let Some(mut ns) = self.namespaces.get_mut(&key) { ns.properties.extend(properties); @@ -95,6 +99,22 @@ impl MemoryStore { Err(anyhow::anyhow!("Namespace not found")) } } + pub(crate) async fn replace_namespace_properties_internal( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + let key = ns_key(tenant_id, catalog_name, &namespace); + + if let Some(mut ns) = self.namespaces.get_mut(&key) { + ns.properties = properties; + Ok(()) + } else { + Err(anyhow::anyhow!("Namespace not found")) + } + } pub(crate) async fn count_namespaces_internal(&self, tenant_id: Uuid) -> Result { // Efficient counting for MemoryStore let count = self diff --git a/pangolin/pangolin_store/src/memory/permissions.rs b/pangolin/pangolin_store/src/memory/permissions.rs index e6ddcf5..38c6155 100644 --- a/pangolin/pangolin_store/src/memory/permissions.rs +++ b/pangolin/pangolin_store/src/memory/permissions.rs @@ -26,13 +26,7 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let permissions = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let permissions = crate::memory::main::paginate_sorted(iter, pagination, |p| p.id); Ok(permissions) } diff --git a/pangolin/pangolin_store/src/memory/roles.rs b/pangolin/pangolin_store/src/memory/roles.rs index fe04fb9..2a4d319 100644 --- a/pangolin/pangolin_store/src/memory/roles.rs +++ b/pangolin/pangolin_store/src/memory/roles.rs @@ -28,13 +28,7 @@ impl MemoryStore { .filter(|r| r.value().tenant_id == tenant_id) .map(|r| r.value().clone()); - let roles = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let roles = crate::memory::main::paginate_sorted(iter, pagination, |r| r.name.clone()); Ok(roles) } pub(crate) async fn update_role_internal(&self, role: Role) -> Result<()> { diff --git a/pangolin/pangolin_store/src/memory/service_users.rs b/pangolin/pangolin_store/src/memory/service_users.rs index 2fc12d2..68c862e 100644 --- a/pangolin/pangolin_store/src/memory/service_users.rs +++ b/pangolin/pangolin_store/src/memory/service_users.rs @@ -27,13 +27,7 @@ impl MemoryStore { .filter(|entry| entry.value().tenant_id == tenant_id) .map(|entry| entry.value().clone()); - let result = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let result = crate::memory::main::paginate_sorted(iter, pagination, |s| s.name.clone()); Ok(result) } /// Record that a service user's API key was just used. diff --git a/pangolin/pangolin_store/src/memory/tags.rs b/pangolin/pangolin_store/src/memory/tags.rs index c410f92..7965d01 100644 --- a/pangolin/pangolin_store/src/memory/tags.rs +++ b/pangolin/pangolin_store/src/memory/tags.rs @@ -42,13 +42,7 @@ impl MemoryStore { }) .map(|r| r.value().clone()); - let tags = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let tags = crate::memory::main::paginate_sorted(iter, pagination, |t| t.name.clone()); Ok(tags) } pub(crate) async fn delete_tag_internal( diff --git a/pangolin/pangolin_store/src/memory/tenants.rs b/pangolin/pangolin_store/src/memory/tenants.rs index 15bb6c5..4d9c28b 100644 --- a/pangolin/pangolin_store/src/memory/tenants.rs +++ b/pangolin/pangolin_store/src/memory/tenants.rs @@ -53,12 +53,41 @@ impl MemoryStore { Err(anyhow::anyhow!("Tenant not found")) } } + /// Delete a tenant and everything scoped to it. + /// + /// B30: the cascade was a `// TODO`. Warehouses (with their cloud + /// credentials), catalogs, namespaces, assets, branches, tags, audit + /// history, permissions and cached tokens all survived tenant deletion on + /// the memory backend - so a "deleted" tenant's storage credentials were + /// still vendable, and a recreated tenant with the same id inherited the old + /// one's data. The retain-based pattern here is the one `delete_catalog` + /// already used. pub(crate) async fn delete_tenant_internal(&self, tenant_id: Uuid) -> Result<()> { - if self.tenants.remove(&tenant_id).is_some() { - // TODO: Cascade delete warehouses and catalogs - Ok(()) - } else { - Err(anyhow::anyhow!("Tenant not found")) + if self.tenants.remove(&tenant_id).is_none() { + return Err(anyhow::anyhow!("Tenant not found")); } + + // Keyed by (tenant, ..) - drop everything whose first key element is + // this tenant. + self.warehouses.retain(|k, _| k.0 != tenant_id); + self.catalogs.retain(|k, _| k.0 != tenant_id); + self.namespaces.retain(|k, _| k.0 != tenant_id); + self.assets.retain(|k, _| k.0 != tenant_id); + self.branches.retain(|k, _| k.0 != tenant_id); + self.tags.retain(|k, _| k.0 != tenant_id); + self.commits.retain(|k, _| k.0 != tenant_id); + self.federated_stats.retain(|k, _| k.0 != tenant_id); + + // Keyed by their own id, with the tenant in the value. + self.assets_by_id.retain(|_, v| v.0 != tenant_id); + self.audit_events.remove(&tenant_id); + self.system_settings.remove(&tenant_id); + self.users.retain(|_, u| u.tenant_id != Some(tenant_id)); + self.roles.retain(|_, r| r.tenant_id != tenant_id); + self.permissions.retain(|_, p| p.tenant_id != tenant_id); + self.service_users.retain(|_, s| s.tenant_id != tenant_id); + self.active_tokens.retain(|_, t| t.tenant_id != tenant_id); + + Ok(()) } } diff --git a/pangolin/pangolin_store/src/memory/tokens.rs b/pangolin/pangolin_store/src/memory/tokens.rs index b62ae8b..dbd73b5 100644 --- a/pangolin/pangolin_store/src/memory/tokens.rs +++ b/pangolin/pangolin_store/src/memory/tokens.rs @@ -32,13 +32,9 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let tokens = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let tokens = crate::memory::main::paginate_sorted(iter, pagination, |t| { + (std::cmp::Reverse(t.expires_at), t.id) + }); Ok(tokens) } diff --git a/pangolin/pangolin_store/src/memory/users.rs b/pangolin/pangolin_store/src/memory/users.rs index 501db25..cdedf6f 100644 --- a/pangolin/pangolin_store/src/memory/users.rs +++ b/pangolin/pangolin_store/src/memory/users.rs @@ -43,13 +43,7 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let users = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let users = crate::memory::main::paginate_sorted(iter, pagination, |u| u.username.clone()); Ok(users) } pub(crate) async fn update_user_internal(&self, user: User) -> Result<()> { diff --git a/pangolin/pangolin_store/src/memory/warehouses.rs b/pangolin/pangolin_store/src/memory/warehouses.rs index 3f5bcb0..f1dbbe2 100644 --- a/pangolin/pangolin_store/src/memory/warehouses.rs +++ b/pangolin/pangolin_store/src/memory/warehouses.rs @@ -36,13 +36,8 @@ impl MemoryStore { .filter(|r| r.key().0 == tenant_id) .map(|r| r.value().clone()); - let warehouses: Vec = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let warehouses: Vec = + crate::memory::main::paginate_sorted(iter, pagination, |w| w.name.clone()); Ok(warehouses) } pub(crate) async fn update_warehouse_internal( diff --git a/pangolin/pangolin_store/src/mongo.rs.bak b/pangolin/pangolin_store/src/mongo.rs.bak deleted file mode 100644 index 4dd3df1..0000000 --- a/pangolin/pangolin_store/src/mongo.rs.bak +++ /dev/null @@ -1,2112 +0,0 @@ -use crate::CatalogStore; -use anyhow::Result; -use async_trait::async_trait; -use futures::stream::TryStreamExt; -use mongodb::{Client, Collection, Database}; -use mongodb::bson::{doc, Document, Bson, Binary}; -use mongodb::bson::spec::BinarySubtype; -use mongodb::options::{ClientOptions, ReplaceOptions}; -use pangolin_core::model::{ - Asset, AssetType, Branch, Catalog, Commit, Namespace, Tag, Tenant, Warehouse, VendingStrategy, - SystemSettings, SyncStats, -}; -use pangolin_core::user::{User, UserRole as CoreUserRole, OAuthProvider}; -use pangolin_core::permission::{Role, Permission, PermissionGrant, UserRole as UserRoleAssignment}; -use pangolin_core::business_metadata::{AccessRequest, RequestStatus, BusinessMetadata}; -use pangolin_core::token::TokenInfo; -use crate::signer::{Signer, Credentials}; -use object_store::ObjectStore; -use object_store::aws::AmazonS3Builder; -use pangolin_core::audit::AuditLogEntry; -use uuid::Uuid; -use std::collections::HashMap; -use chrono::Utc; -use std::sync::Arc; -use object_store::path::Path as ObjPath; - -#[derive(Clone)] -pub struct MongoStore { - client: Client, - db: Database, - object_store_cache: crate::ObjectStoreCache, - metadata_cache: crate::MetadataCache, -} - -impl MongoStore { - pub async fn new(connection_string: &str, database_name: &str) -> Result { - let mut client_options = ClientOptions::parse(connection_string).await?; - - // Configure connection pool from environment variables - if let Ok(max_pool_size) = std::env::var("MONGO_MAX_POOL_SIZE") { - if let Ok(size) = max_pool_size.parse::() { - client_options.max_pool_size = Some(size); - tracing::info!("MongoDB max pool size set to: {}", size); - } - } - - if let Ok(min_pool_size) = std::env::var("MONGO_MIN_POOL_SIZE") { - if let Ok(size) = min_pool_size.parse::() { - client_options.min_pool_size = Some(size); - tracing::info!("MongoDB min pool size set to: {}", size); - } - } - - // Set app name - client_options.app_name = Some("Pangolin".to_string()); - - let client = Client::with_options(client_options)?; - let db = client.database(database_name); - Ok(Self { - client, - db, - object_store_cache: crate::ObjectStoreCache::default(), - metadata_cache: crate::MetadataCache::default(), - }) - } - - fn tenants(&self) -> Collection { - self.db.collection("tenants") - } - - fn warehouses(&self) -> Collection { - self.db.collection("warehouses") - } - - fn catalogs(&self) -> Collection { - self.db.collection("catalogs") - } - - fn namespaces(&self) -> Collection { - self.db.collection("namespaces") - } - - fn assets(&self) -> Collection { - self.db.collection("assets") - } - - fn branches(&self) -> Collection { - self.db.collection("branches") - } - - fn tags(&self) -> Collection { - self.db.collection("tags") - } - - fn commits(&self) -> Collection { - self.db.collection("commits") - } - - fn audit_logs(&self) -> Collection { - self.db.collection("audit_logs") - } - - fn users(&self) -> Collection { - self.db.collection("users") - } - - fn roles(&self) -> Collection { - self.db.collection("roles") - } - - fn user_roles(&self) -> Collection { - self.db.collection("user_roles") - } - - fn permissions(&self) -> Collection { - self.db.collection("permissions") - } - - fn access_requests(&self) -> Collection { - self.db.collection("access_requests") - } - - fn business_metadata(&self) -> Collection { - self.db.collection("business_metadata") - } - - fn active_tokens(&self) -> Collection { - self.db.collection("active_tokens") - } - - fn system_settings(&self) -> Collection { - self.db.collection("system_settings") - } - - fn federated_sync_stats(&self) -> Collection { - self.db.collection("federated_sync_stats") - } - - fn merge_operations(&self) -> Collection { - self.db.collection("merge_operations") - } - - fn merge_conflicts(&self) -> Collection { - self.db.collection("merge_conflicts") - } - - fn service_users(&self) -> Collection { - self.db.collection("service_users") - } -} - -#[async_trait] -impl CatalogStore for MongoStore { - // Tenant Operations - async fn create_tenant(&self, tenant: Tenant) -> Result<()> { - self.tenants().insert_one(tenant).await?; - Ok(()) - } - - async fn get_tenant(&self, id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(id) }; - let tenant = self.tenants().find_one(filter).await?; - Ok(tenant) - } - - async fn list_tenants(&self) -> Result> { - let cursor = self.tenants().find(doc! {}).await?; - let tenants: Vec = cursor.try_collect().await?; - Ok(tenants) - } - - async fn update_tenant(&self, tenant_id: Uuid, updates: pangolin_core::model::TenantUpdate) -> Result { - let filter = doc! { "id": to_bson_uuid(tenant_id) }; - let mut update_doc = doc! {}; - - if let Some(name) = updates.name { - update_doc.insert("name", name); - } - if let Some(properties) = updates.properties { - update_doc.insert("properties", bson::to_bson(&properties)?); - } - - if update_doc.is_empty() { - return self.get_tenant(tenant_id).await? - .ok_or_else(|| anyhow::anyhow!("Tenant not found")); - } - - let update = doc! { "$set": update_doc }; - self.tenants().update_one(filter.clone(), update).await?; - - self.get_tenant(tenant_id).await? - .ok_or_else(|| anyhow::anyhow!("Tenant not found")) - } - - async fn delete_tenant(&self, tenant_id: Uuid) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(tenant_id) }; - let result = self.tenants().delete_one(filter).await?; - - if result.deleted_count == 0 { - return Err(anyhow::anyhow!("Tenant not found")); - } - Ok(()) - } - - // Warehouse Operations - async fn create_warehouse(&self, tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { - // We might want to store tenant_id in the warehouse document if it's not already there - // The Warehouse struct has tenant_id. - self.warehouses().insert_one(warehouse).await?; - Ok(()) - } - - async fn get_warehouse(&self, tenant_id: Uuid, name: String) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": name }; - let warehouse = self.warehouses().find_one(filter).await?; - Ok(warehouse) - } - - async fn list_warehouses(&self, tenant_id: Uuid) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let cursor = self.warehouses().find(filter).await?; - let warehouses: Vec = cursor.try_collect().await?; - Ok(warehouses) - } - - async fn update_warehouse(&self, tenant_id: Uuid, name: String, updates: pangolin_core::model::WarehouseUpdate) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": &name }; - let mut update_doc = doc! {}; - - if let Some(new_name) = &updates.name { - update_doc.insert("name", new_name); - } - if let Some(config) = &updates.storage_config { - update_doc.insert("storage_config", bson::to_bson(config)?); - } - if let Some(use_sts) = updates.use_sts { - update_doc.insert("use_sts", use_sts); - } - if let Some(vending_strategy) = updates.vending_strategy { - update_doc.insert("vending_strategy", bson::to_bson(&vending_strategy)?); - } - - if update_doc.is_empty() { - return self.get_warehouse(tenant_id, name).await? - .ok_or_else(|| anyhow::anyhow!("Warehouse not found")); - } - - let update = doc! { "$set": update_doc }; - self.warehouses().update_one(filter, update).await?; - - let new_name = updates.name.unwrap_or(name); - self.get_warehouse(tenant_id, new_name).await? - .ok_or_else(|| anyhow::anyhow!("Warehouse not found")) - } - - async fn delete_warehouse(&self, tenant_id: Uuid, name: String) -> Result<()> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": &name }; - let result = self.warehouses().delete_one(filter).await?; - - if result.deleted_count == 0 { - return Err(anyhow::anyhow!("Warehouse '{}' not found", name)); - } - Ok(()) - } - - // Catalog Operations - async fn create_catalog(&self, tenant_id: Uuid, catalog: Catalog) -> Result<()> { - // Catalog struct doesn't have tenant_id, so we need to wrap it or add it? - // Wait, Catalog struct in model.rs: - // pub struct Catalog { pub name: String, pub warehouse_name: Option, pub storage_location: Option, pub properties: HashMap } - // It doesn't have tenant_id. - // In Postgres we added a column. In Mongo we can wrap it in a document or just add the field dynamically if we use Document. - // But we are using typed Collection. - // We should probably use a wrapper struct for storage or just use Document. - // Let's use Document for flexibility here since we need to add tenant_id context. - - let mut doc = doc! { - "id": to_bson_uuid(catalog.id), - "tenant_id": to_bson_uuid(tenant_id), - "name": &catalog.name, - "catalog_type": format!("{:?}", catalog.catalog_type), - "properties": mongodb::bson::to_bson(&catalog.properties)? - }; - - // Add optional fields - if let Some(ref warehouse_name) = catalog.warehouse_name { - doc.insert("warehouse_name", warehouse_name); - } - if let Some(ref storage_location) = catalog.storage_location { - doc.insert("storage_location", storage_location); - } - if let Some(ref federated_config) = catalog.federated_config { - doc.insert("federated_config", mongodb::bson::to_bson(federated_config)?); - } - - self.db.collection::("catalogs").insert_one(doc).await?; - Ok(()) - } - - async fn get_catalog(&self, tenant_id: Uuid, name: String) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": name }; - let doc = self.db.collection::("catalogs").find_one(filter).await?; - Ok(doc) - } - - async fn list_catalogs(&self, tenant_id: Uuid) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let cursor = self.db.collection::("catalogs").find(filter).await?; - let catalogs: Vec = cursor.try_collect().await?; - Ok(catalogs) - } - - async fn update_catalog(&self, tenant_id: Uuid, name: String, updates: pangolin_core::model::CatalogUpdate) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": &name }; - let mut update_doc = doc! {}; - - if let Some(warehouse_name) = updates.warehouse_name { - update_doc.insert("warehouse_name", warehouse_name); - } - if let Some(storage_location) = updates.storage_location { - update_doc.insert("storage_location", storage_location); - } - if let Some(properties) = updates.properties { - update_doc.insert("properties", bson::to_bson(&properties)?); - } - - if update_doc.is_empty() { - return self.get_catalog(tenant_id, name).await? - .ok_or_else(|| anyhow::anyhow!("Catalog not found")); - } - - let update = doc! { "$set": update_doc }; - self.db.collection::("catalogs").update_one(filter, update).await?; - - self.get_catalog(tenant_id, name).await? - .ok_or_else(|| anyhow::anyhow!("Catalog not found")) - } - - async fn delete_catalog(&self, tenant_id: Uuid, name: String) -> Result<()> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": &name }; // For catalog - // For children, filter is slightly different (catalog_name property) or similar - let child_filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "catalog_name": &name }; - - // 1. Tags - self.db.collection::("tags").delete_many(child_filter.clone()).await?; - // 2. Branches - self.db.collection::("branches").delete_many(child_filter.clone()).await?; - // 3. Assets - self.db.collection::("assets").delete_many(child_filter.clone()).await?; - // 4. Namespaces - self.db.collection::("namespaces").delete_many(child_filter.clone()).await?; - - // 5. Catalog - let result = self.db.collection::("catalogs").delete_one(filter).await?; - - if result.deleted_count == 0 { - return Err(anyhow::anyhow!("Catalog '{}' not found", name)); - } - Ok(()) - } - - // Namespace Operations - async fn create_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Namespace) -> Result<()> { - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": &namespace.name, // Vec -> Array - "properties": mongodb::bson::to_bson(&namespace.properties)? - }; - self.db.collection::("namespaces").insert_one(doc).await?; - Ok(()) - } - - async fn get_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": namespace - }; - let doc = self.db.collection::("namespaces").find_one(filter).await?; - Ok(doc) - } - - async fn list_namespaces(&self, tenant_id: Uuid, catalog_name: &str, parent: Option) -> Result> { - let mut filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - - if let Some(p) = parent { - // This is tricky. Namespace name is Vec. - // "parent" usually implies a hierarchy. - // If parent is "a.b", we want "a.b.c", "a.b.d". - // We can filter where "name" starts with the parent components. - // But `parent` argument is String (dot joined?). The trait says `parent: Option`. - // In Postgres we did LIKE 'parent%'. - // Here we need to match array prefix. - // Let's assume parent string is dot-separated or something. - // Actually, `Namespace` struct has `name: Vec`. - // If parent is provided, we should convert it to Vec and check if it's a prefix. - // But `parent` is just a string. - // Let's assume for now we just list all and filter in memory or implement prefix match if possible. - // MVP: List all for catalog. - } - - let cursor = self.db.collection::("namespaces").find(filter).await?; - let namespaces: Vec = cursor.try_collect().await?; - Ok(namespaces) - } - - async fn delete_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec) -> Result<()> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": namespace - }; - self.db.collection::("namespaces").delete_one(filter).await?; - Ok(()) - } - - async fn update_namespace_properties(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec, properties: HashMap) -> Result<()> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": namespace - }; - - // We need to merge properties. - // $set: { "properties.key": "value" } - let mut set_doc = doc! {}; - for (k, v) in properties { - set_doc.insert(format!("properties.{}", k), v); - } - - let update = doc! { "$set": set_doc }; - self.db.collection::("namespaces").update_one(filter, update).await?; - Ok(()) - } - - // Asset Operations - async fn create_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, asset: Asset) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": &branch_name, - "namespace": &namespace, - "name": &asset.name - }; - - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": &branch_name, - "namespace": namespace, - "id": to_bson_uuid(asset.id), - "name": &asset.name, - "kind": format!("{:?}", asset.kind), - "location": &asset.location, - "properties": mongodb::bson::to_bson(&asset.properties)? - }; - - let options = ReplaceOptions::builder().upsert(true).build(); - self.db.collection::("assets").replace_one(filter, doc).with_options(options).await?; - Ok(()) - } - - async fn get_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": branch_name, - "namespace": namespace, - "name": name - }; - // Note: Branch support in filter if needed, but usually asset name is unique in namespace? - // Or is it versioned by branch? - // In Postgres we had `active_branch`. - // Let's stick to the filter. - - let doc = self.db.collection::("assets").find_one(filter).await?; - - if let Some(d) = doc { - // Manual deserialization because we stored flattened fields - let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; - - let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; - - let id_bson = d.get("id").ok_or(anyhow::anyhow!("Missing id"))?; - let id = from_bson_uuid(id_bson)?; - - Ok(Some(Asset { - id, - name: d.get_str("name")?.to_string(), - kind, - location: d.get_str("location").unwrap_or("").to_string(), - properties, - })) - } else { - Ok(None) - } - } - - async fn list_assets(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": branch_name, - "namespace": namespace - }; - let cursor = self.db.collection::("assets").find(filter).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut assets = Vec::new(); - for d in docs { - let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; - - let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; - let id = if let Ok(i) = d.get("id").ok_or(anyhow::anyhow!("Missing id")).and_then(|b| from_bson_uuid(b)) { - i - } else { - Uuid::new_v4() // Fallback if old data - }; - - assets.push(Asset { - id, - name: d.get_str("name")?.to_string(), - kind, - location: d.get_str("location").unwrap_or("").to_string(), - properties, - }); - } - Ok(assets) - } - - async fn get_asset_by_id(&self, tenant_id: Uuid, asset_id: Uuid) -> Result)>> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(asset_id) - }; - let d = self.db.collection::("assets").find_one(filter).await?; - - if let Some(d) = d { - let catalog_name = d.get_str("catalog_name")?.to_string(); - let namespace = d.get_array("namespace")?.iter().map(|v| v.as_str().unwrap().to_string()).collect(); - let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; - - let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; - - let asset = Asset { - id: asset_id, - name: d.get_str("name")?.to_string(), - kind, - location: d.get_str("location").unwrap_or("").to_string(), - properties, - }; - - Ok(Some((asset, catalog_name, namespace))) - } else { - Ok(None) - } - } - - async fn delete_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": branch_name, - "namespace": namespace, - "name": name - }; - self.db.collection::("assets").delete_one(filter).await?; - Ok(()) - } - - async fn rename_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, source_namespace: Vec, source_name: String, dest_namespace: Vec, dest_name: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": &branch_name, - "namespace": source_namespace, - "name": source_name - }; - - let update = doc! { - "$set": { - "namespace": dest_namespace, - "name": dest_name - } - }; - - self.db.collection::("assets").update_one(filter, update).await?; - Ok(()) - } - - async fn count_namespaces(&self, tenant_id: Uuid) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let count = self.namespaces().count_documents(filter).await?; - Ok(count as usize) - } - - async fn count_assets(&self, tenant_id: Uuid) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let count = self.assets().count_documents(filter).await?; - Ok(count as usize) - } - - // Branch Operations - async fn create_branch(&self, tenant_id: Uuid, catalog_name: &str, branch: Branch) -> Result<()> { - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": &branch.name, - "head_commit_id": branch.head_commit_id, - "branch_type": format!("{:?}", branch.branch_type), - "assets": &branch.assets - }; - self.db.collection::("branches").insert_one(doc).await?; - Ok(()) - } - - async fn get_branch(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": name - }; - let doc = self.db.collection::("branches").find_one(filter).await?; - - if let Some(d) = doc { - let type_str = d.get_str("branch_type")?; - let branch_type = match type_str { - "Ingest" => pangolin_core::model::BranchType::Ingest, - "Experimental" => pangolin_core::model::BranchType::Experimental, - _ => pangolin_core::model::BranchType::Experimental, - }; - - Ok(Some(Branch { - name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson(d.get("head_commit_id").unwrap().clone())?, - branch_type, - assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, - })) - } else { - Ok(None) - } - } - - async fn list_branches(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - let cursor = self.db.collection::("branches").find(filter).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut branches = Vec::new(); - for d in docs { - let type_str = d.get_str("branch_type")?; - let branch_type = match type_str { - "Ingest" => pangolin_core::model::BranchType::Ingest, - "Experimental" => pangolin_core::model::BranchType::Experimental, - _ => pangolin_core::model::BranchType::Experimental, - }; - - branches.push(Branch { - name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson(d.get("head_commit_id").unwrap().clone())?, - branch_type, - assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, - }); - } - Ok(branches) - } - - async fn merge_branch(&self, tenant_id: Uuid, catalog_name: &str, target_branch: String, source_branch: String) -> Result<()> { - let source = self.get_branch(tenant_id, catalog_name, source_branch.clone()).await? - .ok_or_else(|| anyhow::anyhow!("Source branch not found"))?; - - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": target_branch - }; - - let update = doc! { - "$set": { - "head_commit_id": source.head_commit_id - } - }; - - self.db.collection::("branches").update_one(filter, update).await?; - Ok(()) - } - - // Token Management - async fn list_active_tokens(&self, _tenant_id: Uuid, user_id: Uuid) -> Result> { - let filter = doc! { "user_id": to_bson_uuid(user_id) }; - let cursor = self.active_tokens().find(filter).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut tokens = Vec::new(); - for d in docs { - tokens.push(TokenInfo { - id: from_bson_uuid(d.get("token_id").ok_or(anyhow::anyhow!("Missing token_id"))?)?, - tenant_id: Uuid::default(), // Not stored in mongo active_tokens - user_id: from_bson_uuid(d.get("user_id").ok_or(anyhow::anyhow!("Missing user_id"))?)?, - username: "unknown".to_string(), // Would need join - token: d.get_str("token").ok().map(|s| s.to_string()), - expires_at: mongodb::bson::from_bson(d.get("expires_at").unwrap().clone())?, - created_at: Utc::now(), // Not stored - is_valid: true, - }); - } - Ok(tokens) - } - - async fn store_token(&self, token_info: TokenInfo) -> Result<()> { - let doc = doc! { - "token_id": to_bson_uuid(token_info.id), - "user_id": to_bson_uuid(token_info.user_id), - "token": token_info.token.unwrap_or_default(), - "expires_at": token_info.expires_at - }; - self.active_tokens().insert_one(doc).await?; - Ok(()) - } - - // System Settings - async fn get_system_settings(&self, tenant_id: Uuid) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let doc = self.system_settings().find_one(filter).await?; - - if let Some(d) = doc { - Ok(mongodb::bson::from_bson(d.get("settings").unwrap().clone())?) - } else { - Ok(SystemSettings { - allow_public_signup: None, - default_warehouse_bucket: None, - default_retention_days: None, - smtp_host: None, - smtp_port: None, - smtp_user: None, - smtp_password: None, - }) - } - } - - async fn update_system_settings(&self, tenant_id: Uuid, settings: SystemSettings) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let update = doc! { - "$set": { - "settings": mongodb::bson::to_bson(&settings)? - } - }; - - let options = mongodb::options::UpdateOptions::builder().upsert(true).build(); - self.system_settings().update_one(filter, update).with_options(options).await?; - Ok(settings) - } - - // Service User Methods - async fn create_service_user(&self, service_user: pangolin_core::user::ServiceUser) -> Result<()> { - let doc = mongodb::bson::to_document(&service_user)?; - self.service_users().insert_one(doc).await?; - Ok(()) - } - - async fn get_service_user(&self, id: Uuid) -> Result> { - let filter = doc! { "_id": id.to_string() }; - if let Some(doc) = self.service_users().find_one(filter).await? { - Ok(Some(mongodb::bson::from_document(doc)?)) - } else { - Ok(None) - } - } - - async fn get_service_user_by_api_key_hash(&self, api_key_hash: &str) -> Result> { - let filter = doc! { "api_key_hash": api_key_hash }; - if let Some(doc) = self.service_users().find_one(filter).await? { - Ok(Some(mongodb::bson::from_document(doc)?)) - } else { - Ok(None) - } - } - - async fn list_service_users(&self, tenant_id: Uuid) -> Result> { - let filter = doc! { "tenant_id": tenant_id.to_string() }; - let mut cursor = self.service_users().find(filter).await?; - let mut users = Vec::new(); - - while cursor.advance().await? { - users.push(mongodb::bson::from_document(cursor.deserialize_current()?)?) -; - } - - Ok(users) - } - - async fn update_service_user( - &self, - id: Uuid, - name: Option, - description: Option, - active: Option, - ) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; - let mut update_doc = doc! {}; - - if let Some(n) = name { - update_doc.insert("name", n); - } - if let Some(d) = description { - update_doc.insert("description", d); - } - if let Some(a) = active { - update_doc.insert("active", a); - } - - if !update_doc.is_empty() { - let update = doc! { "$set": update_doc }; - self.service_users().update_one(filter, update).await?; - } - - Ok(()) - } - - async fn delete_service_user(&self, id: Uuid) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; - self.service_users().delete_one(filter).await?; - Ok(()) - } - - async fn update_service_user_last_used(&self, id: Uuid, timestamp: chrono::DateTime) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; - let update = doc! { "$set": { "last_used": timestamp.to_rfc3339() } }; - self.service_users().update_one(filter, update).await?; - Ok(()) - } - - // Merge Operation Methods - async fn create_merge_operation(&self, operation: pangolin_core::model::MergeOperation) -> Result<()> { - let doc = mongodb::bson::to_document(&operation)?; - self.merge_operations().insert_one(doc).await?; - Ok(()) - } - - async fn get_merge_operation(&self, operation_id: Uuid) -> Result> { - let filter = doc! { "_id": operation_id.to_string() }; - if let Some(doc) = self.merge_operations().find_one(filter).await? { - Ok(Some(mongodb::bson::from_document(doc)?)) - } else { - Ok(None) - } - } - - async fn list_merge_operations(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let filter = doc! { - "tenant_id": tenant_id.to_string(), - "catalog_name": catalog_name - }; - let mut cursor = self.merge_operations().find(filter).await?; - let mut operations = Vec::new(); - - while cursor.advance().await? { - operations.push(mongodb::bson::from_document(cursor.deserialize_current()?)?); - } - - Ok(operations) - } - - async fn update_merge_operation_status(&self, operation_id: Uuid, status: pangolin_core::model::MergeStatus) -> Result<()> { - let filter = doc! { "_id": operation_id.to_string() }; - let status_str = format!("{:?}", status); - let update = doc! { "$set": { "status": status_str } }; - self.merge_operations().update_one(filter, update).await?; - Ok(()) - } - - async fn complete_merge_operation(&self, operation_id: Uuid, result_commit_id: Uuid) -> Result<()> { - let filter = doc! { "_id": operation_id.to_string() }; - let update = doc! { - "$set": { - "status": "Completed", - "result_commit_id": result_commit_id.to_string(), - "completed_at": chrono::Utc::now().to_rfc3339() - } - }; - self.merge_operations().update_one(filter, update).await?; - Ok(()) - } - - async fn abort_merge_operation(&self, operation_id: Uuid) -> Result<()> { - let filter = doc! { "_id": operation_id.to_string() }; - let update = doc! { - "$set": { - "status": "Aborted", - "completed_at": chrono::Utc::now().to_rfc3339() - } - }; - self.merge_operations().update_one(filter, update).await?; - Ok(()) - } - - // Merge Conflict Methods - async fn create_merge_conflict(&self, conflict: pangolin_core::model::MergeConflict) -> Result<()> { - let doc = mongodb::bson::to_document(&conflict)?; - self.merge_conflicts().insert_one(doc).await?; - Ok(()) - } - - async fn get_merge_conflict(&self, conflict_id: Uuid) -> Result> { - let filter = doc! { "_id": conflict_id.to_string() }; - if let Some(doc) = self.merge_conflicts().find_one(filter).await? { - Ok(Some(mongodb::bson::from_document(doc)?)) - } else { - Ok(None) - } - } - - async fn list_merge_conflicts(&self, operation_id: Uuid) -> Result> { - let filter = doc! { "merge_operation_id": operation_id.to_string() }; - let mut cursor = self.merge_conflicts().find(filter).await?; - let mut conflicts = Vec::new(); - - while cursor.advance().await? { - conflicts.push(mongodb::bson::from_document(cursor.deserialize_current()?)?); - } - - Ok(conflicts) - } - - async fn resolve_merge_conflict(&self, conflict_id: Uuid, resolution: pangolin_core::model::ConflictResolution) -> Result<()> { - let filter = doc! { "_id": conflict_id.to_string() }; - let resolution_doc = mongodb::bson::to_document(&resolution)?; - let update = doc! { "$set": { "resolution": resolution_doc } }; - self.merge_conflicts().update_one(filter, update).await?; - Ok(()) - } - - async fn add_conflict_to_operation(&self, operation_id: Uuid, conflict_id: Uuid) -> Result<()> { - let filter = doc! { "_id": operation_id.to_string() }; - let update = doc! { "$addToSet": { "conflicts": conflict_id.to_string() } }; - self.merge_operations().update_one(filter, update).await?; - Ok(()) - } - - // Federated Catalog Operations - async fn sync_federated_catalog(&self, tenant_id: Uuid, catalog_name: &str) -> Result<()> { - let stats = SyncStats { - last_synced_at: Some(Utc::now()), - sync_status: "Success".to_string(), - tables_synced: 0, - namespaces_synced: 0, - error_message: None, - }; - - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - let update = doc! { - "$set": { - "stats": mongodb::bson::to_bson(&stats)? - } - }; - - let options = mongodb::options::UpdateOptions::builder().upsert(true).build(); - self.federated_sync_stats().update_one(filter, update).with_options(options).await?; - Ok(()) - } - - async fn get_federated_catalog_stats(&self, tenant_id: Uuid, catalog_name: &str) -> Result { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - let doc = self.federated_sync_stats().find_one(filter).await?; - - if let Some(d) = doc { - Ok(mongodb::bson::from_bson(d.get("stats").unwrap().clone())?) - } else { - Ok(SyncStats { - last_synced_at: None, - sync_status: "Never Synced".to_string(), - tables_synced: 0, - namespaces_synced: 0, - error_message: None, - }) - } - } - - // Commit Operations - async fn create_commit(&self, tenant_id: Uuid, commit: Commit) -> Result<()> { - let mut doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(commit.id), - "timestamp": commit.timestamp, - "author": &commit.author, - "message": &commit.message, - "operations": mongodb::bson::to_bson(&commit.operations)? - }; - if let Some(parent_id) = commit.parent_id { - doc.insert("parent_id", to_bson_uuid(parent_id)); - } else { - doc.insert("parent_id", Bson::Null); - } - self.db.collection::("commits").insert_one(doc).await?; - Ok(()) - } - - async fn get_commit(&self, tenant_id: Uuid, id: Uuid) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(id) - }; - let doc = self.db.collection::("commits").find_one(filter).await?; - - if let Some(d) = doc { - Ok(Some(Commit { - id: mongodb::bson::from_bson(d.get("id").unwrap().clone())?, - parent_id: mongodb::bson::from_bson(d.get("parent_id").unwrap().clone())?, - timestamp: d.get_i64("timestamp")?, - author: d.get_str("author")?.to_string(), - message: d.get_str("message")?.to_string(), - operations: mongodb::bson::from_bson(d.get("operations").unwrap().clone())?, - })) - } else { - Ok(None) - } - } - - // File Operations - - async fn read_file(&self, path: &str) -> Result> { - // Use metadata cache for metadata.json files - if path.ends_with("metadata.json") || path.ends_with(".metadata.json") { - return self.metadata_cache.get_or_fetch(path, || async { - self.read_file_uncached(path).await - }).await; - } - - // Non-metadata files bypass cache - self.read_file_uncached(path).await - } - - - async fn write_file(&self, path: &str, data: Vec) -> Result<()> { - // Invalidate metadata cache on write - if path.ends_with("metadata.json") || path.ends_with(".metadata.json") { - self.metadata_cache.invalidate(path).await; - } - - // Try to look up warehouse credentials first - if let Some(warehouse) = self.get_warehouse_for_location(path).await? { - if path.starts_with("s3://") || path.starts_with("az://") || path.starts_with("gs://") { - // Use cached object store - let cache_key = self.get_object_store_cache_key(&warehouse.storage_config, path); - let store = self.object_store_cache.get_or_insert(cache_key, || { - Arc::new(crate::object_store_factory::create_object_store(&warehouse.storage_config, path).unwrap()) - }); - - // Extract key relative to bucket - let key = if let Some(rest) = path.strip_prefix("s3://").or_else(|| path.strip_prefix("az://")).or_else(|| path.strip_prefix("gs://")) { - rest.split_once('/').map(|(_, k)| k).unwrap_or(rest) - } else { - path - }; - - store.put(&ObjPath::from(key), data.into()).await?; - return Ok(()); - } - } - - // Fallback to existing logic (Global Env Vars) - if let Some(rest) = path.strip_prefix("s3://") { - let (bucket, key) = rest.split_once('/').ok_or_else(|| anyhow::anyhow!("Invalid S3 path"))?; - - let mut builder = AmazonS3Builder::new() - .with_bucket_name(bucket) - .with_allow_http(true); - - if let Ok(endpoint) = std::env::var("S3_ENDPOINT") { - builder = builder.with_endpoint(endpoint); - } - if let Ok(key_id) = std::env::var("AWS_ACCESS_KEY_ID") { - builder = builder.with_access_key_id(key_id); - } - if let Ok(secret) = std::env::var("AWS_SECRET_ACCESS_KEY") { - builder = builder.with_secret_access_key(secret); - } - if let Ok(region) = std::env::var("AWS_REGION") { - builder = builder.with_region(region); - } - - let store = builder.build()?; - let location = ObjPath::from(key); - store.put(&location, data.into()).await?; - Ok(()) - } else { - Err(anyhow::anyhow!("Only s3:// paths are supported in Mongo store")) - } - } - - - // Tag Operations - async fn create_tag(&self, tenant_id: Uuid, catalog_name: &str, tag: Tag) -> Result<()> { - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": &tag.name, - "commit_id": to_bson_uuid(tag.commit_id) - }; - self.db.collection::("tags").insert_one(doc).await?; - Ok(()) - } - - async fn get_tag(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": name - }; - let doc = self.db.collection::("tags").find_one(filter).await?; - - if let Some(d) = doc { - Ok(Some(Tag { - name: d.get_str("name")?.to_string(), - commit_id: mongodb::bson::from_bson(d.get("commit_id").unwrap().clone())?, - })) - } else { - Ok(None) - } - } - - async fn list_tags(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - let cursor = self.db.collection::("tags").find(filter).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut tags = Vec::new(); - for d in docs { - tags.push(Tag { - name: d.get_str("name")?.to_string(), - commit_id: mongodb::bson::from_bson(d.get("commit_id").unwrap().clone())?, - }); - } - Ok(tags) - } - - async fn delete_tag(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result<()> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": name - }; - self.db.collection::("tags").delete_one(filter).await?; - Ok(()) - } - - // Audit Operations - async fn log_audit_event(&self, tenant_id: Uuid, event: AuditLogEntry) -> Result<()> { - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(event.id), - "user_id": event.user_id.map(to_bson_uuid).unwrap_or(Bson::Null), - "username": &event.username, - "action": format!("{:?}", event.action), - "resource_type": format!("{:?}", event.resource_type), - "resource_id": event.resource_id.map(to_bson_uuid).unwrap_or(Bson::Null), - "resource_name": &event.resource_name, - "timestamp": mongodb::bson::DateTime::from_chrono(event.timestamp), - "ip_address": event.ip_address.as_ref().map(|s| s.as_str()).unwrap_or(""), - "user_agent": event.user_agent.as_ref().map(|s| s.as_str()).unwrap_or(""), - "result": format!("{:?}", event.result), - "error_message": event.error_message.as_ref().map(|s| s.as_str()).unwrap_or(""), - "metadata": mongodb::bson::to_bson(&event.metadata)? - }; - self.db.collection::("audit_logs").insert_one(doc).await?; - Ok(()) - } - - async fn list_audit_events(&self, tenant_id: Uuid, filter: Option) -> Result> { - let mut query = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - - // Build filter conditions - if let Some(ref f) = filter { - if let Some(user_id) = f.user_id { - query.insert("user_id", to_bson_uuid(user_id)); - } - if let Some(ref action) = f.action { - query.insert("action", format!("{:?}", action)); - } - if let Some(ref resource_type) = f.resource_type { - query.insert("resource_type", format!("{:?}", resource_type)); - } - if let Some(resource_id) = f.resource_id { - query.insert("resource_id", to_bson_uuid(resource_id)); - } - if let Some(start_time) = f.start_time { - query.insert("timestamp", doc! { "$gte": mongodb::bson::DateTime::from_chrono(start_time) }); - } - if let Some(end_time) = f.end_time { - let existing = query.get_document_mut("timestamp").ok(); - if let Some(existing_doc) = existing { - existing_doc.insert("$lte", mongodb::bson::DateTime::from_chrono(end_time)); - } else { - query.insert("timestamp", doc! { "$lte": mongodb::bson::DateTime::from_chrono(end_time) }); - } - } - if let Some(ref result) = f.result { - query.insert("result", format!("{:?}", result)); - } - } - - // Build options with pagination - let limit = filter.as_ref().and_then(|f| f.limit).unwrap_or(100) as i64; - let skip = filter.as_ref().and_then(|f| f.offset).map(|o| o as u64); - - let mut options = mongodb::options::FindOptions::builder() - .sort(doc! { "timestamp": -1 }) - .limit(limit) - .build(); - - if let Some(skip_val) = skip { - options.skip = Some(skip_val); - } - - let cursor = self.db.collection::("audit_logs") - .find(query) - .with_options(options) - .await?; - let docs: Vec = cursor.try_collect().await?; - - let mut events = Vec::new(); - for d in docs { - // Parse action enum from string - let action_str = d.get_str("action")?; - let action = serde_json::from_str(&format!("\"{}\"" , action_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditAction::CreateCatalog); - - // Parse resource_type enum from string - let resource_type_str = d.get_str("resource_type")?; - let resource_type = serde_json::from_str(&format!("\"{}\"", resource_type_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::ResourceType::Catalog); - - // Parse result enum from string - let result_str = d.get_str("result")?; - let result = serde_json::from_str(&format!("\"{}\"", result_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditResult::Success); - - events.push(AuditLogEntry { - id: mongodb::bson::from_bson(d.get("id").unwrap().clone())?, - tenant_id, - user_id: d.get("user_id").and_then(|b| from_bson_uuid(b).ok()), - username: d.get_str("username")?.to_string(), - action, - resource_type, - resource_id: d.get("resource_id").and_then(|b| from_bson_uuid(b).ok()), - resource_name: d.get_str("resource_name")?.to_string(), - timestamp: d.get_datetime("timestamp")?.to_chrono(), - ip_address: d.get_str("ip_address").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - user_agent: d.get_str("user_agent").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - result, - error_message: d.get_str("error_message").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - metadata: mongodb::bson::from_bson(d.get("metadata").unwrap().clone())?, - }); - } - Ok(events) - } - - async fn get_audit_event(&self, tenant_id: Uuid, event_id: Uuid) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(event_id) - }; - - let doc = self.db.collection::("audit_logs").find_one(filter).await?; - - if let Some(d) = doc { - let action_str = d.get_str("action")?; - let action = serde_json::from_str(&format!("\"{}\"", action_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditAction::CreateCatalog); - - let resource_type_str = d.get_str("resource_type")?; - let resource_type = serde_json::from_str(&format!("\"{}\"", resource_type_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::ResourceType::Catalog); - - let result_str = d.get_str("result")?; - let result = serde_json::from_str(&format!("\"{}\"", result_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditResult::Success); - - Ok(Some(AuditLogEntry { - id: mongodb::bson::from_bson(d.get("id").unwrap().clone())?, - tenant_id, - user_id: d.get("user_id").and_then(|b| from_bson_uuid(b).ok()), - username: d.get_str("username")?.to_string(), - action, - resource_type, - resource_id: d.get("resource_id").and_then(|b| from_bson_uuid(b).ok()), - resource_name: d.get_str("resource_name")?.to_string(), - timestamp: d.get_datetime("timestamp")?.to_chrono(), - ip_address: d.get_str("ip_address").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - user_agent: d.get_str("user_agent").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - result, - error_message: d.get_str("error_message").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - metadata: mongodb::bson::from_bson(d.get("metadata").unwrap().clone())?, - })) - } else { - Ok(None) - } - } - - async fn count_audit_events(&self, tenant_id: Uuid, filter: Option) -> Result { - let mut query = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - - // Build same filter conditions as list_audit_events - if let Some(ref f) = filter { - if let Some(user_id) = f.user_id { - query.insert("user_id", to_bson_uuid(user_id)); - } - if let Some(ref action) = f.action { - query.insert("action", format!("{:?}", action)); - } - if let Some(ref resource_type) = f.resource_type { - query.insert("resource_type", format!("{:?}", resource_type)); - } - if let Some(resource_id) = f.resource_id { - query.insert("resource_id", to_bson_uuid(resource_id)); - } - if let Some(start_time) = f.start_time { - query.insert("timestamp", doc! { "$gte": mongodb::bson::DateTime::from_chrono(start_time) }); - } - if let Some(end_time) = f.end_time { - let existing = query.get_document_mut("timestamp").ok(); - if let Some(existing_doc) = existing { - existing_doc.insert("$lte", mongodb::bson::DateTime::from_chrono(end_time)); - } else { - query.insert("timestamp", doc! { "$lte": mongodb::bson::DateTime::from_chrono(end_time) }); - } - } - if let Some(ref result) = f.result { - query.insert("result", format!("{:?}", result)); - } - } - - let count = self.db.collection::("audit_logs") - .count_documents(query) - .await? as usize; - - Ok(count) - } - - // User Operations - async fn create_user(&self, user: User) -> Result<()> { - self.users().insert_one(user).await?; - Ok(()) - } - - async fn get_user(&self, user_id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(user_id) }; - let user = self.users().find_one(filter).await?; - Ok(user) - } - - async fn get_user_by_username(&self, username: &str) -> Result> { - let filter = doc! { "username": username }; - let user = self.users().find_one(filter).await?; - Ok(user) - } - - async fn list_users(&self, tenant_id: Option) -> Result> { - let filter = if let Some(tid) = tenant_id { - doc! { "tenant_id": to_bson_uuid(tid) } - } else { - doc! {} - }; - let cursor = self.users().find(filter).await?; - let users: Vec = cursor.try_collect().await?; - Ok(users) - } - - async fn update_user(&self, user: User) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(user.id) }; - let mut doc = mongodb::bson::to_document(&user)?; - // Ensure UUIDs are stored as Binary Subtype 0 for consistency with filters - doc.insert("id", to_bson_uuid(user.id)); - if let Some(tid) = user.tenant_id { - doc.insert("tenant-id", to_bson_uuid(tid)); - } - let update = doc! { "$set": doc }; - self.db.collection::("users").update_one(filter, update).await?; - Ok(()) - } - - async fn delete_user(&self, user_id: Uuid) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(user_id) }; - self.users().delete_one(filter).await?; - Ok(()) - } - - // Role Operations - async fn create_role(&self, role: Role) -> Result<()> { - self.roles().insert_one(role).await?; - Ok(()) - } - - async fn get_role(&self, role_id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(role_id) }; - let role = self.roles().find_one(filter).await?; - Ok(role) - } - - async fn list_roles(&self, tenant_id: Uuid) -> Result> { - let filter = doc! { "tenant-id": to_bson_uuid(tenant_id) }; - let cursor = self.roles().find(filter).await?; - let roles: Vec = cursor.try_collect().await?; - Ok(roles) - } - - async fn delete_role(&self, role_id: Uuid) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(role_id) }; - self.roles().delete_one(filter).await?; - Ok(()) - } - - async fn update_role(&self, role: Role) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(role.id) }; - let mut doc = mongodb::bson::to_document(&role)?; - doc.insert("id", to_bson_uuid(role.id)); - doc.insert("tenant-id", to_bson_uuid(role.tenant_id)); - doc.insert("created-by", to_bson_uuid(role.created_by)); - - let update = doc! { "$set": doc }; - self.db.collection::("roles").update_one(filter, update).await?; - Ok(()) - } - - async fn assign_role(&self, user_role: UserRoleAssignment) -> Result<()> { - self.user_roles().insert_one(user_role).await?; - Ok(()) - } - - async fn revoke_role(&self, user_id: Uuid, role_id: Uuid) -> Result<()> { - let filter = doc! { - "user-id": to_bson_uuid(user_id), - "role-id": to_bson_uuid(role_id) - }; - self.user_roles().delete_one(filter).await?; - Ok(()) - } - - async fn get_user_roles(&self, user_id: Uuid) -> Result> { - let filter = doc! { "user-id": to_bson_uuid(user_id) }; - let cursor = self.user_roles().find(filter).await?; - let roles: Vec = cursor.try_collect().await?; - Ok(roles) - } - - // Permission Operations - async fn create_permission(&self, permission: Permission) -> Result<()> { - self.permissions().insert_one(permission).await?; - Ok(()) - } - - async fn revoke_permission(&self, permission_id: Uuid) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(permission_id) }; - self.permissions().delete_one(filter).await?; - Ok(()) - } - - async fn list_user_permissions(&self, user_id: Uuid) -> Result> { - // 1. Fetch direct permissions - let filter = doc! { "user-id": to_bson_uuid(user_id) }; - let cursor = self.permissions().find(filter).await?; - let mut perms: Vec = cursor.try_collect().await?; - - // 2. Fetch role-based permissions - let user_roles = self.get_user_roles(user_id).await?; - for ur in user_roles { - if let Some(role) = self.get_role(ur.role_id).await? { - for grant in role.permissions { - perms.push(Permission { - id: Uuid::new_v4(), // Synthesized ID - user_id, - scope: grant.scope, - actions: grant.actions, - granted_by: role.created_by, - granted_at: role.created_at, - }); - } - } - } - - Ok(perms) - } - - async fn list_permissions(&self, tenant_id: Uuid) -> Result> { - // 1. Get all user IDs for the tenant - let user_filter = doc! { "tenant-id": to_bson_uuid(tenant_id) }; - let user_cursor = self.users().find(user_filter).await?; - let users: Vec = user_cursor.try_collect().await?; - let user_ids: Vec = users.iter().map(|u| to_bson_uuid(u.id)).collect(); - - if user_ids.is_empty() { - return Ok(vec![]); - } - - // 2. Get permissions for those users - let perm_filter = doc! { "user-id": { "$in": user_ids } }; - let perm_cursor = self.permissions().find(perm_filter).await?; - let perms: Vec = perm_cursor.try_collect().await?; - Ok(perms) - } - - // Maintenance Operations - async fn expire_snapshots(&self, _tenant_id: Uuid, _catalog_name: &str, _branch: Option, _namespace: Vec, _table: String, _retention_ms: i64) -> Result<()> { - Ok(()) - } - - // Business Metadata Operations - async fn upsert_business_metadata(&self, metadata: BusinessMetadata) -> Result<()> { - let filter = doc! { "asset-id": to_bson_uuid(metadata.asset_id) }; - let mut doc = mongodb::bson::to_document(&metadata)?; - doc.insert("asset-id", to_bson_uuid(metadata.asset_id)); - self.db.collection::("business_metadata") - .replace_one(filter, doc) - .upsert(true) - .await?; - Ok(()) - } - - async fn get_business_metadata(&self, asset_id: Uuid) -> Result> { - let filter = doc! { "asset-id": to_bson_uuid(asset_id) }; - let meta = self.business_metadata().find_one(filter).await?; - Ok(meta) - } - - async fn delete_business_metadata(&self, asset_id: Uuid) -> Result<()> { - let filter = doc! { "asset-id": to_bson_uuid(asset_id) }; - self.business_metadata().delete_one(filter).await?; - Ok(()) - } - - async fn search_assets(&self, tenant_id: Uuid, query: &str, tags: Option>) -> Result, String, Vec)>> { - let query_regex = mongodb::bson::Regex { - pattern: format!(".*{}.*", regex::escape(query)), - options: "i".to_string(), - }; - - let mut pipeline = vec![ - doc! { "$match": { "tenant_id": to_bson_uuid(tenant_id) } }, - doc! { - "$lookup": { - "from": "business_metadata", - "localField": "id", - "foreignField": "asset_id", - "as": "metadata" - } - }, - doc! { - "$unwind": { - "path": "$metadata", - "preserveNullAndEmptyArrays": true - } - }, - doc! { - "$match": { - "$or": [ - { "name": query_regex.clone() }, - { "metadata.description": query_regex } - ] - } - } - ]; - - if let Some(tag_list) = tags { - if !tag_list.is_empty() { - pipeline.push(doc! { - "$match": { - "metadata.tags": { "$all": tag_list } - } - }); - } - } - - let cursor = self.assets().aggregate(pipeline).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut results = Vec::new(); - - for d in docs { - let metadata_doc = d.get_document("metadata").ok(); - - // Manual deserialization for Asset to ensure we get what we expect, - // though from_document works if struct matches. - // But we need catalog and namespace which are in the doc but not in the struct. - let asset: Asset = mongodb::bson::from_document(d.clone())?; - - let metadata = if let Some(md_doc) = metadata_doc { - if md_doc.is_empty() { None } else { Some(mongodb::bson::from_document(md_doc.clone())?) } - } else { None }; - - let catalog_name = d.get_str("catalog_name")?.to_string(); - let namespace_bson = d.get_array("namespace")?; - let namespace: Vec = namespace_bson.iter() - .map(|b| b.as_str().unwrap_or_default().to_string()) - .collect(); - - results.push((asset, metadata, catalog_name, namespace)); - } - - Ok(results) - } - - async fn search_catalogs(&self, tenant_id: Uuid, query: &str) -> Result> { - let query_regex = mongodb::bson::Regex { - pattern: format!(".*{}.*", regex::escape(query)), - options: "i".to_string(), - }; - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "name": query_regex - }; - let cursor = self.catalogs().find(filter).await?; - let catalogs: Vec = cursor.try_collect().await?; - Ok(catalogs) - } - - async fn search_namespaces(&self, tenant_id: Uuid, query: &str) -> Result> { - // Namespaces search with aggregation to output Docs - let query_regex = mongodb::bson::Regex { - pattern: format!(".*{}.*", regex::escape(query)), - options: "i".to_string(), - }; - - let pipeline = vec![ - doc! { "$match": { "tenant_id": to_bson_uuid(tenant_id) } } - ]; - - // self.namespaces() returns Collection. aggregate returns Cursor. - // BUT we need to call aggregate on collection. self.namespaces() is typed. - // We can call aggregate on typed collection but it returns Cursor. - let cursor = self.namespaces().aggregate(pipeline).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - - for d in docs { - let ns: Namespace = mongodb::bson::from_document(d.clone())?; - let catalog_name = d.get_str("catalog_name")?.to_string(); - - if ns.to_string().to_lowercase().contains(&query_lower) { - results.push((ns, catalog_name)); - } - } - Ok(results) - } - - async fn search_branches(&self, tenant_id: Uuid, query: &str) -> Result> { - // Use aggregate instead of find to get Document cursor easily and access catalog_name - let query_regex = mongodb::bson::Regex { - pattern: format!(".*{}.*", regex::escape(query)), - options: "i".to_string(), - }; - - // Explicitly filter by query in pipeline - let pipeline = vec![ - doc! { "$match": { "tenant_id": to_bson_uuid(tenant_id), "name": query_regex } } - ]; - - let cursor = self.branches().aggregate(pipeline).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut results = Vec::new(); - for d in docs { - let branch: Branch = mongodb::bson::from_document(d.clone())?; - let catalog_name = d.get_str("catalog_name")?.to_string(); - results.push((branch, catalog_name)); - } - Ok(results) - } - - // Access Request Operations - async fn create_access_request(&self, request: AccessRequest) -> Result<()> { - self.access_requests().insert_one(request).await?; - Ok(()) - } - - async fn get_access_request(&self, id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(id) }; - let req = self.access_requests().find_one(filter).await?; - Ok(req) - } - - async fn list_access_requests(&self, tenant_id: Uuid) -> Result> { - // AccessRequests stored with UserID/AssetID but not TenantID directly? - // Struct has: id, user_id, asset_id... - // User has tenant_id. - // To filter by tenant_id, we need a join (lookup) or we store tenant_id denormalized on AccessRequest? - // SQL implementation joins Users. - // Mongo: $lookup. - - // Creating aggregation pipeline: - let pipeline = vec![ - doc! { - "$lookup": { - "from": "users", - "localField": "user-id", - "foreignField": "id", - "as": "user" - } - }, - doc! { "$unwind": "$user" }, - doc! { "$match": { "user.tenant-id": to_bson_uuid(tenant_id) } }, - // Project back to AccessRequest root fields only? - // "replaceRoot"? Or simple map. - doc! { - "$project": { - "user": 0 // remove joined field to match struct - } - } - ]; - - let cursor = self.access_requests().aggregate(pipeline).await?; - // Cursor returns Documents, need to deserialize. - // aggregate returns Cursor. - let docs: Vec = cursor.try_collect().await?; - - let mut reqs = Vec::new(); - for d in docs { - reqs.push(mongodb::bson::from_document(d)?); - } - Ok(reqs) - } - - async fn update_access_request(&self, request: AccessRequest) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(request.id) }; - self.access_requests().replace_one(filter, request).await?; - Ok(()) - } - - async fn remove_orphan_files(&self, _tenant_id: Uuid, _catalog_name: &str, _branch: Option, _namespace: Vec, _table: String, _older_than_ms: i64) -> Result<()> { - Ok(()) - } - - // Metadata IO - async fn get_metadata_location(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String) -> Result> { - let asset = self.get_asset(tenant_id, catalog_name, branch, namespace, table).await?; - if let Some(asset) = asset { - // First check if metadata_location is explicitly set in properties - if let Some(loc) = asset.properties.get("metadata_location") { - return Ok(Some(loc.clone())); - } - // Fall back to the asset's location field - return Ok(Some(asset.location)); - } - Ok(None) - } - - async fn update_metadata_location(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String, expected_location: Option, new_location: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": branch_name, - "namespace": namespace, - "name": table - }; - - // CAS Logic - need to check both location field and properties.metadata_location - // because get_metadata_location falls back to location if metadata_location doesn't exist - let mut query = filter.clone(); - if let Some(expected) = expected_location { - // Match if either properties.metadata_location equals expected OR location equals expected (and metadata_location doesn't exist) - query.insert("$or", vec![ - doc! { "properties.metadata_location": &expected }, - doc! { "location": &expected, "properties.metadata_location": doc! { "$exists": false } } - ]); - } else { - // expected is None, meaning it shouldn't exist or should be null. - query.insert("properties.metadata_location", doc! { "$exists": false }); - } - - let update = doc! { - "$set": { - "properties.metadata_location": new_location - } - }; - - let result = self.db.collection::("assets").update_one(query, update).await?; - - if result.matched_count == 0 { - return Err(anyhow::anyhow!("CAS check failed: Metadata location mismatch or asset not found")); - } - - Ok(()) - } - - // Token Revocation Operations - async fn revoke_token(&self, token_id: Uuid, expires_at: chrono::DateTime, reason: Option) -> Result<()> { - let revoked = pangolin_core::token::RevokedToken::new(token_id, expires_at, reason); - self.db.collection("revoked_tokens").insert_one(revoked).await?; - Ok(()) - } - - async fn is_token_revoked(&self, token_id: Uuid) -> Result { - let filter = doc! { "token_id": to_bson_uuid(token_id) }; - let result = self.db.collection::("revoked_tokens") - .find_one(filter) - .await?; - Ok(result.is_some()) - } - - async fn cleanup_expired_tokens(&self) -> Result { - let now = chrono::Utc::now(); - let filter = doc! { "expires_at": { "$lt": now } }; - let result = self.db.collection::("revoked_tokens") - .delete_many(filter) - .await?; - Ok(result.deleted_count as usize) - } -} - - -#[async_trait] -impl Signer for MongoStore { - async fn get_table_credentials(&self, location: &str) -> Result { - // Attempt to extract bucket/container from location - let (_scheme, container) = if location.starts_with("s3://") { - ("s3", location[5..].split('/').next().unwrap_or("").to_string()) - } else if location.starts_with("az://") { - ("az", location[5..].split('/').next().unwrap_or("").to_string()) - } else if location.starts_with("gs://") { - ("gs", location[5..].split('/').next().unwrap_or("").to_string()) - } else if location.starts_with("abfs://") { - ("abfs", location[7..].split('/').next().unwrap_or("").split('@').next().unwrap_or("").to_string()) - } else { - ("unknown", String::new()) - }; - - // Find warehouse matching this container - let filter = doc! { - "$or": [ - { "storage_config.s3.bucket": &container }, - { "storage_config.azure.container": &container }, - { "storage_config.gcp.bucket": &container } - ] - }; - - let warehouse = self.warehouses().find_one(filter).await? - .ok_or_else(|| anyhow::anyhow!("No warehouse found for location: {}", location))?; - - match &warehouse.vending_strategy { - Some(VendingStrategy::AwsSts { role_arn: _, external_id: _ }) => { - Err(anyhow::anyhow!("AWS STS vending not implemented yet via VendingStrategy in MongoStore")) - } - Some(VendingStrategy::AwsStatic { access_key_id, secret_access_key }) => { - Ok(Credentials::Aws { - access_key_id: access_key_id.clone(), - secret_access_key: secret_access_key.clone(), - session_token: None, - expiration: None, - }) - } - Some(VendingStrategy::AzureSas { account_name, account_key }) => { - let signer = crate::azure_signer::AzureSigner::new(account_name.clone(), account_key.clone()); - let sas_token = signer.generate_sas_token(location).await?; - Ok(Credentials::Azure { - sas_token, - account_name: account_name.clone(), - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - }) - } - Some(VendingStrategy::GcpDownscoped { service_account_email, private_key }) => { - let signer = crate::gcp_signer::GcpSigner::new(service_account_email.clone(), private_key.clone()); - let access_token = signer.generate_downscoped_token(location).await?; - Ok(Credentials::Gcp { - access_token, - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - }) - } - Some(VendingStrategy::None) => Err(anyhow::anyhow!("Vending disabled")), - None => { - // Backward compatibility logic - let access_key = warehouse.storage_config.get("s3.access-key-id") - .ok_or_else(|| anyhow::anyhow!("Missing s3.access-key-id"))?; - let secret_key = warehouse.storage_config.get("s3.secret-access-key") - .ok_or_else(|| anyhow::anyhow!("Missing s3.secret-access-key"))?; - - if warehouse.use_sts { - // Existing STS Logic restored for backward compatibility - let region = warehouse.storage_config.get("s3.region") - .map(|s| s.as_str()) - .unwrap_or("us-east-1"); - - let endpoint = warehouse.storage_config.get("s3.endpoint") - .map(|s| s.as_str()); - - let creds = aws_credential_types::Credentials::new( - access_key.to_string(), - secret_key.to_string(), - None, - None, - "legacy_provider" - ); - - let config_loader = aws_config::from_env() - .region(aws_config::Region::new(region.to_string())) - .credentials_provider(creds); - - let config = if let Some(ep) = endpoint { - config_loader.endpoint_url(ep).load().await - } else { - config_loader.load().await - }; - - let client = aws_sdk_sts::Client::new(&config); - - let role_arn = warehouse.storage_config.get("s3.role-arn").map(|s| s.as_str()); - - if let Some(arn) = role_arn { - let resp = client.assume_role() - .role_arn(arn) - .role_session_name("pangolin-mongo-legacy") - .send() - .await - .map_err(|e| anyhow::anyhow!("STS AssumeRole failed: {}", e))?; - - let c = resp.credentials.ok_or_else(|| anyhow::anyhow!("No credentials in AssumeRole response"))?; - Ok(Credentials::Aws { - access_key_id: c.access_key_id, - secret_access_key: c.secret_access_key, - session_token: Some(c.session_token), - expiration: chrono::DateTime::from_timestamp(c.expiration.secs(), c.expiration.subsec_nanos()), - }) - } else { - let resp = client.get_session_token() - .send() - .await - .map_err(|e| anyhow::anyhow!("STS GetSessionToken failed: {}", e))?; - - let c = resp.credentials.ok_or_else(|| anyhow::anyhow!("No credentials in GetSessionToken response"))?; - Ok(Credentials::Aws { - access_key_id: c.access_key_id, - secret_access_key: c.secret_access_key, - session_token: Some(c.session_token), - expiration: chrono::DateTime::from_timestamp(c.expiration.secs(), c.expiration.subsec_nanos()), - }) - } - } else { - Ok(Credentials::Aws { - access_key_id: access_key.clone(), - secret_access_key: secret_key.clone(), - session_token: None, - expiration: None, - }) - } - } - } - } - - async fn presign_get(&self, _location: &str) -> Result { - Err(anyhow::anyhow!("MongoStore does not support presigning yet")) - } -} - -fn to_bson_uuid(id: Uuid) -> Bson { - Bson::Binary(Binary { - subtype: BinarySubtype::Generic, - bytes: id.as_bytes().to_vec(), - }) -} - -fn from_bson_uuid(bson: &Bson) -> Result { - match bson { - Bson::Binary(Binary { subtype: BinarySubtype::Generic, bytes }) => { - Ok(Uuid::from_slice(bytes)?) - }, - _ => Err(anyhow::anyhow!("Invalid UUID bson")), - } -} - -impl MongoStore { - pub async fn create_user(&self, user: User) -> Result<()> { - self.users().insert_one(user).await?; - Ok(()) - } - - pub async fn get_user(&self, id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(id) }; - let user = self.users().find_one(filter).await?; - Ok(user) - } - - pub async fn list_users(&self, tenant_id: Option) -> Result> { - let filter = if let Some(tid) = tenant_id { - doc! { "tenant-id": to_bson_uuid(tid) } - } else { - doc! {} - }; - let cursor = self.users().find(filter).await?; - let users: Vec = cursor.try_collect().await?; - Ok(users) - } - - async fn get_warehouse_for_location(&self, location: &str) -> Result> { - let cursor = self.warehouses().find(doc! {}).await.map_err(|e| anyhow::anyhow!(e))?; - let warehouses: Vec = cursor.try_collect().await.map_err(|e| anyhow::anyhow!(e))?; - - for warehouse in warehouses { - let s3_match = warehouse.storage_config.get("s3.bucket").or_else(|| warehouse.storage_config.get("bucket")).map(|b| location.contains(b)).unwrap_or(false); - let azure_match = warehouse.storage_config.get("azure.container").map(|c| location.contains(c)).unwrap_or(false); - let gcp_match = warehouse.storage_config.get("gcp.bucket").map(|b| location.contains(b)).unwrap_or(false); - - if s3_match || azure_match || gcp_match { - return Ok(Some(warehouse)); - } - } - - Ok(None) - } - - fn get_object_store_cache_key(&self, config: &HashMap, location: &str) -> String { - let endpoint = config.get("s3.endpoint").or_else(|| config.get("endpoint")).or_else(|| config.get("azure.endpoint")).or_else(|| config.get("gcp.endpoint")).map(|s| s.as_str()).unwrap_or(""); - let bucket = config.get("s3.bucket").or_else(|| config.get("bucket")).or_else(|| config.get("azure.container")).or_else(|| config.get("gcp.bucket")).map(|s| s.as_str()).unwrap_or_else(|| { - location.strip_prefix("s3://").or_else(|| location.strip_prefix("az://")).or_else(|| location.strip_prefix("gs://")).and_then(|s| s.split('/').next()).unwrap_or("") - }); - let access_key = config.get("s3.access-key-id").or_else(|| config.get("access_key_id")).or_else(|| config.get("azure.account-name")).or_else(|| config.get("gcp.service-account-key")).map(|s| s.as_str()).unwrap_or(""); - let region = config.get("s3.region").or_else(|| config.get("region")).or_else(|| config.get("azure.region")).or_else(|| config.get("gcp.region")).map(|s| s.as_str()).unwrap_or(""); - crate::ObjectStoreCache::cache_key(endpoint, &bucket, access_key, region) - } - - async fn read_file_uncached(&self, path: &str) -> Result> { - // Try to look up warehouse credentials first - if let Some(warehouse) = self.get_warehouse_for_location(path).await? { - if path.starts_with("s3://") || path.starts_with("az://") || path.starts_with("gs://") { - // Use cached object store - let cache_key = self.get_object_store_cache_key(&warehouse.storage_config, path); - let store = self.object_store_cache.get_or_insert(cache_key, || { - Arc::new(crate::object_store_factory::create_object_store(&warehouse.storage_config, path).unwrap()) - }); - - // Extract key relative to bucket - let key = if let Some(rest) = path.strip_prefix("s3://").or_else(|| path.strip_prefix("az://")).or_else(|| path.strip_prefix("gs://")) { - rest.split_once('/').map(|(_, k)| k).unwrap_or(rest) - } else { - path - }; - - match store.get(&ObjPath::from(key)).await { - Ok(result) => return Ok(result.bytes().await?.to_vec()), - Err(e) => { - tracing::warn!("Failed to read from warehouse-configured store for {}, falling back to global env: {}", path, e); - } - } - } - } - - if let Some(rest) = path.strip_prefix("s3://") { - let (bucket, key) = rest.split_once('/').ok_or_else(|| anyhow::anyhow!("Invalid S3 path"))?; - - let mut builder = AmazonS3Builder::new() - .with_bucket_name(bucket) - .with_allow_http(true); - - if let Ok(endpoint) = std::env::var("S3_ENDPOINT") { - builder = builder.with_endpoint(endpoint); - } - if let Ok(key_id) = std::env::var("AWS_ACCESS_KEY_ID") { - builder = builder.with_access_key_id(key_id); - } - if let Ok(secret) = std::env::var("AWS_SECRET_ACCESS_KEY") { - builder = builder.with_secret_access_key(secret); - } - if let Ok(region) = std::env::var("AWS_REGION") { - builder = builder.with_region(region); - } - - let store = builder.build()?; - let location = ObjPath::from(key); - let result = store.get(&location).await?; - let bytes = result.bytes().await?; - Ok(bytes.to_vec()) - } else { - Err(anyhow::anyhow!("Only s3:// paths are supported in Mongo store")) - } - } -} diff --git a/pangolin/pangolin_store/src/mongo/assets.rs b/pangolin/pangolin_store/src/mongo/assets.rs index 9b0bfc1..aba0ab6 100644 --- a/pangolin/pangolin_store/src/mongo/assets.rs +++ b/pangolin/pangolin_store/src/mongo/assets.rs @@ -33,7 +33,7 @@ impl MongoStore { "namespace": namespace, "id": to_bson_uuid(asset.id), "name": &asset.name, - "kind": format!("{:?}", asset.kind), + "kind": asset.kind.as_stored_str(), "location": &asset.location, "properties": mongodb::bson::to_bson(&asset.properties)? }; @@ -72,11 +72,8 @@ impl MongoStore { if let Some(d) = doc { let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = AssetType::from_stored_str(kind_str).map_err(|e| anyhow::anyhow!(e))?; let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; @@ -128,11 +125,8 @@ impl MongoStore { let mut assets = Vec::new(); for d in docs { let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = AssetType::from_stored_str(kind_str).map_err(|e| anyhow::anyhow!(e))?; let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; @@ -180,11 +174,8 @@ impl MongoStore { .map(|v| v.as_str().unwrap().to_string()) .collect(); let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = AssetType::from_stored_str(kind_str).map_err(|e| anyhow::anyhow!(e))?; let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; diff --git a/pangolin/pangolin_store/src/mongo/audit.rs b/pangolin/pangolin_store/src/mongo/audit.rs index cd7fa0d..1aa33d1 100644 --- a/pangolin/pangolin_store/src/mongo/audit.rs +++ b/pangolin/pangolin_store/src/mongo/audit.rs @@ -6,6 +6,9 @@ use mongodb::bson::{doc, Bson, Document}; use pangolin_core::audit::{AuditLogEntry, AuditLogFilter}; use uuid::Uuid; +/// Default cap on a listing, matching the SQL backends' `LIMIT 100`. +const DEFAULT_AUDIT_LIMIT: usize = 100; + impl MongoStore { pub async fn log_audit_event(&self, entry: AuditLogEntry) -> Result<()> { let mut doc = mongodb::bson::to_document(&entry)?; @@ -31,8 +34,21 @@ impl MongoStore { Ok(()) } - pub async fn get_audit_event(&self, id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(id) }; + /// Fetch one audit event, scoped to its tenant. + /// + /// B1: the filter was `{ "id": ... }` alone and the caller's `tenant_id` + /// was discarded, so any tenant holding an audit-event UUID could read + /// another tenant's audit record - username, IP, resource names, metadata. + /// Postgres and SQLite both scoped by tenant; only Mongo did not. + pub async fn get_audit_event( + &self, + tenant_id: Uuid, + id: Uuid, + ) -> Result> { + let filter = doc! { + "id": to_bson_uuid(id), + "tenant_id": to_bson_uuid(tenant_id), + }; let doc = self .db .collection::("audit_logs") @@ -60,11 +76,28 @@ impl MongoStore { tenant_id: Uuid, filter: Option, ) -> Result> { + // B23: this applied no sort, no limit and no offset while the SQL + // backends used `ORDER BY timestamp DESC LIMIT 100`. On a busy tenant + // Mongo streamed the entire audit collection into memory and returned + // it in storage order. + let (limit, offset) = filter + .as_ref() + .map(|f| { + ( + f.limit.unwrap_or(DEFAULT_AUDIT_LIMIT), + f.offset.unwrap_or(0), + ) + }) + .unwrap_or((DEFAULT_AUDIT_LIMIT, 0)); + let mongo_filter = self.build_audit_filter(tenant_id, filter)?; let cursor = self .db .collection::("audit_logs") .find(mongo_filter) + .sort(doc! { "timestamp": -1 }) + .skip(offset as u64) + .limit(limit as i64) .await?; let entries: Vec = cursor.try_collect().await?; Ok(entries) @@ -77,15 +110,29 @@ impl MongoStore { ) -> Result { let mut mongo_filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; if let Some(f) = filter { + // B23: these used `format!("{:?}", ..)` - the Debug spelling, + // `"CreateBranch"` - against documents serde wrote in snake_case + // (`"create_branch"`). The filters could never match, so an + // action- or resource-type-filtered listing always returned zero + // rows and `count_audit_events` always returned 0. Going through + // `bson::to_bson` uses the same serde naming as the write path. if let Some(rt) = f.resource_type { - mongo_filter.insert("resource_type", format!("{:?}", rt)); + mongo_filter.insert("resource_type", mongodb::bson::to_bson(&rt)?); } if let Some(ra) = f.action { - mongo_filter.insert("action", format!("{:?}", ra)); + mongo_filter.insert("action", mongodb::bson::to_bson(&ra)?); } if let Some(uid) = f.user_id { mongo_filter.insert("user_id", to_bson_uuid(uid)); } + // B23: `resource_id` and `result` were accepted by the filter type + // and then silently ignored. + if let Some(rid) = f.resource_id { + mongo_filter.insert("resource_id", to_bson_uuid(rid)); + } + if let Some(result) = f.result { + mongo_filter.insert("result", mongodb::bson::to_bson(&result)?); + } if let Some(from) = f.start_time { mongo_filter.insert("timestamp", doc! { "$gte": Bson::DateTime(from.into()) }); } diff --git a/pangolin/pangolin_store/src/mongo/branches.rs b/pangolin/pangolin_store/src/mongo/branches.rs index 2dd10ba..06633c9 100644 --- a/pangolin/pangolin_store/src/mongo/branches.rs +++ b/pangolin/pangolin_store/src/mongo/branches.rs @@ -1,4 +1,4 @@ -use super::main::to_bson_uuid; +use super::main::{read_optional_uuid, to_bson_uuid}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -17,7 +17,10 @@ impl MongoStore { "tenant_id": to_bson_uuid(tenant_id), "catalog_name": catalog_name, "name": &branch.name, - "head_commit_id": branch.head_commit_id, + "head_commit_id": branch + .head_commit_id + .map(to_bson_uuid) + .unwrap_or(mongodb::bson::Bson::Null), "branch_type": format!("{:?}", branch.branch_type), "assets": &branch.assets }; @@ -55,7 +58,7 @@ impl MongoStore { Ok(Some(Branch { name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson(d.get("head_commit_id").unwrap().clone())?, + head_commit_id: read_optional_uuid(&d, "head_commit_id")?, branch_type, assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, })) @@ -100,7 +103,7 @@ impl MongoStore { branches.push(Branch { name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson(d.get("head_commit_id").unwrap().clone())?, + head_commit_id: read_optional_uuid(&d, "head_commit_id")?, branch_type, assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, }); @@ -171,7 +174,10 @@ impl MongoStore { let update = doc! { "$set": { - "head_commit_id": source.head_commit_id + "head_commit_id": source + .head_commit_id + .map(to_bson_uuid) + .unwrap_or(mongodb::bson::Bson::Null) } }; diff --git a/pangolin/pangolin_store/src/mongo/business_metadata.rs b/pangolin/pangolin_store/src/mongo/business_metadata.rs index 30ba040..25e4280 100644 --- a/pangolin/pangolin_store/src/mongo/business_metadata.rs +++ b/pangolin/pangolin_store/src/mongo/business_metadata.rs @@ -1,4 +1,4 @@ -use super::main::{from_bson_uuid, to_bson_uuid}; +use super::main::{from_bson_uuid, read_optional_uuid, to_bson_uuid, with_binary_uuids}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -9,10 +9,25 @@ use std::collections::HashMap; use uuid::Uuid; impl MongoStore { + /// Insert or replace an asset's business metadata. + /// + /// Only `asset-id` used to be rewritten as Binary. The other three UUID + /// fields kept whatever `to_document` produced - a string - so the record + /// could be written and located but never *deserialized*: + /// `get_business_metadata` failed with `invalid type: string "...", + /// expected bytes` on the first field it reached. Writing metadata + /// therefore made an asset's metadata permanently unreadable. pub async fn upsert_business_metadata(&self, metadata: BusinessMetadata) -> Result<()> { let filter = doc! { "asset-id": to_bson_uuid(metadata.asset_id) }; - let mut doc = mongodb::bson::to_document(&metadata)?; - doc.insert("asset-id", to_bson_uuid(metadata.asset_id)); + let doc = with_binary_uuids( + mongodb::bson::to_document(&metadata)?, + &[ + ("id", metadata.id), + ("asset-id", metadata.asset_id), + ("created-by", metadata.created_by), + ("updated-by", metadata.updated_by), + ], + ); let options = mongodb::options::ReplaceOptions::builder() .upsert(true) @@ -211,9 +226,7 @@ impl MongoStore { results.push(( Branch { name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson( - d.get("head_commit_id").unwrap().clone(), - )?, + head_commit_id: read_optional_uuid(&d, "head_commit_id")?, branch_type, assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, }, diff --git a/pangolin/pangolin_store/src/mongo/catalogs.rs b/pangolin/pangolin_store/src/mongo/catalogs.rs index b294d4a..4886b5b 100644 --- a/pangolin/pangolin_store/src/mongo/catalogs.rs +++ b/pangolin/pangolin_store/src/mongo/catalogs.rs @@ -6,6 +6,40 @@ use mongodb::bson::{doc, Document}; use pangolin_core::model::{Catalog, CatalogUpdate}; use uuid::Uuid; +/// Why a transactional catalog delete stopped. +enum CatalogDeleteError { + /// The catalog does not exist. + NotFound, + /// The deployment has no transaction support - a standalone `mongod`. + /// + /// MongoDB reports this as `IllegalOperation` (code 20) with "Transaction + /// numbers are only allowed on a replica set member or mongos", and only + /// once the first operation inside the transaction reaches the server. + TransactionsUnsupported(mongodb::error::Error), + Other(anyhow::Error), +} + +impl CatalogDeleteError { + fn from_mongo(e: mongodb::error::Error) -> Self { + // MongoDB raises `IllegalOperation` for "Transaction numbers are only + // allowed on a replica set member or mongos". The driver replaces that + // server text with its own "does not support retryable writes" wording, + // so matching on the message is unreliable - the code is the stable + // signal. This classifier only ever runs on failures from inside a + // transaction attempt, where an `IllegalOperation` means the deployment + // cannot do transactions; the cost of a false positive is a non-atomic + // delete rather than a wrong result. + const ILLEGAL_OPERATION: i32 = 20; + + if let mongodb::error::ErrorKind::Command(command_error) = &*e.kind { + if command_error.code == ILLEGAL_OPERATION { + return Self::TransactionsUnsupported(e); + } + } + Self::Other(anyhow::anyhow!(e)) + } +} + impl MongoStore { pub async fn create_catalog(&self, tenant_id: Uuid, catalog: Catalog) -> Result<()> { let mut doc = doc! { @@ -128,43 +162,94 @@ impl MongoStore { }; if let Some(session) = session.as_mut() { + // `start_transaction` is a *local* call in the Rust driver: it + // allocates a transaction number and returns without contacting the + // server, so it cannot fail for "this deployment has no transaction + // support". The fallback below was therefore unreachable - the + // topology error surfaced on the first operation *inside* the + // transaction and propagated through `?`, failing the delete + // outright on any standalone `mongod`. Found by running the parity + // suite against a live standalone MongoDB. + // + // The transactional attempt now runs in full and a topology error + // anywhere within it degrades to the sequential path, which is what + // the original comment promised. if let Err(e) = session.start_transaction().await { tracing::warn!( error = %e, - "this MongoDB deployment does not support transactions (a replica set is \ - required); deleting a catalog will not be atomic" + "could not begin a MongoDB transaction; deleting a catalog will not be atomic" ); return self .delete_catalog_unsafe(filter, child_filter, &name) .await; } - for collection in ["tags", "branches", "assets", "namespaces"] { - self.db - .collection::(collection) - .delete_many(child_filter.clone()) - .session(&mut *session) - .await?; + match self + .delete_catalog_in_transaction(session, &filter, &child_filter) + .await + { + Ok(()) => return Ok(()), + Err(CatalogDeleteError::NotFound) => { + let _ = session.abort_transaction().await; + return Err(anyhow::anyhow!("Catalog '{}' not found", name)); + } + Err(CatalogDeleteError::TransactionsUnsupported(e)) => { + tracing::warn!( + error = %e, + "this MongoDB deployment does not support transactions (a replica set \ + is required); deleting a catalog will not be atomic" + ); + let _ = session.abort_transaction().await; + // Falls through to the sequential path below. + } + Err(CatalogDeleteError::Other(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } } + } + + self.delete_catalog_unsafe(filter, child_filter, &name) + .await + } - let result = self - .db - .collection::("catalogs") - .delete_one(filter) + /// The transactional half of [`Self::delete_catalog`]. + /// + /// Split out so a topology error can be told apart from a genuine failure + /// and from "no such catalog" - the caller handles each differently. + async fn delete_catalog_in_transaction( + &self, + session: &mut mongodb::ClientSession, + filter: &Document, + child_filter: &Document, + ) -> std::result::Result<(), CatalogDeleteError> { + for collection in ["tags", "branches", "assets", "namespaces"] { + self.db + .collection::(collection) + .delete_many(child_filter.clone()) .session(&mut *session) - .await?; + .await + .map_err(CatalogDeleteError::from_mongo)?; + } - if result.deleted_count == 0 { - let _ = session.abort_transaction().await; - return Err(anyhow::anyhow!("Catalog '{}' not found", name)); - } + let result = self + .db + .collection::("catalogs") + .delete_one(filter.clone()) + .session(&mut *session) + .await + .map_err(CatalogDeleteError::from_mongo)?; - session.commit_transaction().await?; - return Ok(()); + if result.deleted_count == 0 { + return Err(CatalogDeleteError::NotFound); } - self.delete_catalog_unsafe(filter, child_filter, &name) + session + .commit_transaction() .await + .map_err(CatalogDeleteError::from_mongo)?; + + Ok(()) } /// Non-atomic fallback for deployments without transaction support. @@ -174,6 +259,29 @@ impl MongoStore { child_filter: Document, name: &str, ) -> Result<()> { + // Existence is checked *first*. This path used to delete every matching + // tag, branch, asset and namespace and only then discover the catalog + // did not exist - returning "not found" to a caller who had every reason + // to believe nothing had happened. That is B21, which was fixed for + // SQLite during the roadmap work; the same shape survived here because + // this branch only runs on a deployment without transactions, and + // nothing had ever run it. + // + // Without a transaction the cascade below is still not atomic - a + // failure partway through leaves a partial delete. Checking first at + // least means a *no-op* call destroys nothing, which is the case an + // operator is most likely to hit by typo. + let exists = self + .db + .collection::("catalogs") + .find_one(filter.clone()) + .await? + .is_some(); + + if !exists { + return Err(anyhow::anyhow!("Catalog '{}' not found", name)); + } + for collection in ["tags", "branches", "assets", "namespaces"] { self.db .collection::(collection) @@ -181,15 +289,11 @@ impl MongoStore { .await?; } - let result = self - .db + self.db .collection::("catalogs") .delete_one(filter) .await?; - if result.deleted_count == 0 { - return Err(anyhow::anyhow!("Catalog '{}' not found", name)); - } Ok(()) } } diff --git a/pangolin/pangolin_store/src/mongo/indexes.rs b/pangolin/pangolin_store/src/mongo/indexes.rs new file mode 100644 index 0000000..9667fbd --- /dev/null +++ b/pangolin/pangolin_store/src/mongo/indexes.rs @@ -0,0 +1,401 @@ +//! Index management for the MongoDB backend. +//! +//! MongoDB had two indexes — `commits(parent_id)` and `active_tokens(user_id)` +//! — created at startup with their errors discarded by `.ok()`. Everything else +//! was a collection scan: every catalog lookup, every asset resolution on the +//! Iceberg commit path, every permission check. The other three backends get +//! these for free from primary keys and `UNIQUE` constraints; MongoDB is the +//! only one where the schema is whatever the first write happened to create. +//! +//! Two distinct things are being fixed here. +//! +//! **Performance.** The index set below is derived from the filters the code +//! actually issues, not from what seemed likely — see the field lists in each +//! entry. A missing index on `{tenant_id, name}` does not fail anything; it +//! just makes every catalog lookup read the whole collection, and that only +//! becomes visible under a load nobody ran. +//! +//! **Integrity.** Several of these are `unique`, which is the constraint the +//! SQL backends express as a primary key. Without it MongoDB will happily hold +//! two catalogs with the same name in one tenant, and which one a lookup +//! returns is arbitrary. That is a real parity gap, and it is why +//! `create_catalog` on MongoDB could not detect a duplicate the way Postgres +//! does. +//! +//! ## Failures are reported +//! +//! Creating a unique index on a collection that already contains duplicates +//! fails, and it should: silently continuing would leave the operator believing +//! a constraint exists. Each failure is logged with the collection, the keys and +//! what to do about it, and the count is returned so the caller can decide. +//! Startup does not abort — refusing to boot because an index is missing would +//! turn a performance problem into an outage — but it is impossible to miss in +//! the log. + +use mongodb::bson::{doc, Document}; +use mongodb::{Database, IndexModel}; + +/// One index this backend needs. +struct Index { + collection: &'static str, + keys: Document, + unique: bool, + /// Why it exists, for the log line when creation fails. + reason: &'static str, +} + +fn required_indexes() -> Vec { + vec![ + // ---- Identity and tenancy ---- + Index { + collection: "tenants", + keys: doc! { "id": 1 }, + unique: true, + reason: "every tenant lookup; the SQL backends have this as a primary key", + }, + Index { + collection: "users", + keys: doc! { "id": 1 }, + unique: true, + reason: "user lookup by id on every authenticated request", + }, + Index { + collection: "users", + keys: doc! { "username": 1 }, + unique: false, + reason: "login by username", + }, + Index { + collection: "users", + keys: doc! { "email": 1 }, + unique: false, + reason: "OAuth matches users on the provider-supplied email", + }, + Index { + collection: "users", + keys: doc! { "tenant-id": 1 }, + unique: false, + reason: "listing a tenant's users", + }, + // ---- Catalog objects ---- + Index { + collection: "catalogs", + keys: doc! { "tenant_id": 1, "name": 1 }, + unique: true, + reason: "catalog lookup, and the uniqueness Postgres gets from its primary key", + }, + Index { + collection: "warehouses", + keys: doc! { "tenant_id": 1, "name": 1 }, + unique: true, + reason: "warehouse lookup on every credential-vending request", + }, + Index { + collection: "namespaces", + keys: doc! { "tenant_id": 1, "catalog_name": 1 }, + unique: false, + reason: "listing namespaces in a catalog", + }, + Index { + collection: "assets", + keys: doc! { "tenant_id": 1, "catalog_name": 1, "branch_name": 1 }, + unique: false, + reason: "listing and copying a branch's assets", + }, + Index { + collection: "assets", + keys: doc! { "tenant_id": 1, "id": 1 }, + unique: false, + reason: "get_asset_by_id, used by the business-metadata join", + }, + Index { + collection: "branches", + keys: doc! { "tenant_id": 1, "catalog_name": 1, "name": 1 }, + unique: true, + reason: "branch lookup; two branches of one name in a catalog is corruption", + }, + Index { + collection: "tags", + keys: doc! { "tenant_id": 1, "catalog_name": 1, "name": 1 }, + unique: true, + reason: "tag lookup, and tag names must be unique within a catalog", + }, + Index { + collection: "commits", + keys: doc! { "id": 1 }, + unique: true, + reason: "commit lookup by id", + }, + Index { + collection: "commits", + keys: doc! { "parent_id": 1 }, + unique: false, + reason: "walking commit history", + }, + // ---- Authorization ---- + // + // These are the hot path: every request resolves the caller's roles and + // permissions, so an unindexed scan here is paid on *every* request. + Index { + collection: "roles", + keys: doc! { "id": 1 }, + unique: true, + reason: "role lookup", + }, + Index { + collection: "roles", + keys: doc! { "tenant-id": 1 }, + unique: false, + reason: "listing a tenant's roles", + }, + Index { + collection: "user_roles", + keys: doc! { "user-id": 1 }, + unique: false, + reason: "resolving a caller's roles, on every authorized request", + }, + Index { + collection: "user_roles", + keys: doc! { "role-id": 1 }, + unique: false, + reason: "finding who holds a role, for revocation", + }, + Index { + collection: "permissions", + keys: doc! { "user-id": 1 }, + unique: false, + reason: "resolving direct grants, on every authorized request", + }, + Index { + collection: "permissions", + keys: doc! { "tenant-id": 1 }, + unique: false, + reason: "listing a tenant's grants", + }, + // ---- Credentials and sessions ---- + Index { + collection: "service_users", + keys: doc! { "id": 1 }, + unique: true, + reason: "service-user lookup", + }, + Index { + collection: "service_users", + keys: doc! { "api-key-hash": 1 }, + unique: false, + reason: "API-key authentication resolves the principal by hash", + }, + Index { + collection: "service_users", + keys: doc! { "tenant-id": 1 }, + unique: false, + reason: "API-key auth enumerates a tenant's service users", + }, + Index { + collection: "active_tokens", + keys: doc! { "tenant_id": 1, "user_id": 1 }, + unique: false, + reason: "listing a user's active sessions", + }, + Index { + collection: "active_tokens", + keys: doc! { "token_id": 1 }, + unique: false, + reason: "token lookup by jti", + }, + Index { + collection: "revoked_tokens", + keys: doc! { "token_id": 1 }, + unique: true, + reason: "the revocation check runs on every authenticated request", + }, + Index { + collection: "revoked_tokens", + keys: doc! { "expires_at": 1 }, + unique: false, + reason: "the cleanup job deletes by expiry", + }, + // ---- Everything else ---- + Index { + collection: "audit_logs", + keys: doc! { "tenant_id": 1, "timestamp": -1 }, + unique: false, + reason: "audit listing is tenant-scoped and newest-first", + }, + Index { + collection: "business_metadata", + keys: doc! { "asset-id": 1 }, + unique: true, + reason: "an asset has at most one metadata record", + }, + Index { + collection: "access_requests", + keys: doc! { "id": 1 }, + unique: true, + reason: "access-request lookup", + }, + Index { + collection: "access_requests", + keys: doc! { "user-id": 1 }, + unique: false, + reason: "the listing joins access_requests.user-id to users.id", + }, + Index { + collection: "merge_operations", + keys: doc! { "id": 1 }, + unique: true, + reason: "merge-operation lookup", + }, + Index { + collection: "merge_conflicts", + keys: doc! { "merge_operation_id": 1 }, + unique: false, + reason: "listing an operation's conflicts", + }, + Index { + collection: "system_settings", + keys: doc! { "tenant_id": 1 }, + unique: true, + reason: "one settings document per tenant", + }, + ] +} + +/// Create every index this backend needs. Returns how many could not be made. +/// +/// Idempotent: MongoDB treats `createIndex` for an index that already exists +/// with the same specification as a no-op, so this runs on every startup. +pub(crate) async fn ensure_indexes(db: &Database) -> usize { + let mut failures = 0; + + for index in required_indexes() { + let options = mongodb::options::IndexOptions::builder() + .background(true) + .unique(index.unique) + .build(); + + let model = IndexModel::builder() + .keys(index.keys.clone()) + .options(options) + .build(); + + if let Err(e) = db + .collection::(index.collection) + .create_index(model) + .await + { + failures += 1; + if index.unique { + // Much the most likely cause, and the operator can act on it. + tracing::error!( + collection = index.collection, + keys = %index.keys, + reason = index.reason, + error = %e, + "could not create a unique index. If this is a duplicate-key error, \ + the collection already holds rows that violate the constraint - find \ + and remove them, then restart. Until then this uniqueness is NOT \ + enforced." + ); + } else { + tracing::warn!( + collection = index.collection, + keys = %index.keys, + reason = index.reason, + error = %e, + "could not create an index; queries against this collection will scan it" + ); + } + } + } + + if failures == 0 { + tracing::info!( + count = required_indexes().len(), + "MongoDB indexes are in place" + ); + } else { + tracing::error!( + failures, + total = required_indexes().len(), + "some MongoDB indexes could not be created; see the errors above" + ); + } + + failures +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_authorization_hot_path_is_indexed() { + // Every authenticated request resolves roles and direct grants. An + // unindexed scan here is paid per request, which is the difference + // between a catalog that scales and one that does not. + let indexes = required_indexes(); + for (collection, field) in [ + ("user_roles", "user-id"), + ("permissions", "user-id"), + ("revoked_tokens", "token_id"), + ] { + assert!( + indexes + .iter() + .any(|i| i.collection == collection && i.keys.contains_key(field)), + "{collection}({field}) is on the per-request path and must be indexed" + ); + } + } + + #[test] + fn kebab_case_fields_are_spelled_as_stored() { + // The RBAC collections serialize with `rename_all = "kebab-case"`, so an + // index on `user_id` would index a field that does not exist and silently + // do nothing. This is the same spelling trap that made every role + // assignment unreadable. + let indexes = required_indexes(); + for index in &indexes { + if matches!( + index.collection, + "user_roles" | "permissions" | "service_users" | "business_metadata" + ) { + for key in index.keys.keys() { + assert!( + !key.contains('_') || key == "tenant_id", + "{}: {key} looks snake_case, but this collection stores \ + kebab-case field names", + index.collection + ); + } + } + } + } + + #[test] + fn uniqueness_matches_what_the_sql_backends_enforce() { + let indexes = required_indexes(); + for collection in ["catalogs", "warehouses", "branches", "tags"] { + assert!( + indexes + .iter() + .any(|i| i.collection == collection && i.unique), + "{collection} has a uniqueness constraint in the SQL backends and \ + must have one here, or MongoDB accepts duplicates the others reject" + ); + } + } + + #[test] + fn every_index_explains_itself() { + for index in required_indexes() { + assert!( + !index.reason.is_empty(), + "{} has no stated reason; an index nobody can justify is one \ + nobody can safely remove", + index.collection + ); + } + } +} diff --git a/pangolin/pangolin_store/src/mongo/main.rs b/pangolin/pangolin_store/src/mongo/main.rs index a9a2011..3660dab 100644 --- a/pangolin/pangolin_store/src/mongo/main.rs +++ b/pangolin/pangolin_store/src/mongo/main.rs @@ -43,30 +43,14 @@ impl MongoStore { let client = Client::with_options(client_options)?; let db = client.database(database_name); - // Ensure Indexes - let options = mongodb::options::IndexOptions::builder() - .background(true) - .build(); - - // commits(parent_id) - let commit_index = mongodb::IndexModel::builder() - .keys(doc! { "parent_id": 1 }) - .options(options.clone()) - .build(); - db.collection::("commits") - .create_index(commit_index) - .await - .ok(); - - // active_tokens(user_id) - let token_index = mongodb::IndexModel::builder() - .keys(doc! { "user_id": 1 }) - .options(options) - .build(); - db.collection::("active_tokens") - .create_index(token_index) - .await - .ok(); + // Indexes. There used to be two here - commits(parent_id) and + // active_tokens(user_id) - created with their errors thrown away by + // `.ok()`, so everything else was a collection scan and a failure was + // invisible. `super::indexes` holds the full set, derived from the + // filters this backend actually issues, and reports what it could not + // create. + super::indexes::ensure_indexes(&db).await; + Ok(Self { client, db, @@ -197,12 +181,32 @@ impl MongoStore { .get_asset(tenant_id, catalog_name, branch, namespace, table) .await? { - Ok(asset.properties.get("metadata_location").cloned()) + // Found by running the parity suite against a live MongoDB: this + // read only `properties["metadata_location"]` and had no fallback to + // the asset's own `location`, which is what memory, SQLite and + // Postgres all fall back to. A table created with a location but no + // explicit metadata-location property therefore reported *no* + // metadata location on Mongo alone - so `load_table` could not find + // its metadata and every commit's compare-and-swap was working from + // a different notion of "current" than the read path. + Ok(current_metadata_location(&asset)) } else { Ok(None) } } + /// Publish a new metadata location, but only if the current one still + /// matches `expected_location`. + /// + /// B5: `expected_location` was ignored (`_expected_location`) and the update + /// was an unconditional `$set`. Memory, Postgres and SQLite all enforce the + /// compare-and-swap; on Mongo two concurrent Iceberg commits both + /// "succeeded" and one snapshot was silently lost - the exact failure class + /// the 0.6.0 work fixed at the API layer, still wide open one layer down. + /// + /// Folding the expectation into the *filter* keeps this a single-document + /// atomic update, so it works on a standalone `mongod` with no multi-document + /// transaction required. pub async fn update_metadata_location( &self, tenant_id: Uuid, @@ -210,10 +214,10 @@ impl MongoStore { branch: Option, namespace: Vec, table: String, - _expected_location: Option, + expected_location: Option, new_location: String, ) -> Result<()> { - let filter = doc! { + let mut filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "catalog_name": catalog_name, "branch": branch.unwrap_or_else(|| "main".to_string()), @@ -221,20 +225,106 @@ impl MongoStore { "name": table }; + // The expectation has to be expressed against the *same* notion of + // "current location" that `get_metadata_location` returns - property + // first, then the asset's own `location`. Keeping it inside the filter + // rather than reading-then-comparing preserves the single-document + // atomicity that makes this a real CAS on a standalone mongod. + match &expected_location { + Some(expected) => { + filter.insert( + "$or", + vec![ + doc! { "properties.metadata_location": expected.clone() }, + doc! { "$and": vec![ + doc! { "properties.metadata_location": { "$exists": false } }, + doc! { "location": expected.clone() }, + ]}, + ], + ); + } + // `None` means "there must not be one yet" - the create-path CAS. + None => { + filter.insert( + "$and", + vec![ + doc! { "properties.metadata_location": { "$exists": false } }, + doc! { "$or": vec![ + doc! { "location": { "$exists": false } }, + doc! { "location": "" }, + ]}, + ], + ); + } + } + let update = doc! { "$set": { - "properties.metadata_location": new_location + "properties.metadata_location": &new_location, + "location": &new_location, } }; - self.db + let result = self + .db .collection::("assets") .update_one(filter, update) .await?; + + if result.matched_count == 0 { + return Err(anyhow::anyhow!( + "CAS failure: metadata location did not match {:?}", + expected_location + )); + } Ok(()) } } +/// The metadata location a reader would see for `asset`. +/// +/// Property first, then the asset's own `location` when it is non-empty - the +/// same resolution memory, SQLite and Postgres use. Defined once so the read +/// path and the compare-and-swap cannot drift apart again. +fn current_metadata_location(asset: &pangolin_core::model::Asset) -> Option { + asset + .properties + .get("metadata_location") + .cloned() + .or_else(|| { + if asset.location.is_empty() { + None + } else { + Some(asset.location.clone()) + } + }) +} + +/// Rewrite the named keys of a serde-produced document as BSON Binary UUIDs. +/// +/// `bson::to_document` writes a `Uuid` as a *string*, but the driver's +/// deserializer expects Binary - so a document written through serde alone +/// cannot be read back into a struct with `Uuid` fields ("invalid type: string, +/// expected bytes"), and a filter built with [`to_bson_uuid`] never matches it. +/// +/// That asymmetry is the single cause of the Mongo RBAC failures: role +/// assignments were written by serde and queried as Binary, so `get_user_roles` +/// always returned empty and every role-derived permission silently vanished. +/// The audit-log and token-revocation paths (B1, B2) were the same bug in two +/// other collections. +/// +/// The keys given are the *serialized* names - kebab-case for these types - so +/// the document stays deserializable into its struct. +pub(crate) fn with_binary_uuids( + mut doc: mongodb::bson::Document, + fields: &[(&str, Uuid)], +) -> mongodb::bson::Document { + for (key, value) in fields { + doc.insert(*key, to_bson_uuid(*value)); + } + doc +} + pub(crate) fn to_bson_uuid(id: Uuid) -> Bson { Bson::Binary(Binary { subtype: BinarySubtype::Generic, @@ -242,12 +332,44 @@ pub(crate) fn to_bson_uuid(id: Uuid) -> Bson { }) } +/// Decode a UUID that may have been written by any of the three routes. +/// +/// There are three, and they disagree: +/// +/// 1. [`to_bson_uuid`] - `Binary` with the *generic* subtype; +/// 2. `doc! { "k": some_uuid }` - `Binary` with the *UUID* subtype, via bson's +/// `From for Bson`; +/// 3. `bson::to_document` - a plain `String`. +/// +/// Writes should use `to_bson_uuid` so new data is uniform, but reads have to +/// accept all three: documents written by the other two are already in +/// deployed databases. Being strict here is what made a branch with a head +/// commit unreadable - `create_branch` used route 2 and the reader accepted +/// only route 1. pub(crate) fn from_bson_uuid(bson: &Bson) -> Result { match bson { Bson::Binary(Binary { - subtype: BinarySubtype::Generic, + subtype: BinarySubtype::Generic | BinarySubtype::Uuid, bytes, }) => Ok(Uuid::from_slice(bytes)?), + Bson::String(s) => Ok(Uuid::parse_str(s)?), _ => Err(anyhow::anyhow!("Invalid UUID bson")), } } + +/// Decode an optional UUID field. +/// +/// A missing key and an explicit `null` both mean "absent"; anything else has +/// to decode, because silently returning `None` for a value that is present +/// but unreadable would turn a corrupt record into a plausible-looking one. +/// +/// `bson::from_bson::>` cannot be used for this: handed a +/// `Bson::Binary` it reports `invalid type: map, expected a UUID string`, +/// because the deserializer presents binary data as the extended-JSON map +/// `{"$binary": ...}` while `Uuid`'s `Deserialize` wants a string. +pub(crate) fn read_optional_uuid(doc: &mongodb::bson::Document, key: &str) -> Result> { + match doc.get(key) { + None | Some(Bson::Null) => Ok(None), + Some(value) => from_bson_uuid(value).map(Some), + } +} diff --git a/pangolin/pangolin_store/src/mongo/mod.rs b/pangolin/pangolin_store/src/mongo/mod.rs index 891b7d2..a75226b 100644 --- a/pangolin/pangolin_store/src/mongo/mod.rs +++ b/pangolin/pangolin_store/src/mongo/mod.rs @@ -6,6 +6,7 @@ pub mod business_metadata; pub mod catalogs; pub mod commits; pub mod federated; +mod indexes; pub mod io; pub mod main; pub mod merge; @@ -154,6 +155,17 @@ impl CatalogStore for MongoStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: HashMap, + ) -> Result<()> { + self.replace_namespace_properties(tenant_id, catalog_name, namespace, properties) + .await + } + // Asset Operations async fn create_asset( &self, @@ -460,10 +472,12 @@ impl CatalogStore for MongoStore { } async fn get_audit_event( &self, - _tenant_id: Uuid, + tenant_id: Uuid, id: Uuid, ) -> Result> { - self.get_audit_event(id).await + // B1: `tenant_id` used to be discarded here (`_tenant_id`), which is + // where the cross-tenant audit read came from. + self.get_audit_event(tenant_id, id).await } async fn count_audit_events( &self, @@ -607,6 +621,14 @@ impl CatalogStore for MongoStore { async fn write_file(&self, path: &str, data: Vec) -> Result<()> { self.write_file(path, data).await } + async fn delete_file(&self, path: &str) -> Result<()> { + self.metadata_cache.invalidate(path).await; + let storage_config = self + .get_warehouse_for_location(path) + .await? + .map(|w| w.storage_config); + crate::file_delete::delete_location(storage_config.as_ref(), path).await + } // Access Requests async fn create_access_request(&self, request: AccessRequest) -> Result<()> { diff --git a/pangolin/pangolin_store/src/mongo/namespaces.rs b/pangolin/pangolin_store/src/mongo/namespaces.rs index 6c7d7c7..9cca7aa 100644 --- a/pangolin/pangolin_store/src/mongo/namespaces.rs +++ b/pangolin/pangolin_store/src/mongo/namespaces.rs @@ -120,6 +120,34 @@ impl MongoStore { Ok(()) } + /// Replace a namespace's properties wholesale; see the SQLite twin for why + /// a merge-only method could not implement Iceberg property removals (B16h). + pub async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: HashMap, + ) -> Result<()> { + let filter = doc! { + "tenant_id": to_bson_uuid(tenant_id), + "catalog_name": catalog_name, + "name": namespace + }; + + let props = bson::to_bson(&properties)?; + let update = doc! { "$set": { "properties": props } }; + let result = self + .db + .collection::("namespaces") + .update_one(filter, update) + .await?; + if result.matched_count == 0 { + return Err(anyhow::anyhow!("Namespace not found")); + } + Ok(()) + } + pub async fn count_namespaces(&self, tenant_id: Uuid) -> Result { let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; let count = self.namespaces().count_documents(filter).await?; diff --git a/pangolin/pangolin_store/src/mongo/permissions.rs b/pangolin/pangolin_store/src/mongo/permissions.rs index 9411cbe..dc6cdd5 100644 --- a/pangolin/pangolin_store/src/mongo/permissions.rs +++ b/pangolin/pangolin_store/src/mongo/permissions.rs @@ -1,4 +1,4 @@ -use super::main::to_bson_uuid; +use super::main::{to_bson_uuid, with_binary_uuids}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -9,11 +9,21 @@ use uuid::Uuid; impl MongoStore { pub async fn grant_permission(&self, permission: Permission) -> Result<()> { - let mut doc = mongodb::bson::to_document(&permission)?; - doc.insert("id", to_bson_uuid(permission.id)); - doc.insert("user_id", to_bson_uuid(permission.user_id)); - doc.insert("tenant_id", to_bson_uuid(permission.tenant_id)); - doc.insert("granted_by", to_bson_uuid(permission.granted_by)); + // The overrides used *snake_case* names while `Permission` serializes + // as kebab-case, so this added a second set of fields rather than + // replacing the first: the document carried both `user-id` (a string) + // and `user_id` (Binary). Filters matched the Binary copy, but + // deserializing the record back into `Permission` read the string and + // failed with "invalid type: string, expected bytes". + let doc = with_binary_uuids( + mongodb::bson::to_document(&permission)?, + &[ + ("id", permission.id), + ("user-id", permission.user_id), + ("tenant-id", permission.tenant_id), + ("granted-by", permission.granted_by), + ], + ); self.db .collection::("permissions") @@ -37,7 +47,7 @@ impl MongoStore { pagination: Option, ) -> Result> { // 1. Fetch direct permissions - let filter = doc! { "user_id": to_bson_uuid(user_id) }; + let filter = doc! { "user-id": to_bson_uuid(user_id) }; let cursor = self .db .collection::("permissions") @@ -98,7 +108,7 @@ impl MongoStore { } // 2. Get permissions for those users - let perm_filter = doc! { "user_id": { "$in": user_ids } }; + let perm_filter = doc! { "user-id": { "$in": user_ids } }; let collection = self.db.collection::("permissions"); let mut find = collection.find(perm_filter); diff --git a/pangolin/pangolin_store/src/mongo/roles.rs b/pangolin/pangolin_store/src/mongo/roles.rs index 9ca1062..d24c40b 100644 --- a/pangolin/pangolin_store/src/mongo/roles.rs +++ b/pangolin/pangolin_store/src/mongo/roles.rs @@ -1,4 +1,4 @@ -use super::main::to_bson_uuid; +use super::main::{to_bson_uuid, with_binary_uuids}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -8,7 +8,22 @@ use uuid::Uuid; impl MongoStore { pub async fn create_role(&self, role: Role) -> Result<()> { - self.roles().insert_one(role).await?; + // Written through an explicit document so the UUID fields are Binary. + // `insert_one(role)` went through serde, which writes them as strings - + // and then `get_role`'s Binary filter could never match, so a role could + // be created and never found again. + let doc = with_binary_uuids( + mongodb::bson::to_document(&role)?, + &[ + ("id", role.id), + ("tenant-id", role.tenant_id), + ("created-by", role.created_by), + ], + ); + self.db + .collection::("roles") + .insert_one(doc) + .await?; Ok(()) } @@ -61,13 +76,38 @@ impl MongoStore { pub async fn update_role(&self, role: Role) -> Result<()> { let filter = doc! { "id": to_bson_uuid(role.id) }; - self.roles().replace_one(filter, role).await?; + // Same reason as `create_role`: a serde replacement would put the string + // form back and make the role unfindable again. + let doc = with_binary_uuids( + mongodb::bson::to_document(&role)?, + &[ + ("id", role.id), + ("tenant-id", role.tenant_id), + ("created-by", role.created_by), + ], + ); + self.db + .collection::("roles") + .replace_one(filter, doc) + .await?; Ok(()) } // Role Assignment pub async fn assign_role(&self, assignment: UserRole) -> Result<()> { - let doc = mongodb::bson::to_document(&assignment)?; + // The assignment's UUIDs are written as Binary under their serialized + // (kebab-case) names, matching both `get_user_roles`' filter and the + // deserializer. Previously serde wrote strings, so `get_user_roles` + // returned nothing and every role-derived permission was lost - a user + // with an admin role was authorized as if they had none. + let doc = with_binary_uuids( + mongodb::bson::to_document(&assignment)?, + &[ + ("user-id", assignment.user_id), + ("role-id", assignment.role_id), + ("assigned-by", assignment.assigned_by), + ], + ); self.db .collection::("user_roles") .insert_one(doc) @@ -76,9 +116,10 @@ impl MongoStore { } pub async fn remove_role(&self, user_id: Uuid, role_id: Uuid) -> Result<()> { + // Kebab-case: these are the serialized field names. let filter = doc! { - "user_id": to_bson_uuid(user_id), - "role_id": to_bson_uuid(role_id) + "user-id": to_bson_uuid(user_id), + "role-id": to_bson_uuid(role_id) }; self.db .collection::("user_roles") @@ -88,7 +129,7 @@ impl MongoStore { } pub async fn get_user_roles(&self, user_id: Uuid) -> Result> { - let filter = doc! { "user_id": to_bson_uuid(user_id) }; + let filter = doc! { "user-id": to_bson_uuid(user_id) }; let cursor = self .db .collection::("user_roles") diff --git a/pangolin/pangolin_store/src/mongo/service_users.rs b/pangolin/pangolin_store/src/mongo/service_users.rs index 6709172..868cf4e 100644 --- a/pangolin/pangolin_store/src/mongo/service_users.rs +++ b/pangolin/pangolin_store/src/mongo/service_users.rs @@ -1,4 +1,34 @@ -use super::main::to_bson_uuid; +//! Service users (API-key principals) for the MongoDB backend. +//! +//! Every method here was broken, in three overlapping ways, and none of it was +//! visible from reading a single function: +//! +//! * **The lookup key did not exist.** `create_service_user` inserted the serde +//! document as-is, so Mongo generated an `ObjectId` for `_id`. The four +//! by-id methods then filtered on `{"_id": id.to_string()}`, which matched +//! nothing, ever. Fetching, updating, deleting and touching a service user +//! were all no-ops - `update_service_user` reported "Service user not found" +//! for a user that had just been created. +//! * **The field names were wrong.** `ServiceUser` is `rename_all = +//! "kebab-case"`, so the stored fields are `tenant-id` and `api-key-hash`. +//! The filters used `tenant_id` and `api_key_hash`, so listing returned +//! nothing and - the one that matters - **API-key authentication could never +//! resolve a service user**. It fails closed, so this was a total outage of +//! service-user auth on Mongo rather than a bypass. +//! * **The UUIDs were the wrong type.** The usual asymmetry: `to_document` +//! writes a `Uuid` as a string, while the deserializer and `to_bson_uuid` +//! filters want BSON Binary. +//! +//! Found by the entity round-trip suite, which asserts that every UUID-bearing +//! collection can be written and read back. +//! +//! Note that the `chrono` fields go the *other* way: bson's serializer reports +//! itself human-readable, so `to_document` writes them as RFC3339 strings and +//! that is what `from_document` reads back. `update_service_user_last_used` +//! must therefore write a string too - a `Bson::DateTime` there would be +//! written happily and then break every subsequent read. + +use super::main::{to_bson_uuid, with_binary_uuids}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -8,13 +38,20 @@ use uuid::Uuid; impl MongoStore { pub async fn create_service_user(&self, service_user: ServiceUser) -> Result<()> { - let doc = mongodb::bson::to_document(&service_user)?; + let doc = with_binary_uuids( + mongodb::bson::to_document(&service_user)?, + &[ + ("id", service_user.id), + ("tenant-id", service_user.tenant_id), + ("created-by", service_user.created_by), + ], + ); self.service_users().insert_one(doc).await?; Ok(()) } pub async fn get_service_user(&self, id: Uuid) -> Result> { - let filter = doc! { "_id": id.to_string() }; + let filter = doc! { "id": to_bson_uuid(id) }; if let Some(doc) = self.service_users().find_one(filter).await? { Ok(Some(mongodb::bson::from_document(doc)?)) } else { @@ -22,11 +59,15 @@ impl MongoStore { } } + /// Resolve an API key to its principal. + /// + /// This is the authentication path. The hash is stored verbatim, so an + /// exact match is right; only the field name was wrong. pub async fn get_service_user_by_api_key_hash( &self, api_key_hash: &str, ) -> Result> { - let filter = doc! { "api_key_hash": api_key_hash }; + let filter = doc! { "api-key-hash": api_key_hash }; if let Some(doc) = self.service_users().find_one(filter).await? { Ok(Some(mongodb::bson::from_document(doc)?)) } else { @@ -39,7 +80,7 @@ impl MongoStore { tenant_id: Uuid, pagination: Option, ) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; + let filter = doc! { "tenant-id": to_bson_uuid(tenant_id) }; let collection = self.service_users(); let mut find = collection.find(filter); @@ -69,7 +110,7 @@ impl MongoStore { role: Option, active: Option, ) -> Result { - let filter = doc! { "_id": id.to_string() }; + let filter = doc! { "id": to_bson_uuid(id) }; let mut update_doc = doc! {}; if let Some(n) = name { @@ -79,7 +120,10 @@ impl MongoStore { update_doc.insert("description", d); } if let Some(r) = role { - update_doc.insert("role", format!("{:?}", r)); + // `format!("{:?}", r)` wrote the Rust variant name (`TenantAdmin`). + // `UserRole` is kebab-case, so the record then failed to + // deserialize: changing a service user's role made it unreadable. + update_doc.insert("role", mongodb::bson::to_bson(&r)?); } if let Some(a) = active { update_doc.insert("active", a); @@ -96,14 +140,19 @@ impl MongoStore { } pub async fn delete_service_user(&self, id: Uuid) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; + let filter = doc! { "id": to_bson_uuid(id) }; self.service_users().delete_one(filter).await?; Ok(()) } + /// Stamp the last time this key was used. + /// + /// The field is `last-used`; `last_used_at` was a field no reader ever + /// looked at, so the timestamp shown for every service user stayed at its + /// creation value forever. See the module note on why this is a string. pub async fn update_service_user_last_used(&self, id: Uuid) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; - let update = doc! { "$set": { "last_used_at": chrono::Utc::now() } }; + let filter = doc! { "id": to_bson_uuid(id) }; + let update = doc! { "$set": { "last-used": chrono::Utc::now().to_rfc3339() } }; self.service_users().update_one(filter, update).await?; Ok(()) } diff --git a/pangolin/pangolin_store/src/mongo/tokens.rs b/pangolin/pangolin_store/src/mongo/tokens.rs index 2b92da7..fa5aaf1 100644 --- a/pangolin/pangolin_store/src/mongo/tokens.rs +++ b/pangolin/pangolin_store/src/mongo/tokens.rs @@ -2,10 +2,36 @@ use super::main::{from_bson_uuid, to_bson_uuid}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; -use mongodb::bson::{doc, Document}; +use mongodb::bson::{doc, Bson, Document}; use pangolin_core::token::TokenInfo; use uuid::Uuid; +/// Read a timestamp that may have been written either way. +/// +/// `store_token` builds its document with `doc!`, and `doc!` converts a +/// `chrono::DateTime` into a `Bson::DateTime`. Feeding that back through +/// `bson::from_bson::>` does not work: the deserializer presents +/// a `Bson::DateTime` as the map `{"$date": ...}`, while chrono's `Deserialize` +/// only accepts an RFC3339 string. So `list_active_tokens` failed outright with +/// `invalid type: map, expected an RFC 3339 formatted date and time string` - +/// **listing active tokens was broken for any token that existed**, and the +/// `created_at` arm swallowed the same error and substituted `now()`, so had +/// the first one not aborted the call, every token would have reported the +/// listing time as its creation time. +/// +/// Both encodings are accepted because documents written before this fix are +/// still in the collection, and a token record is not worth a migration - it +/// expires on its own. +fn read_datetime(doc: &Document, key: &str) -> Option> { + match doc.get(key)? { + Bson::DateTime(dt) => Some(dt.to_chrono()), + Bson::String(s) => chrono::DateTime::parse_from_rfc3339(s) + .ok() + .map(|dt| dt.with_timezone(&chrono::Utc)), + _ => None, + } +} + impl MongoStore { pub async fn list_active_tokens( &self, @@ -49,13 +75,9 @@ impl MongoStore { )?, username: d.get_str("username").unwrap_or("unknown").to_string(), token: d.get_str("token").ok().map(|s| s.to_string()), - expires_at: mongodb::bson::from_bson(d.get("expires_at").unwrap().clone())?, - created_at: mongodb::bson::from_bson( - d.get("created_at") - .unwrap_or(&mongodb::bson::Bson::Null) - .clone(), - ) - .unwrap_or_else(|_| chrono::Utc::now()), + expires_at: read_datetime(&d, "expires_at") + .ok_or(anyhow::anyhow!("Missing or unreadable expires_at"))?, + created_at: read_datetime(&d, "created_at").unwrap_or_else(chrono::Utc::now), is_valid: true, }); } @@ -76,16 +98,37 @@ impl MongoStore { Ok(()) } + /// Record a revocation. + /// + /// B2: this used to `insert_one(revoked)` through serde, which wrote the + /// token id as a *string* under the field name `id` (the struct's field), + /// while [`Self::is_token_revoked`] queried `token_id` as a BSON Binary + /// UUID. Neither the field name nor the type matched, so the lookup could + /// never find a revocation and the check returned `false` for every token: + /// on Mongo, revocation - including logout - was a silent no-op and revoked + /// JWTs stayed valid until they expired naturally. + /// + /// Both sides now go through an explicit `doc!` using the same encoding as + /// [`Self::store_token`]. pub async fn revoke_token( &self, token_id: Uuid, expires_at: chrono::DateTime, reason: Option, ) -> Result<()> { - let revoked = pangolin_core::token::RevokedToken::new(token_id, expires_at, reason); + let doc = doc! { + "token_id": to_bson_uuid(token_id), + "expires_at": Bson::DateTime(expires_at.into()), + "reason": reason.map(Bson::String).unwrap_or(Bson::Null), + }; + // Upsert so revoking twice is idempotent rather than accumulating rows. self.db - .collection("revoked_tokens") - .insert_one(revoked) + .collection::("revoked_tokens") + .update_one( + doc! { "token_id": to_bson_uuid(token_id) }, + doc! { "$set": doc }, + ) + .upsert(true) .await?; Ok(()) } @@ -94,18 +137,24 @@ impl MongoStore { let filter = doc! { "token_id": to_bson_uuid(token_id) }; let result = self .db - .collection::("revoked_tokens") + .collection::("revoked_tokens") .find_one(filter) .await?; Ok(result.is_some()) } + /// Drop revocation records whose tokens have expired anyway. + /// + /// B2 (second half): the comparison was `$lt` against a BSON DateTime while + /// serde had written `expires_at` as an RFC3339 *string*, so this deleted + /// nothing and the collection grew without bound. With `revoke_token` + /// writing a real `Bson::DateTime`, the comparison is now type-consistent. pub async fn cleanup_expired_tokens(&self) -> Result { let now = chrono::Utc::now(); - let filter = doc! { "expires_at": { "$lt": now } }; + let filter = doc! { "expires_at": { "$lt": Bson::DateTime(now.into()) } }; let result = self .db - .collection::("revoked_tokens") + .collection::("revoked_tokens") .delete_many(filter) .await?; Ok(result.deleted_count as usize) diff --git a/pangolin/pangolin_store/src/mongo/warehouses.rs b/pangolin/pangolin_store/src/mongo/warehouses.rs index 5ffa51b..2a2bbf1 100644 --- a/pangolin/pangolin_store/src/mongo/warehouses.rs +++ b/pangolin/pangolin_store/src/mongo/warehouses.rs @@ -1,5 +1,6 @@ use super::main::to_bson_uuid; use super::MongoStore; +use crate::secrets; use anyhow::Result; use futures::stream::TryStreamExt; use mongodb::bson::doc; @@ -7,7 +8,9 @@ use pangolin_core::model::{Warehouse, WarehouseUpdate}; use uuid::Uuid; impl MongoStore { - pub async fn create_warehouse(&self, _tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { + pub async fn create_warehouse(&self, _tenant_id: Uuid, mut warehouse: Warehouse) -> Result<()> { + // C-11: see the module note in `crate::secrets`. + secrets::seal(&mut warehouse.storage_config)?; self.warehouses().insert_one(warehouse).await?; Ok(()) } @@ -15,7 +18,13 @@ impl MongoStore { pub async fn get_warehouse(&self, tenant_id: Uuid, name: String) -> Result> { let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": name }; let warehouse = self.warehouses().find_one(filter).await?; - Ok(warehouse) + match warehouse { + Some(mut w) => { + secrets::open(&mut w.storage_config)?; + Ok(Some(w)) + } + None => Ok(None), + } } pub async fn list_warehouses( @@ -37,7 +46,10 @@ impl MongoStore { } let cursor = find.await?; - let warehouses: Vec = cursor.try_collect().await?; + let mut warehouses: Vec = cursor.try_collect().await?; + for warehouse in &mut warehouses { + secrets::open(&mut warehouse.storage_config)?; + } Ok(warehouses) } @@ -54,7 +66,9 @@ impl MongoStore { update_doc.insert("name", new_name); } if let Some(config) = &updates.storage_config { - update_doc.insert("storage_config", mongodb::bson::to_bson(config)?); + let mut sealed = config.clone(); + secrets::seal(&mut sealed)?; + update_doc.insert("storage_config", mongodb::bson::to_bson(&sealed)?); } if let Some(use_sts) = updates.use_sts { update_doc.insert("use_sts", use_sts); diff --git a/pangolin/pangolin_store/src/postgres/assets.rs b/pangolin/pangolin_store/src/postgres/assets.rs index 5e29a96..5ed4120 100644 --- a/pangolin/pangolin_store/src/postgres/assets.rs +++ b/pangolin/pangolin_store/src/postgres/assets.rs @@ -22,7 +22,7 @@ impl PostgresStore { .bind(&branch_name) .bind(&namespace) .bind(&asset.name) - .bind(format!("{:?}", asset.kind)) + .bind(asset.kind.as_stored_str()) .bind(asset.properties.get("metadata_location").unwrap_or(&asset.location)) .bind(serde_json::to_value(&asset.properties)?) .execute(&self.pool) @@ -50,11 +50,9 @@ impl PostgresStore { if let Some(row) = row { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => pangolin_core::model::AssetType::IcebergTable, - "View" => pangolin_core::model::AssetType::View, - _ => pangolin_core::model::AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = pangolin_core::model::AssetType::from_stored_str(&asset_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let a = Asset { id: row.get("id"), @@ -86,11 +84,9 @@ impl PostgresStore { let catalog_name: String = row.get("catalog_name"); let namespace_path: Vec = row.get("namespace_path"); let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => pangolin_core::model::AssetType::IcebergTable, - "View" => pangolin_core::model::AssetType::View, - _ => pangolin_core::model::AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = pangolin_core::model::AssetType::from_stored_str(&asset_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let asset = Asset { id: row.get("id"), @@ -137,11 +133,9 @@ impl PostgresStore { let mut assets = Vec::new(); for row in rows { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => pangolin_core::model::AssetType::IcebergTable, - "View" => pangolin_core::model::AssetType::View, - _ => pangolin_core::model::AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = pangolin_core::model::AssetType::from_stored_str(&asset_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let a = Asset { id: row.get("id"), diff --git a/pangolin/pangolin_store/src/postgres/branches.rs b/pangolin/pangolin_store/src/postgres/branches.rs index 26177d9..eae6a11 100644 --- a/pangolin/pangolin_store/src/postgres/branches.rs +++ b/pangolin/pangolin_store/src/postgres/branches.rs @@ -24,6 +24,107 @@ impl PostgresStore { Ok(()) } + /// Create a branch and copy assets into it in one transaction (A-24). + /// + /// Both statements commit together or neither does. Previously they were + /// issued independently, so a failure between them left a branch that + /// existed with some arbitrary prefix of its assets - and the API returned + /// `200` regardless, because the caller logged the copy error and carried + /// on. + pub async fn create_branch_with_assets( + &self, + tenant_id: Uuid, + catalog_name: &str, + branch: Branch, + src_branch: &str, + assets: Option>, + ) -> Result { + let mut tx = self.pool.begin().await?; + + sqlx::query( + "INSERT INTO branches (tenant_id, catalog_name, name, head_commit_id, branch_type, assets) \ + VALUES ($1, $2, $3, $4, $5, $6) \ + ON CONFLICT (tenant_id, catalog_name, name) DO UPDATE SET \ + head_commit_id = EXCLUDED.head_commit_id, branch_type = EXCLUDED.branch_type, \ + assets = EXCLUDED.assets", + ) + .bind(tenant_id) + .bind(catalog_name) + .bind(&branch.name) + .bind(branch.head_commit_id) + .bind(format!("{:?}", branch.branch_type)) + .bind(&branch.assets) + .execute(&mut *tx) + .await?; + + // `INSERT ... SELECT` rather than reading into the process and writing + // back: the copy is one statement, so there is no window in which some + // rows exist and others do not, and a large branch does not have to fit + // in memory. + // + // `gen_random_uuid()` for the new ids because an asset's primary key is + // unique across branches; reusing the source ids would collide. + let copied = match &assets { + None => sqlx::query( + "INSERT INTO assets (id, tenant_id, catalog_name, namespace_path, name, \ + branch_name, asset_type, metadata_location, properties) \ + SELECT gen_random_uuid(), tenant_id, catalog_name, namespace_path, name, \ + $4, asset_type, metadata_location, properties \ + FROM assets \ + WHERE tenant_id = $1 AND catalog_name = $2 AND branch_name = $3", + ) + .bind(tenant_id) + .bind(catalog_name) + .bind(src_branch) + .bind(&branch.name) + .execute(&mut *tx) + .await? + .rows_affected(), + Some(names) => { + if names.is_empty() { + 0 + } else { + // `namespace.table`, so the last segment is the table and + // everything before it is the namespace path. + let mut wanted: Vec = Vec::with_capacity(names.len()); + for full in names { + let Some((_, table)) = full.rsplit_once('.') else { + // A name with no namespace cannot identify an asset; + // failing is better than silently copying nothing, + // which is what the old loop did with `continue`. + tx.rollback().await.ok(); + return Err(anyhow::anyhow!( + "asset name {full:?} is not in namespace.table form" + )); + }; + wanted.push(table.to_string()); + } + + sqlx::query( + "INSERT INTO assets (id, tenant_id, catalog_name, namespace_path, name, \ + branch_name, asset_type, metadata_location, properties) \ + SELECT gen_random_uuid(), tenant_id, catalog_name, namespace_path, name, \ + $4, asset_type, metadata_location, properties \ + FROM assets \ + WHERE tenant_id = $1 AND catalog_name = $2 AND branch_name = $3 \ + AND array_to_string(namespace_path, '.') || '.' || name = ANY($5)", + ) + .bind(tenant_id) + .bind(catalog_name) + .bind(src_branch) + .bind(&branch.name) + .bind(names) + .execute(&mut *tx) + .await? + .rows_affected() + } + } + }; + + tx.commit().await?; + Ok(copied as usize) + } + pub async fn get_branch( &self, tenant_id: Uuid, @@ -69,7 +170,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = $1 AND catalog_name = $2 LIMIT $3 OFFSET $4") + let rows = sqlx::query("SELECT name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = $1 AND catalog_name = $2 ORDER BY name LIMIT $3 OFFSET $4") .bind(tenant_id) .bind(catalog_name) .bind(limit) diff --git a/pangolin/pangolin_store/src/postgres/business_metadata.rs b/pangolin/pangolin_store/src/postgres/business_metadata.rs new file mode 100644 index 0000000..72dcb6d --- /dev/null +++ b/pangolin/pangolin_store/src/postgres/business_metadata.rs @@ -0,0 +1,95 @@ +//! Business metadata for the Postgres backend. +//! +//! These three methods did not exist. `PostgresStore` inherited the trait's +//! "Operation not supported by this store" defaults, while `search_assets` +//! joined a `business_metadata` table that no migration created - so on +//! Postgres, business metadata could not be written *and* asset search failed +//! with a hard SQL error rather than returning nothing. +//! +//! Found by the cross-backend parity suite on its first run against a live +//! Postgres. The suite asserts tag filtering works identically on all four +//! backends, which is not something a per-backend test can check. + +use super::PostgresStore; +use anyhow::Result; +use pangolin_core::business_metadata::BusinessMetadata; +use sqlx::Row; +use uuid::Uuid; + +impl PostgresStore { + /// Insert or replace an asset's business metadata. + /// + /// Keyed on `asset_id` rather than `id`: an asset has at most one metadata + /// record, and callers construct a fresh `BusinessMetadata` (with a new + /// `id`) when updating. Conflicting on `id` would insert a duplicate row + /// per update and break the `asset_id` unique constraint. + pub async fn upsert_business_metadata(&self, metadata: BusinessMetadata) -> Result<()> { + sqlx::query( + "INSERT INTO business_metadata ( + id, asset_id, description, tags, properties, + discoverable, created_by, created_at, updated_by, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (asset_id) DO UPDATE SET + description = EXCLUDED.description, + tags = EXCLUDED.tags, + properties = EXCLUDED.properties, + discoverable = EXCLUDED.discoverable, + updated_by = EXCLUDED.updated_by, + updated_at = EXCLUDED.updated_at", + ) + .bind(metadata.id) + .bind(metadata.asset_id) + .bind(&metadata.description) + .bind(serde_json::to_value(&metadata.tags)?) + .bind(serde_json::to_value(&metadata.properties)?) + .bind(metadata.discoverable) + .bind(metadata.created_by) + .bind(metadata.created_at) + .bind(metadata.updated_by) + .bind(metadata.updated_at) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get_business_metadata(&self, asset_id: Uuid) -> Result> { + let row = sqlx::query( + "SELECT id, asset_id, description, tags, properties, + discoverable, created_by, created_at, updated_by, updated_at + FROM business_metadata + WHERE asset_id = $1", + ) + .bind(asset_id) + .fetch_optional(&self.pool) + .await?; + + let Some(row) = row else { + return Ok(None); + }; + + Ok(Some(BusinessMetadata { + id: row.get("id"), + asset_id: row.get("asset_id"), + description: row.get("description"), + tags: serde_json::from_value(row.get("tags")).unwrap_or_default(), + properties: serde_json::from_value(row.get("properties")).unwrap_or_default(), + discoverable: row.get("discoverable"), + created_by: row.get("created_by"), + created_at: row.get("created_at"), + updated_by: row.get("updated_by"), + updated_at: row.get("updated_at"), + })) + } + + /// Delete an asset's entire metadata record. + /// + /// Deleting something that is not there is not an error: callers use this + /// to ensure absence. + pub async fn delete_business_metadata(&self, asset_id: Uuid) -> Result<()> { + sqlx::query("DELETE FROM business_metadata WHERE asset_id = $1") + .bind(asset_id) + .execute(&self.pool) + .await?; + Ok(()) + } +} diff --git a/pangolin/pangolin_store/src/postgres/catalogs.rs b/pangolin/pangolin_store/src/postgres/catalogs.rs index fd73273..6978579 100644 --- a/pangolin/pangolin_store/src/postgres/catalogs.rs +++ b/pangolin/pangolin_store/src/postgres/catalogs.rs @@ -62,7 +62,18 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, warehouse_name, storage_location, properties FROM catalogs WHERE tenant_id = $1 LIMIT $2 OFFSET $3") + // B24: the SELECT omitted `catalog_type` and `federated_config`, and the + // loop below hardcoded `Local` / `None`. Every federated catalog looked + // Local in a Postgres listing, so anything branching on `catalog_type` + // over a listing - including the federated-forwarding decision - took + // the wrong path. SQLite and Mongo returned the real values; only + // Postgres invented them. `get_catalog` two functions up decodes both + // correctly, which is the shape mirrored here. + // + // B27: `ORDER BY name` so two pages cover the set exactly once. Without + // it Postgres may return rows in any order between queries, so + // `LIMIT/OFFSET` paging could repeat or skip catalogs. + let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = $1 ORDER BY name LIMIT $2 OFFSET $3") .bind(tenant_id) .bind(limit) .bind(offset) @@ -71,13 +82,19 @@ impl PostgresStore { let mut catalogs = Vec::new(); for row in rows { + let catalog_type_str: String = row.get("catalog_type"); + let catalog_type = match catalog_type_str.as_str() { + "Federated" => pangolin_core::model::CatalogType::Federated, + _ => pangolin_core::model::CatalogType::Local, + }; + catalogs.push(Catalog { id: row.get("id"), name: row.get("name"), - catalog_type: pangolin_core::model::CatalogType::Local, + catalog_type, warehouse_name: row.get("warehouse_name"), storage_location: row.get("storage_location"), - federated_config: None, + federated_config: serde_json::from_value(row.get("federated_config")).ok(), properties: serde_json::from_value(row.get("properties")).unwrap_or_default(), }); } diff --git a/pangolin/pangolin_store/src/postgres/main.rs b/pangolin/pangolin_store/src/postgres/main.rs index cf34019..2147430 100644 --- a/pangolin/pangolin_store/src/postgres/main.rs +++ b/pangolin/pangolin_store/src/postgres/main.rs @@ -256,6 +256,17 @@ impl CatalogStore for PostgresStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + self.replace_namespace_properties(tenant_id, catalog_name, namespace, properties) + .await + } + // Asset Operations async fn create_asset( &self, @@ -387,6 +398,18 @@ impl CatalogStore for PostgresStore { .await } + async fn create_branch_with_assets( + &self, + tenant_id: Uuid, + catalog_name: &str, + branch: pangolin_core::model::Branch, + src_branch: &str, + assets: Option>, + ) -> Result { + self.create_branch_with_assets(tenant_id, catalog_name, branch, src_branch, assets) + .await + } + async fn copy_assets_bulk( &self, tenant_id: Uuid, @@ -414,7 +437,7 @@ impl CatalogStore for PostgresStore { .unwrap_or(0); let rows = if let Some(uid) = user_id { - sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = $1 AND user_id = $2 AND expires_at > $3 LIMIT $4 OFFSET $5") + sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = $1 AND user_id = $2 AND expires_at > $3 ORDER BY expires_at DESC, token_id LIMIT $4 OFFSET $5") .bind(tenant_id) .bind(uid) .bind(Utc::now()) @@ -423,7 +446,7 @@ impl CatalogStore for PostgresStore { .fetch_all(&self.pool) .await? } else { - sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = $1 AND expires_at > $2 LIMIT $3 OFFSET $4") + sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = $1 AND expires_at > $2 ORDER BY expires_at DESC, token_id LIMIT $3 OFFSET $4") .bind(tenant_id) .bind(Utc::now()) .bind(limit) @@ -1153,6 +1176,38 @@ impl CatalogStore for PostgresStore { } } + async fn delete_file(&self, path: &str) -> Result<()> { + self.metadata_cache.invalidate(path).await; + let storage_config = self + .get_warehouse_for_location(path) + .await? + .map(|w| w.storage_config); + crate::file_delete::delete_location(storage_config.as_ref(), path).await + } + + // Business Metadata Operations + // + // These were absent, so `PostgresStore` fell through to the trait defaults + // ("Operation not supported by this store") while `search_assets` joined a + // table no migration created. + async fn upsert_business_metadata( + &self, + metadata: pangolin_core::business_metadata::BusinessMetadata, + ) -> Result<()> { + self.upsert_business_metadata(metadata).await + } + + async fn get_business_metadata( + &self, + asset_id: Uuid, + ) -> Result> { + self.get_business_metadata(asset_id).await + } + + async fn delete_business_metadata(&self, asset_id: Uuid) -> Result<()> { + self.delete_business_metadata(asset_id).await + } + // Tag Operations async fn create_tag(&self, tenant_id: Uuid, catalog_name: &str, tag: Tag) -> Result<()> { self.create_tag(tenant_id, catalog_name, tag).await @@ -1325,10 +1380,11 @@ impl CatalogStore for PostgresStore { m.created_at as meta_created_at, m.updated_by as meta_updated_by, m.updated_at as meta_updated_at FROM assets a LEFT JOIN business_metadata m ON a.id = m.asset_id - WHERE a.tenant_id = $1 AND (a.name ILIKE $2 OR m.description ILIKE $2)" + WHERE a.tenant_id = $1 AND (a.name ILIKE $2 ESCAPE '\\' OR m.description ILIKE $2 ESCAPE '\\')" ); - let query_pattern = format!("%{}%", query); + // B28: unescaped `%`/`_` in the term were LIKE wildcards. + let query_pattern = crate::search::contains_pattern(query); let mut param_index = 3; if let Some(ref tag_list) = tags { @@ -1388,8 +1444,13 @@ impl CatalogStore for PostgresStore { }; let catalog_name: String = row.get("catalog_name"); - let namespace_path: String = row.get("namespace_path"); - let namespace: Vec = namespace_path.split('\x1F').map(String::from).collect(); + // B4: this decoded a `TEXT[]` column as `String`. `sqlx::Row::get` + // *panics* on a decode failure, so any search with at least one hit + // panicked the request - the failure only stayed hidden because a + // search with no results never reached this line. The correct + // decode is already used elsewhere in this file and in + // `postgres/assets.rs`. + let namespace: Vec = row.get("namespace_path"); results.push((asset, metadata, catalog_name, namespace)); } @@ -1398,8 +1459,9 @@ impl CatalogStore for PostgresStore { } async fn search_catalogs(&self, tenant_id: Uuid, query: &str) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = $1 AND name ILIKE $2") + // B28: unescaped `%`/`_` in the term were LIKE wildcards. + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = $1 AND name ILIKE $2 ESCAPE '\\' ORDER BY name") .bind(tenant_id) .bind(&query_pattern) .fetch_all(&self.pool) @@ -1434,9 +1496,10 @@ impl CatalogStore for PostgresStore { tenant_id: Uuid, query: &str, ) -> Result> { - let query_pattern = format!("%{}%", query); + // B28: unescaped `%`/`_` in the term were LIKE wildcards. + let query_pattern = crate::search::contains_pattern(query); // Postgres stores namespace_path as TEXT[] - let rows = sqlx::query("SELECT catalog_name, namespace_path, properties FROM namespaces WHERE tenant_id = $1 AND array_to_string(namespace_path, '.') ILIKE $2") + let rows = sqlx::query("SELECT catalog_name, namespace_path, properties FROM namespaces WHERE tenant_id = $1 AND array_to_string(namespace_path, '.') ILIKE $2 ESCAPE '\\' ORDER BY catalog_name, namespace_path") .bind(tenant_id) .bind(&query_pattern) .fetch_all(&self.pool) @@ -1456,8 +1519,9 @@ impl CatalogStore for PostgresStore { } async fn search_branches(&self, tenant_id: Uuid, query: &str) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT catalog_name, name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = $1 AND name ILIKE $2") + // B28: unescaped `%`/`_` in the term were LIKE wildcards. + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT catalog_name, name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = $1 AND name ILIKE $2 ESCAPE '\\' ORDER BY catalog_name, name") .bind(tenant_id) .bind(&query_pattern) .fetch_all(&self.pool) diff --git a/pangolin/pangolin_store/src/postgres/mod.rs b/pangolin/pangolin_store/src/postgres/mod.rs index 923f662..e25aab6 100644 --- a/pangolin/pangolin_store/src/postgres/mod.rs +++ b/pangolin/pangolin_store/src/postgres/mod.rs @@ -2,6 +2,7 @@ pub mod access_requests; pub mod assets; pub mod audit; pub mod branches; +pub mod business_metadata; pub mod catalogs; pub mod commits; pub mod main; diff --git a/pangolin/pangolin_store/src/postgres/namespaces.rs b/pangolin/pangolin_store/src/postgres/namespaces.rs index 22283ea..9fa5cf1 100644 --- a/pangolin/pangolin_store/src/postgres/namespaces.rs +++ b/pangolin/pangolin_store/src/postgres/namespaces.rs @@ -60,7 +60,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT namespace_path, properties FROM namespaces WHERE tenant_id = $1 AND catalog_name = $2 LIMIT $3 OFFSET $4") + let rows = sqlx::query("SELECT namespace_path, properties FROM namespaces WHERE tenant_id = $1 AND catalog_name = $2 ORDER BY namespace_path LIMIT $3 OFFSET $4") .bind(tenant_id) .bind(catalog_name) .bind(limit) @@ -116,6 +116,28 @@ impl PostgresStore { Ok(()) } + /// Replace a namespace's properties wholesale; see the SQLite twin for why + /// a merge-only method could not implement Iceberg property removals (B16h). + pub async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + let result = sqlx::query("UPDATE namespaces SET properties = $1 WHERE tenant_id = $2 AND catalog_name = $3 AND namespace_path = $4") + .bind(serde_json::to_value(&properties)?) + .bind(tenant_id) + .bind(catalog_name) + .bind(&namespace) + .execute(&self.pool) + .await?; + if result.rows_affected() == 0 { + return Err(anyhow::anyhow!("Namespace not found")); + } + Ok(()) + } + pub async fn count_namespaces(&self, tenant_id: Uuid) -> Result { let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM namespaces WHERE tenant_id = $1") .bind(tenant_id) diff --git a/pangolin/pangolin_store/src/postgres/permissions.rs b/pangolin/pangolin_store/src/postgres/permissions.rs index f77f87a..7899486 100644 --- a/pangolin/pangolin_store/src/postgres/permissions.rs +++ b/pangolin/pangolin_store/src/postgres/permissions.rs @@ -113,7 +113,7 @@ impl PostgresStore { "SELECT id, user_id, tenant_id, scope, actions, granted_by, granted_at FROM permissions WHERE tenant_id = $1 - LIMIT $2 OFFSET $3", + ORDER BY id LIMIT $2 OFFSET $3", ) .bind(tenant_id) .bind(limit) diff --git a/pangolin/pangolin_store/src/postgres/roles.rs b/pangolin/pangolin_store/src/postgres/roles.rs index 8727f13..b8025eb 100644 --- a/pangolin/pangolin_store/src/postgres/roles.rs +++ b/pangolin/pangolin_store/src/postgres/roles.rs @@ -57,7 +57,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, tenant_id, name, description, permissions, created_by, created_at, updated_at FROM roles WHERE tenant_id = $1 LIMIT $2 OFFSET $3") + let rows = sqlx::query("SELECT id, tenant_id, name, description, permissions, created_by, created_at, updated_at FROM roles WHERE tenant_id = $1 ORDER BY name LIMIT $2 OFFSET $3") .bind(tenant_id) .bind(limit) .bind(offset) diff --git a/pangolin/pangolin_store/src/postgres/tags.rs b/pangolin/pangolin_store/src/postgres/tags.rs index 63a4db2..521591d 100644 --- a/pangolin/pangolin_store/src/postgres/tags.rs +++ b/pangolin/pangolin_store/src/postgres/tags.rs @@ -55,7 +55,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT name, commit_id FROM tags WHERE tenant_id = $1 AND catalog_name = $2 LIMIT $3 OFFSET $4") + let rows = sqlx::query("SELECT name, commit_id FROM tags WHERE tenant_id = $1 AND catalog_name = $2 ORDER BY name LIMIT $3 OFFSET $4") .bind(tenant_id) .bind(catalog_name) .bind(limit) diff --git a/pangolin/pangolin_store/src/postgres/tenants.rs b/pangolin/pangolin_store/src/postgres/tenants.rs index df2a7a7..7179cdc 100644 --- a/pangolin/pangolin_store/src/postgres/tenants.rs +++ b/pangolin/pangolin_store/src/postgres/tenants.rs @@ -45,11 +45,13 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, properties FROM tenants LIMIT $1 OFFSET $2") - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; + let rows = sqlx::query( + "SELECT id, name, properties FROM tenants ORDER BY name LIMIT $1 OFFSET $2", + ) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; let mut tenants = Vec::new(); for row in rows { diff --git a/pangolin/pangolin_store/src/postgres/users.rs b/pangolin/pangolin_store/src/postgres/users.rs index f52b27a..c5beb32 100644 --- a/pangolin/pangolin_store/src/postgres/users.rs +++ b/pangolin/pangolin_store/src/postgres/users.rs @@ -98,14 +98,14 @@ impl PostgresStore { .unwrap_or(0); let rows = if let Some(tid) = tenant_id { - sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, active, created_at, updated_at, last_login FROM users WHERE tenant_id = $1 LIMIT $2 OFFSET $3") + sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, active, created_at, updated_at, last_login FROM users WHERE tenant_id = $1 ORDER BY username LIMIT $2 OFFSET $3") .bind(tid) .bind(limit) .bind(offset) .fetch_all(&self.pool) .await? } else { - sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, active, created_at, updated_at, last_login FROM users LIMIT $1 OFFSET $2") + sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, active, created_at, updated_at, last_login FROM users ORDER BY username LIMIT $1 OFFSET $2") .bind(limit) .bind(offset) .fetch_all(&self.pool) diff --git a/pangolin/pangolin_store/src/postgres/warehouses.rs b/pangolin/pangolin_store/src/postgres/warehouses.rs index 61c72de..12414de 100644 --- a/pangolin/pangolin_store/src/postgres/warehouses.rs +++ b/pangolin/pangolin_store/src/postgres/warehouses.rs @@ -1,4 +1,5 @@ use super::PostgresStore; +use crate::secrets; use anyhow::Result; use pangolin_core::model::Warehouse; use sqlx::Row; @@ -6,7 +7,11 @@ use uuid::Uuid; impl PostgresStore { // Warehouse Operations - pub async fn create_warehouse(&self, tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { + pub async fn create_warehouse(&self, tenant_id: Uuid, mut warehouse: Warehouse) -> Result<()> { + // C-11: seal the credentials before they reach the database. Anything + // that can read this table - a backup, a replica, a stray SELECT - sees + // ciphertext rather than every tenant's cloud keys. + secrets::seal(&mut warehouse.storage_config)?; sqlx::query("INSERT INTO warehouses (id, tenant_id, name, use_sts, storage_config, vending_strategy) VALUES ($1, $2, $3, $4, $5, $6)") .bind(warehouse.id) .bind(tenant_id) @@ -32,8 +37,12 @@ impl PostgresStore { name: row.get("name"), tenant_id: row.get("tenant_id"), use_sts: row.try_get("use_sts").unwrap_or(false), - storage_config: serde_json::from_value(row.get("storage_config")) - .unwrap_or_default(), + storage_config: { + let mut config: std::collections::HashMap = + serde_json::from_value(row.get("storage_config")).unwrap_or_default(); + secrets::open(&mut config)?; + config + }, vending_strategy: serde_json::from_value(row.get("vending_strategy")).ok(), })) } else { @@ -53,7 +62,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, tenant_id, use_sts, storage_config, vending_strategy FROM warehouses WHERE tenant_id = $1 LIMIT $2 OFFSET $3") + let rows = sqlx::query("SELECT id, name, tenant_id, use_sts, storage_config, vending_strategy FROM warehouses WHERE tenant_id = $1 ORDER BY name LIMIT $2 OFFSET $3") .bind(tenant_id) .bind(limit) .bind(offset) @@ -67,8 +76,12 @@ impl PostgresStore { name: row.get("name"), tenant_id: row.get("tenant_id"), use_sts: row.try_get("use_sts").unwrap_or(false), - storage_config: serde_json::from_value(row.get("storage_config")) - .unwrap_or_default(), + storage_config: { + let mut config: std::collections::HashMap = + serde_json::from_value(row.get("storage_config")).unwrap_or_default(); + secrets::open(&mut config)?; + config + }, vending_strategy: serde_json::from_value(row.get("vending_strategy")).ok(), }); } @@ -81,6 +94,17 @@ impl PostgresStore { name: String, updates: pangolin_core::model::WarehouseUpdate, ) -> Result { + // Seal before binding, for the same reason as `create_warehouse`. An + // update that rotates a credential must not write the new one in clear. + let sealed_update = match &updates.storage_config { + Some(config) => { + let mut sealed = config.clone(); + secrets::seal(&mut sealed)?; + Some(sealed) + } + None => None, + }; + let mut query = String::from("UPDATE warehouses SET "); let mut set_clauses = Vec::new(); let mut bind_count = 1; @@ -112,7 +136,7 @@ impl PostgresStore { if let Some(use_sts) = &updates.use_sts { q = q.bind(use_sts); } - if let Some(storage_config) = &updates.storage_config { + if let Some(storage_config) = &sealed_update { q = q.bind(serde_json::to_value(storage_config)?); } if let Some(vending_strategy) = &updates.vending_strategy { @@ -127,7 +151,14 @@ impl PostgresStore { name: row.get("name"), tenant_id: row.get("tenant_id"), use_sts: row.try_get("use_sts").unwrap_or(false), - storage_config: serde_json::from_value(row.get("storage_config")).unwrap_or_default(), + // The RETURNING row carries what was just written, which is + // sealed; the caller expects a usable warehouse. + storage_config: { + let mut config: std::collections::HashMap = + serde_json::from_value(row.get("storage_config")).unwrap_or_default(); + secrets::open(&mut config)?; + config + }, vending_strategy: serde_json::from_value(row.get("vending_strategy")).ok(), }) } diff --git a/pangolin/pangolin_store/src/search.rs b/pangolin/pangolin_store/src/search.rs new file mode 100644 index 0000000..7e271c8 --- /dev/null +++ b/pangolin/pangolin_store/src/search.rs @@ -0,0 +1,94 @@ +//! Shared search semantics for the four backends. +//! +//! B28: search behaved four different ways. +//! +//! * **Wildcards.** Postgres and SQLite built their patterns with +//! `format!("%{}%", query)` and no escaping, so a query containing `%` or `_` +//! was interpreted as a LIKE wildcard: searching for `100%` matched +//! everything, and `a_b` matched `axb`. Mongo escaped correctly with +//! `regex::escape`, and the memory backend used a literal `contains`. Four +//! backends, four answers to the same query. +//! * **Tag filters.** Memory and SQLite matched *any* requested tag; Postgres +//! (`@>`) and Mongo (`$all`) required *all* of them. And an empty tag list +//! returned zero results on memory but everything on the others. +//! +//! This module is the single definition of both, so a backend can only diverge +//! by not calling it. + +/// The escape character used with `LIKE ... ESCAPE`. +pub const LIKE_ESCAPE_CHAR: char = '\\'; + +/// Escape LIKE metacharacters in a user-supplied search term. +/// +/// Must be paired with `ESCAPE '\'` in the SQL, which the helpers below embed. +pub fn escape_like(query: &str) -> String { + let mut escaped = String::with_capacity(query.len()); + for ch in query.chars() { + if ch == '%' || ch == '_' || ch == LIKE_ESCAPE_CHAR { + escaped.push(LIKE_ESCAPE_CHAR); + } + escaped.push(ch); + } + escaped +} + +/// Build a `%term%` LIKE pattern with metacharacters escaped. +pub fn contains_pattern(query: &str) -> String { + format!("%{}%", escape_like(query)) +} + +/// The `ESCAPE` clause every `LIKE`/`ILIKE` using [`contains_pattern`] needs. +pub const ESCAPE_CLAUSE: &str = " ESCAPE '\\'"; + +/// Does `tags` satisfy a tag filter? +/// +/// **The chosen semantic is ALL-match**: a result qualifies only if it carries +/// every requested tag. That is what Postgres's `@>` and Mongo's `$all` already +/// did, so aligning on it keeps the two SQL/document backends unchanged and +/// moves memory and SQLite - and it is the semantic faceted filtering wants, +/// where each added tag narrows the result set. +/// +/// An **empty or absent** filter means "no tag constraint", matching everything. +/// Previously an empty list returned nothing on the memory backend and +/// everything elsewhere. +pub fn tags_match(tags: &[String], required: Option<&[String]>) -> bool { + match required { + None => true, + Some(required) => required.iter().all(|want| tags.contains(want)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn like_metacharacters_are_escaped() { + assert_eq!(escape_like("100%"), "100\\%"); + assert_eq!(escape_like("a_b"), "a\\_b"); + assert_eq!(escape_like("back\\slash"), "back\\\\slash"); + assert_eq!(escape_like("plain"), "plain"); + } + + #[test] + fn contains_pattern_wraps_the_escaped_term() { + assert_eq!(contains_pattern("50%"), "%50\\%%"); + } + + #[test] + fn tag_filter_is_all_match() { + let tags = vec!["pii".to_string(), "finance".to_string()]; + + assert!(tags_match(&tags, None), "no filter matches everything"); + assert!(tags_match(&tags, Some(&[])), "an empty filter is no filter"); + assert!(tags_match(&tags, Some(&["pii".to_string()]))); + assert!(tags_match( + &tags, + Some(&["pii".to_string(), "finance".to_string()]) + )); + assert!( + !tags_match(&tags, Some(&["pii".to_string(), "hr".to_string()])), + "every requested tag must be present" + ); + } +} diff --git a/pangolin/pangolin_store/src/secrets.rs b/pangolin/pangolin_store/src/secrets.rs new file mode 100644 index 0000000..462c9c2 --- /dev/null +++ b/pangolin/pangolin_store/src/secrets.rs @@ -0,0 +1,454 @@ +//! Envelope encryption for the secrets inside a warehouse's `storage_config`. +//! +//! C-11. A warehouse holds the credentials Pangolin uses to reach a customer's +//! object storage - AWS secret access keys, Azure account keys, GCP service +//! account JSON. They were stored in the catalog database as plaintext JSON, so +//! anything that could read one row of the `warehouses` table - a backup, a +//! read replica, a SQL injection elsewhere, an operator with analyst access - +//! held every tenant's cloud credentials. +//! +//! What this does and does not protect: +//! +//! * **Does** protect against disclosure of the database contents alone: a +//! dump, a stolen backup, a snapshot, a curious `SELECT`. +//! * **Does not** protect against an attacker who has both the database and +//! the key. The key lives in the server's environment, so a full compromise +//! of a running server still yields the credentials. That is the normal +//! limit of envelope encryption without an HSM or a KMS, and it is worth +//! stating plainly rather than implying more. +//! +//! ## Format +//! +//! A sealed value is `enc:v1:`, AES-256-GCM +//! with a random 96-bit nonce per value. The version is in the string so a +//! future scheme can be introduced without guessing at what old rows contain. +//! +//! ## Reading is deliberately tolerant +//! +//! [`open`] decrypts anything carrying the prefix and passes everything else +//! through untouched. That is what makes this deployable: a database written +//! before this existed is full of plaintext, and refusing to read it would turn +//! a security improvement into an outage. The cost is that a value which was +//! never sealed stays unsealed until something rewrites it - see +//! `docs/operations/encryption.md` for how to force that. + +use anyhow::{anyhow, Context, Result}; +use base64::Engine; +use ring::aead::{Aad, LessSafeKey, Nonce, UnboundKey, AES_256_GCM, NONCE_LEN}; +use ring::rand::{SecureRandom, SystemRandom}; +use std::collections::HashMap; + +/// Marks a sealed value. Also how [`open`] tells sealed from legacy plaintext. +const PREFIX: &str = "enc:v1:"; + +/// The `storage_config` keys whose values are credentials. +/// +/// An allowlist rather than "encrypt everything": the same map carries the +/// bucket name, region and endpoint, which callers compare, log and use to +/// build URLs. Encrypting those would break the object-store factory and the +/// UI for no security gain, because they are not secrets. +/// +/// Both the dotted and undotted spellings appear in real configurations (the UI +/// writes `secret_access_key`, parts of the server read +/// `s3.secret-access-key`), so both are listed. Missing a spelling here means a +/// credential silently stays in plaintext, which is why `is_sensitive` is +/// tested against the exact keys the create-warehouse form writes. +const SENSITIVE_KEYS: &[&str] = &[ + "secret_access_key", + "s3.secret-access-key", + "access_key_id", + "s3.access-key-id", + "session_token", + "s3.session-token", + "account_key", + "azure.account-key", + "adls.account-key", + "client_secret", + "azure.client-secret", + "service_account_json", + "gcs.service-account-key", + "external_id", +]; + +pub fn is_sensitive(key: &str) -> bool { + SENSITIVE_KEYS.contains(&key) +} + +/// The configured data key, or `None` when the operator has not set one. +/// +/// Read on every call rather than cached in a `OnceLock`: the tests set and +/// clear it around individual cases, and a cached key would make the first test +/// to run decide the behaviour of the rest. +fn data_key() -> Result> { + let Some(raw) = std::env::var("PANGOLIN_ENCRYPTION_KEY") + .ok() + .filter(|v| !v.trim().is_empty()) + else { + return Ok(None); + }; + + let bytes = base64::engine::general_purpose::STANDARD + .decode(raw.trim()) + .context( + "PANGOLIN_ENCRYPTION_KEY must be base64. Generate one with: \ + openssl rand -base64 32", + )?; + + if bytes.len() != 32 { + return Err(anyhow!( + "PANGOLIN_ENCRYPTION_KEY decodes to {} bytes; AES-256-GCM needs 32. \ + Generate one with: openssl rand -base64 32", + bytes.len() + )); + } + + let unbound = UnboundKey::new(&AES_256_GCM, &bytes) + .map_err(|_| anyhow!("PANGOLIN_ENCRYPTION_KEY is not a usable AES-256 key"))?; + Ok(Some(LessSafeKey::new(unbound))) +} + +/// Whether at-rest encryption is configured. +pub fn is_enabled() -> bool { + matches!(data_key(), Ok(Some(_))) +} + +fn seal_value(key: &LessSafeKey, plaintext: &str) -> Result { + let rng = SystemRandom::new(); + let mut nonce_bytes = [0u8; NONCE_LEN]; + rng.fill(&mut nonce_bytes) + .map_err(|_| anyhow!("could not draw a nonce from the system RNG"))?; + + let mut buffer = plaintext.as_bytes().to_vec(); + key.seal_in_place_append_tag( + Nonce::assume_unique_for_key(nonce_bytes), + Aad::empty(), + &mut buffer, + ) + .map_err(|_| anyhow!("could not encrypt a warehouse credential"))?; + + let mut payload = Vec::with_capacity(NONCE_LEN + buffer.len()); + payload.extend_from_slice(&nonce_bytes); + payload.extend_from_slice(&buffer); + + Ok(format!( + "{PREFIX}{}", + base64::engine::general_purpose::STANDARD.encode(payload) + )) +} + +fn open_value(key: &LessSafeKey, sealed: &str) -> Result { + let encoded = sealed + .strip_prefix(PREFIX) + .ok_or_else(|| anyhow!("value is not sealed"))?; + let payload = base64::engine::general_purpose::STANDARD + .decode(encoded) + .context("a sealed warehouse credential is not valid base64")?; + + if payload.len() <= NONCE_LEN { + return Err(anyhow!("a sealed warehouse credential is truncated")); + } + + let (nonce_bytes, ciphertext) = payload.split_at(NONCE_LEN); + let mut nonce = [0u8; NONCE_LEN]; + nonce.copy_from_slice(nonce_bytes); + + let mut buffer = ciphertext.to_vec(); + let plaintext = key + .open_in_place( + Nonce::assume_unique_for_key(nonce), + Aad::empty(), + &mut buffer, + ) + .map_err(|_| { + anyhow!( + "could not decrypt a warehouse credential. The most likely cause is \ + that PANGOLIN_ENCRYPTION_KEY is not the key this row was written \ + with." + ) + })?; + + String::from_utf8(plaintext.to_vec()) + .context("a decrypted warehouse credential is not valid UTF-8") +} + +/// Encrypt the credential-bearing entries of a `storage_config`, in place. +/// +/// A no-op when no key is configured, so upgrading without setting one keeps +/// working exactly as before. The server logs a warning at startup in that +/// case; silently doing nothing is the failure mode this whole audit has been +/// about, so it is said out loud there rather than only here. +pub fn seal(config: &mut HashMap) -> Result<()> { + let Some(key) = data_key()? else { + return Ok(()); + }; + seal_with(&key, config) +} + +/// [`seal`] against an explicit key. +/// +/// Split out so the crypto is testable without setting a process-wide +/// environment variable - which the tests were doing, and which raced when they +/// ran in parallel: one test's `unset` decided another's behaviour. +pub(crate) fn seal_with(key: &LessSafeKey, config: &mut HashMap) -> Result<()> { + for (name, value) in config.iter_mut() { + // Already sealed: re-sealing would double-encrypt and the value could + // never be read back. + if !is_sensitive(name) || value.starts_with(PREFIX) || value.is_empty() { + continue; + } + *value = seal_value(key, value)?; + } + Ok(()) +} + +/// Decrypt any sealed entries, in place. Values without the prefix are left +/// alone, which is what lets a database written before 0.8.0 still be read. +pub fn open(config: &mut HashMap) -> Result<()> { + // Nothing sealed: avoid demanding a key just to read a legacy row. + if !config.values().any(|v| v.starts_with(PREFIX)) { + return Ok(()); + } + + let Some(key) = data_key()? else { + return Err(anyhow!( + "this warehouse has encrypted credentials but PANGOLIN_ENCRYPTION_KEY \ + is not set. Set it to the key the credentials were written with; \ + without it they cannot be recovered." + )); + }; + open_with(&key, config) +} + +/// [`open`] against an explicit key. +pub(crate) fn open_with(key: &LessSafeKey, config: &mut HashMap) -> Result<()> { + for value in config.values_mut() { + if value.starts_with(PREFIX) { + *value = open_value(key, value)?; + } + } + Ok(()) +} + +/// True when the map still holds an unencrypted credential. +/// +/// Used by the operations tooling to report what a re-seal would cover, and by +/// the cross-backend test that asserts nothing reaches storage in the clear. +pub fn has_plaintext_secret(config: &HashMap) -> bool { + config + .iter() + .any(|(k, v)| is_sensitive(k) && !v.is_empty() && !v.starts_with(PREFIX)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Sets the key for one test and restores the previous value afterwards. + /// The suite runs in one process, so leaking this would decide unrelated + /// tests' behaviour. + struct KeyGuard(Option); + + impl KeyGuard { + fn set(value: &str) -> Self { + let previous = std::env::var("PANGOLIN_ENCRYPTION_KEY").ok(); + std::env::set_var("PANGOLIN_ENCRYPTION_KEY", value); + Self(previous) + } + fn unset() -> Self { + let previous = std::env::var("PANGOLIN_ENCRYPTION_KEY").ok(); + std::env::remove_var("PANGOLIN_ENCRYPTION_KEY"); + Self(previous) + } + } + + impl Drop for KeyGuard { + fn drop(&mut self) { + match &self.0 { + Some(v) => std::env::set_var("PANGOLIN_ENCRYPTION_KEY", v), + None => std::env::remove_var("PANGOLIN_ENCRYPTION_KEY"), + } + } + } + + /// A key built directly, with no environment variable anywhere near it. + /// Everything that tests the *crypto* uses this; only the handful of cases + /// that test environment handling touch the process env, and those are + /// serialised. + fn key_from(byte: u8) -> LessSafeKey { + LessSafeKey::new(UnboundKey::new(&AES_256_GCM, &[byte; 32]).unwrap()) + } + + fn config_with_secret() -> HashMap { + HashMap::from([ + ("type".to_string(), "s3".to_string()), + ("bucket".to_string(), "customer-data".to_string()), + ("region".to_string(), "us-east-1".to_string()), + ("secret_access_key".to_string(), "SUPER-SECRET".to_string()), + ]) + } + + #[test] + fn a_sealed_value_round_trips() { + let key = key_from(7); + let mut config = config_with_secret(); + seal_with(&key, &mut config).unwrap(); + assert_ne!(config["secret_access_key"], "SUPER-SECRET"); + open_with(&key, &mut config).unwrap(); + assert_eq!(config["secret_access_key"], "SUPER-SECRET"); + } + + #[test] + fn only_credentials_are_encrypted() { + let key = key_from(7); + let mut config = config_with_secret(); + seal_with(&key, &mut config).unwrap(); + + // The object-store factory compares and concatenates these; encrypting + // them would break every storage operation for no security gain. + assert_eq!(config["bucket"], "customer-data"); + assert_eq!(config["region"], "us-east-1"); + assert_eq!(config["type"], "s3"); + assert!(config["secret_access_key"].starts_with(PREFIX)); + } + + #[test] + fn the_ciphertext_does_not_contain_the_plaintext() { + let key = key_from(7); + let mut config = config_with_secret(); + seal_with(&key, &mut config).unwrap(); + let serialized = serde_json::to_string(&config).unwrap(); + assert!( + !serialized.contains("SUPER-SECRET"), + "the serialized form still contains the secret: {serialized}" + ); + } + + #[test] + fn the_same_secret_seals_differently_each_time() { + let key = key_from(7); + let mut a = config_with_secret(); + let mut b = config_with_secret(); + seal_with(&key, &mut a).unwrap(); + seal_with(&key, &mut b).unwrap(); + assert_ne!( + a["secret_access_key"], b["secret_access_key"], + "a fresh nonce per value is what stops an observer seeing that two \ + warehouses share a credential" + ); + } + + #[test] + fn sealing_twice_does_not_double_encrypt() { + let key = key_from(7); + let mut config = config_with_secret(); + seal_with(&key, &mut config).unwrap(); + let once = config["secret_access_key"].clone(); + seal_with(&key, &mut config).unwrap(); + assert_eq!(config["secret_access_key"], once); + open_with(&key, &mut config).unwrap(); + assert_eq!(config["secret_access_key"], "SUPER-SECRET"); + } + + #[test] + fn legacy_plaintext_is_readable() { + // A row written before encryption existed. + let key = key_from(7); + let mut config = config_with_secret(); + open_with(&key, &mut config).unwrap(); + assert_eq!( + config["secret_access_key"], "SUPER-SECRET", + "refusing to read pre-encryption rows would turn this into an outage" + ); + } + + #[test] + #[serial_test::serial(encryption_key_env)] + fn without_a_key_sealing_is_a_no_op() { + let _guard = KeyGuard::unset(); + let mut config = config_with_secret(); + seal(&mut config).unwrap(); + assert_eq!(config["secret_access_key"], "SUPER-SECRET"); + assert!(!is_enabled()); + } + + #[test] + #[serial_test::serial(encryption_key_env)] + fn sealed_data_without_the_key_fails_loudly() { + let mut config = config_with_secret(); + seal_with(&key_from(7), &mut config).unwrap(); + let _guard = KeyGuard::unset(); + let err = open(&mut config).unwrap_err().to_string(); + assert!( + err.contains("PANGOLIN_ENCRYPTION_KEY"), + "the error must name the missing key, not fail obscurely: {err}" + ); + } + + #[test] + fn the_wrong_key_does_not_silently_return_rubbish() { + let mut config = config_with_secret(); + seal_with(&key_from(7), &mut config).unwrap(); + let err = open_with(&key_from(9), &mut config) + .unwrap_err() + .to_string(); + assert!( + err.contains("not the key this row was written with"), + "GCM authentication must reject the wrong key with a usable message: {err}" + ); + } + + #[test] + #[serial_test::serial(encryption_key_env)] + fn a_malformed_key_is_rejected_with_guidance() { + let _guard = KeyGuard::set("not-base64!!"); + let err = data_key().unwrap_err().to_string(); + assert!(err.contains("openssl rand -base64 32"), "got: {err}"); + + let _short = KeyGuard::set(&base64::engine::general_purpose::STANDARD.encode([1u8; 16])); + let err = data_key().unwrap_err().to_string(); + assert!(err.contains("needs 32"), "got: {err}"); + } + + #[test] + fn every_credential_the_ui_writes_is_covered() { + // These are the exact keys pangolin_ui's create-warehouse form writes. + // A spelling missing from SENSITIVE_KEYS means that credential stays in + // plaintext with no error anywhere. + for key in [ + "secret_access_key", + "access_key_id", + "account_key", + "client_secret", + "service_account_json", + "external_id", + ] { + assert!( + is_sensitive(key), + "{key} is a credential and must be sealed" + ); + } + for key in [ + "type", + "bucket", + "region", + "endpoint", + "container", + "account_name", + "project_id", + ] { + assert!( + !is_sensitive(key), + "{key} is not a secret and must stay readable" + ); + } + } + + #[test] + fn plaintext_detection_reports_what_needs_resealing() { + let key = key_from(7); + let mut config = config_with_secret(); + assert!(has_plaintext_secret(&config)); + seal_with(&key, &mut config).unwrap(); + assert!(!has_plaintext_secret(&config)); + } +} diff --git a/pangolin/pangolin_store/src/sqlite/access_requests.rs b/pangolin/pangolin_store/src/sqlite/access_requests.rs index 5cc50f4..b4874dd 100644 --- a/pangolin/pangolin_store/src/sqlite/access_requests.rs +++ b/pangolin/pangolin_store/src/sqlite/access_requests.rs @@ -52,13 +52,14 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = - sqlx::query("SELECT * FROM access_requests WHERE tenant_id = ? LIMIT ? OFFSET ?") - .bind(tenant_id.to_string()) - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; + let rows = sqlx::query( + "SELECT * FROM access_requests WHERE tenant_id = ? ORDER BY id LIMIT ? OFFSET ?", + ) + .bind(tenant_id.to_string()) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; let mut requests = Vec::new(); for row in rows { diff --git a/pangolin/pangolin_store/src/sqlite/assets.rs b/pangolin/pangolin_store/src/sqlite/assets.rs index 9ec47d0..b4dad5f 100644 --- a/pangolin/pangolin_store/src/sqlite/assets.rs +++ b/pangolin/pangolin_store/src/sqlite/assets.rs @@ -24,7 +24,7 @@ impl SqliteStore { .bind(&namespace_path) .bind(&asset.name) .bind(&branch_name) - .bind(format!("{:?}", asset.kind)) + .bind(asset.kind.as_stored_str()) .bind(asset.properties.get("metadata_location").unwrap_or(&asset.location)) .bind(serde_json::to_string(&asset.properties)?) .execute(&self.pool) @@ -139,11 +139,9 @@ impl SqliteStore { if let Some(row) = row { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = + AssetType::from_stored_str(&asset_type_str).map_err(|e| anyhow::anyhow!(e))?; Ok(Some(Asset { id: Uuid::parse_str(&row.get::("id"))?, @@ -178,11 +176,9 @@ impl SqliteStore { serde_json::from_str(&namespace_json).unwrap_or_default(); let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = + AssetType::from_stored_str(&asset_type_str).map_err(|e| anyhow::anyhow!(e))?; let asset = Asset { id: Uuid::parse_str(&row.get::("id"))?, @@ -218,7 +214,7 @@ impl SqliteStore { let namespace_path = serde_json::to_string(&namespace)?; let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let rows = sqlx::query("SELECT id, name, asset_type, metadata_location, properties FROM assets WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND branch_name = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT id, name, asset_type, metadata_location, properties FROM assets WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND branch_name = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(&namespace_path) @@ -231,11 +227,9 @@ impl SqliteStore { let mut assets = Vec::new(); for row in rows { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = + AssetType::from_stored_str(&asset_type_str).map_err(|e| anyhow::anyhow!(e))?; assets.push(Asset { id: Uuid::parse_str(&row.get::("id"))?, @@ -319,16 +313,23 @@ impl SqliteStore { &self, tenant_id: Uuid, catalog_name: &str, - _branch: Option, + branch: Option, namespace: Vec, table: String, ) -> Result> { + // B19: `branch` was discarded (`_branch`) and the query matched rows + // from *every* branch, with `fetch_optional` returning an arbitrary one. + // Reading a table on `dev` could hand back `main`'s metadata pointer - + // silently, and differently depending on row order. Postgres and Mongo + // both scope by branch. + let branch_name = branch.unwrap_or_else(|| "main".to_string()); let namespace_path = serde_json::to_string(&namespace)?; - let row = sqlx::query("SELECT metadata_location, properties FROM assets WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND name = ?") + let row = sqlx::query("SELECT metadata_location, properties FROM assets WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND name = ? AND branch_name = ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(&namespace_path) .bind(&table) + .bind(&branch_name) .fetch_optional(&self.pool) .await?; @@ -388,9 +389,15 @@ impl SqliteStore { )); } - props.insert("metadata_location".to_string(), new_location); + props.insert("metadata_location".to_string(), new_location.clone()); - let update_result = sqlx::query("UPDATE assets SET properties = ? WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND name = ? AND branch_name = ?") + // B20: only `properties` was updated, leaving the `metadata_location` + // *column* stale - and reads populate `Asset.location` from that + // column. On SQLite an asset's `location` was therefore frozen at + // creation time no matter how many Iceberg commits followed. + // Postgres updates both. + let update_result = sqlx::query("UPDATE assets SET metadata_location = ?, properties = ? WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND name = ? AND branch_name = ?") + .bind(&new_location) .bind(serde_json::to_string(&props)?) .bind(tenant_id.to_string()) .bind(catalog_name) diff --git a/pangolin/pangolin_store/src/sqlite/audit_logs.rs b/pangolin/pangolin_store/src/sqlite/audit_logs.rs index 9e6ae43..3cd4254 100644 --- a/pangolin/pangolin_store/src/sqlite/audit_logs.rs +++ b/pangolin/pangolin_store/src/sqlite/audit_logs.rs @@ -19,14 +19,16 @@ impl SqliteStore { .bind(tenant_id.to_string()) .bind(event.user_id.map(|u| u.to_string())) .bind(&event.username) - .bind(format!("{:?}", event.action)) - .bind(format!("{:?}", event.resource_type)) + .bind(pangolin_core::audit::audit_enum_to_stored(&event.action)) + .bind(pangolin_core::audit::audit_enum_to_stored( + &event.resource_type, + )) .bind(event.resource_id.map(|u| u.to_string())) .bind(&event.resource_name) .bind(event.timestamp.timestamp_millis()) .bind(event.ip_address.as_deref().unwrap_or("")) .bind(event.user_agent.as_deref().unwrap_or("")) - .bind(format!("{:?}", event.result)) + .bind(pangolin_core::audit::audit_enum_to_stored(&event.result)) .bind(event.error_message.as_deref().unwrap_or("")) .bind(serde_json::to_string(&event.metadata)?) .execute(&self.pool) @@ -103,18 +105,22 @@ impl SqliteStore { let ts_millis: i64 = row.get("timestamp"); // Parse enums from strings + // B22: these used to lowercase the stored Debug spelling and + // deserialize against snake_case, which never matched, then swallow + // the failure with `.unwrap_or(CreateCatalog)`. Errors now + // propagate: a corrupt audit row is a loud failure, not a + // plausible-looking lie about what happened. let action_str: String = row.get("action"); - let action = serde_json::from_str(&format!("\"{}\"", action_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditAction::CreateCatalog); + let action = pangolin_core::audit::audit_enum_from_stored(&action_str) + .map_err(|e| anyhow::anyhow!(e))?; let resource_type_str: String = row.get("resource_type"); - let resource_type = - serde_json::from_str(&format!("\"{}\"", resource_type_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::ResourceType::Catalog); + let resource_type = pangolin_core::audit::audit_enum_from_stored(&resource_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let result_str: String = row.get("result"); - let result = serde_json::from_str(&format!("\"{}\"", result_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditResult::Success); + let result = pangolin_core::audit::audit_enum_from_stored(&result_str) + .map_err(|e| anyhow::anyhow!(e))?; events.push(AuditLogEntry { id: Uuid::parse_str(&row.get::("id"))?, @@ -183,18 +189,22 @@ impl SqliteStore { if let Some(row) = row { let ts_millis: i64 = row.get("timestamp"); + // B22: these used to lowercase the stored Debug spelling and + // deserialize against snake_case, which never matched, then swallow + // the failure with `.unwrap_or(CreateCatalog)`. Errors now + // propagate: a corrupt audit row is a loud failure, not a + // plausible-looking lie about what happened. let action_str: String = row.get("action"); - let action = serde_json::from_str(&format!("\"{}\"", action_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditAction::CreateCatalog); + let action = pangolin_core::audit::audit_enum_from_stored(&action_str) + .map_err(|e| anyhow::anyhow!(e))?; let resource_type_str: String = row.get("resource_type"); - let resource_type = - serde_json::from_str(&format!("\"{}\"", resource_type_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::ResourceType::Catalog); + let resource_type = pangolin_core::audit::audit_enum_from_stored(&resource_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let result_str: String = row.get("result"); - let result = serde_json::from_str(&format!("\"{}\"", result_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditResult::Success); + let result = pangolin_core::audit::audit_enum_from_stored(&result_str) + .map_err(|e| anyhow::anyhow!(e))?; Ok(Some(AuditLogEntry { id: Uuid::parse_str(&row.get::("id"))?, diff --git a/pangolin/pangolin_store/src/sqlite/branches.rs b/pangolin/pangolin_store/src/sqlite/branches.rs index bcd6e6c..8fc676a 100644 --- a/pangolin/pangolin_store/src/sqlite/branches.rs +++ b/pangolin/pangolin_store/src/sqlite/branches.rs @@ -24,6 +24,104 @@ impl SqliteStore { Ok(()) } + /// Create a branch and copy assets into it in one transaction (A-24). + /// + /// SQLite stores `namespace_path` as a JSON string rather than an array, so + /// the `namespace.table` matching is done on `json_each` output rather than + /// with an array operator as on Postgres. The property that matters is the + /// same: one transaction, so the branch and its assets appear together or + /// not at all. + pub async fn create_branch_with_assets( + &self, + tenant_id: Uuid, + catalog_name: &str, + branch: Branch, + src_branch: &str, + assets: Option>, + ) -> Result { + let mut tx = self.pool.begin().await?; + + sqlx::query( + "INSERT INTO branches (tenant_id, catalog_name, name, head_commit_id, branch_type, assets) \ + VALUES (?, ?, ?, ?, ?, ?)", + ) + .bind(tenant_id.to_string()) + .bind(catalog_name) + .bind(&branch.name) + .bind(branch.head_commit_id.map(|u| u.to_string())) + .bind(format!("{:?}", branch.branch_type)) + .bind(serde_json::to_string(&branch.assets)?) + .execute(&mut *tx) + .await?; + + // Read the source rows inside the transaction so the set cannot change + // between selecting and inserting. + let rows = sqlx::query( + "SELECT namespace_path, name, asset_type, metadata_location, properties \ + FROM assets WHERE tenant_id = ? AND catalog_name = ? AND branch_name = ?", + ) + .bind(tenant_id.to_string()) + .bind(catalog_name) + .bind(src_branch) + .fetch_all(&mut *tx) + .await?; + + let wanted: Option> = match &assets { + Some(names) => { + for full in names { + if !full.contains('.') { + tx.rollback().await.ok(); + return Err(anyhow::anyhow!( + "asset name {full:?} is not in namespace.table form" + )); + } + } + Some(names.iter().cloned().collect()) + } + None => None, + }; + + let mut copied = 0usize; + for row in rows { + let ns_json: String = row.get("namespace_path"); + let name: String = row.get("name"); + let ns: Vec = serde_json::from_str(&ns_json).unwrap_or_default(); + + if let Some(wanted) = &wanted { + let full = format!("{}.{}", ns.join("."), name); + if !wanted.contains(&full) { + continue; + } + } + + let asset_type: String = row.get("asset_type"); + let metadata_location: Option = row.get("metadata_location"); + let properties: String = row.get("properties"); + + // A fresh id: an asset's primary key is unique across branches. + sqlx::query( + "INSERT INTO assets (id, tenant_id, catalog_name, namespace_path, name, \ + branch_name, asset_type, metadata_location, properties) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(Uuid::new_v4().to_string()) + .bind(tenant_id.to_string()) + .bind(catalog_name) + .bind(&ns_json) + .bind(&name) + .bind(&branch.name) + .bind(&asset_type) + .bind(&metadata_location) + .bind(&properties) + .execute(&mut *tx) + .await?; + copied += 1; + } + + tx.commit().await?; + Ok(copied) + } + pub async fn get_branch( &self, tenant_id: Uuid, @@ -71,7 +169,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = ? AND catalog_name = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = ? AND catalog_name = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(limit) @@ -100,19 +198,32 @@ impl SqliteStore { Ok(branches) } + /// Delete a branch and the assets that live on it. + /// + /// B3: two defects, both fatal together. The asset cleanup referenced a + /// column named `branch`, but the schema column is `branch_name` + /// (`sql/sqlite_schema.sql:71`), so the statement failed with "no such + /// column". And because the branch delete had already been committed as its + /// own statement, the branch was gone while its assets survived - orphaned + /// permanently, with no branch to reach them through - and the caller got an + /// error suggesting nothing had happened. Postgres was fixed for exactly + /// this and wraps both statements in a transaction; SQLite was never + /// patched. pub async fn delete_branch( &self, tenant_id: Uuid, catalog_name: &str, name: String, ) -> Result<()> { + let mut tx = self.pool.begin().await?; + let result = sqlx::query( "DELETE FROM branches WHERE tenant_id = ? AND catalog_name = ? AND name = ?", ) .bind(tenant_id.to_string()) .bind(catalog_name) .bind(&name) - .execute(&self.pool) + .execute(&mut *tx) .await?; if result.rows_affected() == 0 { @@ -120,13 +231,16 @@ impl SqliteStore { } // Also delete assets associated with this branch - sqlx::query("DELETE FROM assets WHERE tenant_id = ? AND catalog_name = ? AND branch = ?") - .bind(tenant_id.to_string()) - .bind(catalog_name) - .bind(&name) - .execute(&self.pool) - .await?; + sqlx::query( + "DELETE FROM assets WHERE tenant_id = ? AND catalog_name = ? AND branch_name = ?", + ) + .bind(tenant_id.to_string()) + .bind(catalog_name) + .bind(&name) + .execute(&mut *tx) + .await?; + tx.commit().await?; Ok(()) } diff --git a/pangolin/pangolin_store/src/sqlite/business_metadata.rs b/pangolin/pangolin_store/src/sqlite/business_metadata.rs index 9b8c90d..19c87e0 100644 --- a/pangolin/pangolin_store/src/sqlite/business_metadata.rs +++ b/pangolin/pangolin_store/src/sqlite/business_metadata.rs @@ -81,21 +81,28 @@ impl SqliteStore { m.created_at as meta_created_at, m.updated_by as meta_updated_by, m.updated_at as meta_updated_at FROM assets a LEFT JOIN business_metadata m ON a.id = m.asset_id - WHERE a.tenant_id = ? AND (a.name LIKE ? OR m.description LIKE ?)" + WHERE a.tenant_id = ? AND (a.name LIKE ? ESCAPE '\\' OR m.description LIKE ? ESCAPE '\\')" ); - let query_pattern = format!("%{}%", query); + let query_pattern = crate::search::contains_pattern(query); + // B28: this was an ANY-match (`EXISTS ... value IN (...)`) while + // Postgres (`@>`) and Mongo (`$all`) required *all* the requested tags. + // The chosen semantic - see `crate::search::tags_match` - is ALL-match, + // so counting distinct matches against the requested count is what makes + // SQLite agree with the other three. if let Some(ref tag_list) = tags { if !tag_list.is_empty() { - sql.push_str(" AND EXISTS (SELECT 1 FROM json_each(m.tags) WHERE value IN ("); + sql.push_str( + " AND (SELECT COUNT(DISTINCT value) FROM json_each(m.tags) WHERE value IN (", + ); for (i, _) in tag_list.iter().enumerate() { if i > 0 { sql.push_str(", "); } sql.push('?'); } - sql.push_str("))"); + sql.push_str(&format!(")) = {}", tag_list.len())); } } @@ -117,11 +124,9 @@ impl SqliteStore { for row in rows { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = + AssetType::from_stored_str(&asset_type_str).map_err(|e| anyhow::anyhow!(e))?; let asset = Asset { id: Uuid::parse_str(row.get("id"))?, @@ -153,8 +158,14 @@ impl SqliteStore { }; let catalog_name: String = row.get("catalog_name"); + // B4 (SQLite sibling): `namespace_path` is stored as a JSON array + // (`serde_json::to_string(&namespace)`), so splitting it on 0x1F + // yielded a single element containing raw JSON - a search result's + // namespace came back as `["[\"a\",\"b\"]"]` rather than + // `["a", "b"]`. No panic here, just a silently wrong namespace on + // every search hit. let namespace_path: String = row.get("namespace_path"); - let namespace: Vec = namespace_path.split('\x1F').map(String::from).collect(); + let namespace: Vec = serde_json::from_str(&namespace_path).unwrap_or_default(); results.push((asset, metadata, catalog_name, namespace)); } @@ -163,8 +174,8 @@ impl SqliteStore { } pub async fn search_catalogs(&self, tenant_id: Uuid, query: &str) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = ? AND name LIKE ?") + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = ? AND name LIKE ? ESCAPE '\\' ORDER BY name") .bind(tenant_id.to_string()) .bind(&query_pattern) .fetch_all(&self.pool) @@ -197,8 +208,8 @@ impl SqliteStore { tenant_id: Uuid, query: &str, ) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT catalog_name, namespace_path, properties FROM namespaces WHERE tenant_id = ? AND namespace_path LIKE ?") + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT catalog_name, namespace_path, properties FROM namespaces WHERE tenant_id = ? AND namespace_path LIKE ? ESCAPE '\\' ORDER BY catalog_name, namespace_path") .bind(tenant_id.to_string()) .bind(&query_pattern) .fetch_all(&self.pool) @@ -228,8 +239,8 @@ impl SqliteStore { tenant_id: Uuid, query: &str, ) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT catalog_name, name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = ? AND name LIKE ?") + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT catalog_name, name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = ? AND name LIKE ? ESCAPE '\\' ORDER BY catalog_name, name") .bind(tenant_id.to_string()) .bind(&query_pattern) .fetch_all(&self.pool) diff --git a/pangolin/pangolin_store/src/sqlite/catalogs.rs b/pangolin/pangolin_store/src/sqlite/catalogs.rs index b3d4c5e..3c05307 100644 --- a/pangolin/pangolin_store/src/sqlite/catalogs.rs +++ b/pangolin/pangolin_store/src/sqlite/catalogs.rs @@ -64,7 +64,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(limit) .bind(offset) @@ -143,48 +143,50 @@ impl SqliteStore { .ok_or_else(|| anyhow::anyhow!("Catalog not found")) } + /// Delete a catalog and cascade to its children. + /// + /// B21: this used to run five sequential deletes with no transaction, and + /// the "does the catalog exist?" check was the *last* statement. So + /// `delete_catalog(tenant, "nonexistent")` cheerfully deleted every tag, + /// branch, asset and namespace whose `catalog_name` matched - and only then + /// returned "not found", leaving the caller believing nothing had happened. + /// Any failure part-way through left the same wreckage. Postgres wraps the + /// identical cascade in a transaction; this now matches it, and checks + /// existence first. pub async fn delete_catalog(&self, tenant_id: Uuid, name: String) -> Result<()> { let tid = tenant_id.to_string(); - // Delete cascading children manually (no FK constraints on catalog_name) - // 1. Tags - sqlx::query("DELETE FROM tags WHERE tenant_id = ? AND catalog_name = ?") - .bind(&tid) - .bind(&name) - .execute(&self.pool) - .await?; - - // 2. Branches - sqlx::query("DELETE FROM branches WHERE tenant_id = ? AND catalog_name = ?") - .bind(&tid) - .bind(&name) - .execute(&self.pool) - .await?; - - // 3. Assets - sqlx::query("DELETE FROM assets WHERE tenant_id = ? AND catalog_name = ?") - .bind(&tid) - .bind(&name) - .execute(&self.pool) - .await?; + let mut tx = self.pool.begin().await?; + + // Existence first: nothing is destroyed on behalf of a catalog that is + // not there. + let exists: Option<(i64,)> = + sqlx::query_as("SELECT 1 FROM catalogs WHERE tenant_id = ? AND name = ?") + .bind(&tid) + .bind(&name) + .fetch_optional(&mut *tx) + .await?; + if exists.is_none() { + return Err(anyhow::anyhow!("Catalog '{}' not found", name)); + } - // 4. Namespaces - sqlx::query("DELETE FROM namespaces WHERE tenant_id = ? AND catalog_name = ?") - .bind(&tid) - .bind(&name) - .execute(&self.pool) - .await?; + // Delete cascading children manually (no FK constraints on catalog_name) + for table in ["tags", "branches", "assets", "namespaces"] { + let sql = format!("DELETE FROM {table} WHERE tenant_id = ? AND catalog_name = ?"); + sqlx::query(&sql) + .bind(&tid) + .bind(&name) + .execute(&mut *tx) + .await?; + } - // 5. Catalog - let result = sqlx::query("DELETE FROM catalogs WHERE tenant_id = ? AND name = ?") + sqlx::query("DELETE FROM catalogs WHERE tenant_id = ? AND name = ?") .bind(&tid) .bind(&name) - .execute(&self.pool) + .execute(&mut *tx) .await?; - if result.rows_affected() == 0 { - return Err(anyhow::anyhow!("Catalog '{}' not found", name)); - } + tx.commit().await?; Ok(()) } } diff --git a/pangolin/pangolin_store/src/sqlite/main.rs b/pangolin/pangolin_store/src/sqlite/main.rs index edf6375..9874635 100644 --- a/pangolin/pangolin_store/src/sqlite/main.rs +++ b/pangolin/pangolin_store/src/sqlite/main.rs @@ -9,12 +9,15 @@ use pangolin_core::model::{SyncStats, SystemSettings}; use pangolin_core::permission::{Permission, Role, UserRole as UserRoleAssignment}; use pangolin_core::token::TokenInfo; use pangolin_core::user::{OAuthProvider, User, UserRole}; -use sqlx::sqlite::{SqlitePool, SqlitePoolOptions}; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions}; use sqlx::Row; use uuid::Uuid; /// Version of `sql/sqlite_schema.sql` recorded after a successful migration. -pub const SQLITE_SCHEMA_VERSION: i64 = 1; +/// Bumped to 2: the `audit_logs` table was recreated with the columns the code +/// actually writes. Databases created before this carry the old (actor, +/// resource, details) shape, on which every audit write failed. +pub const SQLITE_SCHEMA_VERSION: i64 = 2; #[derive(Clone)] pub struct SqliteStore { @@ -49,14 +52,20 @@ impl SqliteStore { .and_then(|v| v.parse::().ok()) .unwrap_or(5); + // B18: `PRAGMA foreign_keys` is *per connection*. Running it once via + // `execute(&pool)` configured exactly one arbitrary connection out of + // the pool, so `ON DELETE CASCADE` fired or did not fire depending on + // which connection happened to serve a request - nondeterministically, + // and differently between runs. Setting it through the connect options + // means every connection in the pool is configured, including ones + // created later to grow the pool. + let connect_options = database_url + .parse::()? + .foreign_keys(true); + let pool = SqlitePoolOptions::new() .max_connections(max_connections) - .connect(database_url) - .await?; - - // Enable foreign keys - sqlx::query("PRAGMA foreign_keys = ON") - .execute(&pool) + .connect_with(connect_options) .await?; Ok(Self { @@ -84,6 +93,13 @@ impl SqliteStore { .execute(&self.pool) .await?; + // Structural changes have to run *before* the schema file, because + // `CREATE TABLE IF NOT EXISTS` cannot alter a table that already exists + // - it silently does nothing, which is precisely how bumping + // SQLITE_SCHEMA_VERSION on its own left every upgraded database still + // broken while fresh ones looked fine. + self.migrate_audit_logs_to_v2().await?; + let schema = include_str!("../../sql/sqlite_schema.sql"); self.apply_schema(schema).await?; @@ -103,6 +119,88 @@ impl SqliteStore { Ok(()) } + /// Replace a pre-v2 `audit_logs` table with the current shape. + /// + /// The table was declared with the original `(actor, resource, details)` + /// columns while `sqlite/audit_logs.rs` inserted the full `AuditLogEntry`. + /// The two have been divergent since the SQLite backend was split into + /// modules, so on SQLite *every* audit write has failed with "table + /// audit_logs has no column named user_id" and the backend has never + /// recorded an audit trail. + /// + /// Keyed off table introspection rather than the recorded schema version: + /// some databases predate the version table entirely, and a version number + /// only says what a previous run *claimed*. Asking the table what columns it + /// has is the fact. That also makes this safe to run repeatedly. + /// + /// Any rows present are moved to `audit_logs_pre_v2` rather than discarded + /// or force-fitted. The old shape has no `resource_type`, which is + /// `NOT NULL` and parses as an enum, so there is no honest value to invent + /// for it - and a fabricated entry in an audit log is worse than an absent + /// one. In practice the table is empty, because nothing could ever write + /// to it. + async fn migrate_audit_logs_to_v2(&self) -> Result<()> { + let columns = self.table_columns("audit_logs").await?; + + // No table yet: a fresh database. The schema file creates it. + if columns.is_empty() { + return Ok(()); + } + + // Already current. + if columns.iter().any(|c| c == "user_id") { + return Ok(()); + } + + let row_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM audit_logs") + .fetch_one(&self.pool) + .await + .unwrap_or(0); + + tracing::warn!( + rows = row_count, + "migrating the pre-v2 sqlite audit_logs table; the old table is kept \ + as audit_logs_pre_v2" + ); + + let mut tx = self.pool.begin().await?; + + // A stale backup from an interrupted previous attempt would make the + // rename fail, so clear the way first. + sqlx::query("DROP TABLE IF EXISTS audit_logs_pre_v2") + .execute(&mut *tx) + .await?; + sqlx::query("ALTER TABLE audit_logs RENAME TO audit_logs_pre_v2") + .execute(&mut *tx) + .await?; + + // The index followed the table through the rename and would collide + // with the one the schema file recreates. + sqlx::query("DROP INDEX IF EXISTS idx_audit_logs_tenant_ts") + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(()) + } + + /// Column names of `table`, or an empty vec if it does not exist. + /// + /// Exposed so the migration tests can assert on the *actual* shape of a + /// database rather than on what the recorded version claims - the two + /// disagreeing is exactly the bug the v2 migration exists to fix. + pub async fn table_columns(&self, table: &str) -> Result> { + // `table` is never caller-supplied in a production path, and PRAGMA does + // not accept a bind parameter for a table name. + let rows = sqlx::query(&format!("PRAGMA table_info({table})")) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("name")) + .collect()) + } + /// The schema version currently recorded in the database, if any. pub async fn schema_version(&self) -> Result> { let row = sqlx::query("SELECT MAX(version) as v FROM _pangolin_schema_version") @@ -114,9 +212,16 @@ impl SqliteStore { } pub async fn apply_schema(&self, schema_sql: &str) -> Result<()> { + // Schema creation runs on a single dedicated connection so the + // foreign-key toggle below is scoped to *this* work rather than + // leaking onto whichever pooled connection happened to serve it + // (B18) - the old code could leave `OFF` stuck on a connection the + // matching `ON` never touched. + let mut conn = self.pool.acquire().await?; + // Disable foreign keys during schema creation sqlx::query("PRAGMA foreign_keys = OFF") - .execute(&self.pool) + .execute(&mut *conn) .await?; // Parse statements @@ -151,12 +256,13 @@ impl SqliteStore { } for statement in statements { - sqlx::query(&statement).execute(&self.pool).await?; + sqlx::query(&statement).execute(&mut *conn).await?; } - // Re-enable foreign keys + // Re-enable foreign keys on this connection before returning it to the + // pool. sqlx::query("PRAGMA foreign_keys = ON") - .execute(&self.pool) + .execute(&mut *conn) .await?; Ok(()) } @@ -327,6 +433,17 @@ impl CatalogStore for SqliteStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + self.replace_namespace_properties(tenant_id, catalog_name, namespace, properties) + .await + } + async fn create_asset( &self, tenant_id: Uuid, @@ -399,6 +516,17 @@ impl CatalogStore for SqliteStore { ) .await } + async fn create_branch_with_assets( + &self, + tenant_id: Uuid, + catalog_name: &str, + branch: pangolin_core::model::Branch, + src_branch: &str, + assets: Option>, + ) -> Result { + self.create_branch_with_assets(tenant_id, catalog_name, branch, src_branch, assets) + .await + } async fn copy_assets_bulk( &self, tenant_id: Uuid, @@ -533,6 +661,14 @@ impl CatalogStore for SqliteStore { async fn write_file(&self, path: &str, data: Vec) -> Result<()> { self.write_file(path, data).await } + async fn delete_file(&self, path: &str) -> Result<()> { + self.metadata_cache.invalidate(path).await; + let storage_config = self + .get_warehouse_for_location(path) + .await? + .map(|w| w.storage_config); + crate::file_delete::delete_location(storage_config.as_ref(), path).await + } // Phase 3 & 4 (Access Control, Audit, Settings, etc) async fn create_user(&self, user: User) -> Result<()> { diff --git a/pangolin/pangolin_store/src/sqlite/merge_operations.rs b/pangolin/pangolin_store/src/sqlite/merge_operations.rs index 57b6bc3..92c6933 100644 --- a/pangolin/pangolin_store/src/sqlite/merge_operations.rs +++ b/pangolin/pangolin_store/src/sqlite/merge_operations.rs @@ -97,7 +97,7 @@ impl SqliteStore { let rows = sqlx::query( "SELECT id, tenant_id, catalog_name, source_branch, target_branch, base_commit_id, status, initiated_by, initiated_at, result_commit_id, completed_at - FROM merge_operations WHERE tenant_id = ? AND catalog_name = ? LIMIT ? OFFSET ?" + FROM merge_operations WHERE tenant_id = ? AND catalog_name = ? ORDER BY id LIMIT ? OFFSET ?" ) .bind(tenant_id.to_string()) .bind(catalog_name) @@ -259,7 +259,7 @@ impl SqliteStore { let rows = sqlx::query( "SELECT id, operation_id, conflict_type, asset_id, description, resolution, created_at - FROM merge_conflicts WHERE operation_id = ? LIMIT ? OFFSET ?", + FROM merge_conflicts WHERE operation_id = ? ORDER BY id LIMIT ? OFFSET ?", ) .bind(operation_id.to_string()) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/mod.rs b/pangolin/pangolin_store/src/sqlite/mod.rs index 68435fa..04f916d 100644 --- a/pangolin/pangolin_store/src/sqlite/mod.rs +++ b/pangolin/pangolin_store/src/sqlite/mod.rs @@ -23,5 +23,5 @@ pub mod users; pub mod warehouses; // Re-export SqliteStore -pub use main::SqliteStore; +pub use main::{SqliteStore, SQLITE_SCHEMA_VERSION}; pub mod business_metadata; diff --git a/pangolin/pangolin_store/src/sqlite/namespaces.rs b/pangolin/pangolin_store/src/sqlite/namespaces.rs index d9f9d09..c70742c 100644 --- a/pangolin/pangolin_store/src/sqlite/namespaces.rs +++ b/pangolin/pangolin_store/src/sqlite/namespaces.rs @@ -63,7 +63,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT namespace_path, properties FROM namespaces WHERE tenant_id = ? AND catalog_name = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT namespace_path, properties FROM namespaces WHERE tenant_id = ? AND catalog_name = ? ORDER BY namespace_path LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(limit) @@ -120,4 +120,32 @@ impl SqliteStore { } Ok(()) } + + /// Replace a namespace's properties wholesale. + /// + /// `update_namespace_properties` merges, which cannot express a *removal*. + /// The Iceberg `updateProperties` endpoint takes both `updates` and + /// `removals`, and the handler used to ignore removals entirely while + /// reporting success (B16h); it now computes the resulting map and writes it + /// through here. + pub async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: HashMap, + ) -> Result<()> { + let namespace_path = serde_json::to_string(&namespace)?; + let result = sqlx::query("UPDATE namespaces SET properties = ? WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ?") + .bind(serde_json::to_string(&properties)?) + .bind(tenant_id.to_string()) + .bind(catalog_name) + .bind(&namespace_path) + .execute(&self.pool) + .await?; + if result.rows_affected() == 0 { + return Err(anyhow::anyhow!("Namespace not found")); + } + Ok(()) + } } diff --git a/pangolin/pangolin_store/src/sqlite/permissions.rs b/pangolin/pangolin_store/src/sqlite/permissions.rs index a9a7d0c..a7d70b3 100644 --- a/pangolin/pangolin_store/src/sqlite/permissions.rs +++ b/pangolin/pangolin_store/src/sqlite/permissions.rs @@ -114,7 +114,7 @@ impl SqliteStore { "SELECT id, user_id, tenant_id, scope, actions, granted_by, granted_at FROM permissions WHERE tenant_id = ? - LIMIT ? OFFSET ?", + ORDER BY id LIMIT ? OFFSET ?", ) .bind(tenant_id.to_string()) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/roles.rs b/pangolin/pangolin_store/src/sqlite/roles.rs index 868eb0f..376abad 100644 --- a/pangolin/pangolin_store/src/sqlite/roles.rs +++ b/pangolin/pangolin_store/src/sqlite/roles.rs @@ -58,7 +58,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, tenant_id, name, description, permissions, created_by, created_at, updated_at FROM roles WHERE tenant_id = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT id, tenant_id, name, description, permissions, created_by, created_at, updated_at FROM roles WHERE tenant_id = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(limit) .bind(offset) diff --git a/pangolin/pangolin_store/src/sqlite/service_users.rs b/pangolin/pangolin_store/src/sqlite/service_users.rs index d92f5ad..b745500 100644 --- a/pangolin/pangolin_store/src/sqlite/service_users.rs +++ b/pangolin/pangolin_store/src/sqlite/service_users.rs @@ -128,7 +128,7 @@ impl SqliteStore { let rows = sqlx::query( "SELECT id, name, description, tenant_id, api_key_hash, role, created_at, created_by, last_used, expires_at, active - FROM service_users WHERE tenant_id = ? LIMIT ? OFFSET ?" + FROM service_users WHERE tenant_id = ? ORDER BY name LIMIT ? OFFSET ?" ) .bind(tenant_id.to_string()) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/tags.rs b/pangolin/pangolin_store/src/sqlite/tags.rs index 346081d..906376b 100644 --- a/pangolin/pangolin_store/src/sqlite/tags.rs +++ b/pangolin/pangolin_store/src/sqlite/tags.rs @@ -55,7 +55,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT name, commit_id FROM tags WHERE tenant_id = ? AND catalog_name = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT name, commit_id FROM tags WHERE tenant_id = ? AND catalog_name = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/tenants.rs b/pangolin/pangolin_store/src/sqlite/tenants.rs index 0188c91..83995de 100644 --- a/pangolin/pangolin_store/src/sqlite/tenants.rs +++ b/pangolin/pangolin_store/src/sqlite/tenants.rs @@ -44,11 +44,12 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, properties FROM tenants LIMIT ? OFFSET ?") - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; + let rows = + sqlx::query("SELECT id, name, properties FROM tenants ORDER BY name LIMIT ? OFFSET ?") + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; let mut tenants = Vec::new(); for row in rows { diff --git a/pangolin/pangolin_store/src/sqlite/tokens.rs b/pangolin/pangolin_store/src/sqlite/tokens.rs index f091e98..67e2ffd 100644 --- a/pangolin/pangolin_store/src/sqlite/tokens.rs +++ b/pangolin/pangolin_store/src/sqlite/tokens.rs @@ -7,6 +7,56 @@ use sqlx::Row; use uuid::Uuid; impl SqliteStore { + /// Record a token revocation. + /// + /// Found by the cross-backend parity suite: `SqliteStore` had **no** + /// inherent `revoke_token`, `is_token_revoked` or `cleanup_expired_tokens`, + /// so the trait implementations in `sqlite/main.rs` - written as + /// `self.revoke_token(..)` in the style of every other delegation in that + /// file - resolved to the *trait* method and called themselves. On SQLite, + /// revoking a token (i.e. logging out) recursed until the thread's stack was + /// exhausted and aborted the process: an unauthenticated-adjacent remote + /// crash, not merely a missing feature. + /// + /// The `revoked_tokens` table has existed in the schema since A-27; only the + /// code to use it was missing. + pub async fn revoke_token( + &self, + token_id: Uuid, + expires_at: chrono::DateTime, + reason: Option, + ) -> Result<()> { + sqlx::query( + "INSERT INTO revoked_tokens (token_id, expires_at, reason) VALUES (?, ?, ?) + ON CONFLICT(token_id) DO UPDATE SET + expires_at = excluded.expires_at, + reason = excluded.reason", + ) + .bind(token_id.to_string()) + .bind(expires_at.timestamp_millis()) + .bind(reason) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn is_token_revoked(&self, token_id: Uuid) -> Result { + let row: Option<(i64,)> = sqlx::query_as("SELECT 1 FROM revoked_tokens WHERE token_id = ?") + .bind(token_id.to_string()) + .fetch_optional(&self.pool) + .await?; + Ok(row.is_some()) + } + + /// Drop revocation records whose tokens have expired anyway. + pub async fn cleanup_expired_tokens(&self) -> Result { + let result = sqlx::query("DELETE FROM revoked_tokens WHERE expires_at < ?") + .bind(Utc::now().timestamp_millis()) + .execute(&self.pool) + .await?; + Ok(result.rows_affected() as usize) + } + pub async fn store_token(&self, token_info: TokenInfo) -> Result<()> { sqlx::query("INSERT INTO active_tokens (token_id, user_id, tenant_id, token, expires_at, created_at) VALUES (?, ?, ?, ?, ?, ?)") .bind(token_info.id.to_string()) @@ -76,7 +126,7 @@ impl SqliteStore { .unwrap_or(0); let rows = if let Some(uid) = user_id { - sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = ? AND user_id = ? AND expires_at > ? LIMIT ? OFFSET ?") + sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = ? AND user_id = ? AND expires_at > ? ORDER BY expires_at DESC, token_id LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(uid.to_string()) .bind(Utc::now().timestamp()) @@ -85,7 +135,7 @@ impl SqliteStore { .fetch_all(&self.pool) .await? } else { - sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = ? AND expires_at > ? LIMIT ? OFFSET ?") + sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = ? AND expires_at > ? ORDER BY expires_at DESC, token_id LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(Utc::now().timestamp()) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/users.rs b/pangolin/pangolin_store/src/sqlite/users.rs index 8164d41..013fd3e 100644 --- a/pangolin/pangolin_store/src/sqlite/users.rs +++ b/pangolin/pangolin_store/src/sqlite/users.rs @@ -56,14 +56,14 @@ impl SqliteStore { .unwrap_or(0); let rows = if let Some(tid) = tenant_id { - sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, created_at, updated_at, last_login, active FROM users WHERE tenant_id = ? LIMIT ? OFFSET ?") + sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, created_at, updated_at, last_login, active FROM users WHERE tenant_id = ? ORDER BY username LIMIT ? OFFSET ?") .bind(tid.to_string()) .bind(limit) .bind(offset) .fetch_all(&self.pool) .await? } else { - sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, created_at, updated_at, last_login, active FROM users LIMIT ? OFFSET ?") + sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, created_at, updated_at, last_login, active FROM users ORDER BY username LIMIT ? OFFSET ?") .bind(limit) .bind(offset) .fetch_all(&self.pool) diff --git a/pangolin/pangolin_store/src/sqlite/warehouses.rs b/pangolin/pangolin_store/src/sqlite/warehouses.rs index 78aa470..49c473e 100644 --- a/pangolin/pangolin_store/src/sqlite/warehouses.rs +++ b/pangolin/pangolin_store/src/sqlite/warehouses.rs @@ -1,12 +1,16 @@ /// Warehouse operations for SqliteStore use super::SqliteStore; +use crate::secrets; use anyhow::Result; use pangolin_core::model::{Warehouse, WarehouseUpdate}; use sqlx::Row; use uuid::Uuid; impl SqliteStore { - pub async fn create_warehouse(&self, tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { + pub async fn create_warehouse(&self, tenant_id: Uuid, mut warehouse: Warehouse) -> Result<()> { + // C-11: see the module note in `crate::secrets`. A SQLite file is the + // easiest of all the backends to walk off with. + secrets::seal(&mut warehouse.storage_config)?; sqlx::query("INSERT INTO warehouses (id, tenant_id, name, use_sts, storage_config, vending_strategy) VALUES (?, ?, ?, ?, ?, ?)") .bind(warehouse.id.to_string()) .bind(tenant_id.to_string()) @@ -35,7 +39,12 @@ impl SqliteStore { tenant_id, name: row.get("name"), use_sts: row.get::("use_sts") != 0, - storage_config: serde_json::from_str(&row.get::("storage_config"))?, + storage_config: { + let mut config: std::collections::HashMap = + serde_json::from_str(&row.get::("storage_config"))?; + secrets::open(&mut config)?; + config + }, vending_strategy, })) } else { @@ -55,7 +64,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, use_sts, storage_config, vending_strategy FROM warehouses WHERE tenant_id = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT id, name, use_sts, storage_config, vending_strategy FROM warehouses WHERE tenant_id = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(limit) .bind(offset) @@ -72,7 +81,12 @@ impl SqliteStore { tenant_id, name: row.get("name"), use_sts: row.get::("use_sts") != 0, - storage_config: serde_json::from_str(&row.get::("storage_config"))?, + storage_config: { + let mut config: std::collections::HashMap = + serde_json::from_str(&row.get::("storage_config"))?; + secrets::open(&mut config)?; + config + }, vending_strategy, }); } @@ -116,7 +130,10 @@ impl SqliteStore { q = q.bind(new_name); } if let Some(config) = &updates.storage_config { - q = q.bind(serde_json::to_string(config)?); + // Seal a rotated credential rather than writing the new one clear. + let mut sealed = config.clone(); + secrets::seal(&mut sealed)?; + q = q.bind(serde_json::to_string(&sealed)?); } if let Some(use_sts) = updates.use_sts { q = q.bind(use_sts as i32); diff --git a/pangolin/pangolin_store/src/tests/mod.rs b/pangolin/pangolin_store/src/tests/mod.rs index bceed31..f06b9a3 100644 --- a/pangolin/pangolin_store/src/tests/mod.rs +++ b/pangolin/pangolin_store/src/tests/mod.rs @@ -273,6 +273,8 @@ pub async fn test_dashboard_stats_consistency(store: &S) { pub mod audit_tests; #[cfg(test)] pub mod multi_cloud; +/// Cross-backend parity suite (roadmap improvement #1). +pub mod parity; #[cfg(test)] pub mod postgres_merge_tests; #[cfg(test)] diff --git a/pangolin/pangolin_store/src/tests/parity.rs b/pangolin/pangolin_store/src/tests/parity.rs new file mode 100644 index 0000000..d08c6e3 --- /dev/null +++ b/pangolin/pangolin_store/src/tests/parity.rs @@ -0,0 +1,808 @@ +//! Cross-backend parity suite. +//! +//! Roadmap improvement #1. Nearly half the storage-layer findings in the +//! August audit (B1-B7, B17-B30) were one backend silently diverging from the +//! other three: a tenant filter dropped on Mongo, a CAS skipped in memory, an +//! enum round-tripping to the wrong variant on all the SQL backends, pagination +//! that repeats or skips rows, four different answers to the same search. +//! +//! Each of those is invisible to a per-backend test, because per-backend tests +//! assert what that backend does. What catches them is asserting that all four +//! backends do the *same* thing. Every function here runs against whichever +//! `CatalogStore` it is handed, and `tests/store_integration.rs` runs the whole +//! set against memory, SQLite, Postgres and Mongo. +//! +//! Each assertion names the finding it locks down, so a regression points +//! straight at what it broke. + +use crate::CatalogStore; +use pangolin_core::business_metadata::BusinessMetadata; +use pangolin_core::model::*; +use std::collections::HashMap; +use uuid::Uuid; + +/// Build the tenant -> warehouse -> catalog -> namespace chain the SQL backends' +/// foreign keys require, and return the tenant id. +async fn seed_hierarchy(store: &S, catalog: &str, namespace: &[String]) -> Uuid { + let tenant_id = Uuid::new_v4(); + + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("parity_{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + let _ = store + .create_warehouse( + tenant_id, + Warehouse { + id: Uuid::new_v4(), + name: "wh".to_string(), + tenant_id, + storage_config: HashMap::new(), + use_sts: false, + vending_strategy: None, + }, + ) + .await; + + store + .create_catalog( + tenant_id, + Catalog { + id: Uuid::new_v4(), + name: catalog.to_string(), + catalog_type: CatalogType::Local, + warehouse_name: Some("wh".to_string()), + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }, + ) + .await + .expect("create catalog"); + + store + .create_namespace( + tenant_id, + catalog, + Namespace { + name: namespace.to_vec(), + properties: HashMap::new(), + }, + ) + .await + .expect("create namespace"); + + tenant_id +} + +fn asset(name: &str, kind: AssetType) -> Asset { + Asset { + id: Uuid::new_v4(), + name: name.to_string(), + kind, + location: format!("s3://bucket/{name}"), + properties: HashMap::new(), + } +} + +/// **B7.** Every `AssetType` variant must survive a write/read round trip. +/// +/// All three persistent backends stored the Debug spelling and parsed only +/// `IcebergTable`/`View`, defaulting the other 15 variants to `IcebergTable` - +/// so a `DeltaTable` came back as an Iceberg table with no error anywhere. +pub async fn asset_types_round_trip(store: &S) { + let catalog = "types"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + for kind in AssetType::all() { + let name = format!("asset_{}", kind.as_stored_str().to_lowercase()); + let written = asset(&name, kind.clone()); + + store + .create_asset(tenant_id, catalog, None, namespace.clone(), written.clone()) + .await + .expect("create asset"); + + let read = store + .get_asset(tenant_id, catalog, None, namespace.clone(), name.clone()) + .await + .expect("get asset") + .unwrap_or_else(|| panic!("asset {name} vanished")); + + assert_eq!( + read.kind, kind, + "asset type {kind:?} did not round-trip (B7); it came back as {:?}", + read.kind + ); + } +} + +/// **B27.** Two pages must cover the set exactly once. +/// +/// Every paginated query outside three Postgres call sites ran `LIMIT/OFFSET` +/// with no `ORDER BY`, and the memory backend paged over DashMap iteration +/// order. Both can repeat or skip a row between pages, which is invisible until +/// a client silently misses data. +pub async fn pagination_covers_the_set_exactly_once(store: &S) { + let catalog = "paging"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + const TOTAL: usize = 7; + const PAGE: usize = 3; + + for i in 0..TOTAL { + store + .create_asset( + tenant_id, + catalog, + None, + namespace.clone(), + // Zero-padded so lexical and numeric order agree, and the + // assertion is about paging rather than about collation. + asset(&format!("t{i:03}"), AssetType::IcebergTable), + ) + .await + .expect("create asset"); + } + + let mut seen: Vec = Vec::new(); + let mut offset = 0; + loop { + let page = store + .list_assets( + tenant_id, + catalog, + None, + namespace.clone(), + Some(crate::PaginationParams { + limit: Some(PAGE), + offset: Some(offset), + }), + ) + .await + .expect("list assets"); + + if page.is_empty() { + break; + } + seen.extend(page.iter().map(|a| a.name.clone())); + offset += PAGE; + + // Guard against a backend that ignores offset and loops forever. + assert!(offset <= TOTAL * 4, "pagination did not terminate"); + } + + let mut unique = seen.clone(); + unique.sort(); + unique.dedup(); + + assert_eq!( + unique.len(), + seen.len(), + "paging returned a duplicate row (B27): {seen:?}" + ); + assert_eq!( + unique.len(), + TOTAL, + "paging skipped a row (B27): saw {} of {TOTAL}", + unique.len() + ); +} + +/// **B27.** Repeating the same listing must return the same order. +pub async fn listing_order_is_stable(store: &S) { + let catalog = "stable"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + for name in ["delta", "alpha", "charlie", "bravo"] { + store + .create_asset( + tenant_id, + catalog, + None, + namespace.clone(), + asset(name, AssetType::IcebergTable), + ) + .await + .expect("create asset"); + } + + let first: Vec = store + .list_assets(tenant_id, catalog, None, namespace.clone(), None) + .await + .expect("list assets") + .into_iter() + .map(|a| a.name) + .collect(); + + for _ in 0..3 { + let again: Vec = store + .list_assets(tenant_id, catalog, None, namespace.clone(), None) + .await + .expect("list assets") + .into_iter() + .map(|a| a.name) + .collect(); + assert_eq!(again, first, "listing order is not stable (B27)"); + } +} + +/// **B2 / B0j.** A revoked token must read back as revoked. +/// +/// On Mongo the revocation write and the revocation *check* used different +/// field names and different types, so the check could never match: revocation +/// - including logout - was a silent no-op. +pub async fn revocation_round_trips(store: &S) { + let token_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + + assert!( + !store + .is_token_revoked(token_id) + .await + .expect("check revocation"), + "a fresh token must not be revoked" + ); + + store + .revoke_token(token_id, expires_at, Some("parity test".to_string())) + .await + .expect("revoke token"); + + assert!( + store + .is_token_revoked(token_id) + .await + .expect("check revocation"), + "a revoked token must read back as revoked (B2)" + ); +} + +/// **B1.** An audit event must not be readable from another tenant. +pub async fn audit_events_are_tenant_scoped(store: &S) { + let owner = Uuid::new_v4(); + let stranger = Uuid::new_v4(); + + for tenant in [owner, stranger] { + store + .create_tenant(Tenant { + id: tenant, + name: format!("audit_{tenant}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + } + + let entry = pangolin_core::audit::AuditLogEntry::success( + owner, + Some(Uuid::new_v4()), + "owner".to_string(), + pangolin_core::audit::AuditAction::CreateCatalog, + pangolin_core::audit::ResourceType::Catalog, + Some(Uuid::new_v4()), + "secret_catalog".to_string(), + ); + let event_id = entry.id; + + store + .log_audit_event(owner, entry) + .await + .expect("log audit event"); + + assert!( + store + .get_audit_event(owner, event_id) + .await + .expect("get audit event") + .is_some(), + "the owning tenant must be able to read its own audit event" + ); + + assert!( + store + .get_audit_event(stranger, event_id) + .await + .expect("get audit event") + .is_none(), + "an audit event must not be readable across tenants (B1)" + ); +} + +/// **B22 / B23.** Audit actions must round-trip, not collapse to a default. +/// +/// SQLite persisted the Debug spelling and parsed snake_case, then swallowed the +/// mismatch with `unwrap_or(CreateCatalog)`, so nearly every multi-word action +/// was misattributed. Mongo's filters had the mirror-image problem and always +/// matched zero rows. +pub async fn audit_actions_round_trip(store: &S) { + use pangolin_core::audit::{AuditAction, AuditLogEntry, ResourceType}; + + let tenant_id = Uuid::new_v4(); + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("audit_actions_{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + // Deliberately multi-word: the single-word ones happened to survive. + let actions = [ + AuditAction::CreateBranch, + AuditAction::DeleteNamespace, + AuditAction::CommitTable, + ]; + + for action in &actions { + store + .log_audit_event( + tenant_id, + AuditLogEntry::success( + tenant_id, + Some(Uuid::new_v4()), + "auditor".to_string(), + action.clone(), + ResourceType::Table, + Some(Uuid::new_v4()), + format!("{action:?}_target"), + ), + ) + .await + .expect("log audit event"); + } + + let events = store + .list_audit_events(tenant_id, None) + .await + .expect("list audit events"); + + for action in &actions { + assert!( + events.iter().any(|e| e.action == *action), + "audit action {action:?} did not round-trip (B22); \ + the listing held {:?}", + events.iter().map(|e| &e.action).collect::>() + ); + } +} + +/// **B28.** A search term containing LIKE metacharacters must be literal. +pub async fn search_treats_wildcards_literally(store: &S) { + let catalog = "search"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + for name in ["margin_100pct", "unrelated_table"] { + store + .create_asset( + tenant_id, + catalog, + None, + namespace.clone(), + asset(name, AssetType::IcebergTable), + ) + .await + .expect("create asset"); + } + + // `%` is a LIKE wildcard. Unescaped, this matched everything. + let hits = store + .search_assets(tenant_id, "%", None) + .await + .expect("search assets"); + + assert!( + hits.is_empty(), + "a literal '%' matched {} assets (B28); the term was treated as a wildcard", + hits.len() + ); + + // A genuine substring still matches. + let hits = store + .search_assets(tenant_id, "margin", None) + .await + .expect("search assets"); + assert_eq!( + hits.len(), + 1, + "an ordinary substring search should still find its asset" + ); +} + +/// **B28.** Tag filtering is ALL-match, and an empty filter is no filter. +pub async fn tag_filter_semantics_agree(store: &S) { + let catalog = "tags"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let both = asset("tagged_both", AssetType::IcebergTable); + let one = asset("tagged_one", AssetType::IcebergTable); + + for a in [&both, &one] { + store + .create_asset(tenant_id, catalog, None, namespace.clone(), a.clone()) + .await + .expect("create asset"); + } + + let mut meta_both = BusinessMetadata::new(both.id, Uuid::new_v4()); + meta_both.tags = vec!["pii".to_string(), "finance".to_string()]; + store + .upsert_business_metadata(meta_both) + .await + .expect("upsert metadata"); + + let mut meta_one = BusinessMetadata::new(one.id, Uuid::new_v4()); + meta_one.tags = vec!["pii".to_string()]; + store + .upsert_business_metadata(meta_one) + .await + .expect("upsert metadata"); + + // One tag: both assets carry it. + let hits = store + .search_assets(tenant_id, "tagged", Some(vec!["pii".to_string()])) + .await + .expect("search assets"); + assert_eq!(hits.len(), 2, "single-tag filter should match both assets"); + + // Two tags: ALL-match, so only the asset carrying both. + let hits = store + .search_assets( + tenant_id, + "tagged", + Some(vec!["pii".to_string(), "finance".to_string()]), + ) + .await + .expect("search assets"); + assert_eq!( + hits.len(), + 1, + "the tag filter must be ALL-match (B28): every requested tag has to be present" + ); + + // An empty filter is not a filter. + let hits = store + .search_assets(tenant_id, "tagged", Some(vec![])) + .await + .expect("search assets"); + assert_eq!( + hits.len(), + 2, + "an empty tag list must mean 'no tag filter' (B28)" + ); +} + +/// **B26 / B5.** The compare-and-swap must actually compare. +/// +/// Mongo ignored `expected_location` entirely, so two concurrent commits both +/// "succeeded" and one snapshot was lost; memory skipped the check whenever the +/// expectation was `None`. +pub async fn metadata_cas_rejects_a_stale_writer(store: &S) { + let catalog = "cas"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let table = "committed".to_string(); + let v1 = "s3://bucket/cas/v1.json".to_string(); + let mut a = asset(&table, AssetType::IcebergTable); + a.location = v1.clone(); + a.properties + .insert("metadata_location".to_string(), v1.clone()); + + store + .create_asset(tenant_id, catalog, None, namespace.clone(), a) + .await + .expect("create asset"); + + // Writer A wins. + let v2 = "s3://bucket/cas/v2.json".to_string(); + store + .update_metadata_location( + tenant_id, + catalog, + None, + namespace.clone(), + table.clone(), + Some(v1.clone()), + v2.clone(), + ) + .await + .expect("the first writer's CAS should succeed"); + + // Writer B still believes the table is at v1: it must be refused. + let v3 = "s3://bucket/cas/v3.json".to_string(); + let stale = store + .update_metadata_location( + tenant_id, + catalog, + None, + namespace.clone(), + table.clone(), + Some(v1.clone()), + v3, + ) + .await; + + assert!( + stale.is_err(), + "a stale writer's CAS must fail (B5/B26); it silently overwrote the winner" + ); + + let current = store + .get_metadata_location(tenant_id, catalog, None, namespace, table) + .await + .expect("get metadata location"); + assert_eq!( + current, + Some(v2), + "the losing writer must not have changed the published metadata" + ); +} + +/// **B6 / B30.** Deleting one tenant's data must not disturb another's. +/// +/// Catalog names are per-tenant, but the memory backend's by-id asset index was +/// keyed on catalog name alone, so deleting tenant A's `sales` broke +/// `get_asset_by_id` for tenant B's unrelated `sales`. +pub async fn deleting_a_catalog_does_not_touch_another_tenant(store: &S) { + let catalog = "sales"; + let namespace = vec!["ns".to_string()]; + + let tenant_a = seed_hierarchy(store, catalog, &namespace).await; + let tenant_b = seed_hierarchy(store, catalog, &namespace).await; + + let a_asset = asset("orders", AssetType::IcebergTable); + let b_asset = asset("orders", AssetType::IcebergTable); + + store + .create_asset(tenant_a, catalog, None, namespace.clone(), a_asset.clone()) + .await + .expect("create asset"); + store + .create_asset(tenant_b, catalog, None, namespace.clone(), b_asset.clone()) + .await + .expect("create asset"); + + store + .delete_catalog(tenant_a, catalog.to_string()) + .await + .expect("delete catalog"); + + let survivor = store + .get_asset_by_id(tenant_b, b_asset.id) + .await + .expect("get asset by id"); + + assert!( + survivor.is_some(), + "deleting tenant A's catalog must not evict tenant B's identically-named \ + catalog from the by-id index (B6)" + ); +} + +/// **B21.** Deleting a catalog that does not exist must destroy nothing. +pub async fn deleting_a_missing_catalog_destroys_nothing(store: &S) { + let catalog = "present"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let kept = asset("keep_me", AssetType::IcebergTable); + store + .create_asset(tenant_id, catalog, None, namespace.clone(), kept.clone()) + .await + .expect("create asset"); + + // A namespace under a catalog name that has no catalog row. + // + // This is the state a cascade-before-existence-check destroys, and it is + // reachable: `namespaces` carries a foreign key to `tenants` only, on every + // backend. (An orphaned *asset* is not portable - Postgres has a composite + // key from assets to namespaces, so it refuses one. That strictness is + // worth knowing about; it is also why this uses the child row that every + // backend does allow to be orphaned.) + // + // An earlier version of this test named a catalog with no children at all, + // which passes whatever the ordering is. It is worth being precise about + // what a regression test actually constrains. + let orphan_ns = vec!["orphaned".to_string()]; + let orphan_created = store + .create_namespace( + tenant_id, + "absent", + Namespace { + name: orphan_ns.clone(), + properties: HashMap::new(), + }, + ) + .await + .is_ok(); + + let result = store.delete_catalog(tenant_id, "absent".to_string()).await; + assert!( + result.is_err(), + "deleting a nonexistent catalog should be an error" + ); + + if orphan_created { + // The backend permits orphaned children, so the ordering is observable: + // a cascade that runs before the existence check destroys them and only + // then reports "not found". + let orphan_survived = store + .get_namespace(tenant_id, "absent", orphan_ns) + .await + .expect("get orphaned namespace"); + assert!( + orphan_survived.is_some(), + "a failed delete_catalog destroyed the rows it matched before \ + discovering the catalog did not exist (B21)" + ); + } + // Otherwise the backend enforces referential integrity between namespaces + // and catalogs and the orphaned state is unreachable, so there is nothing + // for a mis-ordered cascade to destroy. Postgres is the one that does; the + // deliberate asymmetry is recorded rather than papered over, because + // "Postgres refuses to create the fixture" and "Postgres passed the + // assertion" are very different facts. + + let still_there = store + .get_asset(tenant_id, catalog, None, namespace, kept.name.clone()) + .await + .expect("get asset"); + assert!( + still_there.is_some(), + "a failed delete_catalog must not have destroyed another catalog's assets (B21)" + ); +} + +/// **B16h.** Namespace property removals must actually remove. +pub async fn namespace_property_replacement_removes_keys(store: &S) { + let catalog = "props"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let mut properties = HashMap::new(); + properties.insert("keep".to_string(), "yes".to_string()); + properties.insert("drop".to_string(), "please".to_string()); + + store + .update_namespace_properties(tenant_id, catalog, namespace.clone(), properties) + .await + .expect("seed properties"); + + let mut remaining = HashMap::new(); + remaining.insert("keep".to_string(), "yes".to_string()); + + store + .replace_namespace_properties(tenant_id, catalog, namespace.clone(), remaining) + .await + .expect("replace properties"); + + let ns = store + .get_namespace(tenant_id, catalog, namespace) + .await + .expect("get namespace") + .expect("namespace should exist"); + + assert_eq!(ns.properties.get("keep").map(String::as_str), Some("yes")); + assert!( + !ns.properties.contains_key("drop"), + "replace_namespace_properties must drop keys the caller left out (B16h)" + ); +} + +/// **B17 / B19.** A multi-level namespace must be usable, not just creatable. +pub async fn nested_namespaces_are_addressable(store: &S) { + let catalog = "nested"; + let namespace = vec!["outer".to_string(), "inner".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let found = store + .get_namespace(tenant_id, catalog, namespace.clone()) + .await + .expect("get namespace"); + assert!( + found.is_some(), + "a nested namespace must be retrievable by its full path (B17)" + ); + + let mut properties = HashMap::new(); + properties.insert("level".to_string(), "two".to_string()); + store + .update_namespace_properties(tenant_id, catalog, namespace.clone(), properties) + .await + .expect("a nested namespace must be updatable (B17)"); + + store + .delete_namespace(tenant_id, catalog, namespace.clone()) + .await + .expect("a nested namespace must be deletable (B17)"); + + assert!( + store + .get_namespace(tenant_id, catalog, namespace) + .await + .expect("get namespace") + .is_none(), + "the namespace should be gone after delete" + ); +} + +/// **B24.** A federated catalog must still look federated in a *listing*. +pub async fn catalog_type_survives_a_listing(store: &S) { + let tenant_id = Uuid::new_v4(); + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("fed_{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + store + .create_catalog( + tenant_id, + Catalog { + id: Uuid::new_v4(), + name: "remote".to_string(), + catalog_type: CatalogType::Federated, + warehouse_name: None, + storage_location: None, + federated_config: Some(FederatedCatalogConfig { + properties: HashMap::new(), + }), + properties: HashMap::new(), + }, + ) + .await + .expect("create catalog"); + + let listed = store + .list_catalogs(tenant_id, None) + .await + .expect("list catalogs"); + + let remote = listed + .iter() + .find(|c| c.name == "remote") + .expect("the federated catalog should be listed"); + + assert_eq!( + remote.catalog_type, + CatalogType::Federated, + "list_catalogs must report the real catalog type (B24); \ + Postgres used to hardcode Local" + ); +} + +/// Run the whole parity suite against one backend. +/// +/// `tests/store_integration.rs` calls this for each of the four, which is what +/// turns "this backend behaves like this" into "all four behave alike". +pub async fn run_all(store: &S) { + asset_types_round_trip(store).await; + pagination_covers_the_set_exactly_once(store).await; + listing_order_is_stable(store).await; + revocation_round_trips(store).await; + audit_events_are_tenant_scoped(store).await; + audit_actions_round_trip(store).await; + search_treats_wildcards_literally(store).await; + tag_filter_semantics_agree(store).await; + metadata_cas_rejects_a_stale_writer(store).await; + deleting_a_catalog_does_not_touch_another_tenant(store).await; + deleting_a_missing_catalog_destroys_nothing(store).await; + namespace_property_replacement_removes_keys(store).await; + nested_namespaces_are_addressable(store).await; + catalog_type_survives_a_listing(store).await; +} diff --git a/pangolin/pangolin_store/tests/branch_atomicity_tests.rs b/pangolin/pangolin_store/tests/branch_atomicity_tests.rs new file mode 100644 index 0000000..dab20a7 --- /dev/null +++ b/pangolin/pangolin_store/tests/branch_atomicity_tests.rs @@ -0,0 +1,312 @@ +//! Creating a branch by copy must be all-or-nothing. +//! +//! A-24's remainder. The branch row and its copied assets used to be written by +//! independent statements, so a failure between them left a branch that existed +//! holding an arbitrary subset of its assets — with no rollback and no repair +//! tool. The API then returned `200`, because the handler logged the copy error +//! and carried on. +//! +//! The interesting assertion is the negative one: after a failed create, the +//! branch must not exist *at all*. A test that only checks the happy path would +//! pass against the old non-transactional code. + +use pangolin_core::model::{ + Asset, AssetType, Branch, BranchType, Catalog, CatalogType, Namespace, Tenant, +}; +use pangolin_store::{CatalogStore, PostgresStore, SqliteStore}; +use std::collections::HashMap; +use uuid::Uuid; + +/// A catalog with two assets on `main`, ready to branch from. +async fn seed(store: &dyn CatalogStore) -> (Uuid, String) { + let tenant_id = Uuid::new_v4(); + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("t-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + store + .create_catalog( + tenant_id, + Catalog { + id: Uuid::new_v4(), + name: "cat".to_string(), + catalog_type: CatalogType::Local, + warehouse_name: None, + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }, + ) + .await + .expect("create catalog"); + + store + .create_namespace( + tenant_id, + "cat", + Namespace { + name: vec!["sales".to_string()], + properties: HashMap::new(), + }, + ) + .await + .expect("create namespace"); + + store + .create_branch( + tenant_id, + "cat", + Branch { + name: "main".to_string(), + head_commit_id: None, + branch_type: BranchType::Experimental, + assets: vec![], + }, + ) + .await + .expect("create main"); + + for name in ["orders", "customers"] { + store + .create_asset( + tenant_id, + "cat", + Some("main".to_string()), + vec!["sales".to_string()], + Asset { + id: Uuid::new_v4(), + name: name.to_string(), + kind: AssetType::IcebergTable, + location: format!("s3://bucket/{name}"), + properties: HashMap::new(), + }, + ) + .await + .expect("create asset"); + } + + (tenant_id, "cat".to_string()) +} + +async fn happy_path(store: &dyn CatalogStore, backend: &str) { + let (tenant_id, catalog) = seed(store).await; + + let copied = store + .create_branch_with_assets( + tenant_id, + &catalog, + Branch { + name: "feature".to_string(), + head_commit_id: None, + branch_type: BranchType::Experimental, + assets: vec![], + }, + "main", + None, + ) + .await + .unwrap_or_else(|e| panic!("{backend}: create_branch_with_assets failed: {e}")); + + assert_eq!(copied, 2, "{backend}: both assets should have been copied"); + + let branch = store + .get_branch(tenant_id, &catalog, "feature".to_string()) + .await + .expect("get branch") + .unwrap_or_else(|| panic!("{backend}: the branch should exist")); + assert_eq!(branch.name, "feature"); + + let assets = store + .list_assets( + tenant_id, + &catalog, + Some("feature".to_string()), + vec!["sales".to_string()], + None, + ) + .await + .expect("list assets"); + assert_eq!( + assets.len(), + 2, + "{backend}: the new branch should carry both assets, saw {assets:?}" + ); + + // The source must be untouched - a copy, not a move. + let source = store + .list_assets( + tenant_id, + &catalog, + Some("main".to_string()), + vec!["sales".to_string()], + None, + ) + .await + .expect("list source assets"); + assert_eq!( + source.len(), + 2, + "{backend}: copying must not empty the source" + ); +} + +/// The assertion that distinguishes this from the old code. +async fn rolls_back(store: &dyn CatalogStore, backend: &str) { + let (tenant_id, catalog) = seed(store).await; + + // A malformed asset name fails after the branch row has been inserted, + // which is exactly the window that used to leave a half-made branch. + let result = store + .create_branch_with_assets( + tenant_id, + &catalog, + Branch { + name: "doomed".to_string(), + head_commit_id: None, + branch_type: BranchType::Experimental, + assets: vec![], + }, + "main", + Some(vec!["this-name-has-no-namespace".to_string()]), + ) + .await; + + assert!( + result.is_err(), + "{backend}: a malformed asset name must fail rather than silently copying nothing" + ); + + let branch = store + .get_branch(tenant_id, &catalog, "doomed".to_string()) + .await + .expect("get branch"); + assert!( + branch.is_none(), + "{backend}: the branch row survived a failed create. The transaction did \ + not roll back, which is the defect this test exists for." + ); +} + +/// Copying a named subset must take only those assets. +async fn selective_copy(store: &dyn CatalogStore, backend: &str) { + let (tenant_id, catalog) = seed(store).await; + + let copied = store + .create_branch_with_assets( + tenant_id, + &catalog, + Branch { + name: "partial".to_string(), + head_commit_id: None, + branch_type: BranchType::Experimental, + assets: vec![], + }, + "main", + Some(vec!["sales.orders".to_string()]), + ) + .await + .unwrap_or_else(|e| panic!("{backend}: selective copy failed: {e}")); + + assert_eq!(copied, 1, "{backend}: only one asset was named"); + + let assets = store + .list_assets( + tenant_id, + &catalog, + Some("partial".to_string()), + vec!["sales".to_string()], + None, + ) + .await + .expect("list assets"); + assert_eq!(assets.len(), 1, "{backend}: saw {assets:?}"); + assert_eq!(assets[0].name, "orders"); +} + +async fn sqlite_store() -> (SqliteStore, tempdir::Guard) { + let guard = tempdir::Guard::new(); + let url = format!( + "sqlite://{}?mode=rwc", + guard.path().join("catalog.db").display() + ); + let store = SqliteStore::new(&url).await.expect("open sqlite"); + store.run_migrations().await.expect("apply the schema"); + (store, guard) +} + +/// A temporary directory that removes itself, so a failing test does not leave +/// databases behind in /tmp. +mod tempdir { + pub struct Guard(std::path::PathBuf); + + impl Guard { + pub fn new() -> Self { + let path = + std::env::temp_dir().join(format!("pangolin-branch-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&path).expect("temp dir"); + Self(path) + } + pub fn path(&self) -> &std::path::Path { + &self.0 + } + } + + impl Drop for Guard { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } +} + +#[tokio::test] +async fn sqlite_creates_a_branch_with_its_assets() { + let (store, _guard) = sqlite_store().await; + happy_path(&store, "sqlite").await; +} + +#[tokio::test] +async fn sqlite_rolls_back_a_failed_create() { + let (store, _guard) = sqlite_store().await; + rolls_back(&store, "sqlite").await; +} + +#[tokio::test] +async fn sqlite_copies_only_the_named_assets() { + let (store, _guard) = sqlite_store().await; + selective_copy(&store, "sqlite").await; +} + +#[tokio::test] +async fn postgres_creates_a_branch_with_its_assets() { + let Some(url) = pangolin_store::test_support::postgres_url() else { + println!("skipping: set PANGOLIN_TEST_POSTGRES_URL to run this test"); + return; + }; + let store = PostgresStore::new(&url).await.expect("open postgres"); + happy_path(&store, "postgres").await; +} + +#[tokio::test] +async fn postgres_rolls_back_a_failed_create() { + let Some(url) = pangolin_store::test_support::postgres_url() else { + println!("skipping: set PANGOLIN_TEST_POSTGRES_URL to run this test"); + return; + }; + let store = PostgresStore::new(&url).await.expect("open postgres"); + rolls_back(&store, "postgres").await; +} + +#[tokio::test] +async fn postgres_copies_only_the_named_assets() { + let Some(url) = pangolin_store::test_support::postgres_url() else { + println!("skipping: set PANGOLIN_TEST_POSTGRES_URL to run this test"); + return; + }; + let store = PostgresStore::new(&url).await.expect("open postgres"); + selective_copy(&store, "postgres").await; +} diff --git a/pangolin/pangolin_store/tests/mongo_audit_tests.rs b/pangolin/pangolin_store/tests/mongo_audit_tests.rs index 477890e..8f1af15 100644 --- a/pangolin/pangolin_store/tests/mongo_audit_tests.rs +++ b/pangolin/pangolin_store/tests/mongo_audit_tests.rs @@ -196,11 +196,21 @@ async fn test_mongo_audit_log_filtering() { // Test 10: Get individual event let event_id = logs[0].id; - let event = store.get_audit_event(event_id).await.unwrap(); + let event = store.get_audit_event(tenant_id, event_id).await.unwrap(); assert!(event.is_some(), "Should find the event"); assert_eq!(event.unwrap().id, event_id); println!("✓ Test 10 passed: Get individual event"); + // B1 regression: an event must not be readable from another tenant, even + // by someone who knows its UUID. + let other_tenant = Uuid::new_v4(); + let leaked = store.get_audit_event(other_tenant, event_id).await.unwrap(); + assert!( + leaked.is_none(), + "an audit event must not be readable across tenants (B1)" + ); + println!("✓ Test 10b passed: Audit events are tenant-scoped"); + println!("\n✅ All MongoDB audit logging tests passed!"); } diff --git a/pangolin/pangolin_store/tests/mongo_index_tests.rs b/pangolin/pangolin_store/tests/mongo_index_tests.rs new file mode 100644 index 0000000..614efe7 --- /dev/null +++ b/pangolin/pangolin_store/tests/mongo_index_tests.rs @@ -0,0 +1,178 @@ +//! The indexes this backend needs must actually exist after startup. +//! +//! `indexes.rs` has unit tests over the *table* of indexes; they prove the list +//! is sensible and say nothing about whether MongoDB accepted any of it. The +//! previous code created two indexes and threw the result away with `.ok()`, so +//! "we create indexes" was true and "the indexes exist" was unverified. +//! +//! Requires a MongoDB; skips with a note when `PANGOLIN_TEST_MONGO_URL` is +//! unset. + +use mongodb::bson::Document; +use pangolin_store::MongoStore; +use uuid::Uuid; + +/// The index keys present on a collection, as `{"field": 1}` documents. +async fn index_keys(db: &mongodb::Database, collection: &str) -> Vec { + let mut cursor = db + .collection::(collection) + .list_indexes() + .await + .expect("list indexes"); + + let mut keys = Vec::new(); + while cursor.advance().await.expect("advance") { + let model = cursor.deserialize_current().expect("deserialize index"); + keys.push(model.keys); + } + keys +} + +fn has_index_on(indexes: &[Document], fields: &[&str]) -> bool { + indexes + .iter() + .any(|keys| keys.len() == fields.len() && fields.iter().all(|f| keys.contains_key(*f))) +} + +macro_rules! store_or_skip { + ($db:ident) => {{ + let Some(url) = pangolin_store::test_support::mongo_url() else { + println!("skipping: set PANGOLIN_TEST_MONGO_URL to run this test"); + return; + }; + let $db = format!("pangolin_idx_{}", Uuid::new_v4().simple()); + let store = MongoStore::new(&url, &$db).await.expect("open mongo"); + let client = mongodb::Client::with_uri_str(&url) + .await + .expect("own client"); + (store, client.database(&$db)) + }}; +} + +#[tokio::test] +async fn the_authorization_hot_path_is_indexed_in_the_database() { + let (_store, db) = store_or_skip!(name); + + // These are consulted on every authenticated request. Before this, each was + // a collection scan. + for (collection, fields) in [ + ("user_roles", vec!["user-id"]), + ("permissions", vec!["user-id"]), + ("revoked_tokens", vec!["token_id"]), + ("service_users", vec!["api-key-hash"]), + ] { + let indexes = index_keys(&db, collection).await; + assert!( + has_index_on(&indexes, &fields), + "{collection} has no index on {fields:?}; this is read on every \ + request. Present: {indexes:?}" + ); + } + + let _ = db.drop().await; +} + +#[tokio::test] +async fn catalog_lookups_are_indexed() { + let (_store, db) = store_or_skip!(name); + + for collection in ["catalogs", "warehouses"] { + let indexes = index_keys(&db, collection).await; + assert!( + has_index_on(&indexes, &["tenant_id", "name"]), + "{collection} has no (tenant_id, name) index. Present: {indexes:?}" + ); + } + + let indexes = index_keys(&db, "assets").await; + assert!( + has_index_on(&indexes, &["tenant_id", "catalog_name", "branch_name"]), + "assets has no branch index; listing or copying a branch scans the \ + whole collection. Present: {indexes:?}" + ); + + let _ = db.drop().await; +} + +/// The uniqueness the SQL backends get from primary keys. +/// +/// Without this MongoDB holds two catalogs of the same name in one tenant and +/// returns an arbitrary one, which is a correctness difference from the other +/// three backends rather than a performance one. +#[tokio::test] +async fn duplicate_catalog_names_are_rejected() { + let Some(url) = pangolin_store::test_support::mongo_url() else { + println!("skipping: set PANGOLIN_TEST_MONGO_URL to run this test"); + return; + }; + let db_name = format!("pangolin_idx_{}", Uuid::new_v4().simple()); + let store = MongoStore::new(&url, &db_name).await.expect("open mongo"); + + use pangolin_core::model::{Catalog, CatalogType, Tenant}; + use pangolin_store::CatalogStore; + use std::collections::HashMap; + + let tenant_id = Uuid::new_v4(); + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("t-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + let catalog = |name: &str| Catalog { + id: Uuid::new_v4(), + name: name.to_string(), + catalog_type: CatalogType::Local, + warehouse_name: None, + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }; + + store + .create_catalog(tenant_id, catalog("dupe")) + .await + .expect("the first create should succeed"); + + let second = store.create_catalog(tenant_id, catalog("dupe")).await; + assert!( + second.is_err(), + "MongoDB accepted a second catalog named 'dupe' in the same tenant. \ + PostgreSQL rejects this with a primary-key violation; without the \ + unique index the two backends disagree about what is valid." + ); + + let client = mongodb::Client::with_uri_str(&url).await.expect("client"); + let _ = client.database(&db_name).drop().await; +} + +/// Running twice must be a no-op, because it runs on every startup. +#[tokio::test] +async fn creating_the_indexes_is_idempotent() { + let Some(url) = pangolin_store::test_support::mongo_url() else { + println!("skipping: set PANGOLIN_TEST_MONGO_URL to run this test"); + return; + }; + let db_name = format!("pangolin_idx_{}", Uuid::new_v4().simple()); + + let _first = MongoStore::new(&url, &db_name).await.expect("first open"); + let client = mongodb::Client::with_uri_str(&url).await.expect("client"); + let db = client.database(&db_name); + let before = index_keys(&db, "catalogs").await.len(); + + // A second server starting against the same database, which is what a + // rolling restart does. + let _second = MongoStore::new(&url, &db_name).await.expect("second open"); + let after = index_keys(&db, "catalogs").await.len(); + + assert_eq!( + before, after, + "a second startup changed the index set; this runs on every boot and \ + must be a no-op" + ); + + let _ = db.drop().await; +} diff --git a/pangolin/pangolin_store/tests/mongo_uuid_round_trip_tests.rs b/pangolin/pangolin_store/tests/mongo_uuid_round_trip_tests.rs new file mode 100644 index 0000000..d42f8d7 --- /dev/null +++ b/pangolin/pangolin_store/tests/mongo_uuid_round_trip_tests.rs @@ -0,0 +1,969 @@ +//! Every MongoDB entity must survive a write/read round trip. +//! +//! There are three ways this codebase turns a `Uuid` into BSON, and they do not +//! agree: +//! +//! | route | produces | +//! |---|---| +//! | `to_bson_uuid` | `Binary`, generic subtype | +//! | `doc! { "k": uuid }` | `Binary`, *UUID* subtype | +//! | `bson::to_document(&value)` | a `String` | +//! +//! And two ways of reading one back, which also disagree: a typed +//! `Collection` deserializes non-human-readably and demands binary, while +//! `bson::from_bson` demands a string. So a write and a read chosen +//! independently - which is how every one of these modules was written - agree +//! only by luck. The failure is never a compile error and usually not a runtime +//! error either: the filter simply matches nothing. +//! +//! What that cost, before this file existed: +//! +//! * **B1** audit events were readable across tenants, because the tenant +//! filter was dropped rather than fixed; +//! * **B2** token revocation was a silent no-op - revoked JWTs stayed valid, +//! and logout did nothing; +//! * role assignments were unreadable, so every role-derived permission +//! silently vanished and **a user holding an admin role was authorized as +//! though they held none**; +//! * permission records could not be deserialized at all; +//! * business metadata could be written but never read back; +//! * **every** service-user method was a no-op, including the API-key lookup, +//! so API-key authentication could not succeed on Mongo at all; +//! * listing active tokens failed outright; +//! * a branch with a head commit - that is, any branch that has been committed +//! to - could not be read. +//! +//! Each was found separately, by a different failing test, months apart. Fixing +//! them one at a time treats eight symptoms of one cause. This file is the +//! cause-level test: it round-trips every entity that carries a `Uuid`, so a +//! new collection whose write and read disagree fails here immediately rather +//! than in whatever feature happens to touch it next. +//! +//! The rule the fixes settled on: **write through `to_bson_uuid`, read through +//! `from_bson_uuid`**, which accepts all three encodings so data already in a +//! deployed database still loads. +//! +//! Requires a MongoDB; skips with a note when `PANGOLIN_TEST_MONGO_URL` is +//! unset. CI sets it for both topologies. + +use chrono::Utc; +use pangolin_core::audit::{AuditAction, AuditLogEntry, ResourceType}; +use pangolin_core::business_metadata::{AccessRequest, BusinessMetadata, RequestStatus}; +use pangolin_core::model::*; +use pangolin_core::permission::{Action, Permission, PermissionScope, Role, UserRole}; +use pangolin_core::token::TokenInfo; +use pangolin_core::user::{ServiceUser, User, UserRole as UserRoleEnum}; +use pangolin_store::{CatalogStore, MongoStore}; +use std::collections::{HashMap, HashSet}; +use uuid::Uuid; + +/// Open a store against a database unique to this test, or skip. +macro_rules! store_or_skip { + () => {{ + let Some(url) = pangolin_store::test_support::mongo_url() else { + println!("skipping: set PANGOLIN_TEST_MONGO_URL to run this test"); + return; + }; + let db = format!("pangolin_roundtrip_{}", Uuid::new_v4().simple()); + match MongoStore::new(&url, &db).await { + Ok(store) => store, + Err(e) => panic!("PANGOLIN_TEST_MONGO_URL is set but unusable: {e}"), + } + }}; +} + +async fn seeded_tenant(store: &MongoStore) -> Uuid { + let tenant_id = Uuid::new_v4(); + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("roundtrip-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + tenant_id +} + +#[tokio::test] +async fn tenant_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let read = store + .get_tenant(tenant_id) + .await + .expect("get tenant") + .expect("tenant should exist"); + assert_eq!(read.id, tenant_id, "tenant id did not round-trip"); +} + +#[tokio::test] +async fn warehouse_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let warehouse = Warehouse { + id: Uuid::new_v4(), + name: "wh".to_string(), + tenant_id, + storage_config: HashMap::new(), + use_sts: false, + vending_strategy: None, + }; + store + .create_warehouse(tenant_id, warehouse.clone()) + .await + .expect("create warehouse"); + + let read = store + .get_warehouse(tenant_id, "wh".to_string()) + .await + .expect("get warehouse") + .expect("warehouse should exist"); + assert_eq!(read.id, warehouse.id, "warehouse id did not round-trip"); + assert_eq!( + read.tenant_id, tenant_id, + "warehouse tenant_id did not round-trip" + ); +} + +#[tokio::test] +async fn catalog_and_asset_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let catalog = Catalog { + id: Uuid::new_v4(), + name: "cat".to_string(), + catalog_type: CatalogType::Local, + warehouse_name: None, + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }; + store + .create_catalog(tenant_id, catalog.clone()) + .await + .expect("create catalog"); + + let read = store + .get_catalog(tenant_id, "cat".to_string()) + .await + .expect("get catalog") + .expect("catalog should exist"); + assert_eq!(read.id, catalog.id, "catalog id did not round-trip"); + + let namespace = vec!["ns".to_string()]; + store + .create_namespace( + tenant_id, + "cat", + Namespace { + name: namespace.clone(), + properties: HashMap::new(), + }, + ) + .await + .expect("create namespace"); + + let asset = Asset { + id: Uuid::new_v4(), + name: "tbl".to_string(), + kind: AssetType::DeltaTable, + location: "s3://bucket/tbl".to_string(), + properties: HashMap::new(), + }; + store + .create_asset(tenant_id, "cat", None, namespace.clone(), asset.clone()) + .await + .expect("create asset"); + + let read = store + .get_asset(tenant_id, "cat", None, namespace, "tbl".to_string()) + .await + .expect("get asset") + .expect("asset should exist"); + assert_eq!(read.id, asset.id, "asset id did not round-trip"); + assert_eq!( + read.kind, + AssetType::DeltaTable, + "asset kind did not round-trip" + ); + + // The by-id index is a separate lookup path with its own encoding. + let by_id = store + .get_asset_by_id(tenant_id, asset.id) + .await + .expect("get asset by id"); + assert!( + by_id.is_some(), + "asset was not findable by id - the id was stored in a form the lookup \ + cannot match" + ); +} + +#[tokio::test] +async fn user_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let user = User { + id: Uuid::new_v4(), + username: format!("rt_{}", Uuid::new_v4().simple()), + email: format!("rt_{}@example.test", Uuid::new_v4().simple()), + password_hash: None, + oauth_provider: None, + oauth_subject: None, + tenant_id: Some(tenant_id), + role: UserRoleEnum::TenantUser, + created_at: Utc::now(), + updated_at: Utc::now(), + last_login: None, + active: true, + }; + store.create_user(user.clone()).await.expect("create user"); + + let read = store + .get_user(user.id) + .await + .expect("get user") + .expect("user should exist"); + assert_eq!(read.id, user.id, "user id did not round-trip"); + assert_eq!( + read.tenant_id, + Some(tenant_id), + "user tenant_id did not round-trip" + ); +} + +/// Roles and role assignments. +/// +/// The assignment is the one that silently mis-authorized users: written by +/// serde as a string, queried as Binary, so `get_user_roles` returned nothing +/// and an admin's grants evaporated. +#[tokio::test] +async fn role_and_assignment_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + let user_id = Uuid::new_v4(); + + let mut role = Role::new("rt-role".to_string(), None, tenant_id, Uuid::new_v4()); + let mut actions = HashSet::new(); + actions.insert(Action::Read); + role.add_permission(PermissionScope::Tenant, actions); + store.create_role(role.clone()).await.expect("create role"); + + let read = store + .get_role(role.id) + .await + .expect("get role") + .expect("role should be findable by id"); + assert_eq!(read.id, role.id, "role id did not round-trip"); + assert_eq!( + read.tenant_id, tenant_id, + "role tenant_id did not round-trip" + ); + assert_eq!( + read.permissions.len(), + 1, + "the role's grants did not round-trip" + ); + + let listed = store.list_roles(tenant_id, None).await.expect("list roles"); + assert!( + listed.iter().any(|r| r.id == role.id), + "the role was not returned by a tenant listing" + ); + + let assignment = UserRole::new(user_id, role.id, Uuid::new_v4()); + store + .assign_role(assignment.clone()) + .await + .expect("assign role"); + + let assignments = store.get_user_roles(user_id).await.expect("get user roles"); + assert_eq!( + assignments.len(), + 1, + "the role assignment was not readable back - this is the defect that \ + silently authorized an admin as though they held no roles" + ); + assert_eq!(assignments[0].role_id, role.id); + assert_eq!(assignments[0].user_id, user_id); + + // The grant must actually reach the permission list, which is what + // authorization consults. + let perms = store + .list_user_permissions(user_id, None) + .await + .expect("list user permissions"); + assert!( + perms.iter().any(|p| p.scope == PermissionScope::Tenant), + "a role-derived permission did not reach list_user_permissions" + ); +} + +#[tokio::test] +async fn direct_permission_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + let user_id = Uuid::new_v4(); + + let mut actions = HashSet::new(); + actions.insert(Action::Write); + let permission = Permission::new( + user_id, + tenant_id, + PermissionScope::Catalog { + catalog_id: Uuid::new_v4(), + }, + actions, + Uuid::new_v4(), + ); + store + .create_permission(permission.clone()) + .await + .expect("create permission"); + + let perms = store + .list_user_permissions(user_id, None) + .await + .expect("list user permissions"); + assert!( + perms.iter().any(|p| p.id == permission.id), + "a direct permission was not readable back" + ); + let found = perms.iter().find(|p| p.id == permission.id).unwrap(); + assert_eq!(found.user_id, user_id, "user_id did not round-trip"); + assert_eq!(found.tenant_id, tenant_id, "tenant_id did not round-trip"); +} + +#[tokio::test] +async fn service_user_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let api_key_hash = format!("hash-{}", Uuid::new_v4().simple()); + let service_user = ServiceUser::new( + "rt-svc".to_string(), + None, + tenant_id, + api_key_hash.clone(), + UserRoleEnum::TenantUser, + Uuid::new_v4(), + None, + ); + store + .create_service_user(service_user.clone()) + .await + .expect("create service user"); + + let read = store + .get_service_user(service_user.id) + .await + .expect("get service user") + .expect("service user should be findable by id"); + assert_eq!( + read.id, service_user.id, + "service user id did not round-trip" + ); + assert_eq!( + read.tenant_id, tenant_id, + "service user tenant_id did not round-trip" + ); + + let listed = store + .list_service_users(tenant_id, None) + .await + .expect("list service users"); + assert!( + listed.iter().any(|s| s.id == service_user.id), + "the service user was not returned by a tenant listing" + ); + + // The authentication path. If this lookup misses, an API key is simply + // rejected - service-user auth is down, not bypassed. + let by_hash = store + .get_service_user_by_api_key_hash(&api_key_hash) + .await + .expect("look up service user by api key hash") + .expect("the api key hash should resolve to its service user"); + assert_eq!( + by_hash.id, service_user.id, + "the api key hash resolved to the wrong principal" + ); + + // Updating the role rewrites a field the reader has to parse back into an + // enum; writing the Rust variant name instead of its serde form made the + // record unreadable from that point on. + let updated = store + .update_service_user( + service_user.id, + None, + Some("promoted".to_string()), + Some(UserRoleEnum::TenantAdmin), + None, + ) + .await + .expect("update service user"); + assert_eq!( + updated.role, + UserRoleEnum::TenantAdmin, + "the updated role did not round-trip" + ); + + store + .update_service_user_last_used(service_user.id) + .await + .expect("touch last_used"); + let touched = store + .get_service_user(service_user.id) + .await + .expect("re-read after touch") + .expect("service user should still exist"); + assert!( + touched.last_used.is_some(), + "last_used was written to a field nothing reads back" + ); + + store + .delete_service_user(service_user.id) + .await + .expect("delete service user"); + assert!( + store + .get_service_user(service_user.id) + .await + .expect("get after delete") + .is_none(), + "the service user survived its own deletion - the delete filter matched nothing" + ); +} + +/// Token issuance and revocation. +/// +/// B2: revocation was written by serde and queried as Binary, so the check +/// never matched and a revoked token - including after logout - stayed valid. +#[tokio::test] +async fn token_and_revocation_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + let user_id = Uuid::new_v4(); + let token_id = Uuid::new_v4(); + + store + .store_token(TokenInfo { + id: token_id, + tenant_id, + user_id, + username: "rt".to_string(), + expires_at: Utc::now() + chrono::Duration::hours(1), + created_at: Utc::now(), + is_valid: true, + token: Some("opaque".to_string()), + }) + .await + .expect("store token"); + + let listed = store + .list_active_tokens(tenant_id, Some(user_id), None) + .await + .expect("list active tokens"); + assert!( + listed.iter().any(|t| t.id == token_id), + "the token was not readable back" + ); + + assert!( + !store + .is_token_revoked(token_id) + .await + .expect("check revocation"), + "a fresh token must not read as revoked" + ); + + store + .revoke_token(token_id, Utc::now() + chrono::Duration::hours(1), None) + .await + .expect("revoke token"); + + assert!( + store + .is_token_revoked(token_id) + .await + .expect("check revocation"), + "a revoked token must read back as revoked (B2)" + ); +} + +/// B1: the audit event's tenant scoping. +#[tokio::test] +async fn audit_event_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + let user_id = Uuid::new_v4(); + let resource_id = Uuid::new_v4(); + + let entry = AuditLogEntry::success( + tenant_id, + Some(user_id), + "rt".to_string(), + AuditAction::CreateBranch, + ResourceType::Branch, + Some(resource_id), + "cat/feature".to_string(), + ); + let event_id = entry.id; + CatalogStore::log_audit_event(&store, tenant_id, entry) + .await + .expect("log audit event"); + + let read = store + .get_audit_event(tenant_id, event_id) + .await + .expect("get audit event") + .expect("audit event should exist"); + assert_eq!(read.id, event_id, "audit event id did not round-trip"); + assert_eq!(read.user_id, Some(user_id), "user_id did not round-trip"); + assert_eq!( + read.resource_id, + Some(resource_id), + "resource_id did not round-trip" + ); + assert_eq!( + read.action, + AuditAction::CreateBranch, + "the action must round-trip, not collapse to a default" + ); + + let listed = store + .list_audit_events(tenant_id, None) + .await + .expect("list audit events"); + assert!( + listed.iter().any(|e| e.id == event_id), + "the audit event was not returned by a listing" + ); +} + +#[tokio::test] +async fn business_metadata_round_trips() { + let store = store_or_skip!(); + let asset_id = Uuid::new_v4(); + let author = Uuid::new_v4(); + + let mut metadata = BusinessMetadata::new(asset_id, author); + metadata.tags = vec!["pii".to_string()]; + metadata.description = Some("round trip".to_string()); + let metadata_id = metadata.id; + + store + .upsert_business_metadata(metadata) + .await + .expect("upsert business metadata"); + + let read = store + .get_business_metadata(asset_id) + .await + .expect("get business metadata") + .expect("metadata should exist"); + assert_eq!(read.id, metadata_id, "metadata id did not round-trip"); + assert_eq!(read.asset_id, asset_id, "asset_id did not round-trip"); + assert_eq!(read.created_by, author, "created_by did not round-trip"); + assert_eq!(read.tags, vec!["pii".to_string()]); +} + +#[tokio::test] +async fn merge_operation_and_conflict_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let operation = MergeOperation::new( + tenant_id, + "cat".to_string(), + "feature".to_string(), + "main".to_string(), + None, + Uuid::new_v4(), + ); + let operation_id = operation.id; + store + .create_merge_operation(operation) + .await + .expect("create merge operation"); + + let read = store + .get_merge_operation(operation_id) + .await + .expect("get merge operation") + .expect("merge operation should be findable by id"); + assert_eq!(read.id, operation_id, "operation id did not round-trip"); + assert_eq!(read.tenant_id, tenant_id, "tenant_id did not round-trip"); + + let listed = store + .list_merge_operations(tenant_id, "cat", None) + .await + .expect("list merge operations"); + assert!( + listed.iter().any(|o| o.id == operation_id), + "the merge operation was not returned by a listing" + ); + + let conflict = MergeConflict::new( + operation_id, + ConflictType::MetadataConflict { + asset_name: "tbl".to_string(), + conflicting_properties: vec!["owner".to_string()], + }, + Some(Uuid::new_v4()), + "owner differs".to_string(), + ); + let conflict_id = conflict.id; + store + .create_merge_conflict(conflict) + .await + .expect("create merge conflict"); + + let conflicts = store + .list_merge_conflicts(operation_id, None) + .await + .expect("list merge conflicts"); + assert!( + conflicts.iter().any(|c| c.id == conflict_id), + "the conflict was not readable back by its operation id" + ); +} + +#[tokio::test] +async fn commit_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let commit = Commit { + id: Uuid::new_v4(), + parent_id: None, + timestamp: Utc::now().timestamp_millis(), + author: "rt".to_string(), + message: "round trip".to_string(), + operations: vec![], + }; + let commit_id = commit.id; + store + .create_commit(tenant_id, commit) + .await + .expect("create commit"); + + let read = store + .get_commit(tenant_id, commit_id) + .await + .expect("get commit") + .expect("commit should be findable by id"); + assert_eq!(read.id, commit_id, "commit id did not round-trip"); +} + +/// Branches and tags. +/// +/// Both are written with a hand-built `doc!` rather than serde, which converts +/// a `Uuid` by a *third* route again (`impl From for Bson`). Whether that +/// agrees with what the read path expects is not something the type system +/// checks, so it is asserted here. +#[tokio::test] +async fn branch_and_tag_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + store + .create_catalog( + tenant_id, + Catalog { + id: Uuid::new_v4(), + name: "cat".to_string(), + catalog_type: CatalogType::Local, + warehouse_name: None, + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }, + ) + .await + .expect("create catalog"); + + let head = Uuid::new_v4(); + store + .create_branch( + tenant_id, + "cat", + Branch { + name: "feature".to_string(), + head_commit_id: Some(head), + branch_type: BranchType::Experimental, + assets: vec!["tbl".to_string()], + }, + ) + .await + .expect("create branch"); + + let read = store + .get_branch(tenant_id, "cat", "feature".to_string()) + .await + .expect("get branch") + .expect("branch should exist"); + assert_eq!( + read.head_commit_id, + Some(head), + "the branch head commit id did not round-trip" + ); + assert_eq!(read.assets, vec!["tbl".to_string()]); + + // A branch with no head is the common case right after creation, and it + // takes a different path through the same conversion. + store + .create_branch( + tenant_id, + "cat", + Branch { + name: "empty".to_string(), + head_commit_id: None, + branch_type: BranchType::Ingest, + assets: vec![], + }, + ) + .await + .expect("create headless branch"); + let headless = store + .get_branch(tenant_id, "cat", "empty".to_string()) + .await + .expect("get headless branch") + .expect("headless branch should exist"); + assert_eq!(headless.head_commit_id, None); + assert!( + matches!(headless.branch_type, BranchType::Ingest), + "the branch type did not round-trip" + ); + + let commit_id = Uuid::new_v4(); + store + .create_tag( + tenant_id, + "cat", + Tag { + name: "v1".to_string(), + commit_id, + }, + ) + .await + .expect("create tag"); + let tag = store + .get_tag(tenant_id, "cat", "v1".to_string()) + .await + .expect("get tag") + .expect("tag should exist"); + assert_eq!( + tag.commit_id, commit_id, + "the tag commit id did not round-trip" + ); + let tags = store + .list_tags(tenant_id, "cat", None) + .await + .expect("list tags"); + assert!( + tags.iter().any(|t| t.name == "v1"), + "the tag was not returned by a listing" + ); +} + +/// Access requests. +/// +/// The listing joins `access_requests.user-id` to `users.id`, so it only +/// returns anything if the two collections encode a UUID the same way - a +/// cross-collection agreement no single-module test can check. +#[tokio::test] +async fn access_request_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let user = User { + id: Uuid::new_v4(), + username: format!("rt_{}", Uuid::new_v4().simple()), + email: format!("rt_{}@example.test", Uuid::new_v4().simple()), + password_hash: None, + oauth_provider: None, + oauth_subject: None, + tenant_id: Some(tenant_id), + role: UserRoleEnum::TenantUser, + created_at: Utc::now(), + updated_at: Utc::now(), + last_login: None, + active: true, + }; + store.create_user(user.clone()).await.expect("create user"); + + let request = AccessRequest::new( + tenant_id, + user.id, + Uuid::new_v4(), + Some("need read".to_string()), + ); + let request_id = request.id; + store + .create_access_request(request) + .await + .expect("create access request"); + + let read = store + .get_access_request(request_id) + .await + .expect("get access request") + .expect("access request should be findable by id"); + assert_eq!(read.user_id, user.id, "user_id did not round-trip"); + assert_eq!(read.asset_id, read.asset_id); + + let listed = store + .list_access_requests(tenant_id, None) + .await + .expect("list access requests"); + assert!( + listed.iter().any(|r| r.id == request_id), + "the access request was not returned by its tenant listing - the join \ + between access_requests.user-id and users.id did not match" + ); + + let mut approved = read; + approved.approve(Uuid::new_v4(), Some("ok".to_string())); + store + .update_access_request(approved.clone()) + .await + .expect("update access request"); + let after = store + .get_access_request(request_id) + .await + .expect("re-read access request") + .expect("access request should still exist"); + assert_eq!( + after.status, + RequestStatus::Approved, + "the approval did not round-trip" + ); + assert!( + after.reviewed_by.is_some(), + "reviewed_by did not round-trip" + ); +} + +/// The merge mutation paths. +/// +/// `create_merge_operation` is covered above, but the status transitions build +/// their update documents by hand - including one that writes a status literal +/// as a Rust `Debug` string. Those writes are never read back by any other +/// test, so a value that no longer deserializes would go unnoticed until a +/// merge was actually completed in production. +#[tokio::test] +async fn merge_status_transitions_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let operation = MergeOperation::new( + tenant_id, + "cat".to_string(), + "feature".to_string(), + "main".to_string(), + Some(Uuid::new_v4()), + Uuid::new_v4(), + ); + let operation_id = operation.id; + store + .create_merge_operation(operation) + .await + .expect("create merge operation"); + + store + .update_merge_operation_status(operation_id, MergeStatus::Conflicted) + .await + .expect("set status"); + let read = store + .get_merge_operation(operation_id) + .await + .expect("re-read after status change") + .expect("operation should still exist"); + assert!( + matches!(read.status, MergeStatus::Conflicted), + "the status write produced a value that does not deserialize back" + ); + + let conflict = MergeConflict::new( + operation_id, + ConflictType::MetadataConflict { + asset_name: "tbl".to_string(), + conflicting_properties: vec!["owner".to_string()], + }, + Some(Uuid::new_v4()), + "owner differs".to_string(), + ); + let conflict_id = conflict.id; + store + .create_merge_conflict(conflict) + .await + .expect("create conflict"); + store + .add_conflict_to_operation(operation_id, conflict_id) + .await + .expect("attach conflict"); + + let with_conflict = store + .get_merge_operation(operation_id) + .await + .expect("re-read after attaching a conflict") + .expect("operation should still exist"); + assert!( + with_conflict.conflicts.contains(&conflict_id), + "the attached conflict id was written in a form the operation cannot \ + read back" + ); + + store + .resolve_merge_conflict( + conflict_id, + ConflictResolution { + conflict_id, + strategy: ResolutionStrategy::TakeSource, + resolved_value: None, + resolved_by: Uuid::new_v4(), + resolved_at: Utc::now(), + }, + ) + .await + .expect("resolve conflict"); + let resolved = store + .get_merge_conflict(conflict_id) + .await + .expect("re-read conflict") + .expect("conflict should still exist"); + assert!( + resolved.resolution.is_some(), + "the resolution did not round-trip" + ); + + let result_commit = Uuid::new_v4(); + store + .complete_merge_operation(operation_id, result_commit) + .await + .expect("complete merge"); + let completed = store + .get_merge_operation(operation_id) + .await + .expect("re-read after completion") + .expect("operation should still exist"); + assert!( + matches!(completed.status, MergeStatus::Completed), + "the completed status did not round-trip" + ); + assert_eq!( + completed.result_commit_id, + Some(result_commit), + "the result commit id did not round-trip" + ); + assert!( + completed.completed_at.is_some(), + "completed_at did not round-trip" + ); +} diff --git a/pangolin/pangolin_store/tests/postgres_tests.rs b/pangolin/pangolin_store/tests/postgres_tests.rs index 486d84a..d493fa6 100644 --- a/pangolin/pangolin_store/tests/postgres_tests.rs +++ b/pangolin/pangolin_store/tests/postgres_tests.rs @@ -138,10 +138,18 @@ async fn test_postgres_access_requests() { .expect("Create tenant"); // Setup: User + // + // The username and email are unique per run. They used to be the literal + // "req_user"/"req_pg@example.com", which is fine against a database created + // fresh for each run and fails on the second run against a persistent one - + // `users_username_key` is a unique constraint. A test that only passes on a + // clean database cannot be re-run, which is exactly what a CI service + // container and a developer's local Postgres both need. + let unique = Uuid::new_v4(); let user = User { id: Uuid::new_v4(), - username: "req_user".to_string(), - email: "req_pg@example.com".to_string(), + username: format!("req_user_{unique}"), + email: format!("req_pg_{unique}@example.com"), password_hash: None, oauth_provider: None, oauth_subject: None, diff --git a/pangolin/pangolin_store/tests/sqlite_migration_tests.rs b/pangolin/pangolin_store/tests/sqlite_migration_tests.rs new file mode 100644 index 0000000..25d7cae --- /dev/null +++ b/pangolin/pangolin_store/tests/sqlite_migration_tests.rs @@ -0,0 +1,206 @@ +//! Upgrade path for an existing SQLite database. +//! +//! The `audit_logs` fix was initially applied by editing `sqlite_schema.sql` and +//! bumping `SQLITE_SCHEMA_VERSION`. That is not a migration: the schema file is +//! written entirely with `CREATE TABLE IF NOT EXISTS`, which does nothing at all +//! when the table already exists. Fresh installs got the new columns and every +//! upgraded database kept the broken ones - with a version number now claiming +//! otherwise, which is worse than no version at all. +//! +//! These tests build a database with the genuine pre-v2 shape and assert the +//! upgrade actually works, which is the only way to tell a real migration from a +//! version bump. + +use pangolin_core::audit::{AuditAction, AuditLogEntry, ResourceType}; +use pangolin_store::{sqlite::SQLITE_SCHEMA_VERSION, SqliteStore}; +use std::collections::HashMap; +use uuid::Uuid; + +/// The `audit_logs` table exactly as it was declared before v2. +const V1_AUDIT_LOGS: &str = "CREATE TABLE IF NOT EXISTS audit_logs ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + timestamp INTEGER NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + resource TEXT NOT NULL, + details TEXT +); +CREATE INDEX IF NOT EXISTS idx_audit_logs_tenant_ts ON audit_logs(tenant_id, timestamp DESC);"; + +/// A unique on-disk database per test; SQLite needs a real file for the +/// rename-and-recreate the migration performs. +fn temp_db_url() -> (std::path::PathBuf, String) { + let dir = std::env::temp_dir().join(format!("pangolin_migration_{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("pangolin.db"); + let url = format!("sqlite://{}?mode=rwc", path.to_string_lossy()); + (dir, url) +} + +/// Stand up a store whose `audit_logs` has the pre-v2 shape, as an upgrading +/// deployment's database does. +async fn legacy_store(url: &str) -> SqliteStore { + let store = SqliteStore::new(url).await.expect("open sqlite"); + + // The tenants table has to exist first: audit_logs carries a foreign key to + // it in the real schema, and the tests below insert a tenant. + store + .apply_schema( + "CREATE TABLE IF NOT EXISTS tenants ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + properties TEXT + );", + ) + .await + .expect("seed tenants"); + + store.apply_schema(V1_AUDIT_LOGS).await.expect("seed v1"); + store +} + +async fn audit_columns(store: &SqliteStore, table: &str) -> Vec { + store.table_columns(table).await.expect("table_info") +} + +/// The core regression: after upgrading, an audit write must succeed. +/// +/// Before the migration existed this failed with "table audit_logs has no +/// column named user_id" - on a database that `run_migrations` had just +/// reported as being at the current version. +#[tokio::test] +async fn upgrading_a_v1_database_makes_audit_logging_work() { + let (_dir, url) = temp_db_url(); + let store = legacy_store(&url).await; + + // Sanity: the fixture really is the old shape. + let before = audit_columns(&store, "audit_logs").await; + assert!( + before.iter().any(|c| c == "actor"), + "fixture should start with the pre-v2 shape, got {before:?}" + ); + assert!( + !before.iter().any(|c| c == "user_id"), + "fixture should not already have the v2 columns" + ); + + store.run_migrations().await.expect("migrate"); + + let after = audit_columns(&store, "audit_logs").await; + for expected in [ + "user_id", + "username", + "resource_type", + "resource_id", + "resource_name", + "ip_address", + "user_agent", + "result", + "error_message", + "metadata", + ] { + assert!( + after.iter().any(|c| c == expected), + "column {expected} missing after migration; got {after:?}" + ); + } + + // The behaviour the columns exist for. + let tenant_id = Uuid::new_v4(); + store + .create_tenant(pangolin_core::model::Tenant { + id: tenant_id, + name: "upgraded".to_string(), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + let entry = AuditLogEntry::success( + tenant_id, + Some(Uuid::new_v4()), + "upgrader".to_string(), + AuditAction::CreateBranch, + ResourceType::Branch, + Some(Uuid::new_v4()), + "cat/feature".to_string(), + ); + + store + .log_audit_event(tenant_id, entry) + .await + .expect("an audit write must succeed after upgrading"); + + let events = store + .list_audit_events(tenant_id, None) + .await + .expect("list audit events"); + assert_eq!(events.len(), 1, "the audit event should be readable back"); + assert_eq!( + events[0].action, + AuditAction::CreateBranch, + "the action must round-trip, not collapse to a default (B22)" + ); +} + +/// The old table is preserved rather than dropped. +#[tokio::test] +async fn upgrading_keeps_the_old_table_as_a_backup() { + let (_dir, url) = temp_db_url(); + let store = legacy_store(&url).await; + + store.run_migrations().await.expect("migrate"); + + let backup = audit_columns(&store, "audit_logs_pre_v2").await; + assert!( + backup.iter().any(|c| c == "actor"), + "the pre-v2 table should be kept as audit_logs_pre_v2, got {backup:?}" + ); +} + +/// Running migrations twice must be a no-op, not a second rename that clobbers +/// the real table. +#[tokio::test] +async fn migrating_twice_is_idempotent() { + let (_dir, url) = temp_db_url(); + let store = legacy_store(&url).await; + + store.run_migrations().await.expect("first migrate"); + store + .run_migrations() + .await + .expect("second migrate must be a no-op"); + + let after = audit_columns(&store, "audit_logs").await; + assert!( + after.iter().any(|c| c == "user_id"), + "audit_logs should still be the v2 shape after a second run, got {after:?}" + ); + assert_eq!( + store.schema_version().await.expect("version"), + Some(SQLITE_SCHEMA_VERSION) + ); +} + +/// A fresh database needs no migration and lands on the current shape. +#[tokio::test] +async fn a_fresh_database_is_created_at_the_current_version() { + let (_dir, url) = temp_db_url(); + let store = SqliteStore::new(&url).await.expect("open sqlite"); + + store.run_migrations().await.expect("migrate"); + + let columns = audit_columns(&store, "audit_logs").await; + assert!(columns.iter().any(|c| c == "user_id")); + assert_eq!( + store.schema_version().await.expect("version"), + Some(SQLITE_SCHEMA_VERSION) + ); + + // Nothing to back up, so no backup table should have been created. + assert!( + audit_columns(&store, "audit_logs_pre_v2").await.is_empty(), + "a fresh database should not produce a backup table" + ); +} diff --git a/pangolin/pangolin_store/tests/store_integration.rs b/pangolin/pangolin_store/tests/store_integration.rs index ce5ee30..edf66e0 100644 --- a/pangolin/pangolin_store/tests/store_integration.rs +++ b/pangolin/pangolin_store/tests/store_integration.rs @@ -1,5 +1,5 @@ use pangolin_store::{ - tests::{test_asset_update_consistency, test_dashboard_stats_consistency}, + tests::{parity, test_asset_update_consistency, test_dashboard_stats_consistency}, MemoryStore, MongoStore, PostgresStore, SqliteStore, }; use std::env; @@ -10,6 +10,7 @@ async fn test_memory_store_regression() { let store = MemoryStore::new(); test_asset_update_consistency(&store).await; test_dashboard_stats_consistency(&store).await; + parity::run_all(&store).await; } #[tokio::test] @@ -32,6 +33,7 @@ async fn test_sqlite_store_regression() { test_asset_update_consistency(&store).await; test_dashboard_stats_consistency(&store).await; + parity::run_all(&store).await; } #[tokio::test] @@ -50,6 +52,7 @@ async fn test_postgres_store_regression() { test_asset_update_consistency(&store).await; test_dashboard_stats_consistency(&store).await; + parity::run_all(&store).await; } #[tokio::test] @@ -70,4 +73,5 @@ async fn test_mongo_store_regression() { test_asset_update_consistency(&store).await; test_dashboard_stats_consistency(&store).await; + parity::run_all(&store).await; } diff --git a/pangolin/pangolin_store/tests/warehouse_encryption_tests.rs b/pangolin/pangolin_store/tests/warehouse_encryption_tests.rs new file mode 100644 index 0000000..e106330 --- /dev/null +++ b/pangolin/pangolin_store/tests/warehouse_encryption_tests.rs @@ -0,0 +1,361 @@ +//! No backend may write a cloud credential to storage in the clear. +//! +//! C-11. `secrets`' unit tests prove the crypto; they say nothing about whether +//! a given backend calls it. This reads what actually landed in the database, +//! through a connection of its own rather than through the store, because the +//! store is the thing under test - asking it to read back its own writes would +//! pass just as happily if `seal` were never called and `open` were a no-op. +//! +//! Each backend gets the same two assertions: +//! +//! 1. the persisted bytes do not contain the plaintext secret; +//! 2. reading through the store still yields the plaintext, so the round trip +//! is usable. +//! +//! A backend added later without wiring `secrets` fails (1) here rather than +//! quietly storing credentials in plaintext, which is exactly how the MongoDB +//! UUID encoding defects survived for so long: no test compared what was +//! written against what was expected. +//! +//! The memory backend is deliberately absent. It keeps warehouses in a +//! `DashMap` and loses everything on restart - there is no "at rest" for it to +//! encrypt, and sealing there would cost work to protect a secret that is in +//! the same process's heap either way. + +use pangolin_core::model::Warehouse; +use pangolin_store::{secrets, CatalogStore, MongoStore, PostgresStore, SqliteStore}; +use serial_test::serial; +use std::collections::HashMap; +use uuid::Uuid; + +const SECRET: &str = "AWS-SECRET-THAT-MUST-NOT-BE-STORED-IN-CLEAR"; + +/// Sets the encryption key for the duration of a test. +struct KeyGuard(Option); + +impl KeyGuard { + fn set() -> Self { + let previous = std::env::var("PANGOLIN_ENCRYPTION_KEY").ok(); + // A fixed key: the test asserts on ciphertext presence, not its value. + // 32 bytes exactly; a shorter one is rejected by the key validator. + std::env::set_var( + "PANGOLIN_ENCRYPTION_KEY", + "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=", + ); + Self(previous) + } +} + +impl Drop for KeyGuard { + fn drop(&mut self) { + match &self.0 { + Some(v) => std::env::set_var("PANGOLIN_ENCRYPTION_KEY", v), + None => std::env::remove_var("PANGOLIN_ENCRYPTION_KEY"), + } + } +} + +fn warehouse_with_secret(tenant_id: Uuid, name: &str) -> Warehouse { + Warehouse { + id: Uuid::new_v4(), + name: name.to_string(), + tenant_id, + storage_config: HashMap::from([ + ("type".to_string(), "s3".to_string()), + ("bucket".to_string(), "customer-data".to_string()), + ("secret_access_key".to_string(), SECRET.to_string()), + ]), + use_sts: false, + vending_strategy: None, + } +} + +/// The shared assertions, given what the database actually holds. +fn assert_sealed(backend: &str, persisted: &str, read_back: &Warehouse) { + assert!( + !persisted.contains(SECRET), + "{backend} wrote the credential in plaintext. What is stored: {persisted}" + ); + assert!( + persisted.contains("enc:v1:"), + "{backend} stored something, but not in the sealed format: {persisted}" + ); + assert!( + persisted.contains("customer-data"), + "{backend} encrypted the bucket name as well; only credentials should be \ + sealed, or the object-store factory cannot build a client" + ); + assert_eq!( + read_back.storage_config.get("secret_access_key").unwrap(), + SECRET, + "{backend} could not read its own credential back" + ); + assert!( + !secrets::has_plaintext_secret(&{ + let mut c = read_back.storage_config.clone(); + let _ = secrets::seal(&mut c); + c + }), + "{backend}: re-sealing the round-tripped config left a plaintext secret" + ); +} + +#[tokio::test] +#[serial(encryption_key_env)] +async fn sqlite_does_not_store_credentials_in_the_clear() { + let _key = KeyGuard::set(); + + let dir = std::env::temp_dir().join(format!("pangolin-enc-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("catalog.db"); + let url = format!("sqlite://{}?mode=rwc", path.display()); + + let store = SqliteStore::new(&url).await.expect("open sqlite"); + store.run_migrations().await.expect("apply the schema"); + let tenant_id = Uuid::new_v4(); + store + .create_tenant(pangolin_core::model::Tenant { + id: tenant_id, + name: format!("t-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + store + .create_warehouse(tenant_id, warehouse_with_secret(tenant_id, "wh")) + .await + .expect("create warehouse"); + + // Read the file through a connection of our own. + let pool = sqlx::SqlitePool::connect(&url).await.expect("own pool"); + let persisted: String = + sqlx::query_scalar("SELECT storage_config FROM warehouses WHERE name = ?") + .bind("wh") + .fetch_one(&pool) + .await + .expect("read the raw row"); + + let read_back = store + .get_warehouse(tenant_id, "wh".to_string()) + .await + .expect("get warehouse") + .expect("warehouse should exist"); + + assert_sealed("sqlite", &persisted, &read_back); + let _ = std::fs::remove_dir_all(&dir); +} + +#[tokio::test] +#[serial(encryption_key_env)] +async fn postgres_does_not_store_credentials_in_the_clear() { + let Some(url) = pangolin_store::test_support::postgres_url() else { + println!("skipping: set PANGOLIN_TEST_POSTGRES_URL to run this test"); + return; + }; + let _key = KeyGuard::set(); + + let store = PostgresStore::new(&url).await.expect("open postgres"); + let tenant_id = Uuid::new_v4(); + store + .create_tenant(pangolin_core::model::Tenant { + id: tenant_id, + name: format!("t-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + store + .create_warehouse(tenant_id, warehouse_with_secret(tenant_id, "wh")) + .await + .expect("create warehouse"); + + let pool = sqlx::PgPool::connect(&url).await.expect("own pool"); + let persisted: serde_json::Value = sqlx::query_scalar( + "SELECT storage_config FROM warehouses WHERE tenant_id = $1 AND name = $2", + ) + .bind(tenant_id) + .bind("wh") + .fetch_one(&pool) + .await + .expect("read the raw row"); + + let read_back = store + .get_warehouse(tenant_id, "wh".to_string()) + .await + .expect("get warehouse") + .expect("warehouse should exist"); + + assert_sealed("postgres", &persisted.to_string(), &read_back); +} + +#[tokio::test] +#[serial(encryption_key_env)] +async fn mongodb_does_not_store_credentials_in_the_clear() { + let Some(url) = pangolin_store::test_support::mongo_url() else { + println!("skipping: set PANGOLIN_TEST_MONGO_URL to run this test"); + return; + }; + let _key = KeyGuard::set(); + + let db_name = format!("pangolin_enc_{}", Uuid::new_v4().simple()); + let store = MongoStore::new(&url, &db_name).await.expect("open mongo"); + let tenant_id = Uuid::new_v4(); + store + .create_tenant(pangolin_core::model::Tenant { + id: tenant_id, + name: format!("t-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + store + .create_warehouse(tenant_id, warehouse_with_secret(tenant_id, "wh")) + .await + .expect("create warehouse"); + + let client = mongodb::Client::with_uri_str(&url) + .await + .expect("own client"); + let raw: mongodb::bson::Document = client + .database(&db_name) + .collection("warehouses") + .find_one(mongodb::bson::doc! { "name": "wh" }) + .await + .expect("read the raw document") + .expect("the document should exist"); + + let read_back = store + .get_warehouse(tenant_id, "wh".to_string()) + .await + .expect("get warehouse") + .expect("warehouse should exist"); + + assert_sealed("mongodb", &raw.to_string(), &read_back); + let _ = client.database(&db_name).drop().await; +} + +/// Rotating a credential must not write the new one in the clear. +/// +/// `create` and `update` are separate code paths in every backend, and sealing +/// only on create would mean the first credential is protected and every +/// rotation after it is not - the worst of both, because the table looks +/// encrypted. +#[tokio::test] +#[serial(encryption_key_env)] +async fn updating_a_credential_seals_the_new_value() { + let _key = KeyGuard::set(); + + let dir = std::env::temp_dir().join(format!("pangolin-enc-upd-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let url = format!("sqlite://{}?mode=rwc", dir.join("catalog.db").display()); + + let store = SqliteStore::new(&url).await.expect("open sqlite"); + store.run_migrations().await.expect("apply the schema"); + let tenant_id = Uuid::new_v4(); + store + .create_tenant(pangolin_core::model::Tenant { + id: tenant_id, + name: format!("t-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + store + .create_warehouse(tenant_id, warehouse_with_secret(tenant_id, "wh")) + .await + .expect("create warehouse"); + + const ROTATED: &str = "ROTATED-SECRET-ALSO-MUST-NOT-BE-CLEAR"; + store + .update_warehouse( + tenant_id, + "wh".to_string(), + pangolin_core::model::WarehouseUpdate { + name: None, + storage_config: Some(HashMap::from([ + ("type".to_string(), "s3".to_string()), + ("bucket".to_string(), "customer-data".to_string()), + ("secret_access_key".to_string(), ROTATED.to_string()), + ])), + use_sts: None, + vending_strategy: None, + }, + ) + .await + .expect("rotate the credential"); + + let pool = sqlx::SqlitePool::connect(&url).await.expect("own pool"); + let persisted: String = + sqlx::query_scalar("SELECT storage_config FROM warehouses WHERE name = ?") + .bind("wh") + .fetch_one(&pool) + .await + .expect("read the raw row"); + + assert!( + !persisted.contains(ROTATED), + "the rotated credential was written in plaintext: {persisted}" + ); + + let read_back = store + .get_warehouse(tenant_id, "wh".to_string()) + .await + .expect("get warehouse") + .expect("warehouse should exist"); + assert_eq!( + read_back.storage_config.get("secret_access_key").unwrap(), + ROTATED + ); + + let _ = std::fs::remove_dir_all(&dir); +} + +/// A database written before encryption existed must still be readable. +/// +/// Deploying this must not turn every existing warehouse into an error. The +/// read path tolerates plaintext precisely so that an upgrade is not an outage. +#[tokio::test] +#[serial(encryption_key_env)] +async fn warehouses_written_before_encryption_still_load() { + let dir = std::env::temp_dir().join(format!("pangolin-enc-legacy-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let url = format!("sqlite://{}?mode=rwc", dir.join("catalog.db").display()); + + let tenant_id = Uuid::new_v4(); + // Written with no key configured, i.e. exactly as an older release would. + { + std::env::remove_var("PANGOLIN_ENCRYPTION_KEY"); + let store = SqliteStore::new(&url).await.expect("open sqlite"); + store.run_migrations().await.expect("apply the schema"); + store + .create_tenant(pangolin_core::model::Tenant { + id: tenant_id, + name: format!("t-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + store + .create_warehouse(tenant_id, warehouse_with_secret(tenant_id, "legacy")) + .await + .expect("create warehouse"); + } + + // Now the operator turns encryption on and restarts. + let _key = KeyGuard::set(); + let store = SqliteStore::new(&url).await.expect("reopen sqlite"); + store.run_migrations().await.expect("apply the schema"); + let read_back = store + .get_warehouse(tenant_id, "legacy".to_string()) + .await + .expect("get warehouse") + .expect("warehouse should exist"); + + assert_eq!( + read_back.storage_config.get("secret_access_key").unwrap(), + SECRET, + "turning encryption on must not make existing warehouses unreadable" + ); + + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/pangolin/scripts/check_env_var_docs.sh b/pangolin/scripts/check_env_var_docs.sh new file mode 100755 index 0000000..8d3ca49 --- /dev/null +++ b/pangolin/scripts/check_env_var_docs.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# Verify the environment-variable reference against the source of truth. +# +# B43: `docs/environment-variables.md` documented `PANGOLIN_HOST`, +# `PANGOLIN_PORT` and `PANGOLIN_STORE_TYPE` - none of which any code reads - and +# omitted roughly twenty variables that are read. Anyone configuring a +# deployment from that page set variables that did nothing and missed the ones +# that mattered, which is exactly how B9's `PANGOLIN_STORE_TYPE` in the compose +# files survived. +# +# Hand-maintained documentation of a machine-readable fact drifts. This script +# re-derives the set from the code and fails when the docs and the code +# disagree, so the drift is caught in CI rather than by an auditor. +# +# Usage: scripts/check_env_var_docs.sh (run from the pangolin/ workspace root) + +set -euo pipefail + +DOC="../docs/environment-variables.md" + +if [[ ! -f "$DOC" ]]; then + echo "error: $DOC not found; run this from the pangolin/ workspace root" >&2 + exit 1 +fi + +# Every PANGOLIN_* name the server actually reads. Test-only knobs +# (PANGOLIN_TEST_*) are deliberately excluded: they configure the test harness, +# not a deployment. +mapfile -t IN_CODE < <( + grep -rhoE 'PANGOLIN_[A-Z0-9_]+' pangolin_api/src pangolin_store/src --include='*.rs' \ + | grep -v '^PANGOLIN_TEST_' \ + | sort -u +) + +# The doc deliberately names a few variables that do *not* exist, to warn +# readers off them. Those lines carry a `` marker so this +# check can tell "documented as real" from "documented as a trap". +mapfile -t IN_DOCS < <( + grep -vF '' "$DOC" \ + | grep -ohE 'PANGOLIN_[A-Z0-9_]+' \ + | grep -v '^PANGOLIN_TEST_' \ + | sort -u +) + +missing=() +for name in "${IN_CODE[@]}"; do + if ! printf '%s\n' "${IN_DOCS[@]}" | grep -qx "$name"; then + missing+=("$name") + fi +done + +phantom=() +for name in "${IN_DOCS[@]}"; do + if ! printf '%s\n' "${IN_CODE[@]}" | grep -qx "$name"; then + phantom+=("$name") + fi +done + +status=0 + +if (( ${#missing[@]} )); then + echo "error: read by the server but absent from $DOC:" >&2 + printf ' %s\n' "${missing[@]}" >&2 + status=1 +fi + +if (( ${#phantom[@]} )); then + echo "error: documented in $DOC but read by nothing:" >&2 + printf ' %s\n' "${phantom[@]}" >&2 + status=1 +fi + +if (( status == 0 )); then + echo "environment-variable reference matches the code (${#IN_CODE[@]} variables)" +fi + +exit "$status" diff --git a/pangolin_ui/.env.example b/pangolin_ui/.env.example index d93c4c7..ae65300 100644 --- a/pangolin_ui/.env.example +++ b/pangolin_ui/.env.example @@ -1,2 +1,12 @@ -VITE_API_URL=http://localhost:8080 -VITE_NO_AUTH=false +# Base URL of the Pangolin API. +# +# B31: this file used to declare VITE_API_URL, the compose files passed +# VITE_API_URL, and the client read PUBLIC_API_URL - three names, none of which +# agreed, so the client's fallback always won and every deployed build called +# `http://localhost:8080`, meaning the *visitor's* machine. +# +# SvelteKit's dynamic public env requires the PUBLIC_ prefix, so PUBLIC_API_URL +# is the correct name and now the only one. +# +# Leave it empty for a same-origin (reverse-proxy) deployment. +PUBLIC_API_URL=http://localhost:8080 diff --git a/pangolin_ui/.gitignore b/pangolin_ui/.gitignore index 3b462cb..5ba696c 100644 --- a/pangolin_ui/.gitignore +++ b/pangolin_ui/.gitignore @@ -21,3 +21,13 @@ Thumbs.db # Vite vite.config.js.timestamp-* vite.config.ts.timestamp-* + +# Test and debug output (B45). +# +# ~260 KB of `check_*.txt` scratch files and Playwright artefacts were tracked +# in git. The ignore patterns did not cover them, and by the time they were +# added the files were already tracked - so `.gitignore` had no effect on them. +test-results/ +playwright-report/ +check_*.txt +node_modules/ diff --git a/pangolin_ui/check_catalogs_list.txt b/pangolin_ui/check_catalogs_list.txt deleted file mode 100644 index dd960f2..0000000 --- a/pangolin_ui/check_catalogs_list.txt +++ /dev/null @@ -1,971 +0,0 @@ - -> pangolin-ui@0.1.0 check -> svelte-kit sync && svelte-check --tsconfig ./tsconfig.json - -Loading svelte-check in workspace: /home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui -Getting Svelte diagnostics... - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/lib/components/ui/Modal.svelte:30:2 -Warn: Elements with the 'dialog' interactive role must have a tabindex value -https://svelte.dev/e/a11y_interactive_supports_focus (svelte) -{#if open} -
e.key === 'Escape' && close()} - role="dialog" - aria-modal="true" - > -
- -
- - -
- - {#if title} -
-

{title}

-
- {/if} - - -
- -
- - -
- - - -
-
-
-
-{/if} - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/lib/components/ui/DataTable.svelte:105:9 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
- Loading... - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/lib/components/ui/Textarea.svelte:18:5 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) - {#if label} - - {/if} - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/+page.svelte:97:5 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- -