acceptance: schema-driven invariant fuzzing for bundle configs

.github/workflows/push.yml

@@ -408,6 +408,76 @@ jobs:
         run: |
           go tool -modfile=tools/task/go.mod task test-sandbox
 
+  test-fuzz:
+    needs:
+      - cleanups
+
+    # Wide rotating seed window with drift on: too slow for every PR, so nightly only and
+    # not part of test-result. The committed acceptance test still checks no-panic per PR.
+    if: ${{ github.event_name == 'schedule' }}
+    name: "task test-fuzz"
+    runs-on:
+      group: databricks-protected-runner-group-large
+      labels: linux-ubuntu-latest-large
+
+    defaults:
+      run:
+        shell: bash
+
+    permissions:
+      id-token: write
+      contents: read
+      # For the failure-reporting step's PR comment.
+      pull-requests: write
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: test-fuzz
+
+      - name: Run tests
+        env:
+          FUZZ_SEED_COUNT: "25"
+        run: |
+          # GITHUB_RUN_NUMBER * COUNT keeps each nightly window non-overlapping, so CI
+          # explores new configs every run.
+          export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT ))
+          go tool -modfile=tools/task/go.mod task test-fuzz
+
+      # Not in test-result, so surface failures by commenting on the PR under test.
+      - name: Report failure
+        if: ${{ failure() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          COMMIT: ${{ github.sha }}
+        run: |
+          body=$(cat <<EOF
+          The nightly schema fuzz invariant job (\`task test-fuzz\`) failed on commit \`$COMMIT\`.
+
+          Run: $RUN_URL
+
+          The failing seed is printed in the job log as \`reproduce with: ...\`.
+          Reproduce locally with:
+
+          \`\`\`
+          FUZZ_SEED_START=<seed> FUZZ_SEED_COUNT=1 task test-fuzz
+          \`\`\`
+          EOF
+          )
+
+          # The commit's pulls endpoint returns the PR that introduced it.
+          pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$COMMIT/pulls" --jq '.[0].number // empty')
+          if [ -n "$pr" ]; then
+            gh pr comment "$pr" --body "$body"
+          else
+            echo "No PR found for commit $COMMIT; skipping failure comment" >&2
+          fi
+
   # This job groups the result of all the above test jobs.
   # It is a required check, so it blocks auto-merge and the merge queue.
   #

Taskfile.yml

@@ -730,6 +730,53 @@ tasks:
           --packages ./acceptance/... \
           -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox"
 
+  test-fuzz:
+    desc: Run schema fuzz invariant tests (random configs, direct engine)
+    # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't see.
+    cmds:
+      - |
+        # Wider window than the committed run, with drift on; a repro narrows it via
+        # FUZZ_SEED_START/COUNT. FUZZ_TIME_BUDGET stops each variant cleanly if 200 seeds
+        # don't fit the Timeout, rather than letting it be force-killed.
+        export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}"
+        export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}"
+        {{.GO_TOOL}} gotestsum \
+          --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \
+          --no-summary=skipped \
+          --packages ./acceptance/... \
+          -- -timeout=${LOCAL_TIMEOUT:-30m} -run "TestAccept/bundle/invariant/fuzz"
+
+  test-fuzz-cover:
+    desc: Run the schema fuzzer under coverage and report which CLI packages it exercises
+    # No `sources:` fingerprint: like test-fuzz, the window depends on FUZZ_* env vars.
+    cmds:
+      - rm -fr ./acceptance/build/cover-fuzz/ ./acceptance/build/cover-fuzz-merged/
+      - mkdir -p ./acceptance/build/cover-fuzz-merged/
+      - |
+        # CLI_GOCOVERDIR makes the harness build a -cover CLI and set GOCOVERDIR per run, so
+        # every fuzzed `bundle` invocation drops aggregatable counter files. Drift off so the
+        # run exercises the full deploy/plan path instead of stopping on the first drift.
+        export CLI_GOCOVERDIR=build/cover-fuzz
+        export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-100}"
+        export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}"
+        unset FUZZ_CHECK_DRIFT || true
+        {{.GO_TOOL}} gotestsum \
+          --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \
+          --no-summary=skipped \
+          --packages ./acceptance/... \
+          -- -timeout=${LOCAL_TIMEOUT:-40m} -run "TestAccept/bundle/invariant/fuzz" || true
+      - "go tool covdata merge -i $(printf '%s,' acceptance/build/cover-fuzz/* | sed 's/,$//') -o acceptance/build/cover-fuzz-merged/"
+      - go tool covdata textfmt -i acceptance/build/cover-fuzz-merged -o coverage-fuzz.txt
+      - |
+        echo "== total CLI coverage exercised by the fuzz corpus =="
+        go tool cover -func=coverage-fuzz.txt | awk '/^total:/{print $NF}'
+        echo
+        echo "== bundle/cmd packages by coverage (ascending; 0.0% = never exercised) =="
+        go tool covdata percent -i=acceptance/build/cover-fuzz-merged \
+          | awk '/coverage:/{p=$3; gsub(/%/,"",p); sub("github.com/databricks/cli/","",$1); printf "%6.1f%%  %s\n", p, $1}' \
+          | grep -E '  (bundle|cmd/bundle)/' \
+          | sort -n
+
   # --- Integration tests ---
 
   integration:

acceptance/bin/check_schema_types.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""
+Assert every `type` in the bundle schema is one gen_fuzz_config.py can generate, so a new
+libs/jsonschema.Type fails loudly here instead of being silently skipped by the fuzzer.
+"""
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from gen_fuzz_config import HANDLED_TYPES
+
+
+def collect_types(node, found):
+    if isinstance(node, dict):
+        t = node.get("type")
+        if isinstance(t, str):
+            found.add(t)
+        elif isinstance(t, list):
+            found.update(x for x in t if isinstance(x, str))
+        for v in node.values():
+            collect_types(v, found)
+    elif isinstance(node, list):
+        for v in node:
+            collect_types(v, found)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--schema", required=True)
+    args = parser.parse_args()
+
+    with open(args.schema) as f:
+        schema = json.load(f)
+
+    found = set()
+    collect_types(schema, found)
+
+    unhandled = sorted(found - HANDLED_TYPES)
+    if unhandled:
+        sys.exit(f"check_schema_types: gen_fuzz_config.py cannot generate schema types {unhandled}")
+
+
+if __name__ == "__main__":
+    main()

acceptance/bin/emit_fuzz_config.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""
+Emit a fuzz databricks.yml on stdout for the current seed, picking the strategy from
+FUZZ_MODE so the invariant scripts don't each duplicate the branch:
+
+  generate (default) - build from scratch by walking `bundle schema` (gen_fuzz_config.py).
+  mutate             - perturb a curated invariant config (mutate_fuzz_config.py).
+
+Reads its inputs from the environment the invariant scripts export: FUZZ_SEED, FUZZ_SCHEMA,
+UNIQUE_NAME, FUZZ_RESOURCES, TESTDIR.
+"""
+
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from envsubst import substitute_variables
+from gen_fuzz_config import gen_config, to_yaml
+from mutate_fuzz_config import load_yaml, mutate
+
+# Curated single-resource configs that deploy standalone (only $UNIQUE_NAME, no init
+# script). All are in the invariant INPUT_CONFIG matrix, so they stay deploy-verified.
+# The seed selects one; mutate_fuzz_config perturbs it.
+MUTATE_BASES = [
+    "catalog",
+    "external_location",
+    "job",
+    "model",
+    "model_serving_endpoint",
+    "pipeline",
+    "registered_model",
+    "schema",
+    "secret_scope",
+    "sql_warehouse",
+    "volume",
+]
+
+
+def generate(seed):
+    with open(os.environ["FUZZ_SCHEMA"]) as f:
+        schema = json.load(f)
+    allowed = {r.strip() for r in os.environ.get("FUZZ_RESOURCES", "").split(",") if r.strip()}
+    unique = f"{os.environ['UNIQUE_NAME']}-{seed}"
+    return to_yaml(gen_config(schema, seed, unique, allowed))
+
+
+def mutate_base(seed):
+    name = MUTATE_BASES[seed % len(MUTATE_BASES)]
+    path = os.path.join(os.environ["TESTDIR"], "..", "configs", name + ".yml.tmpl")
+    with open(path) as f:
+        rendered = substitute_variables(f.read())
+    config = load_yaml(rendered)
+    # The schema lets mutate inject valid optional fields, not just perturb existing ones.
+    with open(os.environ["FUZZ_SCHEMA"]) as f:
+        schema = json.load(f)
+    unique = f"{os.environ['UNIQUE_NAME']}-{seed}"
+    return to_yaml(mutate(config, seed, schema=schema, unique=unique))
+
+
+def main():
+    seed = int(os.environ["FUZZ_SEED"])
+    mode = os.environ.get("FUZZ_MODE", "generate")
+    if mode == "generate":
+        sys.stdout.write(generate(seed))
+    elif mode == "mutate":
+        sys.stdout.write(mutate_base(seed))
+    else:
+        sys.exit(f"emit_fuzz_config: unknown FUZZ_MODE {mode!r}")
+
+
+if __name__ == "__main__":
+    main()