DOC-2235: Document the Visual pipeline editor for Redpanda Connect

.github/workflows/doc-detective-pr-check.yml

@@ -0,0 +1,309 @@
+name: Doc Detective PR check
+
+# Runs the real, browser-driven Doc Detective test(s) associated with any
+# .adoc or spec file a PR touches, against preprod, before merge — the
+# daily cron in doc-detective-screenshots.yml only catches this kind of
+# drift the day *after* it lands. See
+# tests/doc-detective/select-tests-for-changed-files.js for exactly how
+# "associated" is determined (inline `// (test {...})` markup in the .adoc
+# itself, or a spec's `associatedDocs` array).
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '**/*.adoc'
+      - 'tests/doc-detective/specs/**'
+
+permissions:
+  contents: read
+
+jobs:
+  select-tests:
+    # Same-repo PRs only. pull_request events from a fork never receive
+    # this repo's secrets (GitHub withholds them for security), so a fork
+    # PR touching a tested page would just fail on empty credentials —
+    # confusing, not useful. Skip cleanly instead; the daily cron still
+    # catches drift on those pages after merge.
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    outputs:
+      inputs: ${{ steps.select.outputs.inputs }}
+      has-tests: ${{ steps.select.outputs.has-tests }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Determine changed files
+        id: changed
+        run: |
+          git fetch origin "${{ github.event.pull_request.base.sha }}" --depth=1
+          {
+            echo "files<<CHANGED_FILES_EOF"
+            git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}"
+            echo "CHANGED_FILES_EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Select associated tests
+        id: select
+        run: |
+          node tests/doc-detective/select-tests-for-changed-files.js <<'CHANGED_FILES'
+          ${{ steps.changed.outputs.files }}
+          CHANGED_FILES
+
+  test:
+    needs: select-tests
+    if: needs.select-tests.outputs.has-tests == 'true'
+    runs-on: ubuntu-latest
+    # Defense in depth against the doc-detective-CLI-hangs-after-finishing
+    # issue documented on the "Run associated Doc Detective tests" step
+    # below — that step already force-kills the process itself, but this
+    # bounds the whole job in case anything else unexpected hangs. Real runs
+    # observed here finish in ~10-15 minutes; two hung for 100+ minutes each
+    # before a cap existed at all.
+    timeout-minutes: 30
+    # write (not the usual read) — the investigate-on-failure step below
+    # needs to push a new branch and open a PR against main as this job's
+    # own token. It never touches the original PR's own branch (see that
+    # step's comment for why), so this broader scope doesn't weaken the
+    # "never auto-commit to someone else's PR" guarantee the check itself
+    # still keeps.
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # true (not the usual false): the investigate-on-failure step
+          # below needs to `git push`/`gh pr create`/`gh pr comment` as this
+          # job's own token. Safe here because the job's `permissions` block
+          # above already scopes that token to exactly contents/pull-requests
+          # write — nothing broader is being granted by this flag.
+          persist-credentials: true
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Install dependencies
+        run: npm ci
+
+      # visual-editor-quickstart-linux (tests/doc-detective/specs/visual-editor-quickstart.json)
+      # pastes into the pipeline editor via the real OS clipboard, which
+      # needs an actual Chrome + display, not headless Chrome (confirmed by
+      # hand: headless Chrome's synthetic paste silently no-ops rather than
+      # reaching the OS clipboard — a genuine Chrome limitation, not a
+      # doc-detective bug). ubuntu-latest runners have no display at all, so
+      # Xvfb (X Virtual Framebuffer) provides one — Chrome launched under it
+      # behaves exactly like Chrome on a real Linux desktop, including
+      # genuine X11 clipboard access via xclip. Exporting DISPLAY via
+      # $GITHUB_ENV (rather than wrapping just one step in xvfb-run) makes it
+      # available to every later step in this job — harmless for the other,
+      # fully headless tests in the same run, since headless Chrome ignores
+      # DISPLAY entirely. See tests/doc-detective/README.md's "Running the
+      # paste test headless in CI, without a real display" section.
+      # Sized well above the browser viewport requested in config.json
+      # (1600x1000, see there) — confirmed by hand on a real run that a
+      # same-size Xvfb screen isn't enough: Chrome's own window chrome
+      # (tabs, address bar, etc.) eats into it, so a 1280x1024 screen
+      # only left an actual rendered viewport of ~1050x861, well under
+      # the site's responsive breakpoint. That narrower-than-expected
+      # width silently changed the rendered layout enough that several
+      # selectors stopped matching anything — every failure looked like
+      # "element not found," not an obvious size problem.
+      - name: Start Xvfb
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq xvfb xclip
+          Xvfb :99 -screen 0 1920x1200x24 &
+          echo "DISPLAY=:99" >> "$GITHUB_ENV"
+
+      # Preprod only — a PR check must never write test pipelines/secrets
+      # into the shared prod cluster. Prod drift is still covered by the
+      # daily cron in doc-detective-screenshots.yml.
+      - name: Create .env file
+        run: |
+          echo "CLOUD_BASE_URL=https://preprod--redpanda-cloud.netlify.app" >> tests/doc-detective/.env
+          echo "CLOUD_LOGIN_EMAIL=${{ secrets.PREPROD_EMAIL }}" >> tests/doc-detective/.env
+          echo "CLOUD_LOGIN_PASSWORD=${{ secrets.PREPROD_PASSWORD }}" >> tests/doc-detective/.env
+          echo "CLOUD_CLUSTER_ID=${{ vars.PREPROD_CLUSTER_ID }}" >> tests/doc-detective/.env
+
+      # The check itself is still just pass/fail on this PR — it never
+      # commits to or modifies the PR's own branch (see the
+      # investigate-on-failure step below for what happens instead on a real
+      # failure). If the check fails because a screenshot legitimately needs
+      # to change as a result of this PR's own edits, the author can still
+      # regenerate it locally (see tests/doc-detective/README.md) and commit
+      # it themselves, same as any other review feedback — the automated
+      # investigation below is a faster path to the same diagnosis, not a
+      # replacement for it.
+      #
+      # Doesn't rely on the CLI's own process exit at all — confirmed by hand
+      # on real runs that the CLI can finish all tests and print its full
+      # summary (results file written, "Thanks for using Doc Detective"
+      # banner and all), then just hang indefinitely, never actually exiting
+      # the Node process, in this environment. --exit-on-fail (setting
+      # process.exitCode, not calling process.exit()) can't help once that
+      # happens — the exit code is only observed once the event loop drains
+      # naturally, which never happens if something's still holding it open.
+      # Two real runs hung for 100+ minutes each before this was caught (no
+      # step timeout existed yet), burning CI time without ever reaching a
+      # verdict. Instead: run the CLI in the background, poll for the
+      # results file it reliably writes before hanging, force-kill the
+      # process (and anything doc-detective/Chrome-related left running)
+      # once that file appears or a generous cap elapses, then derive
+      # pass/fail directly from the file's own `summary` — same criteria
+      # --exit-on-fail used, but read from disk instead of trusted from the
+      # process. Doesn't fix whatever's keeping the process alive, but the
+      # step can no longer hang past ~12 minutes past real completion
+      # regardless, and the real per-step result is never lost to a forced
+      # kill.
+      - name: Run associated Doc Detective tests
+        run: |
+          set +e
+          rm -f testResults-*.json
+          npx doc-detective --input ${{ needs.select-tests.outputs.inputs }} --config tests/doc-detective/config.json --auto-screenshot &
+          DD_PID=$!
+
+          for i in $(seq 1 90); do
+            if ls testResults-*.json >/dev/null 2>&1; then
+              echo "Results file appeared after ~$((i * 10))s — giving it 15s to finish flushing, then killing the process."
+              break
+            fi
+            sleep 10
+          done
+          sleep 15
+
+          kill -9 "$DD_PID" 2>/dev/null || true
+          pkill -9 -f "doc-detective" 2>/dev/null || true
+          pkill -9 -f "chrome" 2>/dev/null || true
+
+          RESULTS_FILE=$(ls testResults-*.json 2>/dev/null | head -1)
+          if [ -z "$RESULTS_FILE" ]; then
+            echo "No results file appeared within the wait budget — treating as a hard failure."
+            exit 1
+          fi
+
+          node -e "
+            const r = require('./$RESULTS_FILE');
+            console.log(JSON.stringify(r.summary, null, 2));
+
+            // Flatten every test entry across every spec/platform. A given
+            // testId can appear more than once here (one entry per
+            // platform in runOn), so this counts entries, not unique IDs.
+            const allTests = (r.specs || []).flatMap((s) => s.tests || []);
+
+            // Both 'precleanup-*' and '*-retry' tests (see the README's
+            // 'Pre-cleaning before a build test' and 'Retrying a flaky
+            // cleanup step' sections) are DESIGNED to fail cleanly with
+            // 'element not found' in the common, expected case — precleanup
+            // when there's nothing left over from a prior run to clean,
+            // retry when the primary test it retries already succeeded.
+            // Counting either as a real failure would make this check
+            // permanently red even on a fully successful run — confirmed by
+            // hand: a real CI run where the wizard test passed completely
+            // and cluster state was already clean still showed
+            // precleanup-topic-dad-jokes/precleanup-user-connect as FAIL in
+            // the raw counts, exactly as designed. Real failures elsewhere
+            // still gate normally; a precleanup/retry test failing for a
+            // genuinely different reason (found something, then failed to
+            // delete it) isn't distinguished from the expected no-op here —
+            // an accepted, documented low-stakes gap, not silently swept
+            // under the rug.
+            const isExpectedNoOp = (id) => id.startsWith('precleanup-') || id.endsWith('-retry');
+            const realFails = allTests.filter(
+              (t) => t.result === 'FAIL' && !isExpectedNoOp(t.testId)
+            );
+
+            // Cross-platform 'skipped' is expected and correct here, not a
+            // problem: visual-editor-quickstart-mac/-linux each target one
+            // platform via runOn, so exactly one of them is always skipped
+            // on any given runner. What must still fail the build is the
+            // OTHER kind of empty run — nothing could even execute (e.g. no
+            // browser context could start at all) — which shows up as zero
+            // non-precleanup/retry tests reaching a real PASS or FAIL
+            // verdict.
+            const meaningfulRan = allTests.filter(
+              (t) => !isExpectedNoOp(t.testId) && (t.result === 'PASS' || t.result === 'FAIL')
+            );
+
+            if (realFails.length > 0) {
+              console.log('Real failures:', realFails.map((t) => t.testId));
+            }
+            const failed = realFails.length > 0 || meaningfulRan.length === 0;
+            process.exit(failed ? 1 : 0);
+          "
+
+      # Failure diagnostics: without this, a failure on the runner is a
+      # black box — the terminal summary names a failed step but not what
+      # was actually on screen or why. --auto-screenshot (above) captures a
+      # screenshot after every browser step; upload both that and the full
+      # JSON report so a human (or the investigate-with-Claude flow used in
+      # doc-detective-screenshots.yml) can see the real page state instead
+      # of guessing from a step ID alone.
+      - name: Upload Doc Detective artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: doc-detective-results
+          path: |
+            testResults-*.json
+            debug-*.png
+            .doc-detective/
+          retention-days: 7
+
+      # Runs only when the step above failed the job. Investigates the
+      # failure with Claude Code, authenticating directly to Anthropic via
+      # ANTHROPIC_API_KEY (a devprod-issued secret) — same mechanism as the
+      # equivalent step in doc-detective-screenshots.yml, see its comment
+      # for the auth-chain rationale. Runs in the same job right after the
+      # failure, so testResults-*.json/debug-*.png/.doc-detective/ are
+      # already on disk — nothing needs downloading first.
+      #
+      # Deliberately never touches #${{ github.event.pull_request.number }}'s
+      # own branch — that branch belongs to its author, and an unrequested
+      # commit there would be a surprising, unrequested action, same
+      # reasoning as why this check itself never auto-commits. Instead, a
+      # real+fixable diagnosis becomes its own separate PR against `main`
+      # (branched from `main`, not from the PR under test) plus a comment on
+      # the original PR linking to it; anything else (transient, or real but
+      # out of scope for this repo) becomes just a comment stating the
+      # diagnosis. See tests/doc-detective/investigate-pr-failure-prompt-template.md
+      # for the full instructions this step gives Claude, including exactly
+      # what "exactly one of these outcomes" means here.
+      - name: Investigate failure with Claude
+        if: failure()
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_URL: ${{ github.event.pull_request.html_url }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          RUN_ID: ${{ github.run_id }}
+          REPO: ${{ github.repository }}
+        run: |
+          npm install -g @anthropic-ai/claude-code
+
+          sed \
+            -e "s|\${PR_NUMBER}|$PR_NUMBER|g" \
+            -e "s|\${PR_URL}|$PR_URL|g" \
+            -e "s|\${RUN_URL}|$RUN_URL|g" \
+            -e "s|\${RUN_ID}|$RUN_ID|g" \
+            -e "s|\${REPO}|$REPO|g" \
+            tests/doc-detective/investigate-pr-failure-prompt-template.md > /tmp/investigate-pr-prompt.md
+
+          claude -p "$(cat /tmp/investigate-pr-prompt.md)" \
+            --permission-mode bypassPermissions \
+            --effort high \
+            --output-format json \
+            --max-budget-usd 3