fly-deploy.yml

# See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/
#
# Pipeline shape: the deploy is split into a build phase that pushes
# images to registry.fly.io in parallel, and a release phase that points
# each Fly app at the pre-built image. The build phase is the slow part
# (~3 min on Fly's remote builder); decoupling it from release lets us
# build the shared hover/hover-worker image once instead of twice, and
# build the analysis image concurrently rather than after.
#
# Ordering invariant (preserved from the original workflow): hover-analysis
# (lighthouse consumer) must be live before hover-worker (lighthouse
# producer) starts XADDing onto the per-job lighthouse stream. The
# release-worker job declares needs: [release-analysis] to enforce this.

name: Fly Deploy
on:
  push:
    branches:
      - main
    paths-ignore:
      - "**.md"
      - "docs/**"
      - "webflow-designer-extension/**"
      - "webflow-designer-extension-cli/**"
      - "LICENSE"
      - ".gitignore"
      - "CHANGELOG.md"
      - "SECURITY.md"
      - "Roadmap.md"
      - "QandA.md"

# Workflow-level lock: only one full deploy run for this group is in
# flight at a time. cancel-in-progress: false so a second push to main
# queues behind the first rather than aborting it mid-release.
concurrency:
  group: deploy-group
  cancel-in-progress: false

jobs:
  # ───────── Build phase ───────────────────────────────────────────────
  # Two image builds run in parallel. The shared image carries both the
  # API binary (cmd/app) and the worker binary (cmd/worker) — see
  # Dockerfile lines 19–20 — so it is reused by both release-api and
  # release-worker. The analysis image has its own runtime (Debian +
  # Chromium) and is built independently.
  build-shared:
    name: Build hover image (API + worker)
    runs-on: blacksmith-4vcpu-ubuntu-2404
    outputs:
      image: ${{ steps.build.outputs.image }}
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Build and push hover image
        id: build
        env:
          IMAGE_LABEL: deployment-${{ github.run_id }}-${{ github.run_attempt }}
        run: |
          flyctl deploy --build-only --push \
            --image-label "$IMAGE_LABEL" \
            --app hover
          echo "image=registry.fly.io/hover:${IMAGE_LABEL}" >> "$GITHUB_OUTPUT"

  build-analysis:
    name: Build hover-analysis image
    runs-on: blacksmith-4vcpu-ubuntu-2404
    outputs:
      image: ${{ steps.build.outputs.image }}
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Build and push hover-analysis image
        id: build
        env:
          IMAGE_LABEL: deployment-${{ github.run_id }}-${{ github.run_attempt }}
        run: |
          flyctl deploy --build-only --push \
            --image-label "$IMAGE_LABEL" \
            --config fly.analysis.toml \
            --app hover-analysis
          echo "image=registry.fly.io/hover-analysis:${IMAGE_LABEL}" >> "$GITHUB_OUTPUT"

  # ───────── Release phase ─────────────────────────────────────────────
  # release-api and release-analysis fan out in parallel as soon as their
  # builds are ready. release-worker waits on release-analysis so the
  # consumer is healthy before the producer starts publishing.
  release-api:
    name: Release hover (API)
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs: [build-shared]
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Sync secrets to Fly (API app)
        run: |
          flyctl secrets set \
            DATABASE_URL="$DATABASE_URL" \
            DATABASE_DIRECT_URL="$DATABASE_DIRECT_URL" \
            SUPABASE_JWT_SECRET="$SUPABASE_JWT_SECRET" \
            SUPABASE_SERVICE_ROLE_KEY="$SUPABASE_SERVICE_ROLE_KEY" \
            SENTRY_DSN="$SENTRY_DSN" \
            LOOPS_API_KEY="$LOOPS_API_KEY" \
            SLACK_CLIENT_SECRET="$SLACK_CLIENT_SECRET" \
            WEBFLOW_CLIENT_SECRET="$WEBFLOW_CLIENT_SECRET" \
            GOOGLE_CLIENT_SECRET="$GOOGLE_CLIENT_SECRET" \
            OTEL_EXPORTER_OTLP_HEADERS="$OTEL_EXPORTER_OTLP_HEADERS" \
            ARCHIVE_ACCESS_KEY_ID="$ARCHIVE_ACCESS_KEY_ID" \
            ARCHIVE_SECRET_ACCESS_KEY="$ARCHIVE_SECRET_ACCESS_KEY" \
            ARCHIVE_ENDPOINT="$ARCHIVE_ENDPOINT" \
            GRAFANA_CLOUD_USER="$GRAFANA_CLOUD_USER" \
            GRAFANA_CLOUD_API_KEY="$GRAFANA_CLOUD_API_KEY" \
            REDIS_URL="$REDIS_URL" \
            STRIPE_SECRET_KEY="$STRIPE_SECRET_KEY" \
            STRIPE_WEBHOOK_SECRET="$STRIPE_WEBHOOK_SECRET" \
            STRIPE_PUBLISHABLE_KEY="$STRIPE_PUBLISHABLE_KEY" \
            --stage

      - name: Release API app
        env:
          IMAGE: ${{ needs.build-shared.outputs.image }}
        run: |
          flyctl deploy --image "$IMAGE" --app hover
          # Reap stale-image machines left behind by the new release —
          # see PR #369 for why deploy alone isn't enough.
          flyctl scale count 1 --process-group app -a hover --yes

  release-analysis:
    name: Release hover-analysis
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs: [build-analysis]
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Sync secrets to Fly (analysis app)
        run: |
          # hover-analysis consumes the per-job lighthouse streams written
          # by the dispatcher and writes audit metrics back to
          # lighthouse_runs. Only secrets actually referenced by the
          # analysis binary are synced; user-facing auth secrets are
          # deliberately excluded.
          flyctl secrets set \
            --app hover-analysis \
            DATABASE_URL="$DATABASE_URL" \
            DATABASE_DIRECT_URL="$DATABASE_DIRECT_URL" \
            REDIS_URL="$REDIS_URL" \
            SENTRY_DSN="$SENTRY_DSN" \
            OTEL_EXPORTER_OTLP_HEADERS="$OTEL_EXPORTER_OTLP_HEADERS" \
            GRAFANA_CLOUD_USER="$GRAFANA_CLOUD_USER" \
            GRAFANA_CLOUD_API_KEY="$GRAFANA_CLOUD_API_KEY" \
            ARCHIVE_ACCESS_KEY_ID="$ARCHIVE_ACCESS_KEY_ID" \
            ARCHIVE_SECRET_ACCESS_KEY="$ARCHIVE_SECRET_ACCESS_KEY" \
            ARCHIVE_ENDPOINT="$ARCHIVE_ENDPOINT" \
            --stage

      - name: Release analysis app
        env:
          IMAGE: ${{ needs.build-analysis.outputs.image }}
        run: |
          flyctl deploy --image "$IMAGE" \
            --config fly.analysis.toml \
            --app hover-analysis

  release-worker:
    name: Release hover-worker
    runs-on: blacksmith-4vcpu-ubuntu-2404
    # Waits on build-shared for the image, and on release-analysis for
    # the consumer-before-producer invariant.
    needs: [build-shared, release-analysis]
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Sync secrets to Fly (worker app)
        run: |
          # hover-worker consumes Redis streams, executes crawls, persists
          # to Postgres + R2. Only secrets actually referenced by the
          # worker binary are synced here; user-facing auth secrets
          # (Supabase JWT, Slack/Webflow/Google OAuth client secrets) are
          # deliberately excluded.
          flyctl secrets set \
            --app hover-worker \
            DATABASE_URL="$DATABASE_URL" \
            DATABASE_DIRECT_URL="$DATABASE_DIRECT_URL" \
            REDIS_URL="$REDIS_URL" \
            SENTRY_DSN="$SENTRY_DSN" \
            OTEL_EXPORTER_OTLP_HEADERS="$OTEL_EXPORTER_OTLP_HEADERS" \
            GRAFANA_CLOUD_USER="$GRAFANA_CLOUD_USER" \
            GRAFANA_CLOUD_API_KEY="$GRAFANA_CLOUD_API_KEY" \
            ARCHIVE_ACCESS_KEY_ID="$ARCHIVE_ACCESS_KEY_ID" \
            ARCHIVE_SECRET_ACCESS_KEY="$ARCHIVE_SECRET_ACCESS_KEY" \
            ARCHIVE_ENDPOINT="$ARCHIVE_ENDPOINT" \
            --stage

      - name: Release worker app
        # Reuses the hover image from build-shared — Dockerfile compiles
        # both binaries, fly.worker.toml selects the worker entrypoint.
        env:
          IMAGE: ${{ needs.build-shared.outputs.image }}
        run: |
          flyctl deploy --image "$IMAGE" \
            --config fly.worker.toml \
            --app hover-worker

  # ───────── Pool reconcile ───────────────────────────────────────────
  # Pool top-up runs in its own job so downstream autoscaler releases
  # only block on the deploy itself, not on the (much slower) clone-
  # start-stop cycle for each pre-warmed machine. IMAGE_LABEL is derived
  # from the built image tag rather than github.run_attempt — if this
  # job is retried, run_attempt bumps but build-*.outputs.image still
  # points to the prior build's tag. Recomputing IMAGE_LABEL from
  # run_attempt would cause the reconciler to mistake the just-deployed
  # machines for stragglers and destroy them.
  reconcile-analysis-pool:
    name: Reconcile hover-analysis pool
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs: [build-analysis, release-analysis]
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Reap stale-image machines and top up to 10 stopped clones
        env:
          IMAGE: ${{ needs.build-analysis.outputs.image }}
        run: |
          IMAGE_LABEL="${IMAGE##*:}"
          bash scripts/fly-reconcile-pool.sh hover-analysis "$IMAGE_LABEL" 10

  reconcile-worker-pool:
    name: Reconcile hover-worker pool
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs: [build-shared, release-worker]
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Reap stale-image machines and top up to 5 stopped clones
        env:
          IMAGE: ${{ needs.build-shared.outputs.image }}
        run: |
          IMAGE_LABEL="${IMAGE##*:}"
          bash scripts/fly-reconcile-pool.sh hover-worker "$IMAGE_LABEL" 5

  # ───────── Autoscaler release ────────────────────────────────────────
  # fly-autoscaler 0.3.x targets one Fly app per autoscaler instance, so
  # worker and analysis each get their own app. Both jobs run after their
  # target's release succeeds — if the target's pool isn't reconciled,
  # fly-autoscaler has no machines to start/stop. They run in parallel
  # since the two autoscaler apps are independent.
  release-autoscaler-worker:
    name: Release hover-autoscaler-worker
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs: [reconcile-worker-pool]
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Load fly-autoscaler tokens from 1Password
        # Loaded inline (not in the shared fly-setup composite) so review-app
        # CI doesn't need these 1Password fields to exist.
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          FAS_API_TOKEN_WORKER: op://Good Native/hover-fly/FAS_API_TOKEN_WORKER
          FAS_PROMETHEUS_TOKEN: op://Good Native/hover-fly/FAS_PROMETHEUS_TOKEN

      - name: Sync secrets to autoscaler app
        run: |
          # Guard against blank tokens. 1Password's load-secrets-action
          # fails loudly on missing fields, but an empty-but-present field
          # would otherwise stage a blank FAS_API_TOKEN and silently break
          # autoscaler authentication after deploy.
          if [ -z "$FAS_API_TOKEN_WORKER" ]; then
            echo "❌ FAS_API_TOKEN_WORKER is empty. Check op://Good Native/hover-fly/FAS_API_TOKEN_WORKER" >&2
            exit 1
          fi
          if [ -z "$FAS_PROMETHEUS_TOKEN" ]; then
            echo "❌ FAS_PROMETHEUS_TOKEN is empty. Check op://Good Native/hover-fly/FAS_PROMETHEUS_TOKEN" >&2
            exit 1
          fi
          flyctl secrets set \
            --app hover-autoscaler-worker \
            FAS_API_TOKEN="$FAS_API_TOKEN_WORKER" \
            FAS_PROMETHEUS_TOKEN="$FAS_PROMETHEUS_TOKEN" \
            --stage

      - name: Deploy hover-autoscaler-worker
        run: |
          flyctl deploy \
            --config fly.autoscaler-worker.toml \
            --app hover-autoscaler-worker

  release-autoscaler-analysis:
    name: Release hover-autoscaler-analysis
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs: [reconcile-analysis-pool]
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/fly-setup
        with:
          op-service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}

      - name: Load fly-autoscaler tokens from 1Password
        # Loaded inline (not in the shared fly-setup composite) so review-app
        # CI doesn't need these 1Password fields to exist.
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          FAS_API_TOKEN_ANALYSIS:
            op://Good Native/hover-fly/FAS_API_TOKEN_ANALYSIS
          FAS_PROMETHEUS_TOKEN: op://Good Native/hover-fly/FAS_PROMETHEUS_TOKEN

      - name: Sync secrets to autoscaler app
        run: |
          # Guard against blank tokens. 1Password's load-secrets-action
          # fails loudly on missing fields, but an empty-but-present field
          # would otherwise stage a blank FAS_API_TOKEN and silently break
          # autoscaler authentication after deploy.
          if [ -z "$FAS_API_TOKEN_ANALYSIS" ]; then
            echo "❌ FAS_API_TOKEN_ANALYSIS is empty. Check op://Good Native/hover-fly/FAS_API_TOKEN_ANALYSIS" >&2
            exit 1
          fi
          if [ -z "$FAS_PROMETHEUS_TOKEN" ]; then
            echo "❌ FAS_PROMETHEUS_TOKEN is empty. Check op://Good Native/hover-fly/FAS_PROMETHEUS_TOKEN" >&2
            exit 1
          fi
          flyctl secrets set \
            --app hover-autoscaler-analysis \
            FAS_API_TOKEN="$FAS_API_TOKEN_ANALYSIS" \
            FAS_PROMETHEUS_TOKEN="$FAS_PROMETHEUS_TOKEN" \
            --stage

      - name: Deploy hover-autoscaler-analysis
        run: |
          flyctl deploy \
            --config fly.autoscaler-analysis.toml \
            --app hover-autoscaler-analysis

  # ───────── Annotation ────────────────────────────────────────────────
  # if: always() so a partial deploy still produces an annotation for
  # observability. Annotation is best-effort — failures here log a
  # warning but do not fail the workflow.
  annotate:
    name: Post deploy annotation to Grafana
    runs-on: blacksmith-4vcpu-ubuntu-2404
    needs:
      [
        release-api,
        release-analysis,
        release-worker,
        release-autoscaler-worker,
        release-autoscaler-analysis,
      ]
    if: always()
    steps:
      - name: Load Grafana annotation secrets
        continue-on-error: true
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          GRAFANA_URL: op://Good Native/hover-runtime/GRAFANA_URL
          GRAFANA_SA_TOKEN: op://Good Native/hover-runtime/GRAFANA_SA_TOKEN

      - name: Post deploy annotation to Grafana
        # Best-effort: a Grafana outage logs a warning but does not fail
        # the workflow. Commit-message and ref are passed via env vars
        # (not direct ${{ ... }} interpolation) to avoid shell-injection
        # via crafted commit messages — see GitHub Actions injection
        # guidance.
        continue-on-error: true
        env:
          COMMIT_SHA: ${{ github.sha }}
          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
          REPO: ${{ github.repository }}
          REF_NAME: ${{ github.ref_name }}
          WORKFLOW_FILE: fly-deploy.yml
          COMMIT_TAG_CAP: "20"
          GH_TOKEN: ${{ github.token }}
        run: |
          set -o pipefail
          if [ -z "${GRAFANA_URL:-}" ] || [ -z "${GRAFANA_SA_TOKEN:-}" ]; then
            echo "::warning title=Grafana annotation skipped::GRAFANA_URL or GRAFANA_SA_TOKEN missing; skipping annotation."
            exit 0
          fi
          short_sha="${COMMIT_SHA:0:7}"
          commit_url="https://github.com/${REPO}/commit/${COMMIT_SHA}"

          # PR detection — merge commits from GitHub include "(#NNN)" in
          # the subject. Look up the PR via gh api so we get the canonical
          # title rather than whatever shorthand appeared in the commit
          # message.
          pr_number="$(printf '%s' "$COMMIT_MESSAGE" | grep -oE '#[0-9]+' | head -n1 | tr -d '#' || true)"
          pr_section=""
          extra_tags_json='[]'
          if [ -n "${pr_number}" ]; then
            pr_json="$(gh api "repos/${REPO}/pulls/${pr_number}" 2>/dev/null || true)"
            if [ -n "$pr_json" ] && [ "$(printf '%s' "$pr_json" | jq -r '.number // empty')" = "$pr_number" ]; then
              pr_title="$(printf '%s' "$pr_json" | jq -r '.title')"
              pr_url="$(printf '%s' "$pr_json" | jq -r '.html_url')"
              pr_section="$(printf '## PR\n%s\n%s\n\n' "$pr_title" "$pr_url")"
              extra_tags_json="$(jq -cn --arg pr "pr:${pr_number}" '[$pr]')"
            fi
          fi

          # Commit range since previous successful deploy — walk workflow
          # runs for this workflow file on main and pick the most recent
          # success that isn't the current commit.
          prev_sha="$(gh api "repos/${REPO}/actions/workflows/${WORKFLOW_FILE}/runs?branch=main&status=success&per_page=10" \
            --jq "[.workflow_runs[] | select(.head_sha != \"${COMMIT_SHA}\")][0].head_sha" 2>/dev/null || true)"

          commits_body=""
          commit_tags_json='[]'
          if [ -n "$prev_sha" ] && [ "$prev_sha" != "null" ]; then
            compare_json="$(gh api "repos/${REPO}/compare/${prev_sha}...${COMMIT_SHA}" --paginate 2>/dev/null || true)"
            if [ -n "$compare_json" ]; then
              commits_body="$(printf '%s' "$compare_json" | jq -r '.commits[] | "\(.sha[0:7]) \(.commit.message | split("\n")[0])"')"
              commit_tags_json="$(printf '%s' "$compare_json" | jq -c --argjson cap "${COMMIT_TAG_CAP}" '[.commits[].sha[0:7] | "commit:" + .] | .[0:$cap]')"
            fi
          fi

          # Fallback — first deploy, or compare API unavailable: show the
          # head commit only.
          if [ -z "$commits_body" ]; then
            first_line="$(printf '%s' "$COMMIT_MESSAGE" | head -n1)"
            commits_body="$(printf '%s %s' "$short_sha" "$first_line")"
            commit_tags_json="$(jq -cn --arg t "commit:${short_sha}" '[$t]')"
          fi

          # Derive service tags from the files changed in this deploy
          # range. Falls back to the single-commit API when no compare
          # range is available.
          changed_files=""
          if [ -n "$compare_json" ]; then
            changed_files="$(printf '%s' "$compare_json" | jq -r '.files[].filename' 2>/dev/null || true)"
          fi
          if [ -z "$changed_files" ]; then
            changed_files="$(gh api "repos/${REPO}/commits/${COMMIT_SHA}" --jq '.files[].filename' 2>/dev/null || true)"
          fi

          service_tags_json='[]'
          # cmd/app is the API server; cmd/hover is the CLI tool.
          if printf '%s\n' "$changed_files" | grep -qE '^(cmd/app/|internal/(api|archive|auth|cache|loops|notifications|storage|techdetect|util)/|web/)'; then
            service_tags_json="$(jq -cn --argjson t "$service_tags_json" '$t + ["service:hover"]')"
          fi
          if printf '%s\n' "$changed_files" | grep -qE '^cmd/worker/'; then
            service_tags_json="$(jq -cn --argjson t "$service_tags_json" '$t + ["service:hover-worker"]')"
          fi
          # hover-analysis surface: the analysis binary itself, its image
          # config, fly app config, and the lighthouse package modules
          # that only it consumes (runner). The shared lighthouse pieces
          # (sampler, scheduler) are picked up by the broader internal/...
          # clause below since the worker imports them too.
          if printf '%s\n' "$changed_files" | grep -qE '^(cmd/analysis/|Dockerfile\.analysis|fly\.analysis\.toml|\.fly/review_apps\.analysis\.toml|internal/lighthouse/runner\.go)'; then
            service_tags_json="$(jq -cn --argjson t "$service_tags_json" '$t + ["service:hover-analysis"]')"
          fi
          # Shared internal packages used by every service. Lighthouse
          # producer code lives in internal/lighthouse and is imported by
          # the worker; the consumer (analysis) imports the runner.
          if printf '%s\n' "$changed_files" | grep -qE '^internal/(broker|crawler|db|jobs|lighthouse|logging|observability)/'; then
            service_tags_json="$(jq -cn --argjson t "$service_tags_json" '$t + ["service:hover","service:hover-worker","service:hover-analysis"]')"
          fi
          if printf '%s\n' "$changed_files" | grep -qE '^(cmd/hover/|webflow-designer-extension-cli/|npm/)'; then
            service_tags_json="$(jq -cn --argjson t "$service_tags_json" '$t + ["service:hover-cli"]')"
          fi
          if printf '%s\n' "$changed_files" | grep -qE '^(grafana/|\.github/|scripts/)'; then
            service_tags_json="$(jq -cn --argjson t "$service_tags_json" '$t + ["service:grafana"]')"
          fi
          if printf '%s\n' "$changed_files" | grep -qE '^supabase/'; then
            service_tags_json="$(jq -cn --argjson t "$service_tags_json" '$t + ["service:supabase"]')"
          fi
          # Fallback — unrecognised paths: tag every service conservatively.
          if [ "$service_tags_json" = "[]" ]; then
            service_tags_json='["service:hover","service:hover-worker","service:hover-analysis"]'
          fi

          body_text="$(printf '%s## Commits\n%s\n%s' "$pr_section" "$commits_body" "$commit_url")"

          tags_json="$(jq -cn \
            --arg ref "$REF_NAME" \
            --arg sha "$short_sha" \
            --argjson extra "$extra_tags_json" \
            --argjson commits "$commit_tags_json" \
            --argjson services "$service_tags_json" \
            '(["deploy","env:production",
              ("branch:" + $ref), ("commit:" + $sha)] + $services + $extra + $commits) | unique')"

          now_ms=$(($(date +%s) * 1000))
          payload="$(jq -n \
            --argjson time "$now_ms" \
            --arg text "$body_text" \
            --argjson tags "$tags_json" \
            '{time: $time, tags: $tags, text: $text}')"
          if ! curl --fail --silent --show-error \
            --connect-timeout 5 --max-time 15 \
            -X POST "${GRAFANA_URL%/}/api/annotations" \
            -H "Authorization: Bearer ${GRAFANA_SA_TOKEN}" \
            -H "Content-Type: application/json" \
            -d "$payload"; then
            echo "::warning title=Grafana annotation failed::Deploy succeeded but annotation POST failed. Check GRAFANA_URL and GRAFANA_SA_TOKEN in 1Password."
          fi