diff --git a/.github/workflows/build-ts.yml b/.github/workflows/build-ts.yml index 47b5421d8d..f4bf089379 100644 --- a/.github/workflows/build-ts.yml +++ b/.github/workflows/build-ts.yml @@ -44,8 +44,10 @@ jobs: git config --global core.autocrlf false - uses: actions/checkout@v5 with: - # Full history so the changed-files lint can diff against the base. - fetch-depth: 0 + # pull_request checks out the merge commit. Depth 2 is enough for + # merge-base / HEAD^1. push / merge_group keep a full clone so + # `npm run lint -- --ratchet` can still resolve origin/main. + fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} - uses: dorny/paths-filter@v3 id: filter continue-on-error: true @@ -54,36 +56,49 @@ jobs: ts: - "ts/**" - ".github/workflows/build-ts.yml" + # Merge-gate work (install/build/test) still runs on every OS × Node + # cell. Scope only collapses redundant PR ratchets + the base fetch + # onto ubuntu/22. See ts/tools/scripts/prCiScope.mjs. + - name: Decide job scope + id: scope + env: + EVENT_NAME: ${{ github.event_name }} + TS_FILTER: ${{ steps.filter.outputs.ts }} + MATRIX_OS: ${{ matrix.os }} + MATRIX_VERSION: ${{ matrix.version }} + run: node ts/tools/scripts/prCiScope.mjs - uses: pnpm/action-setup@v4 - if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.full == 'true' || steps.scope.outputs.lint == 'true' }} name: Install pnpm with: package_json_file: ts/package.json - uses: actions/setup-node@v5 - if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.full == 'true' || steps.scope.outputs.lint == 'true' }} with: node-version: ${{ matrix.version }} cache: "pnpm" cache-dependency-path: ts/pnpm-lock.yaml - name: Install dependencies - if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.full == 'true' || steps.scope.outputs.lint == 'true' }} working-directory: ts run: | pnpm install --frozen-lockfile --strict-peer-dependencies - name: Build - if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.full == 'true' }} working-directory: ts run: | npm run build + - name: Fetch PR base + if: ${{ steps.scope.outputs.fetch == 'true' }} + run: git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" # On pull requests only changed files are checked (fast); the # format-pr workflow auto-fixes them. Other events check the whole repo. - name: Lint - if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.lint == 'true' }} working-directory: ts shell: bash run: | if [ "${{ github.event_name }}" = "pull_request" ]; then - git fetch --no-tags origin "${{ github.base_ref }}" node tools/scripts/prettier-changed.mjs --base "origin/${{ github.base_ref }}" else npm run lint @@ -96,11 +111,10 @@ jobs: # // code-complexity-allow markers. Tune the thresholds down as hotspots # get refactored. - name: Complexity ratchet - if: ${{ github.event_name == 'pull_request' && steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.ratchet == 'true' }} working-directory: ts shell: bash run: | - git fetch --no-tags origin "${{ github.base_ref }}" npm run code-complexity -- --ratchet --base "origin/${{ github.base_ref }}" \ --cyclomatic 25 --cognitive 30 \ --new-file-cyclomatic 25 --new-file-cognitive 30 @@ -109,45 +123,42 @@ jobs: # base branch. Syntactic rules only, so it is fast; the count can only # trend down. Run `npm run code-lint` locally to see the full report. - name: Lint ratchet - if: ${{ github.event_name == 'pull_request' && steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.ratchet == 'true' }} working-directory: ts shell: bash run: | - git fetch --no-tags origin "${{ github.base_ref }}" npm run code-lint -- --ratchet --base "origin/${{ github.base_ref }}" # Circular-dependency ratchet (PRs only): fail if the change introduces a # runtime import cycle not already present at the base. Builds the cycle # set for HEAD and for the merge base (via a throwaway git worktree), so # this step is heavier than the others (madge runs twice). - name: Circular dependency ratchet - if: ${{ github.event_name == 'pull_request' && steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.ratchet == 'true' }} working-directory: ts shell: bash run: | - git fetch --no-tags origin "${{ github.base_ref }}" npm run code-circular -- --ratchet --base "origin/${{ github.base_ref }}" --exceptions-file tools/scripts/code/circular-baseline-exception.json # Test-debt gate (PRs only): zero tolerance for focused tests # (.only/fit/fdescribe) and no newly skipped tests (.skip/xit/xdescribe) # in changed files. A small, fixable problem -> a hard gate, not a ratchet. - name: Test debt gate - if: ${{ github.event_name == 'pull_request' && steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.ratchet == 'true' }} working-directory: ts shell: bash run: | - git fetch --no-tags origin "${{ github.base_ref }}" npm run code-debt -- --gate --base "origin/${{ github.base_ref }}" - name: Restore better-sqlite3 for Node.js - if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.full == 'true' }} working-directory: ts run: | pnpm run postinstall:better-sqlite3-node-restore - name: Test - if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} + if: ${{ steps.scope.outputs.full == 'true' }} working-directory: ts run: | npm run test:local - name: UI tests (requires display) - if: ${{ (github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false') && runner.os == 'Linux' }} + if: ${{ steps.scope.outputs.full == 'true' && runner.os == 'Linux' }} working-directory: ts run: | Xvfb :99 -screen 0 1600x1200x24 & diff --git a/pipelines/azure-smoke-tests.yml b/pipelines/azure-smoke-tests.yml index 17643ca2a3..b715d01214 100644 --- a/pipelines/azure-smoke-tests.yml +++ b/pipelines/azure-smoke-tests.yml @@ -114,10 +114,11 @@ jobs: pool: vmImage: ubuntu-latest steps: - # Full history so merge-base / parent diffs resolve on both PR merge - # commits and merge-queue branches. + # PR merge commits only need HEAD and HEAD^1 for the diff below. + # CI / merge-queue builds fetch origin main in the script and treat a + # missing merge-base as "run the tests", so a shallow clone is safe. - checkout: self - fetchDepth: 0 + fetchDepth: 2 - bash: | set -uo pipefail # Default to running the tests; only skip when we can positively @@ -169,9 +170,9 @@ jobs: # required status check keeps passing on PRs that do not touch ts/**. dependsOn: detect_changes condition: and(succeeded(), eq(dependencies.detect_changes.outputs['detect.tsChanged'], 'true')) - # Generous cap: the Linux leg can run the shell smoke (60m) + live (60m) - # tests back to back. Requires purchased parallelism for hosted agents. - timeoutInMinutes: 150 + # Shell/CLI only. Live tests run in parallel on live_linux so this job + # no longer waits for test:live. Requires purchased parallelism. + timeoutInMinutes: 90 strategy: # Both legs run independently; one failing does not cancel the other # (equivalent to the GitHub matrix's fail-fast: false). @@ -198,14 +199,22 @@ jobs: displayName: Install libsecret-1-0 condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - - script: | - pnpm exec playwright install --with-deps - displayName: Install Playwright Browsers - workingDirectory: $(buildDirectory)/packages/shell - - - script: | - npm run build - displayName: Build + # Playwright browser download and tsc do not share a write path. + # Overlap them so the longer of the two sets the wait, not the sum. + # + # Scope the work to what shell + CLI smoke actually need: + # * playwright.config.ts only defines a chromium project (Electron + # suites still use the Playwright runner). Chromium alone is enough. + # * fluid-build agent-shell|agent-cli --dep covers the packages the + # smoke/shell steps exercise. Full monorepo build stays on the + # live_linux job and on build_ts. + - bash: | + set -euo pipefail + (cd packages/shell && pnpm exec playwright install --with-deps chromium) & + PW_PID=$! + pnpm exec fluid-build "agent-shell|agent-cli" -t build --dep + wait "$PW_PID" + displayName: Build shell+cli + Playwright chromium (overlapped) workingDirectory: $(buildDirectory) # Single federated (WIF) login for the whole job. addSpnToEnvironment @@ -254,6 +263,8 @@ jobs: # WorkloadIdentityCredential against the now-stale token file, and a failed # assertion exchange is a hard error that stops the chain before it reaches # this task's fresh AzureCliCredential. Clearing them makes it fall through. + # Full shell:test (jest + all Playwright) on every trigger — PR, main, + # and merge-queue. Linux stays on shell:smoke (unchanged from before). - task: AzureCLI@2 displayName: Shell Tests - full (Windows) timeoutInMinutes: 60 @@ -291,14 +302,69 @@ jobs: export DISPLAY=:99 npm run shell:smoke - # Own AzureCLI@2 login with the published AZURE_* vars cleared — same - # rationale as "Shell Tests - full" above. Non-blocking (continueOnError) - # so a late auth issue won't fail the run. + # Remove provisioned secrets even if a prior step failed. + - pwsh: | + node -e "try{require('fs').unlinkSync('./.env');}catch(e){}" + node -e "try{require('fs').unlinkSync('./config.local.yaml');}catch(e){}" + displayName: Clean up Keys + workingDirectory: $(buildDirectory) + condition: always() + + # Live integration tests on every trigger that has ts changes — including + # PullRequest (same suite as main baseline). Own job so shell/CLI legs do + # not serialize behind live; continueOnError matches baseline (a live + # failure does not fail the required pipeline). Parent still waits for + # this job to finish. Build only packages that define test:live (+deps); + # npm run test:live still walks the whole workspace. + - job: live_linux + displayName: Live tests (Linux) + dependsOn: detect_changes + condition: and(succeeded(), eq(dependencies.detect_changes.outputs['detect.tsChanged'], 'true')) + continueOnError: true + timeoutInMinutes: 90 + pool: + vmImage: ubuntu-latest + steps: + - template: include-prepare-repo.yml + parameters: + buildDirectory: $(buildDirectory) + nodeVersion: $(nodeVersion) + registry: $(INSTALL_REGISTRY) + + - script: | + sudo apt install libsecret-1-0 + displayName: Install libsecret-1-0 + + # Packages that ship a test:live script (see ts/packages/*/package.json + # and LIVE_TEST_PACKAGE_FILTER in prCiScope.mjs). --dep pulls their + # workspace dependencies; full monorepo build is redundant for this suite. + - bash: | + set -euo pipefail + FILTER=$(node tools/scripts/prCiScope.mjs --live-package-filter) + echo "livePackageFilter=$FILTER" + pnpm exec fluid-build "$FILTER" -t build --dep + displayName: Build live packages (+deps) + workingDirectory: $(buildDirectory) + + - task: AzureCLI@2 + displayName: Azure login + Get Keys + inputs: + azureSubscription: $(azureSubscription) + scriptType: pscore + scriptLocation: inlineScript + addSpnToEnvironment: true + workingDirectory: $(buildDirectory) + inlineScript: | + $tokenFile = Join-Path "$(Agent.TempDirectory)" "wif-federated-token.txt" + Set-Content -Path $tokenFile -Value "$env:idToken" -NoNewline + Write-Host "##vso[task.setvariable variable=AZURE_CLIENT_ID]$env:servicePrincipalId" + Write-Host "##vso[task.setvariable variable=AZURE_TENANT_ID]$env:tenantId" + Write-Host "##vso[task.setvariable variable=AZURE_FEDERATED_TOKEN_FILE]$tokenFile" + node tools/scripts/getKeys.mjs --vault build-pipeline-kv --commit + - task: AzureCLI@2 displayName: Live Tests (Linux) timeoutInMinutes: 60 - continueOnError: true - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) inputs: azureSubscription: $(azureSubscription) scriptType: bash @@ -308,7 +374,6 @@ jobs: unset AZURE_CLIENT_ID AZURE_TENANT_ID AZURE_FEDERATED_TOKEN_FILE npm run test:live - # Remove provisioned secrets even if a prior step failed. - pwsh: | node -e "try{require('fs').unlinkSync('./.env');}catch(e){}" node -e "try{require('fs').unlinkSync('./config.local.yaml');}catch(e){}" diff --git a/ts/package.json b/ts/package.json index dfec421605..0a240c003d 100644 --- a/ts/package.json +++ b/ts/package.json @@ -82,6 +82,7 @@ "test:keys": "npx tsx tools/scripts/testServiceKeys.ts", "test:live": "pnpm -r ---no-bail -no-sort --stream --workspace-concurrency=1 run test:live", "test:local": "pnpm -r --no-bail --no-sort --stream --workspace-concurrency=3 run test:local", + "test:pr-ci-scope": "node --test tools/scripts/test/prCiScope.spec.mjs", "test:ui": "pnpm -r --no-bail --no-sort --stream --if-present run test:ui" }, "devDependencies": { diff --git a/ts/tools/scripts/code/README.md b/ts/tools/scripts/code/README.md index 3c6a98564b..7153ef2445 100644 --- a/ts/tools/scripts/code/README.md +++ b/ts/tools/scripts/code/README.md @@ -115,12 +115,13 @@ Four code-quality steps run in [`build-ts.yml`](../../../../.github/workflows/build-ts.yml), **on pull requests only**, sequenced after `Build` and before `Test`. They are skipped entirely unless the PR touches `ts/**` or the workflow file itself (a `dorny/paths-filter` -guard), and — like the rest of the job — they run on every matrix cell -(`ubuntu`/`windows`/`macos` × Node 22/24). +guard). The gates are repo-wide, not OS-specific, so they run once on the +`ubuntu-latest` + Node 22 cell (see `ts/tools/scripts/prCiScope.mjs`). -Each step is a **changed-files diff against the PR's base branch**: it first -`git fetch --no-tags origin `, then passes `--base origin/` -so only what the PR actually touches is judged. Two flavors: +Each step is a **changed-files diff against the PR's base branch**: the +workflow fetches `origin/` once, then every gate passes +`--base origin/` so only what the PR actually touches is judged. Two +flavors: - **Ratchet** (`--ratchet`) — _stateless_: the base branch _is_ the baseline (there is no committed baseline file), so the metric can only trend down. diff --git a/ts/tools/scripts/pr-ci-scope.md b/ts/tools/scripts/pr-ci-scope.md new file mode 100644 index 0000000000..18d6fbe52a --- /dev/null +++ b/ts/tools/scripts/pr-ci-scope.md @@ -0,0 +1,107 @@ +# Faster PR pipelines — scope and measurements + +PR CI still **builds, tests, and packages on every required OS × Node cell**. +The cut is redundant work only: the same ratchet running six times, five +identical `git fetch`es of the base, and a full-history clone on cells that +only need `HEAD` to install/build/test. + +## What changed + +- Shared decision helper: `ts/tools/scripts/prCiScope.mjs`. Tests: + `pnpm run test:pr-ci-scope`. +- `build-ts`: install/build/`test:local` (and Linux UI tests) still run on + all 6 cells when `ts/**` changed — same as `main` today. +- `build-ts` PRs: ratchets + one `git fetch` of the base run once (ubuntu/22) + instead of five fetches and four ratchet steps on every cell. The circular + ratchet is the heavy one (madge twice). +- `build-ts` PRs: every cell uses `fetch-depth: 2` (merge commit + parents). + Tests and ratchets do not need the other ~2700 commits. The base ref is + fetched once with `--depth=1`. +- `build-package-shell`: unchanged merge gate — all 3 OS still package. +- `pipelines/azure-smoke-tests.yml`: + - detect job: `fetchDepth: 2` + - Smoke agents overlap Playwright **chromium** install with + `fluid-build agent-shell|agent-cli --dep` (not full monorepo build, + not every browser binary). `playwright.config.ts` only defines + chromium; `shell:smoke` launches Electron. + - `test:live` is a **parallel** Linux job with `continueOnError` (same + blocking semantics as main baseline). It runs on **PullRequest, main, + and merge-queue** — same suite as baseline. Shell/CLI no longer wait + on live serially. Live job builds only packages that define + `test:live` (+deps); still runs `npm run test:live`. + - Windows always runs full `shell:test` (PR, main, merge-queue). Linux + stays on `shell:smoke` (unchanged from before this work). + +## PR suite parity (must match main baseline smoke) + +| Suite | Baseline PR | This branch | +| -------------------- | ----------------------- | ------------------------------------- | +| CLI smoke | yes | yes | +| Linux `shell:smoke` | yes | yes | +| Windows `shell:test` | yes | yes | +| Linux `test:live` | yes (`continueOnError`) | yes (parallel job, `continueOnError`) | + +Allowed cuts are **redundant steps only** (duplicate ratchets, extra fetches, +full-history clones, serial live after shell, full monorepo build where a +scoped `--dep` build covers the suite). Skipping a suite is not allowed. + +## Job counts (from the shipped helper) + +Run `node tools/scripts/prCiScope.mjs --table` from `ts/`: + +| event | ts filter | ts full | ts ratchet | shell package | +| ------------------------- | ------------ | ------- | ---------- | ------------- | +| pull_request (before) | ts changed | 6 / 6 | 6 / 6 | 3 / 3 | +| pull_request (after) | ts changed | 6 / 6 | 1 / 6 | 3 / 3 | +| pull_request (after) | no ts change | 0 / 6 | 0 / 6 | 0 / 3 | +| merge_group / push / main | (ignored) | 6 / 6 | 0 / 6 | 3 / 3 | + +## Required-check span (the 30% bar) + +Span = last required check `completedAt` − first required check `startedAt` +on one SHA. Required names: `Repo Policy Check`, `build_dotnet (Debug|Release)`, +six `build_ts (os, 22|24)`, three `build_package_shell (os, 22)`, +`TypeAgent Smoke Tests`. + +**Baseline — microsoft/TypeAgent#2847** (merged to `main`, SHA of that PR’s +merge; rollup from the PR checks API): + +| | | +| -------------------- | ------------------------------------------------------ | +| First required start | `TypeAgent Smoke Tests` `2026-08-12T16:39:50Z` | +| Last required finish | `build_ts (windows-latest, 24)` `2026-08-12T17:45:44Z` | +| **Baseline span** | **3954 s (65.90 min)** | +| **30% target** | **≤ 2768 s (46.13 min)** | + +Why #2847 is that long: `build_ts (windows-latest, 22)` waited 24.13 min for +a runner, then ran 20.63 min; `windows-24` waited until that finished +(started `17:28:43Z`, 17.02 min). Smoke itself was 39.48 min +(`16:39:50Z`–`17:19:19Z`) and was _not_ the last required check. + +New span for this draft is filled in after a complete required rollup. +Do not treat a still-running rollup as the 30% win. + +## Timed local analog (this clone, file://) + +Repo history at the branch tip: **2689** commits. + +| analog | wall | `.git` size | commits | +| ------------------------------------------- | ----- | ----------- | ------- | +| `git clone --depth 1` (PR non-ratchet cell) | 2.15s | 60 MiB | 1 | +| `git clone` (fetch-depth 0) | 2.90s | 132 MiB | 2689 | +| 1× `git fetch` after clone | 0.03s | — | — | +| 5× `git fetch` after clone | 0.20s | — | — | + +On GitHub-hosted runners the 5 extra fetches and the full-history clone are +network-bound. The circular ratchet comment in `build-ts.yml` says madge +runs twice and is the heaviest gate — that now runs once per PR instead of +six times. + +## Draft PR FYI + +**A draft PR’s pipeline will not run until it is approved.** Azure DevOps PR +validation (`azure-smoke-tests.yml`, required check “TypeAgent Smoke Tests”) +and some GitHub Actions environment / required-workflow gates stay pending +until a reviewer or admin approves the run. Request `/azp run` (or the GitHub +Actions “Approve and run”) after opening the draft; do not treat an empty +check rollup as a YAML failure. diff --git a/ts/tools/scripts/prCiScope.mjs b/ts/tools/scripts/prCiScope.mjs new file mode 100644 index 0000000000..704e9e79a9 --- /dev/null +++ b/ts/tools/scripts/prCiScope.mjs @@ -0,0 +1,243 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Shared PR job-scope decisions used by build-ts.yml. +// +// Merge-gate work (install / build / test:local / UI tests / shell package) +// still runs on every matrix cell that ran it before. This only strips +// *redundant* PR work: the same ratchet on all 6 cells, and five identical +// `git fetch`es of the base. +// +// Usage in Actions: +// EVENT_NAME, TS_FILTER, MATRIX_OS, MATRIX_VERSION -> GITHUB_OUTPUT +// Local table: +// node tools/scripts/prCiScope.mjs --table + +import fs from "node:fs"; +import { pathToFileURL } from "node:url"; + +// dorny/paths-filter writes "true" / "false". The existing workflows treat +// anything other than the string "false" as "run" (including an unset +// output when the action hits continue-on-error). +export function pathFilterAllows(output) { + return output !== "false"; +} + +function isPullRequest(eventName) { + return eventName === "pull_request"; +} + +function nodeVersion(version) { + return Number(version); +} + +/** + * Full install + build + test:local (+ UI tests on Linux). + * Same as main today: every OS × Node cell, unless a PR touches no ts paths. + */ +export function shouldRunBuildTsFull({ eventName, tsFilter }) { + if (!isPullRequest(eventName)) { + return true; + } + return pathFilterAllows(tsFilter); +} + +/** + * Changed-file prettier + complexity / lint / circular / debt gates. + * These are repo-wide (not OS-specific), so they run once on ubuntu + 22. + */ +export function shouldRunBuildTsRatchet({ eventName, tsFilter, os, version }) { + if (!isPullRequest(eventName)) { + return false; + } + if (!pathFilterAllows(tsFilter)) { + return false; + } + return os === "ubuntu-latest" && nodeVersion(version) === 22; +} + +/** + * Lint step: whole-repo lint on non-PR events (every cell, same as today); + * on PRs it is the changed-file prettier check and shares the ratchet cell. + */ +export function shouldRunBuildTsLint(ctx) { + if (!isPullRequest(ctx.eventName)) { + return true; + } + return shouldRunBuildTsRatchet(ctx); +} + +export function shouldFetchPrBase(ctx) { + return shouldRunBuildTsRatchet(ctx); +} + +/** + * Electron shell packaging. Every OS cell still packages when ts changed; + * same as main today. + */ +export function shouldRunShellPackage({ eventName, tsFilter }) { + if (!isPullRequest(eventName)) { + return true; + } + return pathFilterAllows(tsFilter); +} + +/** + * Live integration tests (`test:live`). Same suite on every ADO trigger + * that has ts changes — PullRequest, main, and merge-queue. Baseline ran + * live on the Linux smoke leg with continueOnError; we keep that blocking + * semantics and only move live to a parallel job so shell does not wait + * on it serially. Do not gate this off PullRequest. + */ +export function shouldRunLiveTests(_buildReason) { + return true; +} + +/** + * Package-name regexp for fluid-build of packages that define test:live + * (plus --dep). Kept here so tests fail if the live job drifts to a full + * monorepo build or drops a live package. + */ +export const LIVE_TEST_PACKAGE_FILTER = + "agent-api|default-agent-provider|@typeagent/(aiclient|knowpro|knowledge-processor|azure-ai-foundry)"; + +export function resolveScope(ctx) { + return { + full: shouldRunBuildTsFull(ctx), + ratchet: shouldRunBuildTsRatchet(ctx), + lint: shouldRunBuildTsLint(ctx), + fetch: shouldFetchPrBase(ctx), + package: shouldRunShellPackage(ctx), + }; +} + +export const BUILD_TS_OS = ["ubuntu-latest", "windows-latest", "macos-latest"]; +export const BUILD_TS_VERSIONS = [22, 24]; +export const BUILD_PACKAGE_SHELL_OS = [ + "ubuntu-latest", + "windows-latest", + "macos-latest", +]; + +export function countScope(eventName, tsFilter) { + const tsCells = BUILD_TS_OS.flatMap((os) => + BUILD_TS_VERSIONS.map((version) => ({ os, version })), + ); + const shellCells = BUILD_PACKAGE_SHELL_OS.map((os) => ({ + os, + version: 22, + })); + const ctx = (cell) => ({ eventName, tsFilter, ...cell }); + return { + tsJobs: tsCells.length, + tsFull: tsCells.filter((cell) => shouldRunBuildTsFull(ctx(cell))) + .length, + tsRatchet: tsCells.filter((cell) => shouldRunBuildTsRatchet(ctx(cell))) + .length, + tsLint: tsCells.filter((cell) => shouldRunBuildTsLint(ctx(cell))) + .length, + tsFetch: tsCells.filter((cell) => shouldFetchPrBase(ctx(cell))).length, + shellJobs: shellCells.length, + shellPackage: shellCells.filter((cell) => + shouldRunShellPackage(ctx(cell)), + ).length, + }; +} + +export function formatScopeTable() { + const rows = [ + ["event", "ts filter", "ts full", "ts ratchet", "shell package"], + ["pull_request (before)", "ts changed", "6 / 6", "6 / 6", "3 / 3"], + (() => { + const c = countScope("pull_request", "true"); + return [ + "pull_request (after)", + "ts changed", + `${c.tsFull} / ${c.tsJobs}`, + `${c.tsRatchet} / ${c.tsJobs}`, + `${c.shellPackage} / ${c.shellJobs}`, + ]; + })(), + (() => { + const c = countScope("pull_request", "false"); + return [ + "pull_request (after)", + "no ts change", + `${c.tsFull} / ${c.tsJobs}`, + `${c.tsRatchet} / ${c.tsJobs}`, + `${c.shellPackage} / ${c.shellJobs}`, + ]; + })(), + (() => { + const c = countScope("merge_group", "true"); + return [ + "merge_group / push / main", + "(ignored)", + `${c.tsFull} / ${c.tsJobs}`, + `${c.tsRatchet} / ${c.tsJobs}`, + `${c.shellPackage} / ${c.shellJobs}`, + ]; + })(), + ]; + const widths = rows[0].map((_, i) => + Math.max(...rows.map((row) => row[i].length)), + ); + const line = (row) => + `| ${row.map((cell, i) => cell.padEnd(widths[i])).join(" | ")} |`; + const sep = `| ${widths.map((w) => "-".repeat(w)).join(" | ")} |`; + return [line(rows[0]), sep, ...rows.slice(1).map(line)].join("\n"); +} + +function readCtxFromEnv(env = process.env) { + return { + eventName: env.EVENT_NAME ?? "", + tsFilter: env.TS_FILTER ?? "", + os: env.MATRIX_OS ?? "", + version: env.MATRIX_VERSION ?? "", + }; +} + +export function formatGithubOutput(scope) { + return ( + `full=${scope.full}\n` + + `ratchet=${scope.ratchet}\n` + + `lint=${scope.lint}\n` + + `fetch=${scope.fetch}\n` + + `package=${scope.package}\n` + ); +} + +export function writeGithubOutput(scope, env = process.env) { + const text = formatGithubOutput(scope); + if (env.GITHUB_OUTPUT) { + fs.appendFileSync(env.GITHUB_OUTPUT, text); + } + return text; +} + +function main(argv = process.argv.slice(2), env = process.env) { + if (argv.includes("--table")) { + process.stdout.write(`${formatScopeTable()}\n`); + return 0; + } + if (argv.includes("--run-live-tests")) { + process.stdout.write( + `${shouldRunLiveTests(env.BUILD_REASON ?? env.EVENT_NAME ?? "")}\n`, + ); + return 0; + } + if (argv.includes("--live-package-filter")) { + process.stdout.write(`${LIVE_TEST_PACKAGE_FILTER}\n`); + return 0; + } + const scope = resolveScope(readCtxFromEnv(env)); + process.stdout.write(writeGithubOutput(scope, env)); + return 0; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + process.exit(main()); +} diff --git a/ts/tools/scripts/test/prCiScope.spec.mjs b/ts/tools/scripts/test/prCiScope.spec.mjs new file mode 100644 index 0000000000..95b91fe739 --- /dev/null +++ b/ts/tools/scripts/test/prCiScope.spec.mjs @@ -0,0 +1,401 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { + BUILD_PACKAGE_SHELL_OS, + BUILD_TS_OS, + BUILD_TS_VERSIONS, + countScope, + formatGithubOutput, + resolveScope, + shouldFetchPrBase, + shouldRunBuildTsFull, + shouldRunBuildTsLint, + shouldRunBuildTsRatchet, + shouldRunShellPackage, + shouldRunLiveTests, + LIVE_TEST_PACKAGE_FILTER, +} from "../prCiScope.mjs"; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const scriptPath = path.resolve(scriptDir, "../prCiScope.mjs"); +const repoRoot = path.resolve(scriptDir, "../../../.."); +const buildTsYml = path.join(repoRoot, ".github/workflows/build-ts.yml"); +const buildPackageShellYml = path.join( + repoRoot, + ".github/workflows/build-package-shell.yml", +); +const azureSmokeYml = path.join(repoRoot, "pipelines/azure-smoke-tests.yml"); + +function extractYamlList(yaml, key) { + const match = yaml.match(new RegExp(`${key}:\\s*\\[([^\\]]+)\\]`)); + assert.ok(match, `expected ${key}: [...] in workflow YAML`); + return match[1] + .split(",") + .map((item) => item.replace(/["']/g, "").trim()) + .filter(Boolean); +} + +function runCli(env) { + const outFile = path.join( + os.tmpdir(), + `pr-ci-scope-${process.pid}-${Math.random().toString(16).slice(2)}.txt`, + ); + fs.writeFileSync(outFile, ""); + const stdout = execFileSync(process.execPath, [scriptPath], { + env: { ...process.env, ...env, GITHUB_OUTPUT: outFile }, + encoding: "utf8", + }); + const written = fs.readFileSync(outFile, "utf8"); + fs.unlinkSync(outFile); + return { stdout, written }; +} + +test("PR Node 22 ubuntu does full work plus the single ratchet/fetch", () => { + const ctx = { + eventName: "pull_request", + tsFilter: "true", + os: "ubuntu-latest", + version: 22, + }; + assert.equal(shouldRunBuildTsFull(ctx), true); + assert.equal(shouldRunBuildTsRatchet(ctx), true); + assert.equal(shouldRunBuildTsLint(ctx), true); + assert.equal(shouldFetchPrBase(ctx), true); + assert.equal(shouldRunShellPackage(ctx), true); +}); + +test("PR Node 24 cells still build and test; ratchets stay on ubuntu/22", () => { + for (const os of BUILD_TS_OS) { + const ctx = { + eventName: "pull_request", + tsFilter: "true", + os, + version: 24, + }; + assert.equal(shouldRunBuildTsFull(ctx), true); + assert.equal(shouldRunBuildTsRatchet(ctx), false); + assert.equal(shouldFetchPrBase(ctx), false); + } +}); + +test("PR still packages the shell on every OS", () => { + for (const os of BUILD_PACKAGE_SHELL_OS) { + assert.equal( + shouldRunShellPackage({ + eventName: "pull_request", + tsFilter: "true", + os, + }), + true, + ); + } +}); + +test("merge_group and push keep full matrix work", () => { + for (const eventName of ["merge_group", "push", "workflow_dispatch"]) { + const ts = countScope(eventName, "false"); + assert.equal(ts.tsFull, BUILD_TS_OS.length * BUILD_TS_VERSIONS.length); + assert.equal(ts.tsRatchet, 0); + assert.equal(ts.shellPackage, BUILD_PACKAGE_SHELL_OS.length); + } +}); + +test("PR with no ts change skips expensive work on every cell", () => { + const ts = countScope("pull_request", "false"); + assert.equal(ts.tsFull, 0); + assert.equal(ts.tsRatchet, 0); + assert.equal(ts.shellPackage, 0); +}); + +test("unset path-filter output still allows work (matches != 'false')", () => { + assert.equal( + shouldRunBuildTsFull({ + eventName: "pull_request", + tsFilter: "", + version: 22, + }), + true, + ); +}); + +test("CLI writes GITHUB_OUTPUT for a PR Windows Node 24 cell (full, no ratchet)", () => { + const { stdout, written } = runCli({ + EVENT_NAME: "pull_request", + TS_FILTER: "true", + MATRIX_OS: "windows-latest", + MATRIX_VERSION: "24", + }); + const expected = formatGithubOutput( + resolveScope({ + eventName: "pull_request", + tsFilter: "true", + os: "windows-latest", + version: "24", + }), + ); + assert.equal(written, expected); + assert.equal(stdout, expected); + assert.match(written, /^full=true$/m); + assert.match(written, /^ratchet=false$/m); + assert.match(written, /^package=true$/m); +}); + +test("CLI writes GITHUB_OUTPUT for a full merge_group cell", () => { + const { written } = runCli({ + EVENT_NAME: "merge_group", + TS_FILTER: "false", + MATRIX_OS: "macos-latest", + MATRIX_VERSION: "24", + }); + assert.match(written, /^full=true$/m); + assert.match(written, /^ratchet=false$/m); + assert.match(written, /^lint=true$/m); + assert.match(written, /^package=true$/m); +}); + +test("shipped workflows call prCiScope and keep required matrix names", () => { + const buildTs = fs.readFileSync(buildTsYml, "utf8"); + const buildShell = fs.readFileSync(buildPackageShellYml, "utf8"); + + assert.match(buildTs, /prCiScope\.mjs/); + assert.match(buildTs, /steps\.scope\.outputs\.full/); + assert.match(buildTs, /steps\.scope\.outputs\.ratchet/); + assert.match( + buildShell, + /github\.event_name != 'pull_request' \|\| steps\.filter\.outputs\.ts != 'false'/, + ); + + assert.deepEqual(extractYamlList(buildTs, "os"), BUILD_TS_OS); + assert.deepEqual( + extractYamlList(buildTs, "version").map(Number), + BUILD_TS_VERSIONS, + ); + assert.deepEqual(extractYamlList(buildShell, "os"), BUILD_PACKAGE_SHELL_OS); + + const fetches = + buildTs.match(/git fetch --no-tags --depth=1 origin/g) ?? []; + assert.equal( + fetches.length, + 1, + "PR base must be fetched once, not once per ratchet step", + ); + assert.match( + buildTs, + /fetch-depth: \$\{\{ github\.event_name == 'pull_request' && 2 \|\| 0 \}\}/, + "PR checkout is the merge commit plus parents, not full history", + ); + + const prFull = BUILD_TS_OS.flatMap((os) => + BUILD_TS_VERSIONS.map((version) => + shouldRunBuildTsFull({ + eventName: "pull_request", + tsFilter: "true", + os, + version, + }), + ), + ).filter(Boolean).length; + assert.equal(prFull, BUILD_TS_OS.length * BUILD_TS_VERSIONS.length); + const prRatchet = BUILD_TS_OS.flatMap((os) => + BUILD_TS_VERSIONS.map((version) => + shouldRunBuildTsRatchet({ + eventName: "pull_request", + tsFilter: "true", + os, + version, + }), + ), + ).filter(Boolean).length; + assert.equal(prRatchet, 1); +}); + +test("ADO detect job uses a shallow PR checkout", () => { + const yaml = fs.readFileSync(azureSmokeYml, "utf8"); + assert.match( + yaml, + /job:\s*detect_changes[\s\S]*fetchDepth:\s*2/, + "detect_changes must not clone full history just to diff HEAD^1", + ); +}); + +test("ADO Windows always runs full shell:test (PR, main, merge-queue)", () => { + const yaml = fs.readFileSync(azureSmokeYml, "utf8"); + assert.match(yaml, /Shell Tests - full \(Windows\)/); + assert.match(yaml, /npm run shell:test/); + // No PR-only downgrade to shell:smoke on Windows. + assert.equal( + yaml.includes("--windows-shell-suite"), + false, + "Windows suite must not be switched by prCiScope", + ); + const winAt = yaml.indexOf("Shell Tests - full (Windows)"); + assert.ok(winAt > 0); + const winChunk = yaml.slice(winAt, winAt + 800); + assert.match(winChunk, /npm run shell:test/); + assert.equal( + winChunk.includes("shell:smoke"), + false, + "Windows full step must not call shell:smoke", + ); + // Linux PR path stays smoke-only (pre-existing). + assert.match(yaml, /Shell Tests - smoke \(Linux\)/); + assert.match(yaml, /npm run shell:smoke/); +}); + +test("Windows merge-gate jobs still run full install/build/test/package", () => { + const buildTs = fs.readFileSync(buildTsYml, "utf8"); + const buildShell = fs.readFileSync(buildPackageShellYml, "utf8"); + const smoke = fs.readFileSync(azureSmokeYml, "utf8"); + // Leave Windows Defender at the host default — do not disable it in CI. + assert.equal( + /Defender|MpPreference|ExclusionPath/.test( + buildTs + buildShell + smoke, + ), + false, + "CI must not turn off or exclude Windows Defender", + ); + assert.match(buildTs, /npm run test:local/); + assert.match(buildShell, /pnpm run shell:package/); + assert.match(smoke, /npm run shell:test/); +}); + +test("ADO smoke overlaps Playwright chromium with shell+cli build", () => { + const yaml = fs.readFileSync(azureSmokeYml, "utf8"); + assert.match( + yaml, + /Build shell\+cli \+ Playwright chromium \(overlapped\)/, + ); + assert.match(yaml, /playwright install --with-deps chromium/); + assert.match(yaml, /fluid-build "agent-shell\|agent-cli" -t build --dep/); + assert.equal( + /playwright install --with-deps(?! chromium)/.test(yaml), + false, + "smoke must not download every Playwright browser", + ); + const liveAt = yaml.indexOf("job: live_linux"); + const shellAt = yaml.indexOf("job: shell_and_cli"); + assert.ok(liveAt > shellAt, "live_linux must be its own job"); + const shellChunk = yaml.slice(shellAt, liveAt); + // Shell job must not own a Live Tests step (parallel live_linux does). + assert.equal( + /displayName:\s*Live Tests/.test(shellChunk), + false, + "Linux smoke job must not run Live Tests serially", + ); + assert.equal( + /^\s*npm run test:live\s*$/m.test(shellChunk), + false, + "Linux smoke job must not invoke npm run test:live", + ); + // Live job builds only packages that define test:live (+deps). + const liveChunk = yaml.slice(liveAt); + assert.match(liveChunk, /Build live packages \(\+deps\)/); + assert.match(liveChunk, /npm run test:live/); + assert.match(liveChunk, /--live-package-filter/); + assert.match(liveChunk, /fluid-build "\$FILTER" -t build --dep/); + const filterOut = execFileSync( + process.execPath, + [scriptPath, "--live-package-filter"], + { encoding: "utf8" }, + ).trim(); + assert.equal(filterOut, LIVE_TEST_PACKAGE_FILTER); + // Every package that ships test:live must appear in the filter. + const packagesRoot = path.join(repoRoot, "ts/packages"); + const livePkgNames = []; + for (const ent of fs.readdirSync(packagesRoot, { withFileTypes: true })) { + if (!ent.isDirectory()) continue; + const pkgPath = path.join(packagesRoot, ent.name, "package.json"); + if (!fs.existsSync(pkgPath)) continue; + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); + if (pkg.scripts && pkg.scripts["test:live"]) { + livePkgNames.push(pkg.name); + } + } + assert.ok( + livePkgNames.length >= 1, + "expected at least one test:live package", + ); + for (const name of livePkgNames) { + // Filter is a regexp; bare names and @scope/(a|b|c) forms both match. + const re = new RegExp(LIVE_TEST_PACKAGE_FILTER); + assert.ok( + re.test(name), + `LIVE_TEST_PACKAGE_FILTER must match package ${name}`, + ); + } +}); + +test("ADO live tests run on PullRequest (same suite as main baseline)", () => { + assert.equal(shouldRunLiveTests("PullRequest"), true); + assert.equal(shouldRunLiveTests("IndividualCI"), true); + assert.equal(shouldRunLiveTests("Manual"), true); + const pr = execFileSync( + process.execPath, + [scriptPath, "--run-live-tests"], + { + env: { ...process.env, BUILD_REASON: "PullRequest" }, + encoding: "utf8", + }, + ).trim(); + const ci = execFileSync( + process.execPath, + [scriptPath, "--run-live-tests"], + { + env: { ...process.env, BUILD_REASON: "IndividualCI" }, + encoding: "utf8", + }, + ).trim(); + assert.equal(pr, "true"); + assert.equal(ci, "true"); + const yaml = fs.readFileSync(azureSmokeYml, "utf8"); + assert.match(yaml, /job:\s*live_linux/); + assert.match(yaml, /npm run test:live/); + // Must not gate live off PullRequest (suite parity with baseline). + const liveAt = yaml.indexOf("job: live_linux"); + const liveHead = yaml.slice(liveAt, liveAt + 600); + assert.equal( + /ne\(\s*variables\s*\[\s*['"]Build\.Reason['"]\s*\]\s*,\s*['"]PullRequest['"]\s*\)/.test( + liveHead, + ), + false, + "live_linux must run on PullRequest", + ); + assert.match( + liveHead, + /condition:\s*and\(succeeded\(\),\s*eq\(dependencies\.detect_changes\.outputs\['detect\.tsChanged'\],\s*'true'\)\)/, + ); + // Fail closed: if someone reintroduces a PR skip, this test fails. + assert.equal( + yaml.includes("ne(variables['Build.Reason'], 'PullRequest')"), + false, + "no Build.Reason PullRequest exclusion anywhere in smoke YAML", + ); +}); + +test("PR smoke suite parity with main baseline (CLI, shell smoke/test, live)", () => { + const yaml = fs.readFileSync(azureSmokeYml, "utf8"); + // Linux CLI smoke + assert.match(yaml, /Test CLI - smoke/); + assert.match(yaml, /npm run start:dev/); + // Linux shell:smoke + assert.match(yaml, /Shell Tests - smoke \(Linux\)/); + assert.match(yaml, /npm run shell:smoke/); + // Windows full shell:test + assert.match(yaml, /Shell Tests - full \(Windows\)/); + assert.match(yaml, /npm run shell:test/); + // Linux test:live on PR path (parallel job, not PR-skipped) + assert.match(yaml, /job:\s*live_linux/); + assert.match(yaml, /npm run test:live/); + assert.equal( + yaml.includes("ne(variables['Build.Reason'], 'PullRequest')"), + false, + ); +});