From 6358e08346068337a823f9ef42aa68a0ad0de337 Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 13 Aug 2026 18:10:50 -0700 Subject: [PATCH 1/5] Show test failures summary at end of test step in build-ts --- .github/workflows/build-ts.yml | 2 +- ts/jest.config.js | 10 ++ ts/packages/agents/browser/jest.config.js | 15 +++ .../agents/browserExtension/jest.config.js | 15 +++ ts/packages/agents/onboarding/package.json | 2 +- ts/packages/typeagent-studio/package.json | 2 +- ts/tools/scripts/jestFailureReporter.cjs | 39 ++++++++ ts/tools/scripts/nodeTestFailureReporter.mjs | 42 ++++++++ ts/tools/scripts/runTestLocalWithSummary.mjs | 96 +++++++++++++++++++ ts/tools/scripts/testFailureOutput.cjs | 22 +++++ 10 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 ts/tools/scripts/jestFailureReporter.cjs create mode 100644 ts/tools/scripts/nodeTestFailureReporter.mjs create mode 100644 ts/tools/scripts/runTestLocalWithSummary.mjs create mode 100644 ts/tools/scripts/testFailureOutput.cjs diff --git a/.github/workflows/build-ts.yml b/.github/workflows/build-ts.yml index 47b5421d8d..ac8ceb7609 100644 --- a/.github/workflows/build-ts.yml +++ b/.github/workflows/build-ts.yml @@ -145,7 +145,7 @@ jobs: if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} working-directory: ts run: | - npm run test:local + node tools/scripts/runTestLocalWithSummary.mjs - name: UI tests (requires display) if: ${{ (github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false') && runner.os == 'Linux' }} working-directory: ts diff --git a/ts/jest.config.js b/ts/jest.config.js index bdd5b9ce65..c5961fc0b8 100644 --- a/ts/jest.config.js +++ b/ts/jest.config.js @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +const path = require("node:path"); + /** @type {import('ts-jest').JestConfigWithTsJest} */ module.exports = { testMatch: ["**/dist/test/**/*.(spec|test).js?(x)"], @@ -9,4 +11,12 @@ module.exports = { "^../src/(.*)$": "/dist/$1", }, testTimeout: 90000, + ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined + ? {} + : { + reporters: [ + "default", + path.join(__dirname, "tools/scripts/jestFailureReporter.cjs"), + ], + }), }; diff --git a/ts/packages/agents/browser/jest.config.js b/ts/packages/agents/browser/jest.config.js index dfce758aa6..5e0437aa17 100644 --- a/ts/packages/agents/browser/jest.config.js +++ b/ts/packages/agents/browser/jest.config.js @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { fileURLToPath } from "node:url"; + /** @type {import('ts-jest').JestConfigWithTsJest} */ export default { preset: "ts-jest/presets/default-esm", @@ -39,4 +41,17 @@ export default { // Map .mjs imports from src to .mts files for ts-jest "^(.*)\\.mjs$": "$1.mts", }, + ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined + ? {} + : { + reporters: [ + "default", + fileURLToPath( + new URL( + "../../../tools/scripts/jestFailureReporter.cjs", + import.meta.url, + ), + ), + ], + }), }; diff --git a/ts/packages/agents/browserExtension/jest.config.js b/ts/packages/agents/browserExtension/jest.config.js index e06cce61d7..05ea393085 100644 --- a/ts/packages/agents/browserExtension/jest.config.js +++ b/ts/packages/agents/browserExtension/jest.config.js @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { fileURLToPath } from "node:url"; + /** @type {import('ts-jest').JestConfigWithTsJest} */ export default { preset: "ts-jest/presets/default-esm", @@ -47,4 +49,17 @@ export default { // Map .mjs imports from src to .mts files for ts-jest "^(.*)\\.mjs$": "$1.mts", }, + ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined + ? {} + : { + reporters: [ + "default", + fileURLToPath( + new URL( + "../../../tools/scripts/jestFailureReporter.cjs", + import.meta.url, + ), + ), + ], + }), }; diff --git a/ts/packages/agents/onboarding/package.json b/ts/packages/agents/onboarding/package.json index b22200c6b3..1ece32aaf5 100644 --- a/ts/packages/agents/onboarding/package.json +++ b/ts/packages/agents/onboarding/package.json @@ -43,7 +43,7 @@ "prettier": "prettier --check . --ignore-path ../../../.prettierignore", "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore", "test": "npm run test:local", - "test:local": "tsx --test test/*.spec.ts", + "test:local": "tsx --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=../../../tools/scripts/nodeTestFailureReporter.mjs --test-reporter-destination=stdout test/*.spec.ts", "tsc": "tsc -b" }, "dependencies": { diff --git a/ts/packages/typeagent-studio/package.json b/ts/packages/typeagent-studio/package.json index 0fde5eaabf..6176ec4137 100644 --- a/ts/packages/typeagent-studio/package.json +++ b/ts/packages/typeagent-studio/package.json @@ -23,7 +23,7 @@ "compile": "node esbuild.mjs", "deploy:local": "npm run package && code --install-extension dist-pub/typeagent-studio.vsix --force", "package": "mkdirp dist-pub && vsce package --allow-star-activation --allow-missing-repository --no-dependencies --allow-package-secrets npm -o dist-pub/typeagent-studio.vsix", - "test:local": "tsx --test src/test/*.spec.ts", + "test:local": "tsx --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=../../tools/scripts/nodeTestFailureReporter.mjs --test-reporter-destination=stdout src/test/*.spec.ts", "vscode:prepublish": "npm run compile", "watch": "node esbuild.mjs --watch" }, diff --git a/ts/tools/scripts/jestFailureReporter.cjs b/ts/tools/scripts/jestFailureReporter.cjs new file mode 100644 index 0000000000..2fd19f3604 --- /dev/null +++ b/ts/tools/scripts/jestFailureReporter.cjs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const { writeTestFailures } = require("./testFailureOutput.cjs"); + +class JestFailureReporter { + onRunComplete(_testContexts, aggregatedResult) { + const failures = []; + + for (const testFileResult of aggregatedResult.testResults) { + const failedTests = testFileResult.testResults.filter( + (testResult) => testResult.status === "failed", + ); + + for (const failedTest of failedTests) { + failures.push({ + testFilePath: testFileResult.testFilePath, + fullName: failedTest.fullName, + failureMessages: failedTest.failureMessages, + }); + } + + if ( + failedTests.length === 0 && + typeof testFileResult.failureMessage === "string" + ) { + failures.push({ + testFilePath: testFileResult.testFilePath, + fullName: "Test suite failed to run", + failureMessages: [testFileResult.failureMessage], + }); + } + } + + writeTestFailures(failures); + } +} + +module.exports = JestFailureReporter; diff --git a/ts/tools/scripts/nodeTestFailureReporter.mjs b/ts/tools/scripts/nodeTestFailureReporter.mjs new file mode 100644 index 0000000000..7dd6871d72 --- /dev/null +++ b/ts/tools/scripts/nodeTestFailureReporter.mjs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import failureOutput from "./testFailureOutput.cjs"; + +const { writeTestFailures } = failureOutput; + +export default async function* nodeTestFailureReporter(source) { + const failures = []; + const stderrByTestFile = new Map(); + + try { + for await (const event of source) { + if (event.type === "test:stderr") { + const messages = stderrByTestFile.get(event.data.file) ?? []; + messages.push(event.data.message); + stderrByTestFile.set(event.data.file, messages); + continue; + } + + const error = event.data?.details?.error; + if ( + event.type === "test:fail" && + error?.failureType !== "subtestsFailed" + ) { + const failure = error.cause ?? error; + const failureMessage = + failure === "test failed" + ? (stderrByTestFile.get(event.data.file)?.join("") ?? + failure) + : (failure.stack ?? failure.message ?? String(failure)); + failures.push({ + testFilePath: event.data.file ?? event.data.name, + fullName: event.data.name, + failureMessages: [failureMessage], + }); + } + } + } finally { + writeTestFailures(failures); + } +} diff --git a/ts/tools/scripts/runTestLocalWithSummary.mjs b/ts/tools/scripts/runTestLocalWithSummary.mjs new file mode 100644 index 0000000000..0c264f1528 --- /dev/null +++ b/ts/tools/scripts/runTestLocalWithSummary.mjs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +const failureDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), "typeagent-test-failures-"), +); +const [testCommand, testArguments] = + process.platform === "win32" + ? [ + process.env.ComSpec ?? "cmd.exe", + ["/d", "/s", "/c", "pnpm run test:local"], + ] + : ["pnpm", ["run", "test:local"]]; + +const result = spawnSync(testCommand, testArguments, { + env: { + ...process.env, + TYPEAGENT_TEST_FAILURES_DIR: failureDirectory, + }, + stdio: "inherit", +}); + +try { + printFailureSummary(failureDirectory, result.status); +} finally { + fs.rmSync(failureDirectory, { recursive: true, force: true }); +} + +if (result.error !== undefined) { + throw result.error; +} +if (result.signal !== null) { + process.kill(process.pid, result.signal); +} +process.exitCode = result.status ?? 1; + +function printFailureSummary(directory, exitCode) { + const failures = fs + .readdirSync(directory) + .filter((fileName) => fileName.endsWith(".json")) + .flatMap((fileName) => + JSON.parse(fs.readFileSync(path.join(directory, fileName), "utf8")), + ); + const uniqueFailures = [ + ...new Map( + failures.map((failure) => [JSON.stringify(failure), failure]), + ).values(), + ]; + + if (uniqueFailures.length === 0) { + if (exitCode !== 0) { + console.error( + "\nFAILED TEST SUMMARY\nNo individual test failures were captured. See the test output above.", + ); + } + return; + } + + console.error( + `\n${"=".repeat(80)}\nFAILED TEST SUMMARY (${uniqueFailures.length})\n${"=".repeat(80)}`, + ); + for (const failure of uniqueFailures) { + console.error(`\nFAIL ${normalizePath(failure.testFilePath)}`); + console.error(` ${failure.fullName}`); + for (const message of failure.failureMessages) { + console.error(indent(stripAnsi(message), 4)); + } + } + console.error(`\n${"=".repeat(80)}`); +} + +function normalizePath(filePath) { + return path.relative(process.cwd(), filePath).replaceAll("\\", "/"); +} + +function stripAnsi(value) { + return value.replace( + // eslint-disable-next-line no-control-regex + /[\u001B\u009B][[\]()#;?]*(?:(?:(?:[a-zA-Z\d]*(?:;[-a-zA-Z\d/#&.:=?%@~_]+)*)?\u0007)|(?:(?:\d{1,4}(?:[;:]\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~]))/g, + "", + ); +} + +function indent(value, spaces) { + const prefix = " ".repeat(spaces); + return value + .trimEnd() + .split(/\r?\n/u) + .map((line) => `${prefix}${line}`) + .join("\n"); +} diff --git a/ts/tools/scripts/testFailureOutput.cjs b/ts/tools/scripts/testFailureOutput.cjs new file mode 100644 index 0000000000..b61edeabce --- /dev/null +++ b/ts/tools/scripts/testFailureOutput.cjs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const fs = require("node:fs"); +const path = require("node:path"); +const { randomUUID } = require("node:crypto"); + +function writeTestFailures(failures) { + const outputDirectory = process.env.TYPEAGENT_TEST_FAILURES_DIR; + if (outputDirectory === undefined || failures.length === 0) { + return; + } + + fs.mkdirSync(outputDirectory, { recursive: true }); + const outputPath = path.join( + outputDirectory, + `${process.pid}-${randomUUID()}.json`, + ); + fs.writeFileSync(outputPath, JSON.stringify(failures), "utf8"); +} + +module.exports = { writeTestFailures }; From 71b4f073720eec2e5480321013ed35853a5392b5 Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 13 Aug 2026 22:18:50 -0700 Subject: [PATCH 2/5] Fix the error giving a rachet violation --- ts/tools/scripts/runTestLocalWithSummary.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/ts/tools/scripts/runTestLocalWithSummary.mjs b/ts/tools/scripts/runTestLocalWithSummary.mjs index 0c264f1528..214744c94a 100644 --- a/ts/tools/scripts/runTestLocalWithSummary.mjs +++ b/ts/tools/scripts/runTestLocalWithSummary.mjs @@ -80,7 +80,6 @@ function normalizePath(filePath) { function stripAnsi(value) { return value.replace( - // eslint-disable-next-line no-control-regex /[\u001B\u009B][[\]()#;?]*(?:(?:(?:[a-zA-Z\d]*(?:;[-a-zA-Z\d/#&.:=?%@~_]+)*)?\u0007)|(?:(?:\d{1,4}(?:[;:]\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~]))/g, "", ); From 6087c2e035e4f67a1a79b969a8adeaf6d34f86cc Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 13 Aug 2026 22:22:09 -0700 Subject: [PATCH 3/5] Refactor improvements --- ts/tools/scripts/jestFailureReporter.cjs | 22 ++++++++++++++++++- ts/tools/scripts/runTestLocalWithSummary.mjs | 23 ++++++++++++++++---- ts/tools/scripts/testFailureOutput.cjs | 10 ++++----- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/ts/tools/scripts/jestFailureReporter.cjs b/ts/tools/scripts/jestFailureReporter.cjs index 2fd19f3604..6923ae5670 100644 --- a/ts/tools/scripts/jestFailureReporter.cjs +++ b/ts/tools/scripts/jestFailureReporter.cjs @@ -20,7 +20,16 @@ class JestFailureReporter { }); } - if ( + if (testFileResult.testExecError !== undefined) { + failures.push({ + testFilePath: testFileResult.testFilePath, + fullName: "Test suite failed outside an individual test", + failureMessages: [ + testFileResult.testExecError.stack ?? + testFileResult.testExecError.message, + ], + }); + } else if ( failedTests.length === 0 && typeof testFileResult.failureMessage === "string" ) { @@ -32,6 +41,17 @@ class JestFailureReporter { } } + if (aggregatedResult.runExecError !== undefined) { + failures.push({ + testFilePath: "", + fullName: "Jest failed outside a test suite", + failureMessages: [ + aggregatedResult.runExecError.stack ?? + aggregatedResult.runExecError.message, + ], + }); + } + writeTestFailures(failures); } } diff --git a/ts/tools/scripts/runTestLocalWithSummary.mjs b/ts/tools/scripts/runTestLocalWithSummary.mjs index 214744c94a..a936b11298 100644 --- a/ts/tools/scripts/runTestLocalWithSummary.mjs +++ b/ts/tools/scripts/runTestLocalWithSummary.mjs @@ -43,9 +43,7 @@ function printFailureSummary(directory, exitCode) { const failures = fs .readdirSync(directory) .filter((fileName) => fileName.endsWith(".json")) - .flatMap((fileName) => - JSON.parse(fs.readFileSync(path.join(directory, fileName), "utf8")), - ); + .flatMap((fileName) => readFailureArtifact(directory, fileName)); const uniqueFailures = [ ...new Map( failures.map((failure) => [JSON.stringify(failure), failure]), @@ -74,8 +72,25 @@ function printFailureSummary(directory, exitCode) { console.error(`\n${"=".repeat(80)}`); } +function readFailureArtifact(directory, fileName) { + try { + return JSON.parse( + fs.readFileSync(path.join(directory, fileName), "utf8"), + ); + } catch (error) { + console.error( + `Unable to read test failure artifact ${fileName}: ${error.message}`, + ); + return []; + } +} + function normalizePath(filePath) { - return path.relative(process.cwd(), filePath).replaceAll("\\", "/"); + return ( + path.isAbsolute(filePath) + ? path.relative(process.cwd(), filePath) + : filePath + ).replaceAll("\\", "/"); } function stripAnsi(value) { diff --git a/ts/tools/scripts/testFailureOutput.cjs b/ts/tools/scripts/testFailureOutput.cjs index b61edeabce..a9e6cab507 100644 --- a/ts/tools/scripts/testFailureOutput.cjs +++ b/ts/tools/scripts/testFailureOutput.cjs @@ -12,11 +12,11 @@ function writeTestFailures(failures) { } fs.mkdirSync(outputDirectory, { recursive: true }); - const outputPath = path.join( - outputDirectory, - `${process.pid}-${randomUUID()}.json`, - ); - fs.writeFileSync(outputPath, JSON.stringify(failures), "utf8"); + const outputName = `${process.pid}-${randomUUID()}`; + const temporaryPath = path.join(outputDirectory, `${outputName}.tmp`); + const outputPath = path.join(outputDirectory, `${outputName}.json`); + fs.writeFileSync(temporaryPath, JSON.stringify(failures), "utf8"); + fs.renameSync(temporaryPath, outputPath); } module.exports = { writeTestFailures }; From 6a84bab22299d7c97d41611138dd7f6cb494b42b Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 13 Aug 2026 23:00:10 -0700 Subject: [PATCH 4/5] Add intentional test failure --- .../utils/commonUtils/test/pipelineFailureSummary.spec.ts | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 ts/packages/utils/commonUtils/test/pipelineFailureSummary.spec.ts diff --git a/ts/packages/utils/commonUtils/test/pipelineFailureSummary.spec.ts b/ts/packages/utils/commonUtils/test/pipelineFailureSummary.spec.ts new file mode 100644 index 0000000000..3047df8bd4 --- /dev/null +++ b/ts/packages/utils/commonUtils/test/pipelineFailureSummary.spec.ts @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +describe("TEMP build-ts failure summary validation", () => { + it("surfaces this intentional failure at the end of the test output", () => { + expect("intentional pipeline failure").toBe("remove before merging"); + }); +}); From 8858fa9b61cdc415206dc55f80ec0fa6cc21c693 Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 13 Aug 2026 23:23:29 -0700 Subject: [PATCH 5/5] Remove intentional test failure --- .../utils/commonUtils/test/pipelineFailureSummary.spec.ts | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 ts/packages/utils/commonUtils/test/pipelineFailureSummary.spec.ts diff --git a/ts/packages/utils/commonUtils/test/pipelineFailureSummary.spec.ts b/ts/packages/utils/commonUtils/test/pipelineFailureSummary.spec.ts deleted file mode 100644 index 3047df8bd4..0000000000 --- a/ts/packages/utils/commonUtils/test/pipelineFailureSummary.spec.ts +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -describe("TEMP build-ts failure summary validation", () => { - it("surfaces this intentional failure at the end of the test output", () => { - expect("intentional pipeline failure").toBe("remove before merging"); - }); -});