diff --git a/.github/workflows/build-ts.yml b/.github/workflows/build-ts.yml index 47b5421d8d..ac8ceb7609 100644 --- a/.github/workflows/build-ts.yml +++ b/.github/workflows/build-ts.yml @@ -145,7 +145,7 @@ jobs: if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }} working-directory: ts run: | - npm run test:local + node tools/scripts/runTestLocalWithSummary.mjs - name: UI tests (requires display) if: ${{ (github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false') && runner.os == 'Linux' }} working-directory: ts diff --git a/ts/jest.config.js b/ts/jest.config.js index bdd5b9ce65..c5961fc0b8 100644 --- a/ts/jest.config.js +++ b/ts/jest.config.js @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +const path = require("node:path"); + /** @type {import('ts-jest').JestConfigWithTsJest} */ module.exports = { testMatch: ["**/dist/test/**/*.(spec|test).js?(x)"], @@ -9,4 +11,12 @@ module.exports = { "^../src/(.*)$": "/dist/$1", }, testTimeout: 90000, + ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined + ? {} + : { + reporters: [ + "default", + path.join(__dirname, "tools/scripts/jestFailureReporter.cjs"), + ], + }), }; diff --git a/ts/packages/agents/browser/jest.config.js b/ts/packages/agents/browser/jest.config.js index dfce758aa6..5e0437aa17 100644 --- a/ts/packages/agents/browser/jest.config.js +++ b/ts/packages/agents/browser/jest.config.js @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { fileURLToPath } from "node:url"; + /** @type {import('ts-jest').JestConfigWithTsJest} */ export default { preset: "ts-jest/presets/default-esm", @@ -39,4 +41,17 @@ export default { // Map .mjs imports from src to .mts files for ts-jest "^(.*)\\.mjs$": "$1.mts", }, + ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined + ? {} + : { + reporters: [ + "default", + fileURLToPath( + new URL( + "../../../tools/scripts/jestFailureReporter.cjs", + import.meta.url, + ), + ), + ], + }), }; diff --git a/ts/packages/agents/browserExtension/jest.config.js b/ts/packages/agents/browserExtension/jest.config.js index e06cce61d7..05ea393085 100644 --- a/ts/packages/agents/browserExtension/jest.config.js +++ b/ts/packages/agents/browserExtension/jest.config.js @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { fileURLToPath } from "node:url"; + /** @type {import('ts-jest').JestConfigWithTsJest} */ export default { preset: "ts-jest/presets/default-esm", @@ -47,4 +49,17 @@ export default { // Map .mjs imports from src to .mts files for ts-jest "^(.*)\\.mjs$": "$1.mts", }, + ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined + ? {} + : { + reporters: [ + "default", + fileURLToPath( + new URL( + "../../../tools/scripts/jestFailureReporter.cjs", + import.meta.url, + ), + ), + ], + }), }; diff --git a/ts/packages/agents/onboarding/package.json b/ts/packages/agents/onboarding/package.json index b22200c6b3..1ece32aaf5 100644 --- a/ts/packages/agents/onboarding/package.json +++ b/ts/packages/agents/onboarding/package.json @@ -43,7 +43,7 @@ "prettier": "prettier --check . --ignore-path ../../../.prettierignore", "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore", "test": "npm run test:local", - "test:local": "tsx --test test/*.spec.ts", + "test:local": "tsx --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=../../../tools/scripts/nodeTestFailureReporter.mjs --test-reporter-destination=stdout test/*.spec.ts", "tsc": "tsc -b" }, "dependencies": { diff --git a/ts/packages/typeagent-studio/package.json b/ts/packages/typeagent-studio/package.json index 0fde5eaabf..6176ec4137 100644 --- a/ts/packages/typeagent-studio/package.json +++ b/ts/packages/typeagent-studio/package.json @@ -23,7 +23,7 @@ "compile": "node esbuild.mjs", "deploy:local": "npm run package && code --install-extension dist-pub/typeagent-studio.vsix --force", "package": "mkdirp dist-pub && vsce package --allow-star-activation --allow-missing-repository --no-dependencies --allow-package-secrets npm -o dist-pub/typeagent-studio.vsix", - "test:local": "tsx --test src/test/*.spec.ts", + "test:local": "tsx --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=../../tools/scripts/nodeTestFailureReporter.mjs --test-reporter-destination=stdout src/test/*.spec.ts", "vscode:prepublish": "npm run compile", "watch": "node esbuild.mjs --watch" }, diff --git a/ts/tools/scripts/jestFailureReporter.cjs b/ts/tools/scripts/jestFailureReporter.cjs new file mode 100644 index 0000000000..6923ae5670 --- /dev/null +++ b/ts/tools/scripts/jestFailureReporter.cjs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const { writeTestFailures } = require("./testFailureOutput.cjs"); + +class JestFailureReporter { + onRunComplete(_testContexts, aggregatedResult) { + const failures = []; + + for (const testFileResult of aggregatedResult.testResults) { + const failedTests = testFileResult.testResults.filter( + (testResult) => testResult.status === "failed", + ); + + for (const failedTest of failedTests) { + failures.push({ + testFilePath: testFileResult.testFilePath, + fullName: failedTest.fullName, + failureMessages: failedTest.failureMessages, + }); + } + + if (testFileResult.testExecError !== undefined) { + failures.push({ + testFilePath: testFileResult.testFilePath, + fullName: "Test suite failed outside an individual test", + failureMessages: [ + testFileResult.testExecError.stack ?? + testFileResult.testExecError.message, + ], + }); + } else if ( + failedTests.length === 0 && + typeof testFileResult.failureMessage === "string" + ) { + failures.push({ + testFilePath: testFileResult.testFilePath, + fullName: "Test suite failed to run", + failureMessages: [testFileResult.failureMessage], + }); + } + } + + if (aggregatedResult.runExecError !== undefined) { + failures.push({ + testFilePath: "", + fullName: "Jest failed outside a test suite", + failureMessages: [ + aggregatedResult.runExecError.stack ?? + aggregatedResult.runExecError.message, + ], + }); + } + + writeTestFailures(failures); + } +} + +module.exports = JestFailureReporter; diff --git a/ts/tools/scripts/nodeTestFailureReporter.mjs b/ts/tools/scripts/nodeTestFailureReporter.mjs new file mode 100644 index 0000000000..7dd6871d72 --- /dev/null +++ b/ts/tools/scripts/nodeTestFailureReporter.mjs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import failureOutput from "./testFailureOutput.cjs"; + +const { writeTestFailures } = failureOutput; + +export default async function* nodeTestFailureReporter(source) { + const failures = []; + const stderrByTestFile = new Map(); + + try { + for await (const event of source) { + if (event.type === "test:stderr") { + const messages = stderrByTestFile.get(event.data.file) ?? []; + messages.push(event.data.message); + stderrByTestFile.set(event.data.file, messages); + continue; + } + + const error = event.data?.details?.error; + if ( + event.type === "test:fail" && + error?.failureType !== "subtestsFailed" + ) { + const failure = error.cause ?? error; + const failureMessage = + failure === "test failed" + ? (stderrByTestFile.get(event.data.file)?.join("") ?? + failure) + : (failure.stack ?? failure.message ?? String(failure)); + failures.push({ + testFilePath: event.data.file ?? event.data.name, + fullName: event.data.name, + failureMessages: [failureMessage], + }); + } + } + } finally { + writeTestFailures(failures); + } +} diff --git a/ts/tools/scripts/runTestLocalWithSummary.mjs b/ts/tools/scripts/runTestLocalWithSummary.mjs new file mode 100644 index 0000000000..a936b11298 --- /dev/null +++ b/ts/tools/scripts/runTestLocalWithSummary.mjs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +const failureDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), "typeagent-test-failures-"), +); +const [testCommand, testArguments] = + process.platform === "win32" + ? [ + process.env.ComSpec ?? "cmd.exe", + ["/d", "/s", "/c", "pnpm run test:local"], + ] + : ["pnpm", ["run", "test:local"]]; + +const result = spawnSync(testCommand, testArguments, { + env: { + ...process.env, + TYPEAGENT_TEST_FAILURES_DIR: failureDirectory, + }, + stdio: "inherit", +}); + +try { + printFailureSummary(failureDirectory, result.status); +} finally { + fs.rmSync(failureDirectory, { recursive: true, force: true }); +} + +if (result.error !== undefined) { + throw result.error; +} +if (result.signal !== null) { + process.kill(process.pid, result.signal); +} +process.exitCode = result.status ?? 1; + +function printFailureSummary(directory, exitCode) { + const failures = fs + .readdirSync(directory) + .filter((fileName) => fileName.endsWith(".json")) + .flatMap((fileName) => readFailureArtifact(directory, fileName)); + const uniqueFailures = [ + ...new Map( + failures.map((failure) => [JSON.stringify(failure), failure]), + ).values(), + ]; + + if (uniqueFailures.length === 0) { + if (exitCode !== 0) { + console.error( + "\nFAILED TEST SUMMARY\nNo individual test failures were captured. See the test output above.", + ); + } + return; + } + + console.error( + `\n${"=".repeat(80)}\nFAILED TEST SUMMARY (${uniqueFailures.length})\n${"=".repeat(80)}`, + ); + for (const failure of uniqueFailures) { + console.error(`\nFAIL ${normalizePath(failure.testFilePath)}`); + console.error(` ${failure.fullName}`); + for (const message of failure.failureMessages) { + console.error(indent(stripAnsi(message), 4)); + } + } + console.error(`\n${"=".repeat(80)}`); +} + +function readFailureArtifact(directory, fileName) { + try { + return JSON.parse( + fs.readFileSync(path.join(directory, fileName), "utf8"), + ); + } catch (error) { + console.error( + `Unable to read test failure artifact ${fileName}: ${error.message}`, + ); + return []; + } +} + +function normalizePath(filePath) { + return ( + path.isAbsolute(filePath) + ? path.relative(process.cwd(), filePath) + : filePath + ).replaceAll("\\", "/"); +} + +function stripAnsi(value) { + return value.replace( + /[\u001B\u009B][[\]()#;?]*(?:(?:(?:[a-zA-Z\d]*(?:;[-a-zA-Z\d/#&.:=?%@~_]+)*)?\u0007)|(?:(?:\d{1,4}(?:[;:]\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~]))/g, + "", + ); +} + +function indent(value, spaces) { + const prefix = " ".repeat(spaces); + return value + .trimEnd() + .split(/\r?\n/u) + .map((line) => `${prefix}${line}`) + .join("\n"); +} diff --git a/ts/tools/scripts/testFailureOutput.cjs b/ts/tools/scripts/testFailureOutput.cjs new file mode 100644 index 0000000000..a9e6cab507 --- /dev/null +++ b/ts/tools/scripts/testFailureOutput.cjs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const fs = require("node:fs"); +const path = require("node:path"); +const { randomUUID } = require("node:crypto"); + +function writeTestFailures(failures) { + const outputDirectory = process.env.TYPEAGENT_TEST_FAILURES_DIR; + if (outputDirectory === undefined || failures.length === 0) { + return; + } + + fs.mkdirSync(outputDirectory, { recursive: true }); + const outputName = `${process.pid}-${randomUUID()}`; + const temporaryPath = path.join(outputDirectory, `${outputName}.tmp`); + const outputPath = path.join(outputDirectory, `${outputName}.json`); + fs.writeFileSync(temporaryPath, JSON.stringify(failures), "utf8"); + fs.renameSync(temporaryPath, outputPath); +} + +module.exports = { writeTestFailures };