[CICD] Improve test step to summarize failures at end of step

.github/workflows/build-ts.yml

@@ -145,7 +145,7 @@ jobs:
         if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false' }}
         working-directory: ts
         run: |
-          npm run test:local
+          node tools/scripts/runTestLocalWithSummary.mjs
       - name: UI tests (requires display)
         if: ${{ (github.event_name != 'pull_request' || steps.filter.outputs.ts != 'false') && runner.os == 'Linux' }}
         working-directory: ts

ts/jest.config.js

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+const path = require("node:path");
+
 /** @type {import('ts-jest').JestConfigWithTsJest} */
 module.exports = {
     testMatch: ["**/dist/test/**/*.(spec|test).js?(x)"],
@@ -9,4 +11,12 @@ module.exports = {
         "^../src/(.*)$": "<rootDir>/dist/$1",
     },
     testTimeout: 90000,
+    ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined
+        ? {}
+        : {
+              reporters: [
+                  "default",
+                  path.join(__dirname, "tools/scripts/jestFailureReporter.cjs"),
+              ],
+          }),
 };

ts/packages/agents/browser/jest.config.js

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import { fileURLToPath } from "node:url";
+
 /** @type {import('ts-jest').JestConfigWithTsJest} */
 export default {
     preset: "ts-jest/presets/default-esm",
@@ -39,4 +41,17 @@ export default {
         // Map .mjs imports from src to .mts files for ts-jest
         "^(.*)\\.mjs$": "$1.mts",
     },
+    ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined
+        ? {}
+        : {
+              reporters: [
+                  "default",
+                  fileURLToPath(
+                      new URL(
+                          "../../../tools/scripts/jestFailureReporter.cjs",
+                          import.meta.url,
+                      ),
+                  ),
+              ],
+          }),
 };

ts/packages/agents/browserExtension/jest.config.js

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import { fileURLToPath } from "node:url";
+
 /** @type {import('ts-jest').JestConfigWithTsJest} */
 export default {
     preset: "ts-jest/presets/default-esm",
@@ -47,4 +49,17 @@ export default {
         // Map .mjs imports from src to .mts files for ts-jest
         "^(.*)\\.mjs$": "$1.mts",
     },
+    ...(process.env.TYPEAGENT_TEST_FAILURES_DIR === undefined
+        ? {}
+        : {
+              reporters: [
+                  "default",
+                  fileURLToPath(
+                      new URL(
+                          "../../../tools/scripts/jestFailureReporter.cjs",
+                          import.meta.url,
+                      ),
+                  ),
+              ],
+          }),
 };

ts/packages/agents/onboarding/package.json

@@ -43,7 +43,7 @@
     "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
     "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
     "test": "npm run test:local",
-    "test:local": "tsx --test test/*.spec.ts",
+    "test:local": "tsx --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=../../../tools/scripts/nodeTestFailureReporter.mjs --test-reporter-destination=stdout test/*.spec.ts",
     "tsc": "tsc -b"
   },
   "dependencies": {

ts/packages/typeagent-studio/package.json

@@ -23,7 +23,7 @@
     "compile": "node esbuild.mjs",
     "deploy:local": "npm run package && code --install-extension dist-pub/typeagent-studio.vsix --force",
     "package": "mkdirp dist-pub && vsce package --allow-star-activation --allow-missing-repository --no-dependencies --allow-package-secrets npm -o dist-pub/typeagent-studio.vsix",
-    "test:local": "tsx --test src/test/*.spec.ts",
+    "test:local": "tsx --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=../../tools/scripts/nodeTestFailureReporter.mjs --test-reporter-destination=stdout src/test/*.spec.ts",
     "vscode:prepublish": "npm run compile",
     "watch": "node esbuild.mjs --watch"
   },

ts/tools/scripts/jestFailureReporter.cjs

@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+const { writeTestFailures } = require("./testFailureOutput.cjs");
+
+class JestFailureReporter {
+    onRunComplete(_testContexts, aggregatedResult) {
+        const failures = [];
+
+        for (const testFileResult of aggregatedResult.testResults) {
+            const failedTests = testFileResult.testResults.filter(
+                (testResult) => testResult.status === "failed",
+            );
+
+            for (const failedTest of failedTests) {
+                failures.push({
+                    testFilePath: testFileResult.testFilePath,
+                    fullName: failedTest.fullName,
+                    failureMessages: failedTest.failureMessages,
+                });
+            }
+
+            if (testFileResult.testExecError !== undefined) {
+                failures.push({
+                    testFilePath: testFileResult.testFilePath,
+                    fullName: "Test suite failed outside an individual test",
+                    failureMessages: [
+                        testFileResult.testExecError.stack ??
+                            testFileResult.testExecError.message,
+                    ],
+                });
+            } else if (
+                failedTests.length === 0 &&
+                typeof testFileResult.failureMessage === "string"
+            ) {
+                failures.push({
+                    testFilePath: testFileResult.testFilePath,
+                    fullName: "Test suite failed to run",
+                    failureMessages: [testFileResult.failureMessage],
+                });
+            }
+        }
+
+        if (aggregatedResult.runExecError !== undefined) {
+            failures.push({
+                testFilePath: "<Jest run>",
+                fullName: "Jest failed outside a test suite",
+                failureMessages: [
+                    aggregatedResult.runExecError.stack ??
+                        aggregatedResult.runExecError.message,
+                ],
+            });
+        }
+
+        writeTestFailures(failures);
+    }
+}
+
+module.exports = JestFailureReporter;

ts/tools/scripts/nodeTestFailureReporter.mjs

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import failureOutput from "./testFailureOutput.cjs";
+
+const { writeTestFailures } = failureOutput;
+
+export default async function* nodeTestFailureReporter(source) {
+    const failures = [];
+    const stderrByTestFile = new Map();
+
+    try {
+        for await (const event of source) {
+            if (event.type === "test:stderr") {
+                const messages = stderrByTestFile.get(event.data.file) ?? [];
+                messages.push(event.data.message);
+                stderrByTestFile.set(event.data.file, messages);
+                continue;
+            }
+
+            const error = event.data?.details?.error;
+            if (
+                event.type === "test:fail" &&
+                error?.failureType !== "subtestsFailed"
+            ) {
+                const failure = error.cause ?? error;
+                const failureMessage =
+                    failure === "test failed"
+                        ? (stderrByTestFile.get(event.data.file)?.join("") ??
+                          failure)
+                        : (failure.stack ?? failure.message ?? String(failure));
+                failures.push({
+                    testFilePath: event.data.file ?? event.data.name,
+                    fullName: event.data.name,
+                    failureMessages: [failureMessage],
+                });
+            }
+        }
+    } finally {
+        writeTestFailures(failures);
+    }
+}

ts/tools/scripts/runTestLocalWithSummary.mjs

@@ -0,0 +1,110 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+
+const failureDirectory = fs.mkdtempSync(
+    path.join(os.tmpdir(), "typeagent-test-failures-"),
+);
+const [testCommand, testArguments] =
+    process.platform === "win32"
+        ? [
+              process.env.ComSpec ?? "cmd.exe",
+              ["/d", "/s", "/c", "pnpm run test:local"],
+          ]
+        : ["pnpm", ["run", "test:local"]];
+
+const result = spawnSync(testCommand, testArguments, {
+    env: {
+        ...process.env,
+        TYPEAGENT_TEST_FAILURES_DIR: failureDirectory,
+    },
+    stdio: "inherit",
+});
+
+try {
+    printFailureSummary(failureDirectory, result.status);
+} finally {
+    fs.rmSync(failureDirectory, { recursive: true, force: true });
+}
+
+if (result.error !== undefined) {
+    throw result.error;
+}
+if (result.signal !== null) {
+    process.kill(process.pid, result.signal);
+}
+process.exitCode = result.status ?? 1;
+
+function printFailureSummary(directory, exitCode) {
+    const failures = fs
+        .readdirSync(directory)
+        .filter((fileName) => fileName.endsWith(".json"))
+        .flatMap((fileName) => readFailureArtifact(directory, fileName));
+    const uniqueFailures = [
+        ...new Map(
+            failures.map((failure) => [JSON.stringify(failure), failure]),
+        ).values(),
+    ];
+
+    if (uniqueFailures.length === 0) {
+        if (exitCode !== 0) {
+            console.error(
+                "\nFAILED TEST SUMMARY\nNo individual test failures were captured. See the test output above.",
+            );
+        }
+        return;
+    }
+
+    console.error(
+        `\n${"=".repeat(80)}\nFAILED TEST SUMMARY (${uniqueFailures.length})\n${"=".repeat(80)}`,
+    );
+    for (const failure of uniqueFailures) {
+        console.error(`\nFAIL ${normalizePath(failure.testFilePath)}`);
+        console.error(`  ${failure.fullName}`);
+        for (const message of failure.failureMessages) {
+            console.error(indent(stripAnsi(message), 4));
+        }
+    }
+    console.error(`\n${"=".repeat(80)}`);
+}
+
+function readFailureArtifact(directory, fileName) {
+    try {
+        return JSON.parse(
+            fs.readFileSync(path.join(directory, fileName), "utf8"),
+        );
+    } catch (error) {
+        console.error(
+            `Unable to read test failure artifact ${fileName}: ${error.message}`,
+        );
+        return [];
+    }
+}
+
+function normalizePath(filePath) {
+    return (
+        path.isAbsolute(filePath)
+            ? path.relative(process.cwd(), filePath)
+            : filePath
+    ).replaceAll("\\", "/");
+}
+
+function stripAnsi(value) {
+    return value.replace(
+        /[\u001B\u009B][[\]()#;?]*(?:(?:(?:[a-zA-Z\d]*(?:;[-a-zA-Z\d/#&.:=?%@~_]+)*)?\u0007)|(?:(?:\d{1,4}(?:[;:]\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~]))/g,
+        "",
+    );
+}
+
+function indent(value, spaces) {
+    const prefix = " ".repeat(spaces);
+    return value
+        .trimEnd()
+        .split(/\r?\n/u)
+        .map((line) => `${prefix}${line}`)
+        .join("\n");
+}

ts/tools/scripts/testFailureOutput.cjs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+const fs = require("node:fs");
+const path = require("node:path");
+const { randomUUID } = require("node:crypto");
+
+function writeTestFailures(failures) {
+    const outputDirectory = process.env.TYPEAGENT_TEST_FAILURES_DIR;
+    if (outputDirectory === undefined || failures.length === 0) {
+        return;
+    }
+
+    fs.mkdirSync(outputDirectory, { recursive: true });
+    const outputName = `${process.pid}-${randomUUID()}`;
+    const temporaryPath = path.join(outputDirectory, `${outputName}.tmp`);
+    const outputPath = path.join(outputDirectory, `${outputName}.json`);
+    fs.writeFileSync(temporaryPath, JSON.stringify(failures), "utf8");
+    fs.renameSync(temporaryPath, outputPath);
+}
+
+module.exports = { writeTestFailures };