diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 73f5aafa8f..01fd3d22b9 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -29,6 +29,7 @@ TypeAgent translation bench: catalog, action-parameters grader, simple-action da Workspace: - [@typeagent/action-schema](../../packages/actionSchema/README.md) +- [@typeagent/agent-cache](../../packages/cache/README.md) - [@typeagent/agent-sdk](../../packages/agentSdk/README.md) - [@typeagent/aiclient](../../packages/aiclient/README.md) - [agent-dispatcher](../../packages/dispatcher/dispatcher/README.md) @@ -44,6 +45,7 @@ _None._ - [./src/index.ts](./src/index.ts) - [./src/translationBench/index.ts](./src/translationBench/index.ts) +- [./src/translationBench/runner/index.ts](./src/translationBench/runner/index.ts) - [./src/translationBench/synthesizer/catalogGenerator/index.ts](./src/translationBench/synthesizer/catalogGenerator/index.ts) - [./src/translationBench/synthesizer/goldSchema.ts](./src/translationBench/synthesizer/goldSchema.ts) - [./src/translationBench/synthesizer/index.ts](./src/translationBench/synthesizer/index.ts) @@ -51,11 +53,10 @@ _None._ - [./src/core/paths.ts](./src/core/paths.ts) - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) -- [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts) -- _…and 38 more under `./src/`._ +- _…and 41 more under `./src/`._ --- -_Auto-generated against commit `7c6cbc823caaaf6dfa97f9c42b043ba7065c9472` on `2026-08-13T20:36:56.989Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `fffd6569fbc85af1f6767d66b632d39da623ba06` on `2026-08-14T06:08:36.475Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ diff --git a/ts/packages/benchmarks/local/runs/100-case-probe-run/approve-and-eval.mjs b/ts/packages/benchmarks/local/runs/100-case-probe-run/approve-and-eval.mjs new file mode 100644 index 0000000000..f063653a8c --- /dev/null +++ b/ts/packages/benchmarks/local/runs/100-case-probe-run/approve-and-eval.mjs @@ -0,0 +1,635 @@ +#!/usr/bin/env node +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Dual-root eval launcher: + * - THIS worktree: synthesizer benchmark parse/approve (format matches draft) + * - SIBLING 1k-eval worktree: dispatcher + runner (exports ActionSchemaFileCache etc.) + * Local RUN_DIR only; not part of package src. + */ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Command } from "commander"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const RUN = __dirname; +const THIS_TS = path.resolve(RUN, "../../../../../"); + +// Single config object (commander flags), instead of a pile of TB_* env vars. +// A --config can supply any of the same keys; explicit flags win. +const program = new Command(); +program + .option("--config ", "JSON config file with any of the options below") + .option( + "--runtime-ts ", + "sibling worktree ts/ root for dispatcher+runner", + ) + .option("--draft ", "benchmark draft jsonl") + .option("--approved ", "approved benchmark jsonl output") + .option("--out ", "eval-results.json output") + .option("--html ", "eval-report.html output") + .option("--checkpoint ", "eval checkpoint jsonl") + .option("--gateway-dir ", "publish dir for eval-report.html + results") + .option( + "--index-gateway-dir ", + "publish dir for the landing repo (index + explorers)", + ) + .option("--per-model-concurrency ", "rows in-flight per model", Number) + .option("--model-concurrency ", "models evaluated in parallel", Number) + .option("--max-cases ", "trim to N cases for a smoke eval", Number) + .option("--skip-trust", "skip trust verification for draft approval") + .allowUnknownOption(true); +program.parse(process.argv); +const opts = program.opts(); +const fileCfg = opts.config + ? JSON.parse(fs.readFileSync(path.resolve(opts.config), "utf8")) + : {}; +const cfg = { + runtimeTs: THIS_TS, + draft: path.join(RUN, "artifacts/benchmark-draft-1000.jsonl"), + approved: path.join(RUN, "artifacts/benchmark-approved-1000.jsonl"), + out: path.join(RUN, "artifacts/eval-results.json"), + html: path.join(RUN, "artifacts/eval-report.html"), + checkpoint: path.join(RUN, "artifacts/eval-checkpoint-azure-gpt56.jsonl"), + gatewayDir: path.join(RUN, "published-eval"), + indexGatewayDir: "", + perModelConcurrency: 10, + modelConcurrency: 0, + maxCases: undefined, + skipTrust: false, + ...fileCfg, + ...Object.fromEntries( + Object.entries({ + runtimeTs: opts.runtimeTs, + draft: opts.draft, + approved: opts.approved, + out: opts.out, + html: opts.html, + checkpoint: opts.checkpoint, + gatewayDir: opts.gatewayDir, + indexGatewayDir: opts.indexGatewayDir, + perModelConcurrency: opts.perModelConcurrency, + modelConcurrency: opts.modelConcurrency, + maxCases: opts.maxCases, + skipTrust: opts.skipTrust, + }).filter(([, v]) => v !== undefined), + ), +}; +const SIB_TS = cfg.runtimeTs; + +function loadEnv(file) { + if (!fs.existsSync(file)) return; + for (const line of fs.readFileSync(file, "utf8").split("\n")) { + const m = line.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); + if (!m) continue; + let v = m[2]; + if ( + (v.startsWith('"') && v.endsWith('"')) || + (v.startsWith("'") && v.endsWith("'")) + ) + v = v.slice(1, -1); + if (process.env[m[1]] === undefined) process.env[m[1]] = v; + } +} +loadEnv(path.join(THIS_TS, ".env.real")); +loadEnv(path.join(SIB_TS, ".env.real")); + +const EVAL_MODELS = [ + "azure/gpt-5.6-sol", + "azure/gpt-5.6-terra", + "azure/gpt-5.6-luna", +]; +for (const id of EVAL_MODELS) process.env[`OPENAI_MODEL_${id}`] = id; +process.env.OPENAI_RESPONSE_FORMAT = "1"; + +const PER_MODEL_CONCURRENCY = cfg.perModelConcurrency; +const CONCURRENCY_BY_MODEL = Object.fromEntries( + EVAL_MODELS.map((id) => [id, PER_MODEL_CONCURRENCY]), +); +const CONCURRENCY = PER_MODEL_CONCURRENCY; +const MODEL_CONCURRENCY = cfg.modelConcurrency || EVAL_MODELS.length; +const MAX_CASES = cfg.maxCases; +const PEAK_IN_FLIGHT = Object.values(CONCURRENCY_BY_MODEL).reduce( + (a, b) => a + b, + 0, +); + +const clientPool = String(Math.max(PEAK_IN_FLIGHT, PER_MODEL_CONCURRENCY, 8)); +if (process.env.AZURE_OPENAI_MAX_CONCURRENCY === undefined) { + process.env.AZURE_OPENAI_MAX_CONCURRENCY = clientPool; +} +if (process.env.OPENAI_MAX_CONCURRENCY === undefined) { + process.env.OPENAI_MAX_CONCURRENCY = clientPool; +} +console.log( + `Models=${EVAL_MODELS.join(",")} perModel=${PER_MODEL_CONCURRENCY} modelConcurrency=${MODEL_CONCURRENCY} clientPool=${clientPool}`, +); +console.log(`runtimeTS=${SIB_TS}`); +console.log(`benchmarkTS=${THIS_TS}`); + +const aiclient = await import( + pathToFileURL(path.join(SIB_TS, "packages/aiclient/dist/index.js")).href +); +aiclient.initRuntimeConfigFromProcessEnv(); + +const dap = await import( + pathToFileURL( + path.join(SIB_TS, "packages/defaultAgentProvider/dist/index.js"), + ).href +); +const disp = await import( + pathToFileURL( + path.join(SIB_TS, "packages/dispatcher/dispatcher/dist/internal.js"), + ).href +); +const bmMod = await import( + pathToFileURL( + path.join( + THIS_TS, + "packages/benchmarks/dist/translationBench/synthesizer/benchmark.js", + ), + ).href +); +const srcMod = await import( + pathToFileURL( + path.join( + THIS_TS, + "packages/benchmarks/dist/translationBench/synthesizer/sourceBuilder.js", + ), + ).href +); +const runnerMod = await import( + pathToFileURL( + path.join( + SIB_TS, + "packages/benchmarks/dist/translationBench/runner/runner.js", + ), + ).href +); +const scaleMod = await import( + pathToFileURL( + path.join( + SIB_TS, + "packages/benchmarks/dist/translationBench/runner/scale.js", + ), + ).href +); +const reportMod = await import( + pathToFileURL( + path.join( + SIB_TS, + "packages/benchmarks/dist/translationBench/runner/report.js", + ), + ).href +); +await import( + pathToFileURL( + path.join( + THIS_TS, + "packages/benchmarks/dist/translationBench/synthesizer/adapters/seedQaJsonlAdapter.js", + ), + ).href +); + +/** Local adapter (no cross-branch coverage asserts). */ +function toRunnerLineage(lineage) { + return { + dataset: lineage.dataset, + revision: lineage.revision, + config: lineage.config, + split: lineage.split, + rowIndex: lineage.rowIndex, + rowId: lineage.rowId, + sourceUrl: lineage.sourceUrl, + sourceHash: lineage.canonicalPayloadHash, + sourcePart: lineage.sourcePart, + rawRowHash: lineage.rawRowHash, + sourceSliceHash: lineage.sourceSliceHash, + canonicalPayloadHash: lineage.canonicalPayloadHash, + transformVersion: lineage.transformVersion, + ...(lineage.transformVersion >= 2 ? { derived: true } : {}), + }; +} +function toExplainerProbe(caseId, probe) { + if (probe.selection.role === "seed") { + throw new Error( + `Case '${caseId}' contains a seed in its generalization probes`, + ); + } + return { + id: `${caseId}:${probe.lineage.rowId}:${probe.lineage.sourcePart}${ + probe.lineage.transformVersion >= 2 + ? `:${probe.lineage.canonicalPayloadHash}` + : "" + }`, + role: probe.selection.role, + lineage: toRunnerLineage(probe.lineage), + utterance: probe.utterance, + expectedActions: structuredClone(probe.expectedActions), + order: probe.order, + dimensions: structuredClone(probe.selection.dimensions), + ...(probe.history !== undefined + ? { history: structuredClone(probe.history) } + : {}), + }; +} +function translationBenchBenchmarkToSuite(benchmark) { + if (benchmark.metadata?.approval?.status !== "approved") { + throw new Error( + `Benchmark not approved (status=${benchmark.metadata?.approval?.status})`, + ); + } + const suite = { + version: 1, + name: benchmark.metadata.name, + schemas: structuredClone(benchmark.metadata.schemas), + cases: benchmark.cases.flatMap((evalCase) => { + const primary = { + id: evalCase.id, + lineage: toRunnerLineage(evalCase.seed.lineage), + activeSchemas: structuredClone(evalCase.activeSchemas), + seed: { + utterance: evalCase.seed.utterance, + expectedActions: structuredClone( + evalCase.seed.expectedActions, + ), + order: evalCase.seed.order, + ...(evalCase.seed.history !== undefined + ? { history: structuredClone(evalCase.seed.history) } + : {}), + // Pass through generator soft-match specs (B fix). Without this the + // runner falls back to exact equalNormalizedObject for all params. + ...(evalCase.seed.parameterScore !== undefined + ? { + parameterScore: structuredClone( + evalCase.seed.parameterScore, + ), + } + : {}), + }, + explainer: { + valueInRequest: evalCase.explainer.valueInRequest, + noReferences: evalCase.explainer.noReferences, + probes: evalCase.generalizations.map((probe) => + toExplainerProbe(evalCase.id, probe), + ), + }, + ...(evalCase.dimensions !== undefined + ? { dimensions: structuredClone(evalCase.dimensions) } + : {}), + }; + const translationNegatives = evalCase.generalizations + .filter((probe) => probe.selection.role === "negative") + .map((probe) => ({ + id: `${evalCase.id}:translation-negative:${probe.lineage.rowId}:${probe.lineage.sourcePart}${ + probe.lineage.transformVersion >= 2 + ? `:${probe.lineage.canonicalPayloadHash}` + : "" + }`, + lineage: toRunnerLineage(probe.lineage), + activeSchemas: structuredClone(evalCase.activeSchemas), + seed: { + utterance: probe.utterance, + expectedActions: [], + order: probe.order, + ...(probe.history !== undefined + ? { history: structuredClone(probe.history) } + : {}), + }, + dimensions: structuredClone(probe.selection.dimensions), + })); + return [primary, ...translationNegatives]; + }), + ...(benchmark.metadata.scenarios !== undefined + ? { scenarios: structuredClone(benchmark.metadata.scenarios) } + : {}), + ...(benchmark.metadata.pricing !== undefined + ? { pricing: structuredClone(benchmark.metadata.pricing) } + : {}), + }; + const sourceManifest = { + version: 1, + sources: benchmark.cases.flatMap((evalCase) => [ + toRunnerLineage(evalCase.seed.lineage), + ...evalCase.generalizations.map((probe) => + toRunnerLineage(probe.lineage), + ), + ]), + }; + return { suite, sourceManifest }; +} + +const draftPath = cfg.draft; +const approvedPath = cfg.approved; +const sourcePath = path.join(RUN, "source/anchors-1100.jsonl"); +const manifestPath = path.join(RUN, "source/source-manifest.json"); +const outPath = cfg.out; +const htmlPath = cfg.html; +const checkpointPath = cfg.checkpoint; + +if (!fs.existsSync(draftPath)) throw new Error(`Missing draft: ${draftPath}`); + +const instanceDir = path.join(RUN, "instance-eval"); +fs.mkdirSync(instanceDir, { recursive: true }); +const context = await disp.initializeCommandHandlerContext( + "translation-bench-1k-eval", + { + ...dap.getDefaultDispatcherOptions(), + appAgentProviders: dap.getDefaultAppAgentProviders(instanceDir), + explanationAsynchronousMode: false, + persistSession: false, + metrics: false, + }, +); + +try { + let benchmark; + if (fs.existsSync(approvedPath)) { + benchmark = bmMod.parseTranslationBenchBenchmarkJsonl( + fs.readFileSync(approvedPath, "utf8"), + approvedPath, + ); + console.log("Loaded approved benchmark →", approvedPath); + } else { + benchmark = bmMod.parseTranslationBenchBenchmarkJsonl( + fs.readFileSync(draftPath, "utf8"), + draftPath, + ); + if (MAX_CASES && benchmark.cases.length > MAX_CASES) { + benchmark = { + ...benchmark, + cases: benchmark.cases.slice(0, MAX_CASES), + }; + console.log(`Trimmed to ${MAX_CASES} cases for smoke eval`); + } + if (benchmark.metadata.approval.status === "draft") { + const skipTrust = + cfg.skipTrust || + (MAX_CASES !== undefined && MAX_CASES < benchmark.cases.length); + if (!skipTrust) { + const sourceText = fs.readFileSync(sourcePath, "utf8"); + const sourceManifestFile = JSON.parse( + fs.readFileSync(manifestPath, "utf8"), + ); + srcMod.assertTranslationBenchSourceBenchmarkTrust(benchmark, { + sourceText, + sourceManifest: sourceManifestFile, + provider: context.agents, + }); + } else { + console.log("Skipping source trust assert (trim/skip flag)"); + } + benchmark = bmMod.approveTranslationBenchBenchmark(benchmark, { + reviewedBy: "dom-local-1k-run", + reviewedAt: new Date().toISOString(), + }); + } + fs.writeFileSync( + approvedPath, + bmMod.formatTranslationBenchBenchmarkJsonl(benchmark), + ); + console.log("Approved →", approvedPath); + } + + if (MAX_CASES && benchmark.cases.length > MAX_CASES) { + benchmark = { + ...benchmark, + cases: benchmark.cases.slice(0, MAX_CASES), + }; + console.log(`Eval trimmed to ${MAX_CASES} cases`); + } + + const { suite, sourceManifest } = + translationBenchBenchmarkToSuite(benchmark); + + const asOf = new Date().toISOString().slice(0, 10); + suite.pricing = { + "azure/gpt-5.6-sol": { + inputUsdPerMToken: 5, + cachedInputUsdPerMToken: 2.5, + outputUsdPerMToken: 30, + source: "litellm model_info azure/gpt-5.6-sol", + asOf, + }, + "azure/gpt-5.6-terra": { + inputUsdPerMToken: 2.5, + cachedInputUsdPerMToken: 1.25, + outputUsdPerMToken: 15, + source: "litellm model_info azure/gpt-5.6-terra", + asOf, + }, + "azure/gpt-5.6-luna": { + inputUsdPerMToken: 1, + cachedInputUsdPerMToken: 0.5, + outputUsdPerMToken: 6, + source: "litellm model_info azure/gpt-5.6-luna", + asOf, + }, + }; + + const emptyGold = suite.cases.filter( + (c) => !(c.seed?.expectedActions || []).length, + ).length; + console.log( + `Suite cases=${suite.cases.length} emptyGold=${emptyGold} models=${EVAL_MODELS.length} modelConcurrency=${MODEL_CONCURRENCY} byModel=${JSON.stringify(CONCURRENCY_BY_MODEL)}`, + ); + + const availableModels = await aiclient.getChatModelNames(); + console.log("available models:", availableModels.join(", ")); + const started = Date.now(); + let lastLog = 0; + const noopIO = { + setDisplay() {}, + appendDisplay() {}, + takeAction() {}, + appendDiagnosticData() {}, + }; + const actionContext = { + streamingContext: undefined, + isFromReasoningLoop: false, + activityContext: undefined, + actionIO: noopIO, + sessionContext: { + agentContext: context, + sessionStorage: undefined, + instanceStorage: undefined, + notify() {}, + addAgentNameTag: false, + }, + queueToggleTransientAgent: async () => {}, + }; + + const scenarios = + suite.scenarios ?? + (typeof runnerMod.getDefaultTranslationBenchScenario === "function" + ? [runnerMod.getDefaultTranslationBenchScenario()] + : [{ id: "baseline" }]); + const checkpointSettings = { + kind: "translation-bench-headless-eval", + models: [...EVAL_MODELS], + scenarios: scenarios.map((s) => s.id), + suiteCaseCount: suite.cases.length, + sourceManifestHash: + sourceManifest?.hash ?? + sourceManifest?.sourceManifestHash ?? + JSON.stringify(sourceManifest)?.length, + }; + const runFingerprint = scaleMod.createTranslationBenchRunFingerprint({ + settings: checkpointSettings, + suiteCaseIds: suite.cases.map((c) => c.id), + }); + const checkpointHeader = { + kind: "translation-bench-checkpoint", + version: 1, + runFingerprint, + settings: checkpointSettings, + shardIndex: 0, + shardCount: 1, + }; + fs.mkdirSync(path.dirname(checkpointPath), { recursive: true }); + let checkpoint = scaleMod.appendTranslationBenchCheckpointRows( + checkpointPath, + checkpointHeader, + [], + ); + const seedRows = checkpoint.rows + .filter((row) => row.phase === "translation") + .map((row) => row.value); + const completed = new Set(checkpoint.resumeKeys); + console.log( + `Checkpoint ${checkpointPath}: resumed=${seedRows.length} keys=${completed.size}`, + ); + + const result = await runnerMod.runTranslationBench( + suite, + actionContext, + { + models: EVAL_MODELS, + sourceManifest, + availableModels, + concurrency: CONCURRENCY, + concurrencyByModel: CONCURRENCY_BY_MODEL, + modelConcurrency: MODEL_CONCURRENCY, + seedRows, + isWorkComplete: ({ model, scenarioId, caseId }) => + completed.has( + scaleMod.translationBenchResumeKey({ + phase: "translation", + model, + scenario: scenarioId, + caseId, + }), + ), + onRowComplete: (row) => { + const ckptRow = + scaleMod.createTranslationBenchTranslationCheckpointRow( + row, + ); + checkpoint = scaleMod.appendTranslationBenchCheckpointRows( + checkpointPath, + checkpointHeader, + [ckptRow], + checkpoint, + ); + completed.add(scaleMod.translationBenchResumeKey(ckptRow)); + }, + }, + (done, total) => { + const now = Date.now(); + if (done === total || now - lastLog > 5000) { + lastLog = now; + const elapsed = ((now - started) / 1000).toFixed(0); + const rate = + done > 0 ? (Number(elapsed) / done).toFixed(2) : "?"; + console.log( + `[eval] ${done}/${total} (${((done / total) * 100).toFixed(1)}%) elapsed=${elapsed}s sec_per=${rate} modelC=${MODEL_CONCURRENCY} peak=${PEAK_IN_FLIGHT} ckpt=${completed.size}`, + ); + } + }, + ); + + fs.writeFileSync(outPath, JSON.stringify(result, null, 2)); + // Side outputs follow the eval art dir (dirname of outPath), not the run root — + // so smoke subdirs cannot clobber sibling 1k artifacts. + const artDir = path.dirname(outPath); + fs.mkdirSync(artDir, { recursive: true }); + fs.copyFileSync(checkpointPath, path.join(artDir, "eval-trajectory.jsonl")); + const report = reportMod.createTranslationBenchReport( + suite, + result, + [], + benchmark, + ); + const html = reportMod.renderTranslationBenchHtml(report); + fs.writeFileSync(htmlPath, html); + console.log( + JSON.stringify( + { + outPath, + htmlPath, + elapsedSec: (Date.now() - started) / 1000, + summary: result.summary ?? result.totals ?? Object.keys(result), + }, + null, + 2, + ), + ); + + const gw = cfg.gatewayDir; + fs.mkdirSync(gw, { recursive: true }); + fs.copyFileSync(htmlPath, path.join(gw, "eval-report.html")); + fs.copyFileSync(outPath, path.join(gw, "eval-results.json")); + fs.writeFileSync( + path.join(artDir, "eval-report-by-model.json"), + JSON.stringify(report.byModel ?? [], null, 2), + ); + fs.writeFileSync( + path.join(artDir, "eval-report-summary.json"), + JSON.stringify( + { + suiteName: report.suiteName, + settings: report.settings, + summary: report.summary, + byModel: (report.byModel ?? []).map((m) => ({ + key: m.key, + summary: m.summary, + })), + generatedAt: new Date().toISOString(), + }, + null, + 2, + ), + ); + console.log("gateway →", gw); + console.log("artDir →", artDir); + + // Build + publish the landing "repo": index.html (details + how-each-score-is- + // calculated + one-cell=utterance×model) and the per-cell explorer, pointed at + // this run's artifacts. Robust: never fails the eval if a viz generator errors. + const vizGwDir = + cfg.indexGatewayDir || cfg.gatewayDir || path.join(RUN, "published"); + const { spawnSync } = await import("node:child_process"); + for (const gen of ["update-eval-cases-viz.mjs", "update-index-viz.mjs"]) { + try { + const r = spawnSync( + process.execPath, + [ + path.join(RUN, gen), + "--art-dir", + artDir, + "--gateway-dir", + vizGwDir, + ], + { cwd: RUN, stdio: "inherit" }, + ); + if (r.status !== 0) console.warn(`${gen} exited ${r.status}`); + } catch (e) { + console.warn(`${gen} failed:`, e.message); + } + } + console.log("landing repo →", vizGwDir); +} finally { + await disp.closeCommandHandlerContext(context); +} diff --git a/ts/packages/benchmarks/package.json b/ts/packages/benchmarks/package.json index e64e45e0be..fbf8ebd778 100644 --- a/ts/packages/benchmarks/package.json +++ b/ts/packages/benchmarks/package.json @@ -34,6 +34,7 @@ }, "dependencies": { "@typeagent/action-schema": "workspace:*", + "@typeagent/agent-cache": "workspace:*", "@typeagent/agent-sdk": "workspace:*", "@typeagent/aiclient": "workspace:*", "agent-dispatcher": "workspace:*", diff --git a/ts/packages/benchmarks/src/translationBench/index.ts b/ts/packages/benchmarks/src/translationBench/index.ts index 22cd3309a3..c469267c13 100644 --- a/ts/packages/benchmarks/src/translationBench/index.ts +++ b/ts/packages/benchmarks/src/translationBench/index.ts @@ -4,3 +4,4 @@ export * from "./catalog.js"; export * from "./runConfig.js"; export * from "./synthesizer/index.js"; +export * from "./runner/index.js"; diff --git a/ts/packages/benchmarks/src/translationBench/runner/index.ts b/ts/packages/benchmarks/src/translationBench/runner/index.ts new file mode 100644 index 0000000000..e5690d944b --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/runner/index.ts @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Translation-bench runner scoring surface (E+C empty-gold fairness). + * Full suite execution lives with local eval harnesses until landed separately. + * Single re-export chain: index → runner → scoring. + */ + +export * from "./runner.js"; diff --git a/ts/packages/benchmarks/src/translationBench/runner/runner.ts b/ts/packages/benchmarks/src/translationBench/runner/runner.ts new file mode 100644 index 0000000000..4b19f997a2 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/runner/runner.ts @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Compatibility barrel: scoring lives in `./scoring.js`. + * Local eval harnesses import from `runner/runner.js`; keep that path stable. + */ + +export * from "./scoring.js"; diff --git a/ts/packages/benchmarks/src/translationBench/runner/scoring.ts b/ts/packages/benchmarks/src/translationBench/runner/scoring.ts new file mode 100644 index 0000000000..887d5fd829 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/runner/scoring.ts @@ -0,0 +1,640 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Translation-bench scoring (including E+C empty-gold fairness). + * + * E — dispatcher `unknown` schema-match throw → zero-action PASS on empty gold + * C — filter chat.generateResponse / utility.claudeTask from scored chosen + * + * Shared non-eval IDs: synthesizer/eligibleActions HARDCODED_NON_EVAL_ACTION_IDS. + */ + +import { equalNormalizedObject } from "@typeagent/agent-cache"; +import type { AppAction } from "@typeagent/agent-sdk"; +import type { TranslationBenchOrder } from "../synthesizer/benchmark.js"; +import { HARDCODED_NON_EVAL_ACTION_IDS } from "../synthesizer/eligibleActions.js"; + +/** Clarify schema used as internal abstention (mirrors dispatcherUtils). */ +const DISPATCHER_CLARIFY_NAME = "dispatcher.clarify"; + +/** + * Per-field parameter scoring modes for deterministic soft matching. + * - exact: value must equal expected (default) + * - exists: key must be present on chosen (value ignored) + * - nonempty: key must be present and not empty string/array/null/undefined + * - ignore: field is not scored + */ +export type TranslationBenchParamFieldMode = + | "exact" + | "exists" + | "nonempty" + | "ignore"; + +export interface TranslationBenchParameterScoreSpec { + /** Default mode for fields not listed in `fields` (default: exact). */ + defaultMode?: TranslationBenchParamFieldMode; + /** Per top-level parameter field mode. */ + fields?: Record; +} + +export interface TranslationBenchAction { + schemaName: string; + actionName: string; + parameters?: Record; +} + +export interface TranslationBenchScore { + /** Primary gate: route + parameter score specs (soft when specs present). */ + passed: boolean; + /** Full deep-equal on all parameters, ignoring score specs. */ + exactPassed: boolean; + /** Translator produced parseable actions with no validation error. */ + schemaValid: boolean; + expectedCount: number; + chosenCount: number; + routed: number; + paramMatches: number; + /** Deep-equal parameter matches (always exact). */ + exactParamMatches: number; + isNegative: boolean; + firedOnNegative: boolean; + diagnostics: TranslationBenchDiagnosticCounts; +} + +export interface TranslationBenchDiagnosticCounts { + wrongRouteOrAction: number; + missingRequiredParameter: number; + extraneousParameter: number; + wrongParameterType: number; + wrongValue: number; + invalidJsonOrTranslationFailure: number; +} + +function routeMatches( + a: TranslationBenchAction, + b: TranslationBenchAction, +): boolean { + return a.schemaName === b.schemaName && a.actionName === b.actionName; +} + +function isNonemptyParamValue(value: unknown): boolean { + if (value === undefined || value === null) return false; + if (typeof value === "string") return value.trim().length > 0; + if (Array.isArray(value)) return value.length > 0; + return true; +} + +export function resolveTranslationBenchParamFieldMode( + spec: TranslationBenchParameterScoreSpec | undefined, + field: string, +): TranslationBenchParamFieldMode { + return spec?.fields?.[field] ?? spec?.defaultMode ?? "exact"; +} + +/** + * Deterministic parameter match using optional per-field score specs. + * Specs are typically LLM-authored at dataset generation time and then frozen. + */ +export function parametersMatch( + expected: TranslationBenchAction, + chosen: TranslationBenchAction, + spec?: TranslationBenchParameterScoreSpec, +): boolean { + const expectedParams = expected.parameters ?? {}; + const chosenParams = chosen.parameters ?? {}; + if (spec === undefined) { + return equalNormalizedObject(expectedParams, chosenParams); + } + + const defaultMode = spec.defaultMode ?? "exact"; + for (const key of Object.keys(expectedParams)) { + const mode = resolveTranslationBenchParamFieldMode(spec, key); + if (mode === "ignore") continue; + const hasKey = Object.prototype.hasOwnProperty.call(chosenParams, key); + if (mode === "exists") { + if (!hasKey) return false; + continue; + } + if (mode === "nonempty") { + if (!hasKey || !isNonemptyParamValue(chosenParams[key])) { + return false; + } + continue; + } + // exact + if ( + !hasKey || + !equalNormalizedObject( + { value: expectedParams[key] }, + { value: chosenParams[key] }, + ) + ) { + return false; + } + } + + // Extraneous chosen keys fail under exact default (legacy behavior), + // unless the key is explicitly ignored or only-exists scored. + if (defaultMode === "exact") { + for (const key of Object.keys(chosenParams)) { + const mode = resolveTranslationBenchParamFieldMode(spec, key); + if (mode === "ignore" || mode === "exists" || mode === "nonempty") { + continue; + } + if (!Object.prototype.hasOwnProperty.call(expectedParams, key)) { + return false; + } + } + } + return true; +} + +function parametersMatchExact( + expected: TranslationBenchAction, + chosen: TranslationBenchAction, +): boolean { + return equalNormalizedObject( + expected.parameters ?? {}, + chosen.parameters ?? {}, + ); +} + +interface TranslationBenchAlignment { + routed: number; + paramMatches: number; + exactParamMatches: number; + pairs: { expectedIndex: number; chosenIndex: number }[]; +} + +function alignStrict( + expected: TranslationBenchAction[], + chosen: TranslationBenchAction[], + parameterScore?: Array, +): TranslationBenchAlignment { + let routed = 0; + let paramMatches = 0; + let exactParamMatches = 0; + const pairs: TranslationBenchAlignment["pairs"] = []; + const count = Math.min(expected.length, chosen.length); + for (let i = 0; i < count; i++) { + const e = expected[i]!; + const c = chosen[i]!; + if (routeMatches(e, c)) { + routed++; + pairs.push({ expectedIndex: i, chosenIndex: i }); + if (parametersMatch(e, c, parameterScore?.[i])) paramMatches++; + if (parametersMatchExact(e, c)) exactParamMatches++; + } + } + return { routed, paramMatches, exactParamMatches, pairs }; +} + +function alignAny( + expected: TranslationBenchAction[], + chosen: TranslationBenchAction[], + parameterScore?: Array, +): TranslationBenchAlignment { + const chosenUsed = new Set(); + const expectedUsed = new Set(); + let paramMatches = 0; + let exactParamMatches = 0; + const pairs: TranslationBenchAlignment["pairs"] = []; + + // Prefer soft (or exact) parameter matches first within a route group. + for ( + let expectedIndex = 0; + expectedIndex < expected.length; + expectedIndex++ + ) { + const e = expected[expectedIndex]!; + const match = chosen.findIndex( + (c, index) => + !chosenUsed.has(index) && + routeMatches(e, c) && + parametersMatch(e, c, parameterScore?.[expectedIndex]), + ); + if (match >= 0) { + chosenUsed.add(match); + expectedUsed.add(expectedIndex); + pairs.push({ expectedIndex, chosenIndex: match }); + paramMatches++; + if (parametersMatchExact(e, chosen[match]!)) exactParamMatches++; + } + } + + let routed = paramMatches; + for (let i = 0; i < expected.length; i++) { + const e = expected[i]!; + if (expectedUsed.has(i)) continue; + const match = chosen.findIndex( + (c, index) => !chosenUsed.has(index) && routeMatches(e, c), + ); + if (match >= 0) { + chosenUsed.add(match); + pairs.push({ expectedIndex: i, chosenIndex: match }); + routed++; + if (parametersMatchExact(e, chosen[match]!)) exactParamMatches++; + } + } + return { routed, paramMatches, exactParamMatches, pairs }; +} + +export function createEmptyTranslationBenchDiagnosticCounts(): TranslationBenchDiagnosticCounts { + return { + wrongRouteOrAction: 0, + missingRequiredParameter: 0, + extraneousParameter: 0, + wrongParameterType: 0, + wrongValue: 0, + invalidJsonOrTranslationFailure: 0, + }; +} + +function jsonKind(value: unknown): string { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + return typeof value; +} + +function diagnoseParameterValue( + expected: unknown, + chosen: unknown, + counts: TranslationBenchDiagnosticCounts, +): void { + if (equalNormalizedObject({ value: expected }, { value: chosen })) return; + if (jsonKind(expected) !== jsonKind(chosen)) { + counts.wrongParameterType++; + return; + } + if (Array.isArray(expected) && Array.isArray(chosen)) { + const count = Math.min(expected.length, chosen.length); + const before = Object.values(counts).reduce( + (sum, value) => sum + value, + 0, + ); + for (let index = 0; index < count; index++) { + diagnoseParameterValue(expected[index], chosen[index], counts); + } + counts.missingRequiredParameter += Math.max( + 0, + expected.length - chosen.length, + ); + counts.extraneousParameter += Math.max( + 0, + chosen.length - expected.length, + ); + const after = Object.values(counts).reduce( + (sum, value) => sum + value, + 0, + ); + if (before === after) counts.wrongValue++; + return; + } + if ( + expected !== null && + chosen !== null && + typeof expected === "object" && + typeof chosen === "object" + ) { + const expectedRecord = expected as Record; + const chosenRecord = chosen as Record; + const before = Object.values(counts).reduce( + (sum, value) => sum + value, + 0, + ); + for (const key of Object.keys(expectedRecord)) { + if (!Object.prototype.hasOwnProperty.call(chosenRecord, key)) { + counts.missingRequiredParameter++; + } else { + diagnoseParameterValue( + expectedRecord[key], + chosenRecord[key], + counts, + ); + } + } + for (const key of Object.keys(chosenRecord)) { + if (!Object.prototype.hasOwnProperty.call(expectedRecord, key)) { + counts.extraneousParameter++; + } + } + const after = Object.values(counts).reduce( + (sum, value) => sum + value, + 0, + ); + if (before === after) counts.wrongValue++; + return; + } + counts.wrongValue++; +} + +function diagnoseTranslationError( + error: string, + counts: TranslationBenchDiagnosticCounts, +): void { + const prefix = "JSON validation failed:"; + if (!error.startsWith(prefix)) { + counts.invalidJsonOrTranslationFailure = 1; + return; + } + const primary = error.slice(prefix.length).trimStart().split("\n", 1)[0]!; + if (/^(Missing actionName property|Unknown action name:)/.test(primary)) { + counts.wrongRouteOrAction = 1; + } else if (/^Missing required property /.test(primary)) { + counts.missingRequiredParameter = 1; + } else if (/^Extraneous property /.test(primary)) { + counts.extraneousParameter = 1; + } else if ( + /does not match any union type|should not be null|is not an (?:object|array|string)|is not a (?:number|boolean), got/.test( + primary, + ) + ) { + counts.wrongParameterType = 1; + } else if (/ is not .*?, got .* instead$/.test(primary)) { + counts.wrongValue = 1; + } else { + counts.invalidJsonOrTranslationFailure = 1; + } +} + +function diagnoseParametersWithScoreSpec( + expectedParams: Record, + chosenParams: Record, + counts: TranslationBenchDiagnosticCounts, + spec: TranslationBenchParameterScoreSpec | undefined, +): void { + if (spec === undefined) { + diagnoseParameterValue(expectedParams, chosenParams, counts); + return; + } + + const defaultMode = spec.defaultMode ?? "exact"; + const scoredExpected: Record = {}; + const scoredChosen: Record = {}; + + for (const key of Object.keys(expectedParams)) { + const mode = resolveTranslationBenchParamFieldMode(spec, key); + if (mode === "ignore") continue; + const hasKey = Object.prototype.hasOwnProperty.call(chosenParams, key); + if (mode === "exists") { + if (!hasKey) counts.missingRequiredParameter++; + continue; + } + if (mode === "nonempty") { + if (!hasKey) { + counts.missingRequiredParameter++; + } else if (!isNonemptyParamValue(chosenParams[key])) { + counts.wrongValue++; + } + continue; + } + // exact — defer to structural diagnose for type/value/missing. + scoredExpected[key] = expectedParams[key]; + if (hasKey) scoredChosen[key] = chosenParams[key]; + } + + if (defaultMode === "exact") { + for (const key of Object.keys(chosenParams)) { + const mode = resolveTranslationBenchParamFieldMode(spec, key); + if (mode === "ignore" || mode === "exists" || mode === "nonempty") { + continue; + } + if (!Object.prototype.hasOwnProperty.call(expectedParams, key)) { + scoredChosen[key] = chosenParams[key]; + } + } + } + + diagnoseParameterValue(scoredExpected, scoredChosen, counts); +} + +export function diagnoseTranslationBench( + expected: TranslationBenchAction[], + chosen: TranslationBenchAction[], + order: TranslationBenchOrder, + error?: string, + parameterScore?: Array, +): TranslationBenchDiagnosticCounts { + const counts = createEmptyTranslationBenchDiagnosticCounts(); + if (error !== undefined) { + diagnoseTranslationError(error, counts); + return counts; + } + const alignment = + order === "strict" + ? alignStrict(expected, chosen, parameterScore) + : alignAny(expected, chosen, parameterScore); + counts.wrongRouteOrAction = + Math.max(expected.length, chosen.length) - alignment.routed; + for (const pair of alignment.pairs) { + const spec = parameterScore?.[pair.expectedIndex]; + diagnoseParametersWithScoreSpec( + expected[pair.expectedIndex]!.parameters ?? {}, + chosen[pair.chosenIndex]!.parameters ?? {}, + counts, + spec, + ); + } + return counts; +} + +export function scoreTranslationBench( + expected: TranslationBenchAction[], + chosen: TranslationBenchAction[], + order: TranslationBenchOrder, + abstentionCount = 0, + options?: { + parameterScore?: Array; + /** When false, translator failed validation / threw. Default true. */ + schemaValid?: boolean; + }, +): TranslationBenchScore { + const parameterScore = options?.parameterScore; + const schemaValid = options?.schemaValid ?? true; + const { routed, paramMatches, exactParamMatches } = + order === "strict" + ? alignStrict(expected, chosen, parameterScore) + : alignAny(expected, chosen, parameterScore); + const isNegative = expected.length === 0; + const softPassed = + schemaValid && + expected.length === chosen.length && + paramMatches === expected.length && + !(abstentionCount > 0 && chosen.length > 0); + const exactPassed = + schemaValid && + expected.length === chosen.length && + exactParamMatches === expected.length && + !(abstentionCount > 0 && chosen.length > 0); + return { + passed: softPassed, + exactPassed, + schemaValid: schemaValid && !(abstentionCount > 0 && chosen.length > 0), + expectedCount: expected.length, + chosenCount: chosen.length, + routed, + paramMatches, + exactParamMatches, + isNegative, + firedOnNegative: isNegative && chosen.length > 0, + diagnostics: diagnoseTranslationBench( + expected, + chosen, + order, + undefined, + parameterScore, + ), + }; +} + +function toEvalAction(action: AppAction): TranslationBenchAction { + return { + schemaName: action.schemaName ?? "", + actionName: action.actionName, + ...(action.parameters ? { parameters: action.parameters } : {}), + }; +} + +function isInternalAbstention(action: AppAction): boolean { + // Mirrors dispatcher isUnknownAction + DispatcherClarifyName. + return ( + action.actionName === "unknown" || + action.schemaName === DISPATCHER_CLARIFY_NAME + ); +} + +/** Re-export shared non-eval IDs (single source: synthesizer/eligibleActions). */ +export const TRANSLATION_BENCH_NON_EVAL_ACTION_IDS: ReadonlySet = + HARDCODED_NON_EVAL_ACTION_IDS; + +export function translationBenchActionId(action: { + schemaName?: string; + actionName: string; +}): string { + const schema = action.schemaName ?? ""; + return schema ? `${schema}.${action.actionName}` : action.actionName; +} + +export function isNonEvalTranslationBenchAction(action: { + schemaName?: string; + actionName: string; +}): boolean { + return HARDCODED_NON_EVAL_ACTION_IDS.has(translationBenchActionId(action)); +} + +/** + * Dispatcher throws when the model returns the internal `unknown` abstention + * action (`Unable to match schema name for action unknown`) before the runner + * can filter it via `isInternalAbstention`. That is a correct zero-action + * refusal on empty-gold, not a translation failure. + */ +export function isUnknownActionSchemaMatchError(error: unknown): boolean { + const message = + error instanceof Error ? error.message : String(error ?? ""); + return /Unable to match schema name for action ['"]?unknown['"]?\b/i.test( + message, + ); +} + +/** Drop internal abstentions + non-eval actions from the scored chosen list. */ +export function toScoredTranslationBenchActions( + actions: readonly AppAction[], +): { + rawChosenActions: TranslationBenchAction[]; + chosenActions: TranslationBenchAction[]; + abstentionCount: number; +} { + const rawChosenActions = actions.map(toEvalAction); + const withoutAbstention = actions.filter( + (action) => !isInternalAbstention(action), + ); + const abstentionCount = actions.length - withoutAbstention.length; + const chosenActions = withoutAbstention + .map(toEvalAction) + .filter((action) => !isNonEvalTranslationBenchAction(action)); + return { rawChosenActions, chosenActions, abstentionCount }; +} + +/** + * Build a row score from either a successful translation or a caught error. + * Unknown-schema-match throws are scored as successful zero-action abstention. + */ +export function scoreTranslationBenchTranslationOutcome( + expectedActions: TranslationBenchAction[], + order: TranslationBenchOrder, + outcome: + | { ok: true; actions: readonly AppAction[] } + | { ok: false; error: unknown }, + parameterScore?: Array, +): { + rawChosenActions: TranslationBenchAction[]; + chosenActions: TranslationBenchAction[]; + score: TranslationBenchScore; + error?: string; +} { + const scoreOptions = { + ...(parameterScore !== undefined ? { parameterScore } : {}), + }; + + if (outcome.ok) { + const { rawChosenActions, chosenActions, abstentionCount } = + toScoredTranslationBenchActions(outcome.actions); + return { + rawChosenActions, + chosenActions, + score: scoreTranslationBench( + expectedActions, + chosenActions, + order, + abstentionCount, + { ...scoreOptions, schemaValid: true }, + ), + }; + } + + if (isUnknownActionSchemaMatchError(outcome.error)) { + // Model abstained via `unknown`; dispatcher threw before filter ran. + // schemaName matches post-fix finalize (DispatcherName = "dispatcher"). + const rawChosenActions: TranslationBenchAction[] = [ + { schemaName: "dispatcher", actionName: "unknown" }, + ]; + return { + rawChosenActions, + chosenActions: [], + score: scoreTranslationBench( + expectedActions, + [], + order, + /* abstentionCount */ 1, + { ...scoreOptions, schemaValid: true }, + ), + // No row.error — this is a scored abstention, not a harness failure. + }; + } + + const error = + outcome.error instanceof Error + ? outcome.error.message + : String(outcome.error); + const score = scoreTranslationBench(expectedActions, [], order, 0, { + ...scoreOptions, + schemaValid: false, + }); + score.passed = false; + score.exactPassed = false; + score.schemaValid = false; + score.diagnostics = diagnoseTranslationBench( + expectedActions, + [], + order, + error, + parameterScore, + ); + return { + rawChosenActions: [], + chosenActions: [], + score, + error, + }; +} diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/eligibleActions.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/eligibleActions.ts index 09ae3db6b8..ed671a2190 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/eligibleActions.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/eligibleActions.ts @@ -11,6 +11,22 @@ import { createRequire } from "node:module"; const require = createRequire(import.meta.url); +/** + * Actions we never evaluate in translation bench, regardless of grader + * classification. These are not translatable "tool fires": + * - `chat.generateResponse` is a benign conversational acknowledgment, not a + * tool action; on empty-gold negatives it would otherwise be counted as a + * false fire. + * - `utility.claudeTask` is an internal utility escape hatch, not a targetable + * catalog action. + * Single source of truth for synth scheduling, coverage validation, and the + * runner scored-chosen filter (C). + */ +export const HARDCODED_NON_EVAL_ACTION_IDS: ReadonlySet = new Set([ + "chat.generateResponse", + "utility.claudeTask", +]); + let cachedPackagedLlmJudgeExcludedActions: ReadonlySet | undefined; function isPlainObject(value: unknown): value is Record { @@ -60,9 +76,10 @@ export function getPackagedLlmJudgeExcludedActions(): ReadonlySet { `Unsupported or corrupt packaged action-parameters grader at ${graderPath}`, ); } - cachedPackagedLlmJudgeExcludedActions = new Set( - listLlmAsAJudgeExcludedActionIds(raw.byAction), - ); + cachedPackagedLlmJudgeExcludedActions = new Set([ + ...listLlmAsAJudgeExcludedActionIds(raw.byAction), + ...HARDCODED_NON_EVAL_ACTION_IDS, + ]); } return cachedPackagedLlmJudgeExcludedActions; } diff --git a/ts/packages/benchmarks/test/translationBench.runnerScoring.spec.ts b/ts/packages/benchmarks/test/translationBench.runnerScoring.spec.ts new file mode 100644 index 0000000000..0df49bcf23 --- /dev/null +++ b/ts/packages/benchmarks/test/translationBench.runnerScoring.spec.ts @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + isNonEvalTranslationBenchAction, + isUnknownActionSchemaMatchError, + scoreTranslationBench, + scoreTranslationBenchTranslationOutcome, + toScoredTranslationBenchActions, + TRANSLATION_BENCH_NON_EVAL_ACTION_IDS, +} from "../src/translationBench/runner/runner.js"; +import { HARDCODED_NON_EVAL_ACTION_IDS } from "../src/translationBench/synthesizer/eligibleActions.js"; +import type { AppAction } from "@typeagent/agent-sdk"; + +describe("translationBench runner scoring fairness (E + C)", () => { + it("recognizes dispatcher unknown schema-match errors", () => { + expect( + isUnknownActionSchemaMatchError( + new Error( + "Internal Error: Unable to match schema name for action unknown", + ), + ), + ).toBe(true); + expect( + isUnknownActionSchemaMatchError( + "Internal Error: Unable to match schema name for action 'unknown'", + ), + ).toBe(true); + expect( + isUnknownActionSchemaMatchError( + new Error("JSON validation failed: Missing required property"), + ), + ).toBe(false); + }); + + it("treats unknown schema-match throw as zero-action PASS on empty gold", () => { + const { chosenActions, score, error, rawChosenActions } = + scoreTranslationBenchTranslationOutcome([], "any", { + ok: false, + error: new Error( + "Internal Error: Unable to match schema name for action unknown", + ), + }); + expect(error).toBeUndefined(); + expect(chosenActions).toEqual([]); + expect(rawChosenActions).toEqual([ + { schemaName: "dispatcher", actionName: "unknown" }, + ]); + expect(score.passed).toBe(true); + expect(score.exactPassed).toBe(true); + expect(score.schemaValid).toBe(true); + expect(score.isNegative).toBe(true); + expect(score.firedOnNegative).toBe(false); + expect(score.diagnostics.invalidJsonOrTranslationFailure).toBe(0); + }); + + it("unknown schema-match throw still FAILs when gold expects actions", () => { + const { score, error } = scoreTranslationBenchTranslationOutcome( + [ + { + schemaName: "browser", + actionName: "goBack", + parameters: {}, + }, + ], + "any", + { + ok: false, + error: new Error( + "Internal Error: Unable to match schema name for action unknown", + ), + }, + ); + expect(error).toBeUndefined(); // abstention scored, not harness error + expect(score.passed).toBe(false); + expect(score.exactPassed).toBe(false); + expect(score.schemaValid).toBe(true); + expect(score.isNegative).toBe(false); + expect(score.chosenCount).toBe(0); + expect(score.expectedCount).toBe(1); + }); + + it("runner non-eval IDs are the shared generator set (no drift)", () => { + expect([...TRANSLATION_BENCH_NON_EVAL_ACTION_IDS].sort()).toEqual( + [...HARDCODED_NON_EVAL_ACTION_IDS].sort(), + ); + expect(TRANSLATION_BENCH_NON_EVAL_ACTION_IDS).toBe( + HARDCODED_NON_EVAL_ACTION_IDS, + ); + }); + + it("success-path unknown action is filtered; sibling tool fire remains", () => { + const r = toScoredTranslationBenchActions([ + { + schemaName: "browser", + actionName: "closeWebPage", + parameters: {}, + } as AppAction, + { schemaName: "dispatcher", actionName: "unknown" } as AppAction, + ]); + expect(r.abstentionCount).toBe(1); + const score = scoreTranslationBench( + [], + r.chosenActions, + "any", + r.abstentionCount, + { schemaValid: true }, + ); + expect(r.chosenActions).toHaveLength(1); + expect(r.chosenActions[0]?.actionName).toBe("closeWebPage"); + expect(score.passed).toBe(false); + expect(score.firedOnNegative).toBe(true); + }); + + it("still FAILs real translation errors on empty gold", () => { + const { score, error } = scoreTranslationBenchTranslationOutcome( + [], + "any", + { + ok: false, + error: new Error( + "JSON validation failed: Missing required property 'parameters.requests'", + ), + }, + ); + expect(error).toMatch(/JSON validation failed/); + expect(score.passed).toBe(false); + expect(score.schemaValid).toBe(false); + // Missing-required is classified under missingRequiredParameter, not invalidJson. + expect(score.diagnostics.missingRequiredParameter).toBe(1); + expect(score.diagnostics.invalidJsonOrTranslationFailure).toBe(0); + }); + + it("filters unknown abstention from successful translations", () => { + const actions = [{ actionName: "unknown" } as AppAction]; + const { chosenActions, abstentionCount, rawChosenActions } = + toScoredTranslationBenchActions(actions); + expect(abstentionCount).toBe(1); + expect(chosenActions).toEqual([]); + expect(rawChosenActions[0]?.actionName).toBe("unknown"); + const score = scoreTranslationBench([], chosenActions, "any", 1, { + schemaValid: true, + }); + expect(score.passed).toBe(true); + expect(score.firedOnNegative).toBe(false); + }); + + it("does not count chat.generateResponse / utility.claudeTask as fires", () => { + expect( + TRANSLATION_BENCH_NON_EVAL_ACTION_IDS.has("chat.generateResponse"), + ).toBe(true); + expect( + TRANSLATION_BENCH_NON_EVAL_ACTION_IDS.has("utility.claudeTask"), + ).toBe(true); + expect( + isNonEvalTranslationBenchAction({ + schemaName: "chat", + actionName: "generateResponse", + }), + ).toBe(true); + + const { chosenActions, score, error } = + scoreTranslationBenchTranslationOutcome([], "any", { + ok: true, + actions: [ + { + schemaName: "chat", + actionName: "generateResponse", + parameters: { text: "ok" }, + } as AppAction, + ], + }); + expect(error).toBeUndefined(); + expect(chosenActions).toEqual([]); + expect(score.passed).toBe(true); + expect(score.firedOnNegative).toBe(false); + expect(score.chosenCount).toBe(0); + }); + + it("still counts real tool fires on empty gold as FAIL", () => { + const { chosenActions, score } = + scoreTranslationBenchTranslationOutcome([], "any", { + ok: true, + actions: [ + { + schemaName: "browser", + actionName: "closeWebPage", + parameters: {}, + } as AppAction, + ], + }); + expect(chosenActions).toHaveLength(1); + expect(score.passed).toBe(false); + expect(score.firedOnNegative).toBe(true); + }); + + it("keeps real actions when mixed with non-eval chat ack", () => { + const { chosenActions, score } = + scoreTranslationBenchTranslationOutcome( + [ + { + schemaName: "browser", + actionName: "goBack", + parameters: {}, + }, + ], + "any", + { + ok: true, + actions: [ + { + schemaName: "browser", + actionName: "goBack", + parameters: {}, + } as AppAction, + { + schemaName: "chat", + actionName: "generateResponse", + parameters: { text: "done" }, + } as AppAction, + ], + }, + ); + expect(chosenActions.map((a) => a.actionName)).toEqual(["goBack"]); + expect(score.passed).toBe(true); + }); +}); diff --git a/ts/packages/dispatcher/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts b/ts/packages/dispatcher/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts index 46f507239f..be8d5ad046 100644 --- a/ts/packages/dispatcher/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts +++ b/ts/packages/dispatcher/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts @@ -233,9 +233,11 @@ async function canTranslateWithoutContext( } else { newAction = newTranslatedActions; } - const newSchemaName = usedTranslators - .get(schemaName)! - .getSchemaName(newAction.actionName); + const newSchemaName = isUnknownAction(newAction) + ? DispatcherName + : usedTranslators + .get(schemaName)! + .getSchemaName(newAction.actionName); if (newSchemaName === undefined) { // Should not happen throw new Error( diff --git a/ts/packages/dispatcher/dispatcher/src/translation/translateRequest.ts b/ts/packages/dispatcher/dispatcher/src/translation/translateRequest.ts index e18170209f..e542343451 100644 --- a/ts/packages/dispatcher/dispatcher/src/translation/translateRequest.ts +++ b/ts/packages/dispatcher/dispatcher/src/translation/translateRequest.ts @@ -1045,6 +1045,20 @@ async function finalizeAction( currentAction = unknownAction; } + // UnknownAction is a deliberate abstention ("no matching tool"). It is not + // registered on translator action maps, so getSchemaName would miss it and + // historically threw — wiping any sibling actions already finalized in a + // MultipleAction. Return it on the dispatcher schema so callers can filter + // via isUnknownAction without losing the rest of the batch. + if (isUnknownAction(currentAction)) { + return createExecutableAction( + DispatcherName, + currentAction.actionName, + currentAction.parameters, + resultEntityId, + ); + } + // A translator may combine actions from multiple schemas (inline, selected actions) const currentActionSchemaName = currentTranslator.getSchemaName( currentAction.actionName, diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml index 9b6741018c..60b92aa859 100644 --- a/ts/pnpm-lock.yaml +++ b/ts/pnpm-lock.yaml @@ -4075,6 +4075,9 @@ importers: '@typeagent/action-schema': specifier: workspace:* version: link:../actionSchema + '@typeagent/agent-cache': + specifier: workspace:* + version: link:../cache '@typeagent/agent-sdk': specifier: workspace:* version: link:../agentSdk