From 6d7ae0dd2405fd3e52ae093c23682bfee514da07 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Mon, 10 Aug 2026 15:55:16 -0700 Subject: [PATCH 1/3] feat(benchmarks): add negative-case translation repro script - New reproNegatives.mjs drives the real TypeAgent translator over 20 hardcoded negation utterances ("Do not X"), each expecting no action. - Offers only actions from eligible-gold-actions.generated.json, so non-gold actions (e.g. browser.executeAdHocScript, excluded as llmAsAJudge) can never be produced; an offLimits guard catches leaks. - Bounded-concurrency worker pool capped by the model's maxConcurrency. - commander CLI: --models/--limit/--only/--repeat/--concurrency plus --api-key/--endpoint so credentials are passed in, not baked in. - Export createTypeAgentTranslatorForSelectedActions from agent-dispatcher/internal so benchmarks can scope the offered actions. --- .../eligible-gold-actions.generated.json | 300 ++++++++++++++++++ .../scripts/reproNegatives.mjs | 284 +++++++++++++++++ .../dispatcher/dispatcher/src/internal.ts | 1 + 3 files changed, 585 insertions(+) create mode 100644 ts/packages/benchmarks/src/translationBench/eligible-gold-actions.generated.json create mode 100644 ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs diff --git a/ts/packages/benchmarks/src/translationBench/eligible-gold-actions.generated.json b/ts/packages/benchmarks/src/translationBench/eligible-gold-actions.generated.json new file mode 100644 index 000000000..9ef1e8a54 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/eligible-gold-actions.generated.json @@ -0,0 +1,300 @@ +{ + "version": 1, + "catalogVersion": "2026-08-09", + "policyHash": "d1c784e87ff8a92ee66d5d79e3a74c02e9e8da47df9937cf89b8b27c469a1413", + "graderRulesFingerprint": "4dbb8db2f9413485", + "generatedAt": "2026-08-09T13:27:20.514Z", + "model": "azure/gpt-5.6-sol", + "allowlist": [ + "browser.captureScreenshot", + "browser.changeSearchProvider", + "browser.closeAllWebPages", + "browser.closeWebPage", + "browser.external.addToBookmarks", + "browser.external.closeTab", + "browser.external.closeWindow", + "browser.external.openFromBookmarks", + "browser.external.openFromHistory", + "browser.external.openTab", + "browser.external.switchToTabByText", + "browser.followLinkByText", + "browser.goBack", + "browser.goForward", + "browser.openWebPage", + "browser.readPageContent", + "browser.reloadPage", + "browser.scrollDown", + "browser.scrollUp", + "browser.stopReadPageContent", + "browser.zoomReset", + "calendar.findEvents", + "calendar.findThisWeeksEvents", + "calendar.findTodaysEvents", + "calendar.scheduleEvent", + "code.changeColorScheme", + "code.changeEditorLayout", + "code.code-debug.removeAllBreakpoints", + "code.code-debug.setBreakpoint", + "code.code-debug.showDebugPanel", + "code.code-debug.showHover", + "code.code-debug.startDebugging", + "code.code-debug.step", + "code.code-debug.stopDebugging", + "code.code-debug.toggleBreakpoint", + "code.code-display.closeEditor", + "code.code-display.fontZoomReset", + "code.code-display.openMarkdownPreview", + "code.code-display.openMarkdownPreviewToSide", + "code.code-display.openSettings", + "code.code-display.replaceInFiles", + "code.code-display.showExplorer", + "code.code-display.showOutputPanel", + "code.code-display.showSearch", + "code.code-display.showSourceControl", + "code.code-display.toggleSearchDetails", + "code.code-display.zenMode", + "code.code-editor.saveAllFiles", + "code.code-editor.saveCurrentFile", + "code.code-extension.installExtension", + "code.code-extension.showExtensions", + "code.code-general.gotoFileOrLineOrSymbol", + "code.code-general.showCommandPalette", + "code.code-general.showKeyboardShortcuts", + "code.code-general.showUserSettings", + "code.launchVSCode", + "code.newTextFile", + "code.splitEditor", + "desktop.AdjustScreenBrightness", + "desktop.AdjustVolume", + "desktop.ApplyTheme", + "desktop.BluetoothToggle", + "desktop.CloseProgram", + "desktop.CreateDesktop", + "desktop.DisconnectWifi", + "desktop.EnableWifi", + "desktop.LaunchProgram", + "desktop.Maximize", + "desktop.Minimize", + "desktop.Mute", + "desktop.NextDesktop", + "desktop.PreviousDesktop", + "desktop.RestartService", + "desktop.SetScreenResolution", + "desktop.SetTextSize", + "desktop.SetThemeMode", + "desktop.SwitchTo", + "desktop.ToggleNotifications", + "desktop.desktop-display.AdjustColorTemperature", + "desktop.desktop-display.AdjustScreenOrientation", + "desktop.desktop-display.DisplayScaling", + "desktop.desktop-display.EnableBlueLightFilterSchedule", + "desktop.desktop-display.RotationLock", + "desktop.desktop-input.AdjustMousePointerSize", + "desktop.desktop-input.CursorTrail", + "desktop.desktop-input.EnableTouchPad", + "desktop.desktop-input.EnhancePointerPrecision", + "desktop.desktop-input.MouseCursorSpeed", + "desktop.desktop-input.MousePointerCustomization", + "desktop.desktop-input.MouseWheelScrollLines", + "desktop.desktop-input.SetPrimaryMouseButton", + "desktop.desktop-input.ToggleMouseSonar", + "desktop.desktop-input.TouchpadCursorSpeed", + "desktop.desktop-personalization.ApplyColorToTitleBar", + "desktop.desktop-personalization.EnableTransparency", + "desktop.desktop-personalization.SystemThemeMode", + "desktop.desktop-power.BatterySaverActivationLevel", + "desktop.desktop-power.SetPowerModeOnBattery", + "desktop.desktop-power.SetPowerModePluggedIn", + "desktop.desktop-privacy.ManageCameraAccess", + "desktop.desktop-privacy.ManageLocationAccess", + "desktop.desktop-privacy.ManageMicrophoneAccess", + "desktop.desktop-system.AutomaticDSTAdjustment", + "desktop.desktop-system.AutomaticTimeSettingAction", + "desktop.desktop-system.EnableFilterKeysAction", + "desktop.desktop-system.EnableGameMode", + "desktop.desktop-system.EnableMagnifier", + "desktop.desktop-system.EnableMeteredConnections", + "desktop.desktop-system.EnableNarratorAction", + "desktop.desktop-system.EnableQuietHours", + "desktop.desktop-system.EnableStickyKeys", + "desktop.desktop-system.MinimizeWindowsOnMonitorDisconnectAction", + "desktop.desktop-system.MonoAudioToggle", + "desktop.desktop-system.RememberWindowLocations", + "desktop.desktop-system.ShowFileExtensions", + "desktop.desktop-system.ShowHiddenAndSystemFiles", + "desktop.desktop-taskbar.AutoHideTaskbar", + "desktop.desktop-taskbar.DisplaySecondsInSystrayClock", + "desktop.desktop-taskbar.DisplayTaskbarOnAllMonitors", + "desktop.desktop-taskbar.ShowBadgesOnTaskbar", + "desktop.desktop-taskbar.TaskViewVisibility", + "desktop.desktop-taskbar.TaskbarAlignment", + "desktop.desktop-taskbar.ToggleWidgetsButtonVisibility", + "discord.addThreadMember", + "discord.createChannelInvite", + "discord.createMessage", + "discord.followAnnouncementChannel", + "discord.groupDmAddRecipient", + "discord.groupDmRemoveRecipient", + "discord.joinThread", + "discord.leaveThread", + "discord.removeThreadMember", + "discord.startThreadFromMessage", + "discord.startThreadInForumOrMediaChannel", + "discord.startThreadWithoutMessage", + "discord.triggerTypingIndicator", + "github-cli.authLogin", + "github-cli.authLogout", + "github-cli.browseIssue", + "github-cli.browsePr", + "github-cli.browseRepo", + "github-cli.cacheDelete", + "github-cli.cacheList", + "github-cli.codespaceCreate", + "github-cli.codespaceDelete", + "github-cli.extensionInstall", + "github-cli.gistDelete", + "github-cli.issueAddLabel", + "github-cli.issueClose", + "github-cli.issueDelete", + "github-cli.issueList", + "github-cli.issueReopen", + "github-cli.issueView", + "github-cli.labelCreate", + "github-cli.myAssignedIssues", + "github-cli.myPullRequests", + "github-cli.orgList", + "github-cli.orgView", + "github-cli.prCheckout", + "github-cli.prChecks", + "github-cli.prClose", + "github-cli.prList", + "github-cli.prMerge", + "github-cli.prMergedStatus", + "github-cli.prView", + "github-cli.projectDelete", + "github-cli.projectList", + "github-cli.releaseDelete", + "github-cli.releaseList", + "github-cli.repoClone", + "github-cli.repoDelete", + "github-cli.repoFork", + "github-cli.repoView", + "github-cli.runView", + "github-cli.secretCreate", + "github-cli.starRepo", + "github-cli.variableCreate", + "github-cli.workflowView", + "ipconfig.displayDHCPClassIDs", + "ipconfig.displayDNSResolverCacheContents", + "ipconfig.displayFullConfigurationInformation", + "ipconfig.displayIPv6DHCPClassIDs", + "ipconfig.modifyDHCPClassID", + "ipconfig.modifyIPv6DHCPClassID", + "ipconfig.purgeDNSResolverCache", + "ipconfig.refreshDHCPLeasesAndReRegisterDNSNames", + "ipconfig.releaseIPv4Address", + "ipconfig.releaseIPv6Address", + "ipconfig.renewIPv4Address", + "ipconfig.renewIPv6Address", + "list.addItems", + "list.clearList", + "list.createList", + "list.removeItems", + "localPlayer.addToQueue", + "localPlayer.clearQueue", + "localPlayer.listFiles", + "localPlayer.mute", + "localPlayer.playFile", + "localPlayer.playFolder", + "localPlayer.playFromQueue", + "localPlayer.repeat", + "localPlayer.resume", + "localPlayer.searchFiles", + "localPlayer.setMusicFolder", + "localPlayer.showMusicFolder", + "localPlayer.showQueue", + "markdown.createDocument", + "markdown.openDocument", + "montage.changeTitle", + "montage.clearSelectedPhotos", + "montage.createNewMontage", + "montage.deleteAllMontages", + "montage.deleteMontage", + "montage.listMontages", + "montage.openMontage", + "montage.showSearchParameters", + "montage.startSlideShow", + "player.addCurrentTrackToPlaylist", + "player.createPlaylist", + "player.deletePlaylist", + "player.findMusic", + "player.getFavorites", + "player.getPlaylist", + "player.getQueue", + "player.listDevices", + "player.listPlaylists", + "player.playFromCurrentTrackList", + "player.playMusic", + "player.playPlaylist", + "player.resumePlayback", + "player.selectDevice", + "player.setMaxVolume", + "player.showSelectedDevice", + "screencapture.startRecording", + "screencapture.stopRecording", + "screencapture.takeScreenshot", + "system.config.exitAgentPriorityMode", + "system.config.toggleDeveloperMode", + "system.config.toggleExplanation", + "system.conversation.deleteConversation", + "system.conversation.newConversation", + "system.conversation.nextConversation", + "system.conversation.prevConversation", + "system.conversation.renameConversation", + "system.conversation.switchConversation", + "system.history.clearHistory", + "system.notify.clearNotifications", + "system.settings.setAutoComplete", + "system.settings.setConversationResume", + "system.settings.setIdleTimeout", + "system.settings.setServerHidden", + "timer.cancelReminder", + "timer.listReminders", + "timer.repeatReminder", + "timer.setReminder", + "visualStudio.addBreakpoint", + "visualStudio.break", + "visualStudio.build", + "visualStudio.clean", + "visualStudio.closeAll", + "visualStudio.debug", + "visualStudio.findInFiles", + "visualStudio.findText", + "visualStudio.go", + "visualStudio.gotoLine", + "visualStudio.openFile", + "visualStudio.redo", + "visualStudio.run", + "visualStudio.saveAll", + "visualStudio.stepInto", + "visualStudio.stepOut", + "visualStudio.stepOver", + "visualStudio.undo", + "weather.getAlerts", + "weather.getForecast", + "windowsClock.addWorldClock", + "windowsClock.createAlarm", + "windowsClock.navigateToAlarmTab", + "windowsClock.navigateToFocusTab", + "windowsClock.navigateToStopwatchTab", + "windowsClock.navigateToTimerTab", + "windowsClock.navigateToWorldClockTab", + "windowsClock.recordLap", + "windowsClock.renameTimer", + "windowsClock.setAlarmEnabled", + "windowsClock.setFocusSessionRunning", + "windowsClock.setStopwatchRunning", + "windowsClock.setTimerViewMode", + "windowsClock.startTimer" + ] +} diff --git a/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs b/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs new file mode 100644 index 000000000..c615f8b12 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// +// Reproduce translation-bench NEGATIVE failures (BAD_NEGATIVE) across the +// gpt-5.6 sol/terra/luna models, using the real TypeAgent translator. +// +// A negative utterance ("Do not X", "Don't X") must NOT produce the target +// action. The dispatcher translator is expected to answer `unknown` (or fail +// to produce the target action). Producing the target action = BAD_NEGATIVE. +// +// Only actions in eligible-gold-actions.generated.json are eligible, and each +// case activates only its target schema (the dispatcher activates a small +// schema set per request; sending all ~290 actions at once exceeds the +// provider's tool-array limit). +// +// Usage: +// node reproNegatives.mjs --api-key --models sol --limit 20 --concurrency 6 + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { Command } from "commander"; +import { getChatModelMaxConcurrency } from "@typeagent/aiclient"; +import { + getAllActionConfigProvider, + createTypeAgentTranslatorForSelectedActions, +} from "agent-dispatcher/internal"; +import { getInstanceDir } from "agent-dispatcher/helpers/data"; +import { getDefaultAppAgentProviders } from "default-agent-provider"; + +/** Run `tasks` with at most `limit` in flight, preserving result order. */ +async function mapConcurrent(tasks, limit) { + const results = new Array(tasks.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, tasks.length) }, async () => { + while (true) { + const i = next++; + if (i >= tasks.length) return; + results[i] = await tasks[i](); + } + }); + await Promise.all(workers); + return results; +} + +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); +const benchDir = path.resolve(MODULE_DIR, ".."); + +const MODEL_MAP = { + sol: "azure/gpt-5.6-sol", + terra: "azure/gpt-5.6-terra", + luna: "azure/gpt-5.6-luna", +}; + +const DEFAULT_ENDPOINT = "http://127.0.0.1:4627/v1/chat/completions"; + +const csv = (v) => + v + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + +const opts = new Command() + .name("reproNegatives") + .description( + "Reproduce translation-bench BAD_NEGATIVE failures via the real TypeAgent translator.", + ) + .option("--models ", "comma-separated model keys", csv, [ + "sol", + "terra", + "luna", + ]) + .option("--limit ", "max negative cases to run", Number, 20) + .option( + "--only ", + "comma-separated action-id filter, e.g. browser.goForward", + csv, + [], + ) + .option("--repeat ", "repeat each cell N times", Number, 1) + .option("--concurrency ", "max in-flight translations", Number, 4) + .option("--api-key ", "API key for the completions endpoint") + .option("--endpoint ", "chat completions endpoint", DEFAULT_ENDPOINT) + .parse() + .opts(); + +// aiclient resolves credentials from the environment; feed it the CLI values. +const KEY_VAR = "OPENAI_API_KEY"; +process.env.OPENAI_ENDPOINT = opts.endpoint; +if (opts.apiKey) { + process.env[KEY_VAR] = opts.apiKey; +} +if (!process.env[KEY_VAR]) { + throw new Error(`missing credential: pass --api-key or set ${KEY_VAR}`); +} + +const models = opts.models; +const limit = opts.limit; +const only = opts.only; +const repeat = opts.repeat; +// Bounded parallelism, capped by the model's configured maxConcurrency so we +// don't outrun provider rate limits. +const concurrency = getChatModelMaxConcurrency(opts.concurrency, undefined, 4); + +// ---- eligible-gold allowlist (the only permitted schemas/actions) -------- +const allowlist = new Set( + JSON.parse( + readFileSync( + path.join(benchDir, "eligible-gold-actions.generated.json"), + "utf8", + ), + ).allowlist, +); + +// ---- hardcoded negative utterances -------------------------------------- +// 20 fixed imperative-negation cases. `id` is the allowlisted action that must +// NOT be produced; a negative expects no action (`expectedActions: []`). +const NEGATIVE_CASES = [ + { id: "browser.goForward", utterance: "Do not navigate forward in the browser.", expectedActions: [] }, + { id: "browser.goBack", utterance: "Don't go back to the previous page.", expectedActions: [] }, + { id: "browser.scrollDown", utterance: "Do not scroll down this page.", expectedActions: [] }, + { id: "browser.scrollUp", utterance: "Please don't scroll up.", expectedActions: [] }, + { id: "browser.stopReadPageContent", utterance: "Don't stop the browser's ongoing page-reading session.", expectedActions: [] }, + { id: "browser.readPageContent", utterance: "Do not read the contents of this page aloud.", expectedActions: [] }, + { id: "browser.closeWebPage", utterance: "Don't close the current web page.", expectedActions: [] }, + { id: "browser.closeAllWebPages", utterance: "Leave all my shopping, banking, news, email, and calendar browser tabs alone; don't close any of them.", expectedActions: [] }, + { id: "browser.reloadPage", utterance: "Please don't reload the page.", expectedActions: [] }, + { id: "browser.captureScreenshot", utterance: "Do not take a screenshot of the browser.", expectedActions: [] }, + { id: "browser.zoomReset", utterance: "Don't reset the browser zoom level.", expectedActions: [] }, + { id: "browser.followLinkByText", utterance: "Do not follow the 'Learn more' link on this page.", expectedActions: [] }, + { id: "browser.changeSearchProvider", utterance: "Don't change my default search provider.", expectedActions: [] }, + { id: "browser.openWebPage", utterance: "Do not open a new web page right now.", expectedActions: [] }, + { id: "desktop.AdjustScreenBrightness", utterance: "Do not adjust the screen brightness.", expectedActions: [] }, + { id: "desktop.AdjustVolume", utterance: "Don't change the system volume.", expectedActions: [] }, + { id: "desktop.ApplyTheme", utterance: "Do not apply a new desktop theme.", expectedActions: [] }, + { id: "desktop.CloseProgram", utterance: "Please don't close any running programs.", expectedActions: [] }, + { id: "list.clearList", utterance: "Do not clear my shopping list.", expectedActions: [] }, + { id: "timer.cancelReminder", utterance: "Don't cancel my reminder.", expectedActions: [] }, +]; + +for (const c of NEGATIVE_CASES) { + if (!allowlist.has(c.id)) { + throw new Error(`negative case action not in eligible-gold allowlist: ${c.id}`); + } +} + +/** Flatten a translated action (possibly a multiple-action wrapper) to ids. */ +function producedActionIds(data, schemaOf) { + if (data === null || typeof data !== "object") return []; + if (Array.isArray(data.actions)) { + return data.actions.flatMap((a) => producedActionIds(a, schemaOf)); + } + // Multiple-action entries wrap the real action. + if (data.action !== undefined) return producedActionIds(data.action, schemaOf); + + const actionName = data.actionName; + if (typeof actionName !== "string") return []; + // `unknown` is the translator's no-actionable-intent answer -> a PASS. + if (actionName === "unknown") return []; + const schemaName = data.translatorName ?? schemaOf(actionName) ?? ""; + return [schemaName ? `${schemaName}.${actionName}` : actionName]; +} + +async function main() { + + + const { provider } = await getAllActionConfigProvider( + getDefaultAppAgentProviders(getInstanceDir()), + ); + + const selected = only.length > 0 + ? NEGATIVE_CASES.filter((c) => only.includes(c.id)) + : NEGATIVE_CASES; + const cases = selected.slice(0, Math.min(limit, selected.length)); + console.log( + `negatives=${cases.length} models=${models.join(",")} concurrency=${concurrency} (real TypeAgent translator)\n`, + ); + + const summary = {}; + for (const m of models) summary[m] = { pass: 0, fail: 0, err: 0 }; + + // One unit of work per (case, model); run them through a bounded pool. + const jobs = []; + for (const c of cases) { + // Activate only the target action's schema for this request, matching + // how the dispatcher scopes translation per request. + const schemaName = c.id.slice(0, c.id.lastIndexOf(".")); + const actionConfig = provider.tryGetActionConfig(schemaName); + if (actionConfig === undefined) { + console.log(`SKIP ${c.id}: schema '${schemaName}' not available`); + continue; + } + // Offer ONLY the eligible-gold allowlisted actions for this schema. + // Activating the whole schema would expose non-gold actions (e.g. + // browser.executeAdHocScript, excluded as llmAsAJudge), letting the + // model "pass" a negative by firing an ineligible action. + const schemaFile = provider.getActionSchemaFileForConfig(actionConfig); + const allDefs = schemaFile.parsedActionSchema.actionSchemas; + const definitions = []; + for (const [actionName, def] of allDefs) { + if (allowlist.has(`${schemaName}.${actionName}`)) { + definitions.push(def); + } + } + if (definitions.length === 0) { + console.log(`SKIP ${c.id}: no allowlisted actions in '${schemaName}'`); + continue; + } + for (const m of models) { + for (let r = 0; r < repeat; r++) { + jobs.push({ c, m, actionConfig, definitions }); + } + } + } + + const outcomes = await mapConcurrent( + jobs.map(({ c, m, actionConfig, definitions }) => async () => { + const model = MODEL_MAP[m] ?? m; + try { + const translator = createTypeAgentTranslatorForSelectedActions( + definitions, + actionConfig, + [], + [], + provider, + undefined, + model, + ); + const result = await translator.translate(c.utterance); + if (!result.success) { + // Translator refusing to produce an action is the correct + // behavior for a negative. + return { m, kind: "pass" }; + } + const produced = producedActionIds(result.data, (a) => + translator.getSchemaName(a), + ); + const bad = produced.length !== c.expectedActions.length; + // Guard: anything produced must be allowlisted. If not, the + // offered action set leaked non-gold actions and the result is + // not trustworthy. + const offLimits = produced.filter((p) => !allowlist.has(p)); + return { m, c, produced, offLimits, kind: bad ? "fail" : "pass" }; + } catch (e) { + return { m, c, kind: "err", message: String(e?.message ?? e) }; + } + }), + concurrency, + ); + + // Report in deterministic case order, not completion order. + for (const o of outcomes) { + summary[o.m][o.kind]++; + if (o.kind === "fail") { + console.log(`FAIL BAD_NEGATIVE ${o.m.padEnd(5)} | target=${o.c.id}`); + console.log(` utterance: ${o.c.utterance}`); + console.log(` expectedActions: ${JSON.stringify(o.c.expectedActions)}`); + console.log(` produced: ${JSON.stringify(o.produced)}\n`); + } else if (o.kind === "err") { + console.log(`ERR ${o.m} ${o.c.id}: ${o.message.slice(0, 200)}`); + } + } + + console.log("\n=== summary (negative cases; PASS = no action produced) ==="); + let anyScored = false; + for (const m of models) { + const s = summary[m]; + const scored = s.pass + s.fail; + if (scored > 0) anyScored = true; + const rate = scored ? `${((100 * s.fail) / scored).toFixed(1)}%` : "n/a (nothing scored)"; + console.log( + `${m.padEnd(6)} pass=${s.pass} fail=${s.fail} err=${s.err} bad-negative-rate=${rate}`, + ); + } + if (!anyScored) { + console.log( + "\nNO RESULTS: every call errored — the numbers above are not a signal.", + ); + process.exitCode = 1; + } +} + +await main(); diff --git a/ts/packages/dispatcher/dispatcher/src/internal.ts b/ts/packages/dispatcher/dispatcher/src/internal.ts index cf2964d0f..44db4aacd 100644 --- a/ts/packages/dispatcher/dispatcher/src/internal.ts +++ b/ts/packages/dispatcher/dispatcher/src/internal.ts @@ -44,6 +44,7 @@ export { getFullSchemaText, getAppAgentName, loadAgentJsonTranslator, + createTypeAgentTranslatorForSelectedActions, } from "./translation/agentTranslators.js"; export type { TypeAgentTranslator, From e70065a71dc4b8184e4dbf9a705ccd68d721c088 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Mon, 10 Aug 2026 16:02:34 -0700 Subject: [PATCH 2/3] refactor(benchmarks): strip comments from reproNegatives script --- .../scripts/reproNegatives.mjs | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs b/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs index c615f8b12..5a254d2d0 100644 --- a/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs +++ b/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs @@ -1,21 +1,3 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -// -// Reproduce translation-bench NEGATIVE failures (BAD_NEGATIVE) across the -// gpt-5.6 sol/terra/luna models, using the real TypeAgent translator. -// -// A negative utterance ("Do not X", "Don't X") must NOT produce the target -// action. The dispatcher translator is expected to answer `unknown` (or fail -// to produce the target action). Producing the target action = BAD_NEGATIVE. -// -// Only actions in eligible-gold-actions.generated.json are eligible, and each -// case activates only its target schema (the dispatcher activates a small -// schema set per request; sending all ~290 actions at once exceeds the -// provider's tool-array limit). -// -// Usage: -// node reproNegatives.mjs --api-key --models sol --limit 20 --concurrency 6 - import { readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -29,7 +11,6 @@ import { import { getInstanceDir } from "agent-dispatcher/helpers/data"; import { getDefaultAppAgentProviders } from "default-agent-provider"; -/** Run `tasks` with at most `limit` in flight, preserving result order. */ async function mapConcurrent(tasks, limit) { const results = new Array(tasks.length); let next = 0; @@ -85,7 +66,6 @@ const opts = new Command() .parse() .opts(); -// aiclient resolves credentials from the environment; feed it the CLI values. const KEY_VAR = "OPENAI_API_KEY"; process.env.OPENAI_ENDPOINT = opts.endpoint; if (opts.apiKey) { @@ -99,11 +79,8 @@ const models = opts.models; const limit = opts.limit; const only = opts.only; const repeat = opts.repeat; -// Bounded parallelism, capped by the model's configured maxConcurrency so we -// don't outrun provider rate limits. const concurrency = getChatModelMaxConcurrency(opts.concurrency, undefined, 4); -// ---- eligible-gold allowlist (the only permitted schemas/actions) -------- const allowlist = new Set( JSON.parse( readFileSync( @@ -113,9 +90,6 @@ const allowlist = new Set( ).allowlist, ); -// ---- hardcoded negative utterances -------------------------------------- -// 20 fixed imperative-negation cases. `id` is the allowlisted action that must -// NOT be produced; a negative expects no action (`expectedActions: []`). const NEGATIVE_CASES = [ { id: "browser.goForward", utterance: "Do not navigate forward in the browser.", expectedActions: [] }, { id: "browser.goBack", utterance: "Don't go back to the previous page.", expectedActions: [] }, @@ -145,26 +119,21 @@ for (const c of NEGATIVE_CASES) { } } -/** Flatten a translated action (possibly a multiple-action wrapper) to ids. */ function producedActionIds(data, schemaOf) { if (data === null || typeof data !== "object") return []; if (Array.isArray(data.actions)) { return data.actions.flatMap((a) => producedActionIds(a, schemaOf)); } - // Multiple-action entries wrap the real action. if (data.action !== undefined) return producedActionIds(data.action, schemaOf); const actionName = data.actionName; if (typeof actionName !== "string") return []; - // `unknown` is the translator's no-actionable-intent answer -> a PASS. if (actionName === "unknown") return []; const schemaName = data.translatorName ?? schemaOf(actionName) ?? ""; return [schemaName ? `${schemaName}.${actionName}` : actionName]; } async function main() { - - const { provider } = await getAllActionConfigProvider( getDefaultAppAgentProviders(getInstanceDir()), ); @@ -180,21 +149,14 @@ async function main() { const summary = {}; for (const m of models) summary[m] = { pass: 0, fail: 0, err: 0 }; - // One unit of work per (case, model); run them through a bounded pool. const jobs = []; for (const c of cases) { - // Activate only the target action's schema for this request, matching - // how the dispatcher scopes translation per request. const schemaName = c.id.slice(0, c.id.lastIndexOf(".")); const actionConfig = provider.tryGetActionConfig(schemaName); if (actionConfig === undefined) { console.log(`SKIP ${c.id}: schema '${schemaName}' not available`); continue; } - // Offer ONLY the eligible-gold allowlisted actions for this schema. - // Activating the whole schema would expose non-gold actions (e.g. - // browser.executeAdHocScript, excluded as llmAsAJudge), letting the - // model "pass" a negative by firing an ineligible action. const schemaFile = provider.getActionSchemaFileForConfig(actionConfig); const allDefs = schemaFile.parsedActionSchema.actionSchemas; const definitions = []; @@ -229,17 +191,12 @@ async function main() { ); const result = await translator.translate(c.utterance); if (!result.success) { - // Translator refusing to produce an action is the correct - // behavior for a negative. return { m, kind: "pass" }; } const produced = producedActionIds(result.data, (a) => translator.getSchemaName(a), ); const bad = produced.length !== c.expectedActions.length; - // Guard: anything produced must be allowlisted. If not, the - // offered action set leaked non-gold actions and the result is - // not trustworthy. const offLimits = produced.filter((p) => !allowlist.has(p)); return { m, c, produced, offLimits, kind: bad ? "fail" : "pass" }; } catch (e) { @@ -249,7 +206,6 @@ async function main() { concurrency, ); - // Report in deterministic case order, not completion order. for (const o of outcomes) { summary[o.m][o.kind]++; if (o.kind === "fail") { From 5bd768795ab0c5d8dbd9ed0209492d78df16eac4 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Mon, 10 Aug 2026 23:05:43 +0000 Subject: [PATCH 3/3] style: apply prettier formatting and policy fixes --- .../scripts/reproNegatives.mjs | 183 ++++++++++++++---- 1 file changed, 145 insertions(+), 38 deletions(-) diff --git a/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs b/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs index 5a254d2d0..f4af5607f 100644 --- a/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs +++ b/ts/packages/benchmarks/src/translationBench/scripts/reproNegatives.mjs @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + import { readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -14,13 +17,16 @@ import { getDefaultAppAgentProviders } from "default-agent-provider"; async function mapConcurrent(tasks, limit) { const results = new Array(tasks.length); let next = 0; - const workers = Array.from({ length: Math.min(limit, tasks.length) }, async () => { - while (true) { - const i = next++; - if (i >= tasks.length) return; - results[i] = await tasks[i](); - } - }); + const workers = Array.from( + { length: Math.min(limit, tasks.length) }, + async () => { + while (true) { + const i = next++; + if (i >= tasks.length) return; + results[i] = await tasks[i](); + } + }, + ); await Promise.all(workers); return results; } @@ -91,31 +97,114 @@ const allowlist = new Set( ); const NEGATIVE_CASES = [ - { id: "browser.goForward", utterance: "Do not navigate forward in the browser.", expectedActions: [] }, - { id: "browser.goBack", utterance: "Don't go back to the previous page.", expectedActions: [] }, - { id: "browser.scrollDown", utterance: "Do not scroll down this page.", expectedActions: [] }, - { id: "browser.scrollUp", utterance: "Please don't scroll up.", expectedActions: [] }, - { id: "browser.stopReadPageContent", utterance: "Don't stop the browser's ongoing page-reading session.", expectedActions: [] }, - { id: "browser.readPageContent", utterance: "Do not read the contents of this page aloud.", expectedActions: [] }, - { id: "browser.closeWebPage", utterance: "Don't close the current web page.", expectedActions: [] }, - { id: "browser.closeAllWebPages", utterance: "Leave all my shopping, banking, news, email, and calendar browser tabs alone; don't close any of them.", expectedActions: [] }, - { id: "browser.reloadPage", utterance: "Please don't reload the page.", expectedActions: [] }, - { id: "browser.captureScreenshot", utterance: "Do not take a screenshot of the browser.", expectedActions: [] }, - { id: "browser.zoomReset", utterance: "Don't reset the browser zoom level.", expectedActions: [] }, - { id: "browser.followLinkByText", utterance: "Do not follow the 'Learn more' link on this page.", expectedActions: [] }, - { id: "browser.changeSearchProvider", utterance: "Don't change my default search provider.", expectedActions: [] }, - { id: "browser.openWebPage", utterance: "Do not open a new web page right now.", expectedActions: [] }, - { id: "desktop.AdjustScreenBrightness", utterance: "Do not adjust the screen brightness.", expectedActions: [] }, - { id: "desktop.AdjustVolume", utterance: "Don't change the system volume.", expectedActions: [] }, - { id: "desktop.ApplyTheme", utterance: "Do not apply a new desktop theme.", expectedActions: [] }, - { id: "desktop.CloseProgram", utterance: "Please don't close any running programs.", expectedActions: [] }, - { id: "list.clearList", utterance: "Do not clear my shopping list.", expectedActions: [] }, - { id: "timer.cancelReminder", utterance: "Don't cancel my reminder.", expectedActions: [] }, + { + id: "browser.goForward", + utterance: "Do not navigate forward in the browser.", + expectedActions: [], + }, + { + id: "browser.goBack", + utterance: "Don't go back to the previous page.", + expectedActions: [], + }, + { + id: "browser.scrollDown", + utterance: "Do not scroll down this page.", + expectedActions: [], + }, + { + id: "browser.scrollUp", + utterance: "Please don't scroll up.", + expectedActions: [], + }, + { + id: "browser.stopReadPageContent", + utterance: "Don't stop the browser's ongoing page-reading session.", + expectedActions: [], + }, + { + id: "browser.readPageContent", + utterance: "Do not read the contents of this page aloud.", + expectedActions: [], + }, + { + id: "browser.closeWebPage", + utterance: "Don't close the current web page.", + expectedActions: [], + }, + { + id: "browser.closeAllWebPages", + utterance: + "Leave all my shopping, banking, news, email, and calendar browser tabs alone; don't close any of them.", + expectedActions: [], + }, + { + id: "browser.reloadPage", + utterance: "Please don't reload the page.", + expectedActions: [], + }, + { + id: "browser.captureScreenshot", + utterance: "Do not take a screenshot of the browser.", + expectedActions: [], + }, + { + id: "browser.zoomReset", + utterance: "Don't reset the browser zoom level.", + expectedActions: [], + }, + { + id: "browser.followLinkByText", + utterance: "Do not follow the 'Learn more' link on this page.", + expectedActions: [], + }, + { + id: "browser.changeSearchProvider", + utterance: "Don't change my default search provider.", + expectedActions: [], + }, + { + id: "browser.openWebPage", + utterance: "Do not open a new web page right now.", + expectedActions: [], + }, + { + id: "desktop.AdjustScreenBrightness", + utterance: "Do not adjust the screen brightness.", + expectedActions: [], + }, + { + id: "desktop.AdjustVolume", + utterance: "Don't change the system volume.", + expectedActions: [], + }, + { + id: "desktop.ApplyTheme", + utterance: "Do not apply a new desktop theme.", + expectedActions: [], + }, + { + id: "desktop.CloseProgram", + utterance: "Please don't close any running programs.", + expectedActions: [], + }, + { + id: "list.clearList", + utterance: "Do not clear my shopping list.", + expectedActions: [], + }, + { + id: "timer.cancelReminder", + utterance: "Don't cancel my reminder.", + expectedActions: [], + }, ]; for (const c of NEGATIVE_CASES) { if (!allowlist.has(c.id)) { - throw new Error(`negative case action not in eligible-gold allowlist: ${c.id}`); + throw new Error( + `negative case action not in eligible-gold allowlist: ${c.id}`, + ); } } @@ -124,7 +213,8 @@ function producedActionIds(data, schemaOf) { if (Array.isArray(data.actions)) { return data.actions.flatMap((a) => producedActionIds(a, schemaOf)); } - if (data.action !== undefined) return producedActionIds(data.action, schemaOf); + if (data.action !== undefined) + return producedActionIds(data.action, schemaOf); const actionName = data.actionName; if (typeof actionName !== "string") return []; @@ -138,9 +228,10 @@ async function main() { getDefaultAppAgentProviders(getInstanceDir()), ); - const selected = only.length > 0 - ? NEGATIVE_CASES.filter((c) => only.includes(c.id)) - : NEGATIVE_CASES; + const selected = + only.length > 0 + ? NEGATIVE_CASES.filter((c) => only.includes(c.id)) + : NEGATIVE_CASES; const cases = selected.slice(0, Math.min(limit, selected.length)); console.log( `negatives=${cases.length} models=${models.join(",")} concurrency=${concurrency} (real TypeAgent translator)\n`, @@ -166,7 +257,9 @@ async function main() { } } if (definitions.length === 0) { - console.log(`SKIP ${c.id}: no allowlisted actions in '${schemaName}'`); + console.log( + `SKIP ${c.id}: no allowlisted actions in '${schemaName}'`, + ); continue; } for (const m of models) { @@ -198,7 +291,13 @@ async function main() { ); const bad = produced.length !== c.expectedActions.length; const offLimits = produced.filter((p) => !allowlist.has(p)); - return { m, c, produced, offLimits, kind: bad ? "fail" : "pass" }; + return { + m, + c, + produced, + offLimits, + kind: bad ? "fail" : "pass", + }; } catch (e) { return { m, c, kind: "err", message: String(e?.message ?? e) }; } @@ -209,22 +308,30 @@ async function main() { for (const o of outcomes) { summary[o.m][o.kind]++; if (o.kind === "fail") { - console.log(`FAIL BAD_NEGATIVE ${o.m.padEnd(5)} | target=${o.c.id}`); + console.log( + `FAIL BAD_NEGATIVE ${o.m.padEnd(5)} | target=${o.c.id}`, + ); console.log(` utterance: ${o.c.utterance}`); - console.log(` expectedActions: ${JSON.stringify(o.c.expectedActions)}`); + console.log( + ` expectedActions: ${JSON.stringify(o.c.expectedActions)}`, + ); console.log(` produced: ${JSON.stringify(o.produced)}\n`); } else if (o.kind === "err") { console.log(`ERR ${o.m} ${o.c.id}: ${o.message.slice(0, 200)}`); } } - console.log("\n=== summary (negative cases; PASS = no action produced) ==="); + console.log( + "\n=== summary (negative cases; PASS = no action produced) ===", + ); let anyScored = false; for (const m of models) { const s = summary[m]; const scored = s.pass + s.fail; if (scored > 0) anyScored = true; - const rate = scored ? `${((100 * s.fail) / scored).toFixed(1)}%` : "n/a (nothing scored)"; + const rate = scored + ? `${((100 * s.fail) / scored).toFixed(1)}%` + : "n/a (nothing scored)"; console.log( `${m.padEnd(6)} pass=${s.pass} fail=${s.fail} err=${s.err} bad-negative-rate=${rate}`, );