diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/utteranceDisambiguation.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/utteranceDisambiguation.ts index 3629bbd32..4b34ea55c 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/utteranceDisambiguation.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/utteranceDisambiguation.ts @@ -109,6 +109,26 @@ const KNOWN_CONFUSABLE_PAIRS: ReadonlyArray< { schemaName: "browser", actionName: "openWebPage" }, "list flows for domain hostname vs navigate to that hostname", ], + [ + { schemaName: "github-cli", actionName: "browseIssue" }, + { schemaName: "browser", actionName: "openWebPage" }, + "GitHub issue via gh/browseIssue vs generic browser open of a URL/page; prefer 'GitHub issue #N', 'gh browse issue', 'in this repo' — avoid bare 'open in the browser' / 'web page'", + ], + [ + { schemaName: "github-cli", actionName: "browsePr" }, + { schemaName: "browser", actionName: "openWebPage" }, + "GitHub PR via gh/browsePr vs generic browser open; prefer 'pull request #N', 'gh browse pr', 'in this repo'", + ], + [ + { schemaName: "github-cli", actionName: "browseRepo" }, + { schemaName: "browser", actionName: "openWebPage" }, + "GitHub repo via gh/browseRepo vs generic browser open; prefer 'this GitHub repo', 'gh browse', owner/name", + ], + [ + { schemaName: "browser.external", actionName: "openTab" }, + { schemaName: "browser", actionName: "openWebPage" }, + "external/system browser tab vs in-app openWebPage; prefer 'external browser', 'system browser', 'outside the app', 'default browser app' — avoid bare 'new tab' / 'in my browser'", + ], ]; /** diff --git a/ts/packages/benchmarks/test/translationBench.utteranceDisambiguation.spec.ts b/ts/packages/benchmarks/test/translationBench.utteranceDisambiguation.spec.ts index 2a40cb524..8abd48e20 100644 --- a/ts/packages/benchmarks/test/translationBench.utteranceDisambiguation.spec.ts +++ b/ts/packages/benchmarks/test/translationBench.utteranceDisambiguation.spec.ts @@ -102,6 +102,104 @@ describe("translation bench confusable siblings", () => { ]), ); }); + + it("finds curated github-cli.browseIssue ↔ browser.openWebPage pair", () => { + const catalog: TranslationBenchBenchmarkSchema[] = [ + ...browserCatalog(), + { + schemaName: "github-cli", + description: "github cli", + tools: [ + { + type: "function", + function: { + name: "browseIssue", + description: "Browse a GitHub issue", + parameters: { + type: "object", + properties: { number: { type: "number" } }, + }, + }, + }, + ], + typeAgent: { + sourceHash: `github-${HASH}`, + schemaType: "GithubAction", + parsedActionSchema: toJSONParsedActionSchema( + parseToolsJsonSchema([ + { + name: "browseIssue", + description: "Browse a GitHub issue", + inputSchema: { + type: "object", + properties: { + number: { type: "number" }, + }, + additionalProperties: false, + }, + }, + ]), + ), + }, + }, + ]; + const siblings = findTranslationBenchConfusableSiblings( + { schemaName: "github-cli", actionName: "browseIssue" }, + catalog, + ); + expect(siblings.map((s) => `${s.schemaName}.${s.actionName}`)).toEqual( + expect.arrayContaining(["browser.openWebPage"]), + ); + }); + + it("finds curated browser.external.openTab ↔ browser.openWebPage pair", () => { + const catalog: TranslationBenchBenchmarkSchema[] = [ + ...browserCatalog(), + { + schemaName: "browser.external", + description: "external browser", + tools: [ + { + type: "function", + function: { + name: "openTab", + description: "Open external browser tab", + parameters: { + type: "object", + properties: { url: { type: "string" } }, + }, + }, + }, + ], + typeAgent: { + sourceHash: `external-${HASH}`, + schemaType: "ExternalAction", + parsedActionSchema: toJSONParsedActionSchema( + parseToolsJsonSchema([ + { + name: "openTab", + description: "Open external browser tab", + inputSchema: { + type: "object", + properties: { + url: { type: "string" }, + }, + additionalProperties: false, + }, + }, + ]), + ), + }, + }, + ]; + const siblings = findTranslationBenchConfusableSiblings( + { schemaName: "browser.external", actionName: "openTab" }, + catalog, + ); + expect(siblings.map((s) => `${s.schemaName}.${s.actionName}`)).toEqual( + expect.arrayContaining(["browser.openWebPage"]), + ); + }); }); describe("translation bench utterance disambiguation", () => {