Skip to content

Add github-cli and external-browser confusable pairs #2867

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Dominic Nguyen (datduyng) merged 1 commit into from
Aug 13, 2026
+118 −0
Merged

Add github-cli and external-browser confusable pairs #2867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions ts/packages/benchmarks/src/translationBench/synthesizer/utteranceDisambiguation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,26 @@ const KNOWN_CONFUSABLE_PAIRS: ReadonlyArray<
{ schemaName: "browser", actionName: "openWebPage" },
"list flows for domain hostname vs navigate to that hostname",
],
[
{ schemaName: "github-cli", actionName: "browseIssue" },
{ schemaName: "browser", actionName: "openWebPage" },
"GitHub issue via gh/browseIssue vs generic browser open of a URL/page; prefer 'GitHub issue #N', 'gh browse issue', 'in this repo' — avoid bare 'open in the browser' / 'web page'",
],
[
{ schemaName: "github-cli", actionName: "browsePr" },
{ schemaName: "browser", actionName: "openWebPage" },
"GitHub PR via gh/browsePr vs generic browser open; prefer 'pull request #N', 'gh browse pr', 'in this repo'",
],
[
{ schemaName: "github-cli", actionName: "browseRepo" },
{ schemaName: "browser", actionName: "openWebPage" },
"GitHub repo via gh/browseRepo vs generic browser open; prefer 'this GitHub repo', 'gh browse', owner/name",
],
[
{ schemaName: "browser.external", actionName: "openTab" },
{ schemaName: "browser", actionName: "openWebPage" },
"external/system browser tab vs in-app openWebPage; prefer 'external browser', 'system browser', 'outside the app', 'default browser app' — avoid bare 'new tab' / 'in my browser'",
],
];

/**
Expand Down
98 changes: 98 additions & 0 deletions ts/packages/benchmarks/test/translationBench.utteranceDisambiguation.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,104 @@ describe("translation bench confusable siblings", () => {
]),
);
});

it("finds curated github-cli.browseIssue ↔ browser.openWebPage pair", () => {
const catalog: TranslationBenchBenchmarkSchema[] = [
...browserCatalog(),
{
schemaName: "github-cli",
description: "github cli",
tools: [
{
type: "function",
function: {
name: "browseIssue",
description: "Browse a GitHub issue",
parameters: {
type: "object",
properties: { number: { type: "number" } },
},
},
},
],
typeAgent: {
sourceHash: `github-${HASH}`,
schemaType: "GithubAction",
parsedActionSchema: toJSONParsedActionSchema(
parseToolsJsonSchema([
{
name: "browseIssue",
description: "Browse a GitHub issue",
inputSchema: {
type: "object",
properties: {
number: { type: "number" },
},
additionalProperties: false,
},
},
]),
),
},
},
];
const siblings = findTranslationBenchConfusableSiblings(
{ schemaName: "github-cli", actionName: "browseIssue" },
catalog,
);
expect(siblings.map((s) => `${s.schemaName}.${s.actionName}`)).toEqual(
expect.arrayContaining(["browser.openWebPage"]),
);
});

it("finds curated browser.external.openTab ↔ browser.openWebPage pair", () => {
const catalog: TranslationBenchBenchmarkSchema[] = [
...browserCatalog(),
{
schemaName: "browser.external",
description: "external browser",
tools: [
{
type: "function",
function: {
name: "openTab",
description: "Open external browser tab",
parameters: {
type: "object",
properties: { url: { type: "string" } },
},
},
},
],
typeAgent: {
sourceHash: `external-${HASH}`,
schemaType: "ExternalAction",
parsedActionSchema: toJSONParsedActionSchema(
parseToolsJsonSchema([
{
name: "openTab",
description: "Open external browser tab",
inputSchema: {
type: "object",
properties: {
url: { type: "string" },
},
additionalProperties: false,
},
},
]),
),
},
},
];
const siblings = findTranslationBenchConfusableSiblings(
{ schemaName: "browser.external", actionName: "openTab" },
catalog,
);
expect(siblings.map((s) => `${s.schemaName}.${s.actionName}`)).toEqual(
expect.arrayContaining(["browser.openWebPage"]),
);
});
});

describe("translation bench utterance disambiguation", () => {
Expand Down
Loading