Skip to content

Keep empty parameters object after gold placeholder strip #2866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Dominic Nguyen (datduyng) merged 2 commits into from
Aug 14, 2026
+171 −2
Merged

Keep empty parameters object after gold placeholder strip #2866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8b725ab
Keep empty parameters object after gold placeholder strip
datduyng Aug 13, 2026
5885b29
ci: retrigger after electron install flake
datduyng Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,13 +289,18 @@ function stripEmptyGoldPlaceholdersFromActions(
actions: TranslationBenchBenchmarkAction[],
): TranslationBenchBenchmarkAction[] {
return actions.map((action) => {
if (action.parameters === undefined) {
return action;
}
const { parameters } = stripEmptyGoldPlaceholders(action.parameters);
if (parameters === action.parameters) {
return action;
}
if (parameters === undefined) {
const { parameters: _drop, ...rest } = action;
return rest;
// Keep parameters:{} when nested empties strip to nothing. Schemas
// like code.getSelection / desktop.ListThemes require the key
// (empty object type); dropping it fails validateAction.
return { ...action, parameters: {} };
}
return { ...action, parameters };
});
Expand Down
164 changes: 164 additions & 0 deletions ts/packages/benchmarks/test/translationBench.keepEmptyParams.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { describe, expect, it } from "@jest/globals";

import { stripEmptyGoldPlaceholdersFromCandidate } from "../src/translationBench/synthesizer/generationCandidate.js";

describe("keep empty parameters after gold placeholder strip", () => {
it("keeps parameters:{} when nested empties strip to nothing", () => {
const stripped = stripEmptyGoldPlaceholdersFromCandidate({
seed: {
utterance: "get the current selection",
expectedActions: [
{
schemaName: "code",
actionName: "getSelection",
parameters: {
unused: "",
nested: { flag: null },
},
},
],
order: "strict",
},
genCases: [
{
id: "pos-1",
role: "positive",
utterance: "what text is selected",
expectedActions: [
{
schemaName: "code",
actionName: "getSelection",
parameters: { unused: " " },
},
],
order: "strict",
dimensions: { variation: "paraphrase" },
},
{
id: "neg-1",
role: "negative",
utterance: "leave my editor alone",
expectedActions: [],
order: "strict",
dimensions: { negativeKind: "pure_refusal" },
},
],
});

expect(stripped.seed.expectedActions[0]).toEqual({
schemaName: "code",
actionName: "getSelection",
parameters: {},
});
expect(stripped.genCases[0]!.expectedActions[0]).toEqual({
schemaName: "code",
actionName: "getSelection",
parameters: {},
});
// Negatives are not rewritten by the strip helper.
expect(stripped.genCases[1]!.expectedActions).toEqual([]);
});

it("does not invent parameters when gold omitted the key", () => {
const stripped = stripEmptyGoldPlaceholdersFromCandidate({
seed: {
utterance: "list themes",
expectedActions: [
{
schemaName: "desktop",
actionName: "ListThemes",
},
],
order: "strict",
},
genCases: [
{
id: "pos-1",
role: "positive",
utterance: "show desktop themes",
expectedActions: [
{
schemaName: "desktop",
actionName: "ListThemes",
},
],
order: "strict",
dimensions: { variation: "paraphrase" },
},
{
id: "neg-1",
role: "negative",
utterance: "do not list themes",
expectedActions: [],
order: "strict",
dimensions: { negativeKind: "pure_refusal" },
},
],
});

expect(stripped.seed.expectedActions[0]).toEqual({
schemaName: "desktop",
actionName: "ListThemes",
});
expect(stripped.seed.expectedActions[0]).not.toHaveProperty(
"parameters",
);
});

it("still strips empty nested fields while keeping non-empty siblings", () => {
const stripped = stripEmptyGoldPlaceholdersFromCandidate({
seed: {
utterance: "open apple.com",
expectedActions: [
{
schemaName: "browser",
actionName: "openWebPage",
parameters: {
site: "apple.com",
tab: "",
extras: {},
},
},
],
order: "strict",
},
genCases: [
{
id: "pos-1",
role: "positive",
utterance: "navigate to apple.com",
expectedActions: [
{
schemaName: "browser",
actionName: "openWebPage",
parameters: {
site: "apple.com",
tab: null,
},
},
],
order: "strict",
dimensions: { variation: "paraphrase" },
},
{
id: "neg-1",
role: "negative",
utterance: "leave browser alone",
expectedActions: [],
order: "strict",
dimensions: { negativeKind: "pure_refusal" },
},
],
});

expect(stripped.seed.expectedActions[0]!.parameters).toEqual({
site: "apple.com",
});
expect(stripped.genCases[0]!.expectedActions[0]!.parameters).toEqual({
site: "apple.com",
});
});
});
Loading