From f74ed3ce0a3e0edee3262495549cb41f30e944af Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Thu, 13 Aug 2026 01:00:18 -0700 Subject: [PATCH 1/4] Add translation-bench gold schema optionality helper Rewrite OpenAI all-required tool JSON schemas to TypeAgent field.optional and strip optional false gold booleans for single-action labeling. --- .../synthesizer/goldSchema.ts | 252 ++++++++++++++++++ .../src/translationBench/synthesizer/index.ts | 1 + .../test/translationBench.goldSchema.spec.ts | 157 +++++++++++ 3 files changed, 410 insertions(+) create mode 100644 ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts create mode 100644 ts/packages/benchmarks/test/translationBench.goldSchema.spec.ts diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts new file mode 100644 index 000000000..3fc4c85a1 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + resolveTypeReference, + type ActionParamObject, + type SchemaType, +} from "@typeagent/action-schema"; + +export const TRANSLATION_BENCH_GOLD_OPTIONALITY_RULE = + "TypeAgent optionality is the source of truth for gold. The OpenAI tool " + + "JSON schema lists every property in required[] (translator convention) " + + "and does NOT make optional TypeAgent fields required. parameterScore / " + + "nonempty scores a present value; it is not a presence requirement. Omit " + + "optional fields the utterance does not support, including optional " + + "false booleans and empty arrays."; + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function listGoldParameterFields(type: SchemaType): { + required: string[]; + optional: string[]; +} { + const resolved = resolveTypeReference(type); + if (resolved?.type !== "object") { + return { required: [], optional: [] }; + } + const required: string[] = []; + const optional: string[] = []; + for (const [name, field] of Object.entries(resolved.fields)) { + if (field.optional) { + optional.push(name); + } else { + required.push(name); + } + } + return { required, optional }; +} + +function requiredFieldNames(type: ActionParamObject): string[] { + return Object.entries(type.fields) + .filter(([, field]) => field.optional !== true) + .map(([name]) => name); +} + +function rewriteNode( + json: unknown, + type: SchemaType, + defs: Record | undefined, + seenDefs: Set, +): void { + if (!isPlainObject(json)) { + return; + } + if (typeof json.$ref === "string" && json.$ref.startsWith("#/$defs/")) { + const name = json.$ref.slice("#/$defs/".length); + if ( + defs !== undefined && + isPlainObject(defs[name]) && + !seenDefs.has(name) + ) { + seenDefs.add(name); + rewriteNode(defs[name], type, defs, seenDefs); + } + return; + } + const resolved = resolveTypeReference(type) ?? type; + if (resolved.type === "object") { + json.required = requiredFieldNames(resolved); + const properties = json.properties; + if (isPlainObject(properties)) { + for (const [name, field] of Object.entries(resolved.fields)) { + if (properties[name] !== undefined) { + rewriteNode(properties[name], field.type, defs, seenDefs); + } + } + } + return; + } + if (resolved.type === "array") { + rewriteNode(json.items, resolved.elementType, defs, seenDefs); + return; + } + if (resolved.type === "type-union" && Array.isArray(json.anyOf)) { + const variants = json.anyOf; + resolved.types.forEach((member, index) => { + rewriteNode(variants[index], member, defs, seenDefs); + }); + } +} + +/** + * Rewrite an OpenAI-style tool parameters JSON schema so `required` matches + * TypeAgent field.optional. generateActionActionFunctionJsonSchemas marks + * every property required (strict-mode convention); gold labeling must not. + */ +export function rewriteJsonSchemaRequiredForGold( + schema: Record, + type: SchemaType, +): Record { + const clone = structuredClone(schema); + const defs = isPlainObject(clone.$defs) ? clone.$defs : undefined; + rewriteNode(clone, type, defs, new Set()); + return clone; +} + +export function applyGoldOptionalityToToolParameters( + parameters: Record | undefined, + parameterType: SchemaType | undefined, +): Record | undefined { + if (parameters === undefined || parameterType === undefined) { + return parameters; + } + return rewriteJsonSchemaRequiredForGold(parameters, parameterType); +} + +function matchingUnionObject( + type: SchemaType, + value: Record, +): ActionParamObject | undefined { + const resolved = resolveTypeReference(type) ?? type; + if (resolved.type === "object") { + return resolved; + } + if (resolved.type !== "type-union") { + return undefined; + } + const keys = Object.keys(value); + for (const member of resolved.types) { + const objectType = resolveTypeReference(member); + if (objectType?.type !== "object") continue; + if (keys.every((key) => objectType.fields[key] !== undefined)) { + return objectType as ActionParamObject; + } + } + return undefined; +} + +function isOptionalBooleanField(type: ActionParamObject, key: string): boolean { + const field = type.fields[key]; + if (field === undefined || !field.optional) { + return false; + } + const resolved = resolveTypeReference(field.type); + return resolved?.type === "boolean"; +} + +function stripOptionalFalseValue( + value: unknown, + type: SchemaType, + path: string, + removed: string[], +): { kept: true; value: unknown } | { kept: false } { + if (Array.isArray(value)) { + const resolved = resolveTypeReference(type) ?? type; + if (resolved.type !== "array") { + return { kept: true, value }; + } + const next: unknown[] = []; + let changed = false; + for (let i = 0; i < value.length; i += 1) { + const child = stripOptionalFalseValue( + value[i], + resolved.elementType, + `${path}[${i}]`, + removed, + ); + if (!child.kept) { + changed = true; + continue; + } + if (child.value !== value[i]) { + changed = true; + } + next.push(child.value); + } + if (next.length === 0 && value.length > 0) { + removed.push(path); + return { kept: false }; + } + return { kept: true, value: changed ? next : value }; + } + if (!isPlainObject(value)) { + return { kept: true, value }; + } + const objectType = matchingUnionObject(type, value); + if (objectType === undefined) { + return { kept: true, value }; + } + const next: Record = {}; + let changed = false; + for (const [key, childValue] of Object.entries(value)) { + const childPath = path === "" ? key : `${path}.${key}`; + if (childValue === false && isOptionalBooleanField(objectType, key)) { + removed.push(childPath); + changed = true; + continue; + } + const field = objectType.fields[key]; + if (field === undefined) { + next[key] = childValue; + continue; + } + const child = stripOptionalFalseValue( + childValue, + field.type, + childPath, + removed, + ); + if (!child.kept) { + changed = true; + continue; + } + if (child.value !== childValue) { + changed = true; + } + next[key] = child.value; + } + if (Object.keys(next).length === 0) { + if (path !== "") { + removed.push(path); + } + return { kept: false }; + } + return { kept: true, value: changed ? next : value }; +} + +export function stripOptionalFalseGoldBooleans( + parameters: Record | undefined, + type: SchemaType, +): { + parameters: Record | undefined; + removed: string[]; +} { + if (parameters === undefined) { + return { parameters: undefined, removed: [] }; + } + const removed: string[] = []; + const stripped = stripOptionalFalseValue(parameters, type, "", removed); + if (!stripped.kept) { + return { parameters: undefined, removed }; + } + if (stripped.value === parameters) { + return { parameters, removed: [] }; + } + return { + parameters: stripped.value as Record, + removed, + }; +} diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts index 3ea9db106..9ac5315c2 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts @@ -18,3 +18,4 @@ export { seedQaJsonlAdapter } from "./adapters/seedQaJsonlAdapter.js"; export * from "./emptyGoldUtterance.js"; export * from "./goldParameterHygiene.js"; export * from "./actionValidation.js"; +export * from "./goldSchema.js"; diff --git a/ts/packages/benchmarks/test/translationBench.goldSchema.spec.ts b/ts/packages/benchmarks/test/translationBench.goldSchema.spec.ts new file mode 100644 index 000000000..809ab2349 --- /dev/null +++ b/ts/packages/benchmarks/test/translationBench.goldSchema.spec.ts @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "@jest/globals"; +import type { SchemaType } from "@typeagent/action-schema"; + +import { + listGoldParameterFields, + rewriteJsonSchemaRequiredForGold, + stripOptionalFalseGoldBooleans, +} from "../src/translationBench/synthesizer/goldSchema.js"; + +const montageParameters: SchemaType = { + type: "object", + fields: { + title: { type: { type: "string" } }, + search_filters: { + optional: true, + type: { type: "array", elementType: { type: "string" } }, + }, + files: { + optional: true, + type: { type: "array", elementType: { type: "string" } }, + }, + }, +}; + +const fileTarget: SchemaType = { + type: "object", + fields: { + language: { type: { type: "string" } }, + file: { + optional: true, + type: { + type: "object", + fields: { + fileName: { optional: true, type: { type: "string" } }, + createIfNotExists: { + optional: true, + type: { type: "boolean" }, + }, + fallbackToActiveFile: { + optional: true, + type: { type: "boolean" }, + }, + }, + }, + }, + focus: { type: { type: "boolean" } }, + }, +}; + +describe("gold schema optionality", () => { + it("lists required vs optional TypeAgent parameter fields", () => { + expect(listGoldParameterFields(montageParameters)).toEqual({ + required: ["title"], + optional: ["search_filters", "files"], + }); + }); + + it("rewrites OpenAI all-required JSON schema to TypeAgent optionality", () => { + const openaiStyle = { + type: "object", + properties: { + title: { type: "string" }, + search_filters: { type: "array", items: { type: "string" } }, + files: { type: "array", items: { type: "string" } }, + }, + required: ["title", "search_filters", "files"], + additionalProperties: false, + }; + const rewritten = rewriteJsonSchemaRequiredForGold( + openaiStyle, + montageParameters, + ); + expect(rewritten.required).toEqual(["title"]); + expect(openaiStyle.required).toEqual([ + "title", + "search_filters", + "files", + ]); + }); + + it("rewrites nested FileTarget required arrays", () => { + const openaiStyle = { + type: "object", + properties: { + language: { type: "string" }, + file: { + type: "object", + properties: { + fileName: { type: "string" }, + createIfNotExists: { type: "boolean" }, + fallbackToActiveFile: { type: "boolean" }, + }, + required: [ + "fileName", + "createIfNotExists", + "fallbackToActiveFile", + ], + }, + focus: { type: "boolean" }, + }, + required: ["language", "file", "focus"], + }; + const rewritten = rewriteJsonSchemaRequiredForGold( + openaiStyle, + fileTarget, + ); + expect(rewritten.required).toEqual(["language", "focus"]); + const file = rewritten.properties as Record; + expect((file.file as { required: string[] }).required).toEqual([]); + }); + + it("strips optional false booleans and keeps required false / optional true", () => { + const stripped = stripOptionalFalseGoldBooleans( + { + language: "python", + file: { + fileName: "server.py", + createIfNotExists: false, + fallbackToActiveFile: false, + }, + focus: false, + }, + fileTarget, + ); + expect(stripped.parameters).toEqual({ + language: "python", + file: { fileName: "server.py" }, + focus: false, + }); + expect(stripped.removed.sort()).toEqual([ + "file.createIfNotExists", + "file.fallbackToActiveFile", + ]); + }); + + it("drops a nested file object that only held optional false flags", () => { + const stripped = stripOptionalFalseGoldBooleans( + { + language: "python", + file: { + createIfNotExists: false, + fallbackToActiveFile: false, + }, + focus: true, + }, + fileTarget, + ); + expect(stripped.parameters).toEqual({ + language: "python", + focus: true, + }); + expect(stripped.removed).toEqual(expect.arrayContaining(["file"])); + }); +}); From dce31e4f6285c528829aeb4737de0ce67e615e5b Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Thu, 13 Aug 2026 08:09:13 +0000 Subject: [PATCH 2/4] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 9ad6c8094..b1d0ab760 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -45,17 +45,17 @@ _None._ - [./src/index.ts](./src/index.ts) - [./src/translationBench/index.ts](./src/translationBench/index.ts) - [./src/translationBench/synthesizer/catalogGenerator/index.ts](./src/translationBench/synthesizer/catalogGenerator/index.ts) +- [./src/translationBench/synthesizer/goldSchema.ts](./src/translationBench/synthesizer/goldSchema.ts) - [./src/translationBench/synthesizer/index.ts](./src/translationBench/synthesizer/index.ts) - [./src/core/model-prices.generated.json](./src/core/model-prices.generated.json) - [./src/core/paths.ts](./src/core/paths.ts) - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/types.ts](./src/core/types.ts) -- [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) -- _…and 35 more under `./src/`._ +- _…and 36 more under `./src/`._ --- -_Auto-generated against commit `01feca686ce14ae8b00182f75c305cf40e92c415` on `2026-08-13T03:34:32.783Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `f74ed3ce0a3e0edee3262495549cb41f30e944af` on `2026-08-13T08:06:45.967Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From caa6e419283ea5b0df3e060df625aa039775eac4 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Thu, 13 Aug 2026 01:09:42 -0700 Subject: [PATCH 3/4] Reduce stripOptionalFalseValue cognitive complexity under CI cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split array/object walkers into helpers so cognitive stays ≤30 - Keeps gold optional-false boolean strip behavior unchanged --- .../synthesizer/goldSchema.ts | 82 ++++++++++++------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts index 3fc4c85a1..2a5e9110c 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts @@ -147,44 +147,49 @@ function isOptionalBooleanField(type: ActionParamObject, key: string): boolean { return resolved?.type === "boolean"; } -function stripOptionalFalseValue( - value: unknown, +type StripResult = { kept: true; value: unknown } | { kept: false }; + +function stripOptionalFalseArray( + value: unknown[], type: SchemaType, path: string, removed: string[], -): { kept: true; value: unknown } | { kept: false } { - if (Array.isArray(value)) { - const resolved = resolveTypeReference(type) ?? type; - if (resolved.type !== "array") { - return { kept: true, value }; - } - const next: unknown[] = []; - let changed = false; - for (let i = 0; i < value.length; i += 1) { - const child = stripOptionalFalseValue( - value[i], - resolved.elementType, - `${path}[${i}]`, - removed, - ); - if (!child.kept) { - changed = true; - continue; - } - if (child.value !== value[i]) { - changed = true; - } - next.push(child.value); +): StripResult { + const resolved = resolveTypeReference(type) ?? type; + if (resolved.type !== "array") { + return { kept: true, value }; + } + const next: unknown[] = []; + let changed = false; + for (let i = 0; i < value.length; i += 1) { + const child = stripOptionalFalseValue( + value[i], + resolved.elementType, + `${path}[${i}]`, + removed, + ); + if (!child.kept) { + changed = true; + continue; } - if (next.length === 0 && value.length > 0) { - removed.push(path); - return { kept: false }; + if (child.value !== value[i]) { + changed = true; } - return { kept: true, value: changed ? next : value }; + next.push(child.value); } - if (!isPlainObject(value)) { - return { kept: true, value }; + if (next.length === 0 && value.length > 0) { + removed.push(path); + return { kept: false }; } + return { kept: true, value: changed ? next : value }; +} + +function stripOptionalFalseObject( + value: Record, + type: SchemaType, + path: string, + removed: string[], +): StripResult { const objectType = matchingUnionObject(type, value); if (objectType === undefined) { return { kept: true, value }; @@ -227,6 +232,21 @@ function stripOptionalFalseValue( return { kept: true, value: changed ? next : value }; } +function stripOptionalFalseValue( + value: unknown, + type: SchemaType, + path: string, + removed: string[], +): StripResult { + if (Array.isArray(value)) { + return stripOptionalFalseArray(value, type, path, removed); + } + if (!isPlainObject(value)) { + return { kept: true, value }; + } + return stripOptionalFalseObject(value, type, path, removed); +} + export function stripOptionalFalseGoldBooleans( parameters: Record | undefined, type: SchemaType, From c98afa938750bf0905b65af9fd97284b2f13eb5b Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Thu, 13 Aug 2026 08:23:55 +0000 Subject: [PATCH 4/4] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index b1d0ab760..fb9ca4803 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -34,7 +34,7 @@ Workspace: - [agent-dispatcher](../../packages/dispatcher/dispatcher/README.md) - [default-agent-provider](../../packages/defaultAgentProvider/README.md) -External: `commander`, `js-yaml`, `zod` +External: `commander`, `gpt-tokenizer`, `js-yaml`, `zod` ### Used by @@ -51,11 +51,11 @@ _None._ - [./src/core/paths.ts](./src/core/paths.ts) - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) -- [./src/core/types.ts](./src/core/types.ts) -- _…and 36 more under `./src/`._ +- [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts) +- _…and 37 more under `./src/`._ --- -_Auto-generated against commit `f74ed3ce0a3e0edee3262495549cb41f30e944af` on `2026-08-13T08:06:45.967Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `ec5d9161876ae305ea1c253d6e038fa7d364fa62` on `2026-08-13T08:21:46.174Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._