diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index ea88afa50..fb9ca4803 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -45,17 +45,17 @@ _None._ - [./src/index.ts](./src/index.ts) - [./src/translationBench/index.ts](./src/translationBench/index.ts) - [./src/translationBench/synthesizer/catalogGenerator/index.ts](./src/translationBench/synthesizer/catalogGenerator/index.ts) +- [./src/translationBench/synthesizer/goldSchema.ts](./src/translationBench/synthesizer/goldSchema.ts) - [./src/translationBench/synthesizer/index.ts](./src/translationBench/synthesizer/index.ts) - [./src/core/model-prices.generated.json](./src/core/model-prices.generated.json) - [./src/core/paths.ts](./src/core/paths.ts) - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts) -- [./src/core/types.ts](./src/core/types.ts) -- _…and 36 more under `./src/`._ +- _…and 37 more under `./src/`._ --- -_Auto-generated against commit `ced33869a1369ff7c998a3517bf17bc9739a05e5` on `2026-08-13T05:25:43.204Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `ec5d9161876ae305ea1c253d6e038fa7d364fa62` on `2026-08-13T08:21:46.174Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts new file mode 100644 index 000000000..2a5e9110c --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/goldSchema.ts @@ -0,0 +1,272 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + resolveTypeReference, + type ActionParamObject, + type SchemaType, +} from "@typeagent/action-schema"; + +export const TRANSLATION_BENCH_GOLD_OPTIONALITY_RULE = + "TypeAgent optionality is the source of truth for gold. The OpenAI tool " + + "JSON schema lists every property in required[] (translator convention) " + + "and does NOT make optional TypeAgent fields required. parameterScore / " + + "nonempty scores a present value; it is not a presence requirement. Omit " + + "optional fields the utterance does not support, including optional " + + "false booleans and empty arrays."; + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function listGoldParameterFields(type: SchemaType): { + required: string[]; + optional: string[]; +} { + const resolved = resolveTypeReference(type); + if (resolved?.type !== "object") { + return { required: [], optional: [] }; + } + const required: string[] = []; + const optional: string[] = []; + for (const [name, field] of Object.entries(resolved.fields)) { + if (field.optional) { + optional.push(name); + } else { + required.push(name); + } + } + return { required, optional }; +} + +function requiredFieldNames(type: ActionParamObject): string[] { + return Object.entries(type.fields) + .filter(([, field]) => field.optional !== true) + .map(([name]) => name); +} + +function rewriteNode( + json: unknown, + type: SchemaType, + defs: Record | undefined, + seenDefs: Set, +): void { + if (!isPlainObject(json)) { + return; + } + if (typeof json.$ref === "string" && json.$ref.startsWith("#/$defs/")) { + const name = json.$ref.slice("#/$defs/".length); + if ( + defs !== undefined && + isPlainObject(defs[name]) && + !seenDefs.has(name) + ) { + seenDefs.add(name); + rewriteNode(defs[name], type, defs, seenDefs); + } + return; + } + const resolved = resolveTypeReference(type) ?? type; + if (resolved.type === "object") { + json.required = requiredFieldNames(resolved); + const properties = json.properties; + if (isPlainObject(properties)) { + for (const [name, field] of Object.entries(resolved.fields)) { + if (properties[name] !== undefined) { + rewriteNode(properties[name], field.type, defs, seenDefs); + } + } + } + return; + } + if (resolved.type === "array") { + rewriteNode(json.items, resolved.elementType, defs, seenDefs); + return; + } + if (resolved.type === "type-union" && Array.isArray(json.anyOf)) { + const variants = json.anyOf; + resolved.types.forEach((member, index) => { + rewriteNode(variants[index], member, defs, seenDefs); + }); + } +} + +/** + * Rewrite an OpenAI-style tool parameters JSON schema so `required` matches + * TypeAgent field.optional. generateActionActionFunctionJsonSchemas marks + * every property required (strict-mode convention); gold labeling must not. + */ +export function rewriteJsonSchemaRequiredForGold( + schema: Record, + type: SchemaType, +): Record { + const clone = structuredClone(schema); + const defs = isPlainObject(clone.$defs) ? clone.$defs : undefined; + rewriteNode(clone, type, defs, new Set()); + return clone; +} + +export function applyGoldOptionalityToToolParameters( + parameters: Record | undefined, + parameterType: SchemaType | undefined, +): Record | undefined { + if (parameters === undefined || parameterType === undefined) { + return parameters; + } + return rewriteJsonSchemaRequiredForGold(parameters, parameterType); +} + +function matchingUnionObject( + type: SchemaType, + value: Record, +): ActionParamObject | undefined { + const resolved = resolveTypeReference(type) ?? type; + if (resolved.type === "object") { + return resolved; + } + if (resolved.type !== "type-union") { + return undefined; + } + const keys = Object.keys(value); + for (const member of resolved.types) { + const objectType = resolveTypeReference(member); + if (objectType?.type !== "object") continue; + if (keys.every((key) => objectType.fields[key] !== undefined)) { + return objectType as ActionParamObject; + } + } + return undefined; +} + +function isOptionalBooleanField(type: ActionParamObject, key: string): boolean { + const field = type.fields[key]; + if (field === undefined || !field.optional) { + return false; + } + const resolved = resolveTypeReference(field.type); + return resolved?.type === "boolean"; +} + +type StripResult = { kept: true; value: unknown } | { kept: false }; + +function stripOptionalFalseArray( + value: unknown[], + type: SchemaType, + path: string, + removed: string[], +): StripResult { + const resolved = resolveTypeReference(type) ?? type; + if (resolved.type !== "array") { + return { kept: true, value }; + } + const next: unknown[] = []; + let changed = false; + for (let i = 0; i < value.length; i += 1) { + const child = stripOptionalFalseValue( + value[i], + resolved.elementType, + `${path}[${i}]`, + removed, + ); + if (!child.kept) { + changed = true; + continue; + } + if (child.value !== value[i]) { + changed = true; + } + next.push(child.value); + } + if (next.length === 0 && value.length > 0) { + removed.push(path); + return { kept: false }; + } + return { kept: true, value: changed ? next : value }; +} + +function stripOptionalFalseObject( + value: Record, + type: SchemaType, + path: string, + removed: string[], +): StripResult { + const objectType = matchingUnionObject(type, value); + if (objectType === undefined) { + return { kept: true, value }; + } + const next: Record = {}; + let changed = false; + for (const [key, childValue] of Object.entries(value)) { + const childPath = path === "" ? key : `${path}.${key}`; + if (childValue === false && isOptionalBooleanField(objectType, key)) { + removed.push(childPath); + changed = true; + continue; + } + const field = objectType.fields[key]; + if (field === undefined) { + next[key] = childValue; + continue; + } + const child = stripOptionalFalseValue( + childValue, + field.type, + childPath, + removed, + ); + if (!child.kept) { + changed = true; + continue; + } + if (child.value !== childValue) { + changed = true; + } + next[key] = child.value; + } + if (Object.keys(next).length === 0) { + if (path !== "") { + removed.push(path); + } + return { kept: false }; + } + return { kept: true, value: changed ? next : value }; +} + +function stripOptionalFalseValue( + value: unknown, + type: SchemaType, + path: string, + removed: string[], +): StripResult { + if (Array.isArray(value)) { + return stripOptionalFalseArray(value, type, path, removed); + } + if (!isPlainObject(value)) { + return { kept: true, value }; + } + return stripOptionalFalseObject(value, type, path, removed); +} + +export function stripOptionalFalseGoldBooleans( + parameters: Record | undefined, + type: SchemaType, +): { + parameters: Record | undefined; + removed: string[]; +} { + if (parameters === undefined) { + return { parameters: undefined, removed: [] }; + } + const removed: string[] = []; + const stripped = stripOptionalFalseValue(parameters, type, "", removed); + if (!stripped.kept) { + return { parameters: undefined, removed }; + } + if (stripped.value === parameters) { + return { parameters, removed: [] }; + } + return { + parameters: stripped.value as Record, + removed, + }; +} diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts index 3ea9db106..9ac5315c2 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/index.ts @@ -18,3 +18,4 @@ export { seedQaJsonlAdapter } from "./adapters/seedQaJsonlAdapter.js"; export * from "./emptyGoldUtterance.js"; export * from "./goldParameterHygiene.js"; export * from "./actionValidation.js"; +export * from "./goldSchema.js"; diff --git a/ts/packages/benchmarks/test/translationBench.goldSchema.spec.ts b/ts/packages/benchmarks/test/translationBench.goldSchema.spec.ts new file mode 100644 index 000000000..809ab2349 --- /dev/null +++ b/ts/packages/benchmarks/test/translationBench.goldSchema.spec.ts @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "@jest/globals"; +import type { SchemaType } from "@typeagent/action-schema"; + +import { + listGoldParameterFields, + rewriteJsonSchemaRequiredForGold, + stripOptionalFalseGoldBooleans, +} from "../src/translationBench/synthesizer/goldSchema.js"; + +const montageParameters: SchemaType = { + type: "object", + fields: { + title: { type: { type: "string" } }, + search_filters: { + optional: true, + type: { type: "array", elementType: { type: "string" } }, + }, + files: { + optional: true, + type: { type: "array", elementType: { type: "string" } }, + }, + }, +}; + +const fileTarget: SchemaType = { + type: "object", + fields: { + language: { type: { type: "string" } }, + file: { + optional: true, + type: { + type: "object", + fields: { + fileName: { optional: true, type: { type: "string" } }, + createIfNotExists: { + optional: true, + type: { type: "boolean" }, + }, + fallbackToActiveFile: { + optional: true, + type: { type: "boolean" }, + }, + }, + }, + }, + focus: { type: { type: "boolean" } }, + }, +}; + +describe("gold schema optionality", () => { + it("lists required vs optional TypeAgent parameter fields", () => { + expect(listGoldParameterFields(montageParameters)).toEqual({ + required: ["title"], + optional: ["search_filters", "files"], + }); + }); + + it("rewrites OpenAI all-required JSON schema to TypeAgent optionality", () => { + const openaiStyle = { + type: "object", + properties: { + title: { type: "string" }, + search_filters: { type: "array", items: { type: "string" } }, + files: { type: "array", items: { type: "string" } }, + }, + required: ["title", "search_filters", "files"], + additionalProperties: false, + }; + const rewritten = rewriteJsonSchemaRequiredForGold( + openaiStyle, + montageParameters, + ); + expect(rewritten.required).toEqual(["title"]); + expect(openaiStyle.required).toEqual([ + "title", + "search_filters", + "files", + ]); + }); + + it("rewrites nested FileTarget required arrays", () => { + const openaiStyle = { + type: "object", + properties: { + language: { type: "string" }, + file: { + type: "object", + properties: { + fileName: { type: "string" }, + createIfNotExists: { type: "boolean" }, + fallbackToActiveFile: { type: "boolean" }, + }, + required: [ + "fileName", + "createIfNotExists", + "fallbackToActiveFile", + ], + }, + focus: { type: "boolean" }, + }, + required: ["language", "file", "focus"], + }; + const rewritten = rewriteJsonSchemaRequiredForGold( + openaiStyle, + fileTarget, + ); + expect(rewritten.required).toEqual(["language", "focus"]); + const file = rewritten.properties as Record; + expect((file.file as { required: string[] }).required).toEqual([]); + }); + + it("strips optional false booleans and keeps required false / optional true", () => { + const stripped = stripOptionalFalseGoldBooleans( + { + language: "python", + file: { + fileName: "server.py", + createIfNotExists: false, + fallbackToActiveFile: false, + }, + focus: false, + }, + fileTarget, + ); + expect(stripped.parameters).toEqual({ + language: "python", + file: { fileName: "server.py" }, + focus: false, + }); + expect(stripped.removed.sort()).toEqual([ + "file.createIfNotExists", + "file.fallbackToActiveFile", + ]); + }); + + it("drops a nested file object that only held optional false flags", () => { + const stripped = stripOptionalFalseGoldBooleans( + { + language: "python", + file: { + createIfNotExists: false, + fallbackToActiveFile: false, + }, + focus: true, + }, + fileTarget, + ); + expect(stripped.parameters).toEqual({ + language: "python", + focus: true, + }); + expect(stripped.removed).toEqual(expect.arrayContaining(["file"])); + }); +});