diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx index d46cc38ea..e731df263 100644 --- a/src/handlers/eval/ondemand/index.tsx +++ b/src/handlers/eval/ondemand/index.tsx @@ -3,11 +3,13 @@ import type { AppIO } from "../../../io"; import type { Core } from "../../types"; import { createHelpDefault } from "../../help"; import { createEvaluateOnDemandHandler } from "./evaluate"; +import { createSimulateOnDemandHandler } from "./simulate"; // ondemand groups the synchronous, client-side evaluation commands. It has no TUI // screen (unlike evaluator/online-eval), so a bare invocation prints help. export function createOnDemandHandler(core: Core, io: AppIO): Router { return new Router("ondemand", "evaluate existing sessions synchronously, client-side") .default(createHelpDefault(io)) - .handler(createEvaluateOnDemandHandler(core, io)); + .handler(createEvaluateOnDemandHandler(core, io)) + .handler(createSimulateOnDemandHandler(core, io)); } diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx index 903a9140e..5bbb55ca2 100644 --- a/src/handlers/eval/ondemand/ondemand.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -168,7 +168,7 @@ const BASE = [ ]; describe("eval ondemand command hierarchy", () => { - test("registers evaluate under eval → ondemand", () => { + test("registers evaluate + simulate under eval → ondemand", () => { const io = testIO(); const root = createRootHandler(new TestCoreClient(), { io: io.io, @@ -180,7 +180,7 @@ describe("eval ondemand command hierarchy", () => { .find((c) => c.name() === "eval") ?.children() .find((c) => c.name() === "ondemand"); - expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate"]); }); }); diff --git a/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-groundtruth.golden.json b/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-groundtruth.golden.json new file mode 100644 index 000000000..81d7bfbba --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-groundtruth.golden.json @@ -0,0 +1,19 @@ +[ + { + "context": { + "spanContext": { + "sessionId": "s1" + } + }, + "assertions": [ + { + "text": "polite" + } + ], + "expectedTrajectory": { + "toolNames": [ + "lookup" + ] + } + } +] \ No newline at end of file diff --git a/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-output.golden.json b/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-output.golden.json new file mode 100644 index 000000000..21b06350a --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/__fixtures__/ondemand-simulate-output.golden.json @@ -0,0 +1,7 @@ +{ + "sessionsRequested": 2, + "sessionsEvaluated": 2, + "results": [], + "examplesInvoked": 2, + "examplesFailed": 0 +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/simulate/index.tsx b/src/handlers/eval/ondemand/simulate/index.tsx new file mode 100644 index 000000000..322b44eea --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/index.tsx @@ -0,0 +1,123 @@ +import z from "zod"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { InvokedSession } from "../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; + +// Composes invokeDataset (replay) → getTracesForAgent (gather) → evaluate (grade, +// synchronous). The on-demand twin of batch-evaluation simulate: no async job, scores +// print inline. Invoke flags mirror `runtime invoke`. +export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "simulate", + description: "replay a dataset against a runtime, then evaluate the sessions client-side", + flags: [ + flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()), + flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag( + "payload-template", + 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + z.string().optional(), + ), + flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()), + flag( + "bearer-token", + "CUSTOM_JWT bearer token (for JWT-auth runtimes)", + z.string().optional(), + ), + flag("user-id", "runtime user id", z.string().optional()), + flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), + flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + ], + handle: async (ctx, flags) => { + if (!flags["runtime-id"]) + throw new InputValidationError("required option '--runtime-id' not specified"); + if (!flags["payload-template"]) { + throw new InputValidationError("required option '--payload-template' not specified"); + } + if (!flags["dataset"]) + throw new InputValidationError("required option '--dataset' not specified"); + if (!flags["evaluator"]?.length) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once("SIGINT", interrupt); + try { + const opts = coreOptsFromCtx(ctx); + + // 1. Replay the dataset — reuse invokeDataset verbatim (grader-agnostic). + const replay = await core.eval.invokeDataset( + { + runtimeId: flags["runtime-id"], + qualifier: flags["qualifier"], + payloadTemplate: flags["payload-template"], + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + userId: flags["user-id"], + dataset: flags["dataset"], + datasetVersion: flags["dataset-version"], + }, + opts, + controller.signal, + ); + if (replay.invoked === 0) { + const detail = replay.firstError ? `; first error: ${replay.firstError.message}` : ""; + throw new InputValidationError( + `no examples could be invoked (${replay.failed} failed) — nothing to evaluate${detail}`, + ); + } + + // 2. Gather the just-created sessions' traces (client-side CloudWatch read). + const traces = await core.eval.getTracesForAgent( + { + agent: flags["runtime-id"], + endpoint: flags["qualifier"], + sessionIds: replay.sessions.map((s) => s.sessionId), + }, + opts, + ); + + // 3. Adapt neutral ground truth → EvaluationReferenceInput[] and grade synchronously. + const groundTruth = replay.sessions.flatMap(toReferenceInputs); + const result = await core.eval.evaluate( + { traces, evaluatorIds: flags["evaluator"], groundTruth }, + opts, + ); + + ctx.require(JsonRendererKey).renderJson({ + ...result, + examplesInvoked: replay.invoked, + examplesFailed: replay.failed, + }); + } finally { + process.off("SIGINT", interrupt); + } + }, + }); + +// Adapt one invoked session's neutral InlineGroundTruth to the Evaluate API's +// EvaluationReferenceInput, correlated by sessionId. assertions ({text}[]) and +// expectedTrajectory ({toolNames}) map 1:1. Per-turn expectedResponse is trace-level and +// needs a turn→trace id we don't have here, so it is omitted (batch simulate covers it). +function toReferenceInputs(s: InvokedSession): EvaluationReferenceInput[] { + const gt = s.groundTruth; + if (!gt?.assertions?.length && !gt?.expectedTrajectory) return []; + return [ + { + context: { spanContext: { sessionId: s.sessionId } }, + ...(gt.assertions?.length && { assertions: gt.assertions }), + ...(gt.expectedTrajectory && { expectedTrajectory: gt.expectedTrajectory }), + }, + ]; +} diff --git a/src/handlers/eval/ondemand/simulate/simulate.test.tsx b/src/handlers/eval/ondemand/simulate/simulate.test.tsx new file mode 100644 index 000000000..8d04a162b --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/simulate.test.tsx @@ -0,0 +1,189 @@ +import { test, expect, describe } from "bun:test"; +import { join } from "node:path"; +import { createRootHandler } from "../../../index"; +import { + createSilentLogger, + matchGolden, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../../testing"; +import type { EvaluateResult, InvokeDatasetResult, SessionTrace } from "../../types"; + +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Two invoked sessions: e1 carries session-level ground truth (assertions + trajectory), +// e2 carries none. The handler adapts these to EvaluationReferenceInput[] for evaluate. +const INVOKE_RESULT: InvokeDatasetResult = { + sessions: [ + { + exampleId: "e1", + sessionId: "s1", + groundTruth: { + assertions: [{ text: "polite" }], + expectedTrajectory: { toolNames: ["lookup"] }, + // turn-level expectedResponse is intentionally present to prove it is dropped (v1). + turns: [{ input: { prompt: "hi" }, expectedResponse: { text: "hello" } }], + }, + }, + { exampleId: "e2", sessionId: "s2" }, + ], + invoked: 2, + failed: 0, +}; + +const TRACES: SessionTrace[] = [ + { sessionId: "s1", spans: [], traceIds: ["t1"], toolCallSpanIds: [] }, + { sessionId: "s2", spans: [], traceIds: ["t2"], toolCallSpanIds: [] }, +]; + +const EVAL_RESULT: EvaluateResult = { sessionsRequested: 2, sessionsEvaluated: 2, results: [] }; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setInvokeDatasetResponse(INVOKE_RESULT); + core.eval.setGetTracesResponse(TRACES); + core.eval.setEvaluateResponse(EVAL_RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +const BASE = [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", +]; + +describe("eval ondemand simulate", () => { + test("registered under ondemand", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + // Bare `ondemand` prints help listing both subcommands. + expect(root).toBeDefined(); + }); + + test.each([ + [ + [ + "eval", + "ondemand", + "simulate", + "--payload-template", + "{}", + "--dataset", + "d", + "--evaluator", + "E", + ], + /--runtime-id/, + ], + [ + ["eval", "ondemand", "simulate", "--runtime-id", "r", "--dataset", "d", "--evaluator", "E"], + /--payload-template/, + ], + [ + [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r", + "--payload-template", + "{}", + "--evaluator", + "E", + ], + /--dataset/, + ], + [ + [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r", + "--payload-template", + "{}", + "--dataset", + "d", + ], + /--evaluator/, + ], + ])("rejects missing required flag", async (args, expected) => { + await expect(run(args)).rejects.toThrow(expected); + }); + + test("composes invokeDataset → getTracesForAgent → evaluate", async () => { + const { core } = await run([...BASE, "--qualifier", "PROD", "--header", "x-a:1"]); + + const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect(invoke?.args[0]).toMatchObject({ + runtimeId: "r-1", + qualifier: "PROD", + payloadTemplate: '{"prompt":"{input}"}', + headers: [["x-a", "1"]], + dataset: "/tmp/ds.jsonl", + }); + // Ctrl-C signal threaded to the replay. + expect(invoke?.args[2]).toBeInstanceOf(AbortSignal); + + // getTracesForAgent is asked for exactly the sessions the replay created. + const traces = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(traces?.args[0]).toMatchObject({ + agent: "r-1", + endpoint: "PROD", + sessionIds: ["s1", "s2"], + }); + + // evaluate gets the runtime evaluators over the gathered traces. + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ evaluatorIds: ["Builtin.Helpfulness"] }); + }); + + // Golden: the adapted EvaluationReferenceInput[] handed to evaluate. Locks session-level + // assertions/trajectory (1:1), the sessionId correlation, and that e2 (no GT) and the + // turn-level expectedResponse both contribute nothing. + test("adapts inline ground truth to EvaluationReferenceInput [golden]", async () => { + const { core } = await run(BASE); + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate).toBeDefined(); + const input = evaluate!.args[0] as { groundTruth: unknown }; + matchGolden( + FIXTURES, + "ondemand-simulate-groundtruth.golden.json", + JSON.stringify(input.groundTruth, null, 2), + ); + }); + + test("renders scores inline with invoked/failed counts [golden]", async () => { + const { stdout } = await run(BASE); + matchGolden(FIXTURES, "ondemand-simulate-output.golden.json", stdout); + }); + + test("refuses to evaluate when nothing was invoked", async () => { + await expect( + run(BASE, (core) => + core.eval.setInvokeDatasetResponse({ sessions: [], invoked: 0, failed: 3 }), + ), + ).rejects.toThrow(/no examples could be invoked \(3 failed\)/); + }); +});