From 2ec82daff0d57620aa62979a997344114bafa54a Mon Sep 17 00:00:00 2001 From: Nicolas Borges Date: Fri, 21 Aug 2026 15:19:55 -0400 Subject: [PATCH] feat: add imperative batch-insights command --- src/core/batchInsights.test.tsx | 58 +++++ src/core/eval.tsx | 18 ++ .../eval/batch-evaluation/evaluate/index.tsx | 132 +---------- ...tAgentRuntimeCommand.9f77333d1b9dcf5d.json | 47 ++++ ...tchEvaluationCommand.78dd8df2fc94e379.json | 36 +++ ...chEvaluationsCommand.23f97c9dcdd6350b.json | 110 +++++++++ ...tchEvaluationCommand.7af4b4e0e2cc1c3d.json | 21 ++ .../__fixtures__/get.golden.json | 32 +++ .../__fixtures__/list.golden.json | 18 ++ .../__fixtures__/run.golden.json | 19 ++ .../batch-insights.fixture.test.tsx | 91 ++++++++ .../batch-insights/batch-insights.test.tsx | 221 ++++++++++++++++++ .../eval/batch-insights/get/index.tsx | 27 +++ src/handlers/eval/batch-insights/index.tsx | 15 ++ .../eval/batch-insights/list/index.tsx | 30 +++ .../eval/batch-insights/run/index.tsx | 47 ++++ src/handlers/eval/index.tsx | 2 + src/handlers/eval/sessionSource.tsx | 126 ++++++++++ src/handlers/eval/types.tsx | 15 ++ src/testing/TestCoreClient.tsx | 10 + 20 files changed, 948 insertions(+), 127 deletions(-) create mode 100644 src/core/batchInsights.test.tsx create mode 100644 src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json create mode 100644 src/handlers/eval/batch-insights/__fixtures__/GetBatchEvaluationCommand.78dd8df2fc94e379.json create mode 100644 src/handlers/eval/batch-insights/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json create mode 100644 src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.7af4b4e0e2cc1c3d.json create mode 100644 src/handlers/eval/batch-insights/__fixtures__/get.golden.json create mode 100644 src/handlers/eval/batch-insights/__fixtures__/list.golden.json create mode 100644 src/handlers/eval/batch-insights/__fixtures__/run.golden.json create mode 100644 src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx create mode 100644 src/handlers/eval/batch-insights/batch-insights.test.tsx create mode 100644 src/handlers/eval/batch-insights/get/index.tsx create mode 100644 src/handlers/eval/batch-insights/index.tsx create mode 100644 src/handlers/eval/batch-insights/list/index.tsx create mode 100644 src/handlers/eval/batch-insights/run/index.tsx create mode 100644 src/handlers/eval/sessionSource.tsx diff --git a/src/core/batchInsights.test.tsx b/src/core/batchInsights.test.tsx new file mode 100644 index 000000000..e0789d6ac --- /dev/null +++ b/src/core/batchInsights.test.tsx @@ -0,0 +1,58 @@ +import { describe, expect, mock, test } from "bun:test"; +import { StartBatchEvaluationCommand } from "@aws-sdk/client-bedrock-agentcore"; +import { EvalClient } from "./eval"; +import type { AwsClients } from "./types"; + +describe("EvalClient.startBatchInsights", () => { + test("maps the semantic input to StartBatchEvaluation insights", async () => { + const send = mock(async (_command: unknown) => ({ + batchEvaluationId: "bi-1", + status: "RUNNING", + })); + const clients = { + data: () => ({ send }), + control: () => { + throw new Error("unexpected control client call"); + }, + iam: () => { + throw new Error("unexpected IAM client call"); + }, + logs: () => { + throw new Error("unexpected Logs client call"); + }, + } as unknown as AwsClients; + const client = new EvalClient(clients); + const dataSourceConfig = { + onlineEvaluationConfigSource: { + onlineEvaluationConfigArn: "arn:aws:bedrock-agentcore:us-west-2:123:online-evaluation/oe-1", + }, + }; + + await client.startBatchInsights( + { + name: "insights_run", + description: "description", + insightIds: ["Builtin.Insight.FailureAnalysis", "Builtin.Insight.UserIntent"], + evaluatorIds: ["Builtin.Helpfulness"], + source: { origin: "raw", dataSourceConfig }, + kmsKeyArn: "arn:aws:kms:us-west-2:123:key/abc", + }, + { region: "us-west-2" }, + ); + + expect(send).toHaveBeenCalledTimes(1); + const command = send.mock.calls[0]?.[0]; + expect(command).toBeInstanceOf(StartBatchEvaluationCommand); + expect((command as StartBatchEvaluationCommand).input).toEqual({ + batchEvaluationName: "insights_run", + description: "description", + insights: [ + { insightId: "Builtin.Insight.FailureAnalysis" }, + { insightId: "Builtin.Insight.UserIntent" }, + ], + evaluators: [{ evaluatorId: "Builtin.Helpfulness" }], + dataSourceConfig, + kmsKeyArn: "arn:aws:kms:us-west-2:123:key/abc", + }); + }); +}); diff --git a/src/core/eval.tsx b/src/core/eval.tsx index aef88d11d..61a4b366f 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -109,6 +109,7 @@ import type { SessionSourceValue, SessionTrace, SpanRecord, + StartBatchInsightsInput, StartBatchEvaluationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, @@ -379,6 +380,23 @@ export class EvalClient implements CoreEvalClient { ); } + async startBatchInsights( + input: StartBatchInsightsInput, + options: CoreOptions, + ): Promise { + const dataSourceConfig = await this.dataSourceConfigForSource(input.source, options); + return this.clients.data(toClientConfig(options)).send( + new StartBatchEvaluationCommand({ + batchEvaluationName: input.name, + description: input.description, + insights: input.insightIds.map((insightId) => ({ insightId })), + evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + dataSourceConfig, + kmsKeyArn: input.kmsKeyArn, + }), + ); + } + // dataSourceConfigForSource maps a resolved SessionSourceValue to the data-plane // dataSourceConfig union. The agent arm reuses the same runtime resolution + // log-group derivation the control-plane agentDataSource uses, then attaches the diff --git a/src/handlers/eval/batch-evaluation/evaluate/index.tsx b/src/handlers/eval/batch-evaluation/evaluate/index.tsx index 561a1cf22..5e54ac2f5 100644 --- a/src/handlers/eval/batch-evaluation/evaluate/index.tsx +++ b/src/handlers/eval/batch-evaluation/evaluate/index.tsx @@ -4,50 +4,16 @@ import { InputValidationError } from "../../../../errors"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; -import type { SessionMetadataShape, DataSourceConfig } from "@aws-sdk/client-bedrock-agentcore"; +import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import type { SessionSourceValue, SessionWindow } from "../../types"; +import { resolveSessionSource, sessionSourceFlags } from "../../sessionSource"; export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => createHandler({ name: "evaluate", description: "evaluate existing sessions service-side (async; returns a job id)", flags: [ - flag( - "agent", - "source: harness id or runtime id whose sessions to evaluate", - z.string().optional(), - ), - flag( - "endpoint", - "runtime endpoint qualifier (default DEFAULT; only with --agent)", - z.string().optional(), - ), - flag( - "online-eval", - "source: evaluate sessions an online-eval config already sampled", - z.string().optional(), - ), - flag( - "data-source-config", - "source: raw DataSourceConfig JSON (inline, file://, or -); escape hatch", - z.string().optional(), - ), - flag( - "start-time", - "time filter: window start (ISO-8601, with --end-time)", - z.string().optional(), - ), - flag( - "end-time", - "time filter: window end (ISO-8601, with --start-time)", - z.string().optional(), - ), - flag( - "session-ids", - "filter: specific session ids (only with --agent)", - z.array(z.string()).optional(), - ), + ...sessionSourceFlags, flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), flag( "ground-truth", @@ -68,13 +34,9 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => ); } - const resolver = new SourceResolver({ stdin: io.stdin }); - const rawDataSourceConfig = parseJsonFlag( - "data-source-config", - await resolver.resolveText("data-source-config", flags["data-source-config"]), - ); - const source = resolveDataSource(flags, rawDataSourceConfig); + const source = await resolveSessionSource(flags, io); + const resolver = new SourceResolver({ stdin: io.stdin }); const groundTruth = parseJsonFlag( "ground-truth", await resolver.resolveText("ground-truth", flags["ground-truth"]), @@ -94,87 +56,3 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => ctx.require(JsonRendererKey).renderJson(response); }, }); - -// DataSourceFlags is hand-listed and can drift from the flag declarations above. -// When insights lands as a second consumer, promote this type + resolveDataSource -// into a static SessionSource class instead of keeping them as loose siblings. -type DataSourceFlags = { - agent?: string; - endpoint?: string; - "online-eval"?: string; - "start-time"?: string; - "end-time"?: string; - "session-ids"?: string[]; -}; - -function resolveDataSource( - flags: DataSourceFlags, - rawDataSourceConfig: DataSourceConfig | undefined, -): SessionSourceValue { - const hasAgent = flags["agent"] !== undefined; - const hasOnlineEval = flags["online-eval"] !== undefined; - const hasRaw = rawDataSourceConfig !== undefined; - - const armCount = [hasAgent, hasOnlineEval, hasRaw].filter(Boolean).length; - if (armCount !== 1) { - throw new InputValidationError( - "specify exactly one source: '--agent', '--online-eval', or '--data-source-config'", - ); - } - - const hasIds = !!flags["session-ids"]?.length; - - if (hasRaw) { - // The raw config is self-contained; the ergonomic filter flags don't apply. - if ( - flags["start-time"] !== undefined || - flags["end-time"] !== undefined || - hasIds || - flags["endpoint"] !== undefined - ) { - throw new InputValidationError( - "filter flags cannot be combined with '--data-source-config' (put them in the JSON)", - ); - } - return { origin: "raw", dataSourceConfig: rawDataSourceConfig! }; - } - - const window = resolveWindow(flags); - - if (hasOnlineEval) { - // The online-eval arm has no sessionIds filter and no endpoint. - if (hasIds) - throw new InputValidationError("'--session-ids' cannot be used with '--online-eval'"); - if (flags["endpoint"]) - throw new InputValidationError("'--endpoint' can only be used with '--agent'"); - return { origin: "online-eval", onlineEvaluationConfigId: flags["online-eval"]!, window }; - } - - return { - origin: "agent", - agent: flags["agent"]!, - endpoint: flags["endpoint"], - window, - sessionIds: hasIds ? flags["session-ids"] : undefined, - }; -} - -// resolveWindow validates the explicit time window: both halves must come -// together and start must precede end. -function resolveWindow(flags: DataSourceFlags): SessionWindow | undefined { - const hasStart = flags["start-time"] !== undefined; - const hasEnd = flags["end-time"] !== undefined; - if (!hasStart && !hasEnd) return undefined; // no time filter — all available sessions - if (!hasStart || !hasEnd) { - throw new InputValidationError("--start-time and --end-time must be provided together"); - } - const startTime = new Date(flags["start-time"]!); - const endTime = new Date(flags["end-time"]!); - if (Number.isNaN(+startTime) || Number.isNaN(+endTime)) { - throw new InputValidationError("--start-time and --end-time must be ISO-8601 timestamps"); - } - if (+startTime >= +endTime) { - throw new InputValidationError("--start-time must be before --end-time"); - } - return { startTime, endTime }; -} diff --git a/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json new file mode 100644 index 000000000..2d3b5e713 --- /dev/null +++ b/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json @@ -0,0 +1,47 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeName": "asdf_MyAgent", + "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeVersion": "1", + "createdAt": { + "$date": "2026-04-23T21:17:21.895Z" + }, + "lastUpdatedAt": { + "$date": "2026-04-23T21:17:35.159Z" + }, + "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: asdf_MyAgent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2", + "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip" + } + }, + "runtime": "PYTHON_3_13", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE", + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/GetBatchEvaluationCommand.78dd8df2fc94e379.json b/src/handlers/eval/batch-insights/__fixtures__/GetBatchEvaluationCommand.78dd8df2fc94e379.json new file mode 100644 index 000000000..78e411a1a --- /dev/null +++ b/src/handlers/eval/batch-insights/__fixtures__/GetBatchEvaluationCommand.78dd8df2fc94e379.json @@ -0,0 +1,36 @@ +{ + "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", + "batchEvaluationName": "golden_batch_insights_fixture", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-21T19:27:51.136Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "asdf_MyAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT" + ] + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 2, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [] + }, + "description": "Golden batch insights fixture", + "updatedAt": { + "$date": "2026-08-21T19:28:56.756Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/batch-insights/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json new file mode 100644 index 000000000..1e421c913 --- /dev/null +++ b/src/handlers/eval/batch-insights/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json @@ -0,0 +1,110 @@ +{ + "batchEvaluations": [ + { + "batchEvaluationId": "golden_batch_evaluate-b957bb900a", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_evaluate-b957bb900a", + "batchEvaluationName": "golden_batch_evaluate", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-11T21:18:32.724Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": { + "$date": "2026-08-11T21:18:35.488Z" + } + }, + { + "batchEvaluationId": "golden_batch_evaluate_fixture685-d2967f9a21", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_evaluate_fixture685-d2967f9a21", + "batchEvaluationName": "golden_batch_evaluate_fixture685", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-11T21:19:58.199Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": { + "$date": "2026-08-11T21:20:01.334Z" + } + }, + { + "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", + "batchEvaluationName": "golden_batch_insights_fixture", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-21T19:27:51.136Z" + }, + "description": "Golden batch insights fixture", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "updatedAt": { + "$date": "2026-08-21T19:28:56.756Z" + } + }, + { + "batchEvaluationId": "sim_live_1f9e-cbcc074b38", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/sim_live_1f9e-cbcc074b38", + "batchEvaluationName": "sim_live_1f9e", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-13T22:26:21.948Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": { + "$date": "2026-08-13T22:27:25.296Z" + } + }, + { + "batchEvaluationId": "sim_live_6275-c9338a9ec5", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/sim_live_6275-c9338a9ec5", + "batchEvaluationName": "sim_live_6275", + "status": "FAILED", + "createdAt": { + "$date": "2026-08-13T22:18:02.147Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-08-13T22:19:05.784Z" + } + }, + { + "batchEvaluationId": "sim_mt_2755-7517683065", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/sim_mt_2755-7517683065", + "batchEvaluationName": "sim_mt_2755", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-08-14T19:49:59.317Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "updatedAt": { + "$date": "2026-08-14T19:51:02.750Z" + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.7af4b4e0e2cc1c3d.json b/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.7af4b4e0e2cc1c3d.json new file mode 100644 index 000000000..506ae0f3f --- /dev/null +++ b/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.7af4b4e0e2cc1c3d.json @@ -0,0 +1,21 @@ +{ + "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", + "batchEvaluationName": "golden_batch_insights_fixture", + "status": "PENDING", + "createdAt": { + "$date": "2026-08-21T19:27:51.136Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-golden_batch_insights_fixture-cd634815b4" + } + }, + "description": "Golden batch insights fixture" +} diff --git a/src/handlers/eval/batch-insights/__fixtures__/get.golden.json b/src/handlers/eval/batch-insights/__fixtures__/get.golden.json new file mode 100644 index 000000000..1291f4976 --- /dev/null +++ b/src/handlers/eval/batch-insights/__fixtures__/get.golden.json @@ -0,0 +1,32 @@ +{ + "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", + "batchEvaluationName": "golden_batch_insights_fixture", + "status": "COMPLETED", + "createdAt": "2026-08-21T19:27:51.136Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "asdf_MyAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT" + ] + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 2, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [] + }, + "description": "Golden batch insights fixture", + "updatedAt": "2026-08-21T19:28:56.756Z" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/list.golden.json b/src/handlers/eval/batch-insights/__fixtures__/list.golden.json new file mode 100644 index 000000000..288e5fcbc --- /dev/null +++ b/src/handlers/eval/batch-insights/__fixtures__/list.golden.json @@ -0,0 +1,18 @@ +{ + "batchEvaluations": [ + { + "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", + "batchEvaluationName": "golden_batch_insights_fixture", + "status": "COMPLETED", + "createdAt": "2026-08-21T19:27:51.136Z", + "description": "Golden batch insights fixture", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "updatedAt": "2026-08-21T19:28:56.756Z" + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/run.golden.json b/src/handlers/eval/batch-insights/__fixtures__/run.golden.json new file mode 100644 index 000000000..d07171893 --- /dev/null +++ b/src/handlers/eval/batch-insights/__fixtures__/run.golden.json @@ -0,0 +1,19 @@ +{ + "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", + "batchEvaluationName": "golden_batch_insights_fixture", + "status": "PENDING", + "createdAt": "2026-08-21T19:27:51.136Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-golden_batch_insights_fixture-cd634815b4" + } + }, + "description": "Golden batch insights fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx b/src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx new file mode 100644 index 000000000..e253a5c98 --- /dev/null +++ b/src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx @@ -0,0 +1,91 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); +const FIXTURE_JOB_ID = "golden_batch_insights_fixture-cd634815b4"; +const FIXTURE_NAME = "golden_batch_insights_fixture"; +const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; + +// Record with: +// RECORD=1 bun test src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx +// +// `get` pins a pre-existing completed insights job. `run` is a write and creates +// another durable service job, so use a unique FIXTURE_NAME before re-recording. +// Do not repeat record mode with the same name: the recorded conflict replaces +// the successful StartBatchEvaluation fixture. +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["bun", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +describe("eval batch-insights (fixture-backed)", () => { + test("get returns the service-side insights job directly", async () => { + const stdout = await run(["eval", "batch-insights", "get", "--id", FIXTURE_JOB_ID, "--json"]); + + matchGolden(FIXTURES, "get.golden.json", stdout); + const detail = JSON.parse(stdout); + expect(detail.status).toBe("COMPLETED"); + expect(detail.insights).toEqual([{ insightId: "Builtin.Insight.FailureAnalysis" }]); + expect(detail.results).toBeUndefined(); + }); + + test("list filters evaluator-only jobs from the shared service page", async () => { + const stdout = await run(["eval", "batch-insights", "list", "--json"]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + const page = JSON.parse(stdout); + expect(page.batchEvaluations.length).toBeGreaterThan(0); + expect( + page.batchEvaluations.every((job: { insights?: unknown[] }) => job.insights?.length), + ).toBe(true); + expect(page.batchEvaluations[0].batchEvaluationId).toBe(FIXTURE_JOB_ID); + }); + + test("run submits an insights batch job", async () => { + const stdout = await run([ + "eval", + "batch-insights", + "run", + "--name", + FIXTURE_NAME, + "--description", + "Golden batch insights fixture", + "--agent", + FIXTURE_AGENT, + "--json", + ]); + + matchGolden(FIXTURES, "run.golden.json", stdout); + const job = JSON.parse(stdout); + expect(job.batchEvaluationId).toBeTruthy(); + expect(job.insights).toEqual([{ insightId: "Builtin.Insight.FailureAnalysis" }]); + }); +}); diff --git a/src/handlers/eval/batch-insights/batch-insights.test.tsx b/src/handlers/eval/batch-insights/batch-insights.test.tsx new file mode 100644 index 000000000..c1df87306 --- /dev/null +++ b/src/handlers/eval/batch-insights/batch-insights.test.tsx @@ -0,0 +1,221 @@ +import { describe, expect, test } from "bun:test"; +import type { + GetBatchEvaluationResponse, + ListBatchEvaluationsResponse, +} from "@aws-sdk/client-bedrock-agentcore"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["bun", "agentcore", ...args, "--region", REGION]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +describe("eval batch-insights command hierarchy", () => { + test("registers only run, get, and list", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((child) => child.name() === "eval") + ?.children() + .find((child) => child.name() === "batch-insights"); + + expect(group?.children().map((child) => child.name())).toEqual(["run", "get", "list"]); + }); + + test("prints help for a bare invocation without a Core call", async () => { + const { core, stdout } = await run(["eval", "batch-insights", "--json"]); + expect(stdout).toContain("Usage: agentcore eval batch-insights"); + expect(core.eval.calls).toHaveLength(0); + }); +}); + +describe("eval batch-insights run", () => { + test("requires --name and exactly one session source", async () => { + await expect(run(["eval", "batch-insights", "run", "--agent", "agent-1"])).rejects.toThrow( + /--name/, + ); + await expect(run(["eval", "batch-insights", "run", "--name", "insights_run"])).rejects.toThrow( + /exactly one source/, + ); + await expect( + run([ + "eval", + "batch-insights", + "run", + "--name", + "insights_run", + "--agent", + "agent-1", + "--online-eval", + "online-1", + ]), + ).rejects.toThrow(/exactly one source/); + }); + + test("uses failure analysis by default and passes the resolved agent source", async () => { + const { core } = await run([ + "eval", + "batch-insights", + "run", + "--name", + "insights_run", + "--agent", + "agent-1", + "--endpoint", + "prod", + "--start-time", + "2026-08-01T00:00:00Z", + "--end-time", + "2026-08-02T00:00:00Z", + "--session-ids", + "s1", + "s2", + "--description", + "Analyze failures", + "--kms-key-arn", + "arn:aws:kms:us-west-2:123:key/abc", + "--json", + ]); + + expect(core.eval.calls).toEqual([ + { + method: "startBatchInsights", + args: [ + { + name: "insights_run", + description: "Analyze failures", + insightIds: ["Builtin.Insight.FailureAnalysis"], + evaluatorIds: undefined, + source: { + origin: "agent", + agent: "agent-1", + endpoint: "prod", + window: { + startTime: new Date("2026-08-01T00:00:00Z"), + endTime: new Date("2026-08-02T00:00:00Z"), + }, + sessionIds: ["s1", "s2"], + }, + kmsKeyArn: "arn:aws:kms:us-west-2:123:key/abc", + }, + { region: REGION }, + ], + }, + ]); + }); + + test("accepts explicit insights and optional evaluator chaining", async () => { + const { core } = await run([ + "eval", + "batch-insights", + "run", + "--name", + "insights_run", + "--online-eval", + "online-1", + "--insight", + "Builtin.Insight.UserIntent", + "Builtin.Insight.ExecutionSummary", + "--evaluator", + "Builtin.Helpfulness", + "--json", + ]); + + expect(core.eval.calls[0]?.method).toBe("startBatchInsights"); + expect(core.eval.calls[0]?.args[0]).toMatchObject({ + insightIds: ["Builtin.Insight.UserIntent", "Builtin.Insight.ExecutionSummary"], + evaluatorIds: ["Builtin.Helpfulness"], + source: { + origin: "online-eval", + onlineEvaluationConfigId: "online-1", + }, + }); + }); +}); + +describe("eval batch-insights get", () => { + test("returns insight reports without reading CloudWatch scores", async () => { + const response = { + batchEvaluationId: "bi-1", + status: "COMPLETED", + insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], + failureAnalysisResult: { failures: [] }, + } as unknown as GetBatchEvaluationResponse; + const { core, stdout } = await run( + ["eval", "batch-insights", "get", "--id", "bi-1", "--json"], + (client) => client.eval.setBatchEvalGetResponse(response), + ); + + expect(JSON.parse(stdout)).toMatchObject({ + batchEvaluationId: "bi-1", + failureAnalysisResult: { failures: [] }, + }); + expect(JSON.parse(stdout).consoleUrl).toBeUndefined(); + expect(core.eval.calls[0]?.args[2]).toEqual({ includeResults: false }); + }); + + test("rejects an evaluator-only batch evaluation", async () => { + const response = { + batchEvaluationId: "be-1", + evaluators: [{ evaluatorId: "Builtin.Helpfulness" }], + } as unknown as GetBatchEvaluationResponse; + + await expect( + run(["eval", "batch-insights", "get", "--id", "be-1"], (client) => + client.eval.setBatchEvalGetResponse(response), + ), + ).rejects.toThrow(/not a batch insights run/); + }); +}); + +describe("eval batch-insights list", () => { + test("filters mixed service results and preserves pagination", async () => { + const response = { + batchEvaluations: [ + { + batchEvaluationId: "bi-1", + insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], + }, + { + batchEvaluationId: "be-1", + evaluators: [{ evaluatorId: "Builtin.Helpfulness" }], + }, + { batchEvaluationId: "bi-2", insights: [{ insightId: "Builtin.Insight.UserIntent" }] }, + ], + nextToken: "next-page", + } as unknown as ListBatchEvaluationsResponse; + const { core, stdout } = await run( + ["eval", "batch-insights", "list", "--next-token", "page-1", "--max-results", "20", "--json"], + (client) => client.eval.setBatchEvalListResponse(response, "page-1"), + ); + const output = JSON.parse(stdout); + + expect(output.nextToken).toBe("next-page"); + expect( + output.batchEvaluations.map((item: { batchEvaluationId: string }) => item.batchEvaluationId), + ).toEqual(["bi-1", "bi-2"]); + expect(output.batchEvaluations[0].consoleUrl).toBeUndefined(); + expect(core.eval.calls[0]?.args).toEqual(["page-1", 20, { region: REGION }]); + }); +}); diff --git a/src/handlers/eval/batch-insights/get/index.tsx b/src/handlers/eval/batch-insights/get/index.tsx new file mode 100644 index 000000000..065c43222 --- /dev/null +++ b/src/handlers/eval/batch-insights/get/index.tsx @@ -0,0 +1,27 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetBatchInsightsHandler = (core: Core) => + createHandler({ + name: "get", + description: "get a batch insights run and its reports by id", + flags: [flag("id", "the ID of the batch insights run", z.string().optional())], + handle: async (ctx, flags) => { + const id = flags["id"]; + if (!id) throw new InputValidationError("required option '--id ' not specified"); + + const opts = coreOptsFromCtx(ctx); + const { detail } = await core.eval.getBatchEvaluation(id, opts, { + includeResults: false, + }); + if (!detail.insights?.length) { + throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); + } + + ctx.require(JsonRendererKey).renderJson(detail); + }, + }); diff --git a/src/handlers/eval/batch-insights/index.tsx b/src/handlers/eval/batch-insights/index.tsx new file mode 100644 index 000000000..ddcada174 --- /dev/null +++ b/src/handlers/eval/batch-insights/index.tsx @@ -0,0 +1,15 @@ +import type { AppIO } from "../../../io"; +import { Router } from "../../../router"; +import { createHelpDefault } from "../../help"; +import type { Core } from "../../types"; +import { createGetBatchInsightsHandler } from "./get"; +import { createListBatchInsightsHandler } from "./list"; +import { createRunBatchInsightsHandler } from "./run"; + +export function createBatchInsightsHandler(core: Core, io: AppIO): Router { + return new Router("batch-insights", "run and inspect batch insights") + .default(createHelpDefault(io)) + .handler(createRunBatchInsightsHandler(core, io)) + .handler(createGetBatchInsightsHandler(core)) + .handler(createListBatchInsightsHandler(core)); +} diff --git a/src/handlers/eval/batch-insights/list/index.tsx b/src/handlers/eval/batch-insights/list/index.tsx new file mode 100644 index 000000000..7b37e9030 --- /dev/null +++ b/src/handlers/eval/batch-insights/list/index.tsx @@ -0,0 +1,30 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListBatchInsightsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list batch insights runs", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of service items to inspect", z.number().optional()), + ], + handle: async (ctx, flags) => { + const opts = coreOptsFromCtx(ctx); + const response = await core.eval.listBatchEvaluations( + flags["next-token"], + flags["max-results"], + opts, + ); + + ctx.require(JsonRendererKey).renderJson({ + ...response, + batchEvaluations: (response.batchEvaluations ?? []).filter( + (evaluation) => evaluation.insights?.length, + ), + }); + }, + }); diff --git a/src/handlers/eval/batch-insights/run/index.tsx b/src/handlers/eval/batch-insights/run/index.tsx new file mode 100644 index 000000000..e941cb6ca --- /dev/null +++ b/src/handlers/eval/batch-insights/run/index.tsx @@ -0,0 +1,47 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import type { AppIO } from "../../../../io"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { resolveSessionSource, sessionSourceFlags } from "../../sessionSource"; + +const DEFAULT_INSIGHT = "Builtin.Insight.FailureAnalysis"; + +export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => + createHandler({ + name: "run", + description: "start an asynchronous batch insights run over existing sessions", + flags: [ + ...sessionSourceFlags, + flag("insight", "insight id(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT])), + flag( + "evaluator", + "optional evaluator id(s) to run alongside the insights", + z.array(z.string()).optional(), + ), + flag("name", "batch insights name (must be unique in the account)", z.string().optional()), + flag("description", "optional description", z.string().optional()), + flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) { + throw new InputValidationError("required option '--name ' not specified"); + } + + const source = await resolveSessionSource(flags, io); + const response = await core.eval.startBatchInsights( + { + name: flags["name"], + description: flags["description"], + insightIds: flags["insight"], + evaluatorIds: flags["evaluator"], + source, + kmsKeyArn: flags["kms-key-arn"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index add16936e..2e41a78f5 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -7,6 +7,7 @@ import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; import { createDatasetHandler } from "./dataset"; import { createBatchEvaluationHandler } from "./batch-evaluation"; +import { createBatchInsightsHandler } from "./batch-insights"; import { createOnDemandHandler } from "./ondemand"; import { createConfigBundleHandler } from "./config-bundle"; @@ -18,6 +19,7 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createOnlineEvalHandler(core, io)) .handler(createDatasetHandler(core, io)) .handler(createBatchEvaluationHandler(core, io)) + .handler(createBatchInsightsHandler(core, io)) .handler(createOnDemandHandler(core, io)) .handler(createConfigBundleHandler(core, io)); } diff --git a/src/handlers/eval/sessionSource.tsx b/src/handlers/eval/sessionSource.tsx new file mode 100644 index 000000000..cb80d8f40 --- /dev/null +++ b/src/handlers/eval/sessionSource.tsx @@ -0,0 +1,126 @@ +import type { DataSourceConfig } from "@aws-sdk/client-bedrock-agentcore"; +import z from "zod"; +import { InputValidationError } from "../../errors"; +import { SourceResolver, type AppIO } from "../../io"; +import { flag, type Flag } from "../../router"; +import { parseJsonFlag } from "../utils"; +import type { SessionSourceValue, SessionWindow } from "./types"; + +export const sessionSourceFlags = [ + flag("agent", "source: harness id or runtime id whose sessions to use", z.string().optional()), + flag( + "endpoint", + "runtime endpoint qualifier (default DEFAULT; only with --agent)", + z.string().optional(), + ), + flag( + "online-eval", + "source: use sessions an online-eval config already sampled", + z.string().optional(), + ), + flag( + "data-source-config", + "source: raw DataSourceConfig JSON (inline, file://, or -); escape hatch", + z.string().optional(), + ), + flag( + "start-time", + "time filter: window start (ISO-8601, with --end-time)", + z.string().optional(), + ), + flag("end-time", "time filter: window end (ISO-8601, with --start-time)", z.string().optional()), + flag( + "session-ids", + "filter: specific session ids (only with --agent)", + z.array(z.string()).optional(), + ), +] as const; + +export type SessionSourceFlags = { + [F in (typeof sessionSourceFlags)[number] as F["name"]]: F extends Flag + ? T + : never; +}; + +export async function resolveSessionSource( + flags: SessionSourceFlags, + io: AppIO, +): Promise { + const resolver = new SourceResolver({ stdin: io.stdin }); + const rawDataSourceConfig = parseJsonFlag( + "data-source-config", + await resolver.resolveText("data-source-config", flags["data-source-config"]), + ); + return resolveDataSource(flags, rawDataSourceConfig); +} + +function resolveDataSource( + flags: SessionSourceFlags, + rawDataSourceConfig: DataSourceConfig | undefined, +): SessionSourceValue { + const hasAgent = flags["agent"] !== undefined; + const hasOnlineEval = flags["online-eval"] !== undefined; + const hasRaw = rawDataSourceConfig !== undefined; + + const armCount = [hasAgent, hasOnlineEval, hasRaw].filter(Boolean).length; + if (armCount !== 1) { + throw new InputValidationError( + "specify exactly one source: '--agent', '--online-eval', or '--data-source-config'", + ); + } + + const hasIds = !!flags["session-ids"]?.length; + + if (hasRaw) { + if ( + flags["start-time"] !== undefined || + flags["end-time"] !== undefined || + hasIds || + flags["endpoint"] !== undefined + ) { + throw new InputValidationError( + "filter flags cannot be combined with '--data-source-config' (put them in the JSON)", + ); + } + return { origin: "raw", dataSourceConfig: rawDataSourceConfig }; + } + + const window = resolveWindow(flags); + + if (hasOnlineEval) { + // The online-eval arm has no sessionIds filter and no endpoint. + if (hasIds) + throw new InputValidationError("'--session-ids' cannot be used with '--online-eval'"); + if (flags["endpoint"]) + throw new InputValidationError("'--endpoint' can only be used with '--agent'"); + return { origin: "online-eval", onlineEvaluationConfigId: flags["online-eval"]!, window }; + } + + return { + origin: "agent", + agent: flags["agent"]!, + endpoint: flags["endpoint"], + window, + sessionIds: hasIds ? flags["session-ids"] : undefined, + }; +} + +// resolveWindow validates the explicit time window: both halves must come +// together and start must precede end. +function resolveWindow(flags: SessionSourceFlags): SessionWindow | undefined { + const hasStart = flags["start-time"] !== undefined; + const hasEnd = flags["end-time"] !== undefined; + if (!hasStart && !hasEnd) return undefined; + if (!hasStart || !hasEnd) { + throw new InputValidationError("--start-time and --end-time must be provided together"); + } + const startTime = new Date(flags["start-time"]!); + const endTime = new Date(flags["end-time"]!); + if (Number.isNaN(+startTime) || Number.isNaN(+endTime)) { + throw new InputValidationError("--start-time and --end-time must be ISO-8601 timestamps"); + } + if (+startTime >= +endTime) { + throw new InputValidationError("--start-time must be before --end-time"); + } + return { startTime, endTime }; +} diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 226c17f81..52aa9fb28 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -205,6 +205,17 @@ export type StartBatchEvaluationInput = { kmsKeyArn?: string; }; +// Batch insights use the same service job API as batch evaluations, but remain +// a distinct Core operation so each command keeps its own required fields. +export type StartBatchInsightsInput = { + name: string; + description?: string; + insightIds: string[]; + evaluatorIds?: string[]; + source: SessionSourceValue; + kmsKeyArn?: string; +}; + export type SpanRecord = Record; export type SessionTrace = { @@ -300,6 +311,10 @@ export interface CoreEvalClient { input: StartBatchEvaluationInput, options: CoreOptions, ): Promise; + startBatchInsights( + input: StartBatchInsightsInput, + options: CoreOptions, + ): Promise; // getTracesForAgent resolves the agent to its runtime log group and reads the // target sessions' traces client-side (CloudWatch Logs Insights), grouped by diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 50047db83..5c82156cd 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -137,6 +137,7 @@ import type { GetTracesInput, LlmAsAJudgeUpdate, SessionTrace, + StartBatchInsightsInput, StartBatchEvaluationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, @@ -1699,6 +1700,15 @@ export class TestEvalClient implements CoreEvalClient { return this.startBatchEvalResponse; } + async startBatchInsights( + input: StartBatchInsightsInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "startBatchInsights", args: [input, options] }); + if (this.error) throw this.error; + return this.startBatchEvalResponse; + } + // setGetTracesResponse sets what getTracesForAgent resolves to (when not // erroring). setGetTracesResponse(traces: SessionTrace[]): this {