From 59e81dd6d0ce966b557ae31766e31e194cf5a8a8 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 12 Aug 2026 17:40:47 +0000 Subject: [PATCH 01/11] feat(eval): add ondemand evaluate (synchronous, client-side) --- src/core/eval.tsx | 338 ++++++++++ src/core/types.tsx | 2 +- src/handlers/eval/index.tsx | 4 +- .../EvaluateCommand.76a9b708fd699093.json | 23 + .../EvaluateCommand.c0c383465d2d11bc.json | 23 + ...tAgentRuntimeCommand.9f77333d1b9dcf5d.json | 47 ++ .../GetEvaluatorCommand.716589b0884f35c0.json | 51 ++ ...tQueryResultsCommand.19164cb5cd9b9b70.json | 586 ++++++++++++++++++ ...tQueryResultsCommand.275f4a33f37e8fc7.json | 212 +++++++ .../StartQueryCommand.1623083efa971538.json | 3 + .../StartQueryCommand.8981323eb27080a.json | 3 + .../__fixtures__/evaluate.golden.json | 44 ++ src/handlers/eval/ondemand/evaluate/index.tsx | 135 ++++ src/handlers/eval/ondemand/index.tsx | 13 + .../eval/ondemand/ondemand.fixture.test.tsx | 88 +++ src/handlers/eval/ondemand/ondemand.test.tsx | 178 ++++++ src/handlers/eval/types.tsx | 60 ++ src/testing/TestCoreClient.tsx | 35 ++ 18 files changed, 1843 insertions(+), 2 deletions(-) create mode 100644 src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json create mode 100644 src/handlers/eval/ondemand/evaluate/index.tsx create mode 100644 src/handlers/eval/ondemand/index.tsx create mode 100644 src/handlers/eval/ondemand/ondemand.fixture.test.tsx create mode 100644 src/handlers/eval/ondemand/ondemand.test.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index c7e1baaa8..d83a9eed3 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -33,19 +33,32 @@ import { type DataSourceConfig, type ListOnlineEvaluationConfigsResponse, type Rule, + type EvaluatorLevel, type UpdateEvaluatorResponse, type UpdateOnlineEvaluationConfigResponse, type BedrockAgentCoreControlClient, } from "@aws-sdk/client-bedrock-agentcore-control"; import { + EvaluateCommand, GetBatchEvaluationCommand, ListBatchEvaluationsCommand, StartBatchEvaluationCommand, + type EvaluationReferenceInput, + type EvaluationResultContent, + type EvaluationTarget, type ListBatchEvaluationsResponse, type StartBatchEvaluationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, type CloudWatchFilterConfig, } from "@aws-sdk/client-bedrock-agentcore"; +import { + GetQueryResultsCommand, + ResourceNotFoundException, + StartQueryCommand, + type CloudWatchLogsClient, + type ResultField, +} from "@aws-sdk/client-cloudwatch-logs"; +import type { DocumentType } from "@smithy/types"; import { Transform } from "node:stream"; import { FileWriteError, InputValidationError, NetworkingError } from "../errors"; import type { @@ -55,9 +68,14 @@ import type { CoreEvalClient, CreateDatasetInput, CreateOnlineEvalInput, + EvaluateInput, + EvaluateResult, GetBatchEvaluationResult, + GetTracesInput, LlmAsAJudgeUpdate, SessionSourceValue, + SessionTrace, + SpanRecord, StartBatchEvaluationInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -77,6 +95,19 @@ import { const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; +// The shared, account-level OTel span log group. +const SPANS_LOG_GROUP = "aws/spans"; + +// Default discovery window when no explicit --start/--end or --lookback-days is +// given. Mirrors the batch service's now-7d default. +const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000; + +// A hard Evaluate limit: at most 10 trace/span ids per request. +const EVALUATE_TARGET_BATCH = 10; + +// CloudWatch Logs Insights hard ceiling: a query returns at most 100k rows. +const INSIGHTS_MAX_ROWS = 100_000; + // noopLogger is the default for the optional logger arg so callers that don't // need batch-evaluation result-log diagnostics (e.g. dataset-only tests) can // omit it. Production (src/core/index.tsx) injects a real child logger. @@ -339,6 +370,98 @@ export class EvalClient implements CoreEvalClient { }; } + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { + const qualifier = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; + + const { runtimeId, runtimeName } = await resolveAgentToNameAndId( + input.agent, + this.clients, + options, + ); + const logGroupName = runtimeLogGroup(runtimeId, qualifier); + const serviceName = runtimeServiceName(runtimeName, qualifier); + + // CloudWatch Insights takes epoch seconds. Discovery defaults to now-7d when + // no explicit window is given (matches the batch service's default). + const endMs = input.window ? +input.window.endTime : Date.now(); + const startMs = input.window ? +input.window.startTime : endMs - SEVEN_DAYS_MS; + const startSec = Math.floor(startMs / 1000); + const endSec = Math.floor(endMs / 1000); + + const logs = this.clients.logs(toClientConfig(options)); + const queryString = buildSpanQuery(serviceName, input.sessionIds, input.traceId); + + // Runtime group required (missing = agent has no traces); aws/spans optional now + // https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html#observability-configure-unified-traces + const [runtimeRows, sharedRows] = await Promise.all([ + runInsightsQuery(logs, [logGroupName], queryString, startSec, endSec).catch((error) => { + if (error instanceof ResourceNotFoundException) { + throw new InputValidationError( + `No telemetry found for agent "${input.agent}": its runtime log group ${logGroupName} ` + + `does not exist. Ensure the agent has been invoked and emits traces.`, + { meta: { agent: input.agent, logGroupName } }, + ); + } + throw error; + }), + runInsightsQuery(logs, [SPANS_LOG_GROUP], queryString, startSec, endSec).catch((error) => { + if (error instanceof ResourceNotFoundException) return []; + throw error; + }), + ]); + const traces = groupSpansBySession([...sharedRows, ...runtimeRows]); + + // Warn when explicitly requested sessions never showed up in the logs (aged + // out, wrong id, or never emitted) so a caller isn't misled by a partial run. + if (input.sessionIds?.length) { + const found = new Set(traces.map((t) => t.sessionId)); + const missing = input.sessionIds.filter((id) => !found.has(id)); + if (missing.length > 0) { + this.logger.warn(`requested sessions not found in logs: ${missing.join(", ")}`); + } + } + return traces; + } + + async evaluate(input: EvaluateInput, options: CoreOptions): Promise { + const data = this.clients.data(toClientConfig(options)); + const control = this.clients.control(toClientConfig(options)); + const levels = await resolveEvaluatorLevels(input.evaluatorIds, control, options); + const refsBySession = groupRefsBySession(input.groundTruth); + + const results: EvaluationResultContent[] = []; + // Sessions that actually produced Evaluate results — distinct from the sessions + // handed in, since a TRACE/TOOL_CALL session with no matching ids makes no call. + const evaluatedSessions = new Set(); + for (const evaluatorId of input.evaluatorIds) { + const level = levels.get(evaluatorId) ?? "SESSION"; + for (const trace of input.traces) { + // TRACE/TOOL_CALL sessions with no ids at that level contribute no calls + // (empty batch list); SESSION always makes one call with no target. + for (const target of targetBatches(level, trace)) { + const response = await data.send( + new EvaluateCommand({ + evaluatorId, + // SpanRecord is Record for ergonomic reads; the API + // wants DocumentType[] (assignable to it, so a single cast suffices). + evaluationInput: { sessionSpans: trace.spans as DocumentType[] }, + evaluationTarget: target, + evaluationReferenceInputs: refsBySession.get(trace.sessionId), + }), + ); + const evaluationResults = response.evaluationResults ?? []; + if (evaluationResults.length > 0) evaluatedSessions.add(trace.sessionId); + results.push(...evaluationResults); + } + } + } + return { + sessionsRequested: input.traces.length, + sessionsEvaluated: evaluatedSessions.size, + results, + }; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, @@ -787,6 +910,221 @@ async function agentDataSource( }; } +// sanitizeQueryValue strips single quotes so an id can't break out of the quoted +// Insights filter literal it is interpolated into (matches the old CLI). +function sanitizeQueryValue(value: string): string { + return value.replace(/'/g, ""); +} + +// buildSpanQuery is the single-phase Insights query: scope to one runtime by its +// OTel service.name, optionally narrow to specific sessions and/or one trace, and +// select the full span JSON (@message) plus the session id to group by. It does +// NOT over-filter on ispresent(kind) — that span-only predicate is what forced the +// old CLI's second query for log records; the looser scope returns everything for +// the session in one pass. +function buildSpanQuery(serviceName: string, sessionIds?: string[], traceId?: string): string { + let query = `fields @message, attributes.session.id as sessionId, traceId, spanId + | filter resource.attributes.service.name in ['${sanitizeQueryValue(serviceName)}']`; + if (sessionIds && sessionIds.length > 0) { + const ids = sessionIds.map((id) => `'${sanitizeQueryValue(id)}'`).join(", "); + query += `\n | filter attributes.session.id in [${ids}]`; + } + if (traceId) { + query += `\n | filter traceId = '${sanitizeQueryValue(traceId)}'`; + } + query += `\n | sort @timestamp asc\n | limit ${INSIGHTS_MAX_ROWS}`; + return query; +} + +// runInsightsQuery starts a CloudWatch Logs Insights query, waits for it to finish, +// then drains all result pages. GetQueryResults returns <=10k rows per call, so a +// long session's spans span multiple pages (nextToken); dropping any would score a +// partial conversation. Fails fast if the ceiling is hit — that belongs in batch. +async function runInsightsQuery( + logs: CloudWatchLogsClient, + logGroupNames: string[], + queryString: string, + startSec: number, + endSec: number, +): Promise { + const started = await logs.send( + new StartQueryCommand({ logGroupNames, queryString, startTime: startSec, endTime: endSec }), + ); + const queryId = started.queryId; + + // Phase 1: wait for completion. A large scan can take minutes, so the deadline is + // generous; each poll costs one cheap GetQueryResults call. + let status = "Running"; + for (let i = 0; i < 300 && status !== "Complete"; i++) { + const result = await logs.send(new GetQueryResultsCommand({ queryId })); + status = result.status ?? "Unknown"; + if (status === "Failed" || status === "Cancelled" || status === "Timeout") { + throw new NetworkingError(`CloudWatch Logs Insights query ${status.toLowerCase()}`, { + meta: { queryId }, + }); + } + if (status !== "Complete") await new Promise((resolve) => setTimeout(resolve, 1000)); + } + if (status !== "Complete") { + throw new NetworkingError("CloudWatch Logs Insights query did not finish in time", { + meta: { queryId }, + }); + } + + // Phase 2: drain pages. Terminates on nextToken; total is bounded by the query's + // `| limit INSIGHTS_MAX_ROWS`. + const rows: ResultField[][] = []; + let nextToken: string | undefined; + do { + const result = await logs.send(new GetQueryResultsCommand({ queryId, nextToken })); + rows.push(...(result.results ?? [])); + nextToken = result.nextToken; + } while (nextToken); + + if (rows.length >= INSIGHTS_MAX_ROWS) { + throw new InputValidationError( + `Too many spans in scope (>= ${INSIGHTS_MAX_ROWS}). Narrow --session-ids or the time ` + + `window, or use 'eval batch-evaluation' for large jobs.`, + ); + } + return rows; +} + +// Group parsed @message docs by session, keeping only sessions with >=1 span +// (Evaluate rejects log-only sessions), and derive each session's trace/tool ids. +function groupSpansBySession(rows: ResultField[][]): SessionTrace[] { + const docsBySession = new Map(); + const sessionsWithSpans = new Set(); + for (const row of rows) { + const message = row.find((f) => f.field === "@message")?.value; + const sessionId = row.find((f) => f.field === "sessionId")?.value; + // Drop orphan records with no session id (e.g. system logs) — an + // unidentifiable session can't be evaluated. + if (!message || !sessionId) continue; + let doc: SpanRecord; + try { + doc = JSON.parse(message) as SpanRecord; + } catch { + continue; + } + const list = docsBySession.get(sessionId); + if (list) list.push(doc); + else docsBySession.set(sessionId, [doc]); + // distinquish between log record and span (which has "kind") + if ("kind" in doc) sessionsWithSpans.add(sessionId); + } + return [...docsBySession] + .filter(([sessionId]) => sessionsWithSpans.has(sessionId)) + .map(([sessionId, spans]) => ({ + sessionId, + spans, + traceIds: extractTraceIds(spans), + toolCallSpanIds: extractToolCallSpanIds(spans), + })); +} + +// extractTraceIds pulls the distinct trace ids out of a session's spans, preserving +// first-seen order (ported from the old CLI's span-collector). +function extractTraceIds(spans: SpanRecord[]): string[] { + const seen = new Set(); + const traceIds: string[] = []; + for (const span of spans) { + const traceId = span.traceId; + // Skip empty ids: log records not tied to a trace carry traceId "", and the + // Evaluate API rejects any target id that isn't a 32-char trace id. + if (typeof traceId === "string" && traceId.length > 0 && !seen.has(traceId)) { + seen.add(traceId); + traceIds.push(traceId); + } + } + return traceIds; +} + +// isToolSpan classifies a tool-execution span by the framework markers the AgentCore +// evaluation SDK uses (Strands / OpenInference / Traceloop). Exact, case-sensitive — +// OpenInference is "TOOL" (upper), Traceloop is "tool" (lower). +function isToolSpan(attrs: Record): boolean { + return ( + attrs["gen_ai.operation.name"] === "execute_tool" || + attrs["openinference.span.kind"] === "TOOL" || + attrs["traceloop.span.kind"] === "tool" + ); +} + +// extractToolCallSpanIds pulls the span ids of tool-execution spans for TOOL_CALL +// evaluators. A tool name attribute alone is unreliable (Traceloop names tools via +// traceloop.entity.name, not tool.name), so classify by span kind instead. +function extractToolCallSpanIds(spans: SpanRecord[]): string[] { + const spanIds: string[] = []; + for (const span of spans) { + const spanId = span.spanId; + if (typeof spanId !== "string" || spanId.length === 0) continue; + if (isToolSpan((span.attributes ?? {}) as Record)) spanIds.push(spanId); + } + return spanIds; +} + +// resolveEvaluatorLevels maps each evaluator id to its evaluation level via +// GetEvaluator — authoritative for both builtins (e.g. Builtin.Helpfulness ⇒ TRACE) +// and custom evaluators, so the level (which decides whether the Evaluate call +// targets trace ids, tool-call span ids, or the whole session) is never guessed. +// GetEvaluator errors propagate: a failed lookup (e.g. AccessDenied, not-found) +// must surface, not silently degrade to SESSION and submit the wrong scope. +async function resolveEvaluatorLevels( + evaluatorIds: string[], + control: BedrockAgentCoreControlClient, + _options: CoreOptions, +): Promise> { + const levels = new Map(); + for (const id of new Set(evaluatorIds)) { + const evaluator = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + // level is a required response field (SDK types it `| undefined` without `?`). + levels.set(id, evaluator.level!); + } + return levels; +} + +// groupRefsBySession indexes ground-truth reference inputs by the session they +// apply to (context.spanContext.sessionId), so each Evaluate call attaches only its +// own session's references — the synchronous Evaluate shape, not batch's job-level +// evaluationMetadata. +function groupRefsBySession( + groundTruth: EvaluationReferenceInput[] | undefined, +): Map { + const map = new Map(); + for (const ref of groundTruth ?? []) { + const sessionId = + ref.context && "spanContext" in ref.context ? ref.context.spanContext?.sessionId : undefined; + if (!sessionId) continue; + const list = map.get(sessionId); + if (list) list.push(ref); + else map.set(sessionId, [ref]); + } + return map; +} + +// targetBatches splits a session's evaluation targets into <=10-id Evaluate calls. +// SESSION evaluators make a single call with no target; TRACE/TOOL_CALL sessions +// with no ids at that level make none (the session is skipped for that evaluator). +function targetBatches( + level: EvaluatorLevel, + trace: SessionTrace, +): (EvaluationTarget | undefined)[] { + if (level === "TRACE") { + return chunk(trace.traceIds, EVALUATE_TARGET_BATCH).map((traceIds) => ({ traceIds })); + } + if (level === "TOOL_CALL") { + return chunk(trace.toolCallSpanIds, EVALUATE_TARGET_BATCH).map((spanIds) => ({ spanIds })); + } + return [undefined]; +} + +function chunk(items: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)); + return out; +} + // A just-written role or inline policy is not visible to the service immediately // (IAM is eventually consistent), and the service validates both when the config // is created. It surfaces as one of two messages depending on which part has not diff --git a/src/core/types.tsx b/src/core/types.tsx index 98a2f338d..9b1a2d4a8 100644 --- a/src/core/types.tsx +++ b/src/core/types.tsx @@ -37,7 +37,7 @@ export type CoreFetch = ( // full ClientConfig so callers can request any client customization (region, // endpoint, ...). export interface AwsClients { - control(config: ClientConfig): BedrockAgentCoreControlClient; + control(config: ClientConfig): BedrockAgentCoreControlClient data(config: ClientConfig): BedrockAgentCoreClient; iam(config: ClientConfig): IAMClient; // logs reads the CloudWatch Logs streams AgentCore writes batch-evaluation diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index da3e6c686..0ecef2055 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -7,6 +7,7 @@ import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; import { createDatasetHandler } from "./dataset"; import { createBatchEvaluationHandler } from "./batch-evaluation"; +import { createOnDemandHandler } from "./ondemand"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") @@ -15,7 +16,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createEvaluatorHandler(core, io)) .handler(createOnlineEvalHandler(core, io)) .handler(createDatasetHandler(core, io)) - .handler(createBatchEvaluationHandler(core, io)); + .handler(createBatchEvaluationHandler(core, io)) + .handler(createOnDemandHandler(core, io)); } export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json new file mode 100644 index 000000000..31796ea2d --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json @@ -0,0 +1,23 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a", + "traceId": "6a7cabfa3bfe9a7348415c7b507648a8" + } + }, + "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 941, + "outputTokens": 118, + "totalTokens": 1059 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json new file mode 100644 index 000000000..f0b13e303 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json @@ -0,0 +1,23 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", + "traceId": "6a7cac0e3fa42bfe5437eef070d1231c" + } + }, + "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.", + "value": 1, + "label": "Above And Beyond", + "tokenUsage": { + "inputTokens": 859, + "outputTokens": 154, + "totalTokens": 1013 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json new file mode 100644 index 000000000..2d3b5e713 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json @@ -0,0 +1,47 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeName": "asdf_MyAgent", + "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeVersion": "1", + "createdAt": { + "$date": "2026-04-23T21:17:21.895Z" + }, + "lastUpdatedAt": { + "$date": "2026-04-23T21:17:35.159Z" + }, + "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: asdf_MyAgent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2", + "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip" + } + }, + "runtime": "PYTHON_3_13", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE", + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json new file mode 100644 index 000000000..1825df228 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json @@ -0,0 +1,51 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorConfig": { + "llmAsAJudge": { + "ratingScale": { + "numerical": [ + { + "value": 0, + "label": "Not helpful at all" + }, + { + "value": 1, + "label": "Very unhelpful" + }, + { + "value": 2, + "label": "Somewhat unhelpful" + }, + { + "value": 3, + "label": "Neutral/Mixed" + }, + { + "value": 4, + "label": "Somewhat helpful" + }, + { + "value": 5, + "label": "Very helpful" + }, + { + "value": 6, + "label": "Above and beyond" + } + ] + } + } + }, + "level": "TRACE", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "lockedForModification": true +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json new file mode 100644 index 000000000..f5626155b --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json @@ -0,0 +1,586 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"1d3bcfe3653d81e4\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392103874218,\"endTimeUnixNano\":1786555392105289657,\"durationNano\":1415439,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "1d3bcfe3653d81e4" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"11655ee4cdde1872\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392105841503,\"endTimeUnixNano\":1786555392106502618,\"durationNano\":661115,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "11655ee4cdde1872" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"210fa6dea654dcbc\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392106868305,\"endTimeUnixNano\":1786555392107751277,\"durationNano\":882972,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "210fa6dea654dcbc" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"cde6ff0cb3289288\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392241890106,\"endTimeUnixNano\":1786555392285594042,\"durationNano\":43703936,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "cde6ff0cb3289288" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"9d3e945db9599945\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392289575652,\"endTimeUnixNano\":1786555392316954181,\"durationNano\":27378529,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "9d3e945db9599945" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"31efc7027314ac76\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392318607342,\"endTimeUnixNano\":1786555392369734375,\"durationNano\":51127033,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "31efc7027314ac76" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"53a0cd3ac657fb83\",\"parentSpanId\":\"db6cce231beea7f0\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392389258299,\"endTimeUnixNano\":1786555394009438471,\"durationNano\":1620180172,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"c1b5339e-9b84-45b0-a4ad-cfd818de8fca\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"tool_use\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1210,\"aws.genai.token_count_total\":1280,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH3RTZMMT4\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "53a0cd3ac657fb83" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"db6cce231beea7f0\",\"parentSpanId\":\"7f719fd629b95bca\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392374099914,\"endTimeUnixNano\":1786555394010008258,\"durationNano\":1635908344,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1210,\"gen_ai.usage.output_tokens\":70,\"gen_ai.server.request.duration\":1601,\"gen_ai.usage.total_tokens\":1280,\"gen_ai.usage.completion_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.374107+00:00\",\"gen_ai.server.time_to_first_token\":1613,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.009969+00:00\",\"gen_ai.usage.input_tokens\":1210,\"aws.genai.token_count_total\":1280,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "db6cce231beea7f0" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"26952cbd9a43fe35\",\"parentSpanId\":\"7f719fd629b95bca\",\"flags\":256,\"name\":\"execute_tool add_numbers\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394010521473,\"endTimeUnixNano\":1786555394011315403,\"durationNano\":793930,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.tool.status\":\"success\",\"gen_ai.tool.call.id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\",\"gen_ai.tool.description\":\"Return the sum of two numbers\",\"gen_ai.tool.json_schema\":\"{\\\"properties\\\": {\\\"a\\\": {\\\"description\\\": \\\"Parameter a\\\", \\\"type\\\": \\\"integer\\\"}, \\\"b\\\": {\\\"description\\\": \\\"Parameter b\\\", \\\"type\\\": \\\"integer\\\"}}, \\\"required\\\": [\\\"a\\\", \\\"b\\\"], \\\"type\\\": \\\"object\\\"}\",\"aws.genai.span_kind\":\"TOOL\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.010538+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"execute_tool\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.011292+00:00\",\"gen_ai.tool.name\":\"add_numbers\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "26952cbd9a43fe35" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7f719fd629b95bca\",\"parentSpanId\":\"a5e83c7e772efdf6\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392373929874,\"endTimeUnixNano\":1786555394011480146,\"durationNano\":1637550272,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.011468+00:00\",\"event_loop.cycle_id\":\"23f63c39-82fe-4159-8035-ff8e6208b88f\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.373944+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7f719fd629b95bca" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"0abec106a8ff39b7\",\"parentSpanId\":\"7576c2384130fece\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555394012454655,\"endTimeUnixNano\":1786555397051836921,\"durationNano\":3039382266,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"8926bd3b-32aa-42a7-a234-5bafc1cd8537\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1294,\"aws.genai.token_count_total\":1306,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH3RTZMMT4\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "0abec106a8ff39b7" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7576c2384130fece\",\"parentSpanId\":\"a1d0f9231c5db33d\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394011912861,\"endTimeUnixNano\":1786555397052514047,\"durationNano\":3040601186,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1294,\"gen_ai.usage.output_tokens\":12,\"gen_ai.server.request.duration\":3034,\"gen_ai.usage.total_tokens\":1306,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.011919+00:00\",\"gen_ai.server.time_to_first_token\":2804,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.052476+00:00\",\"gen_ai.usage.input_tokens\":1294,\"aws.genai.token_count_total\":1306,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7576c2384130fece" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a1d0f9231c5db33d\",\"parentSpanId\":\"a5e83c7e772efdf6\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394011746512,\"endTimeUnixNano\":1786555397052873967,\"durationNano\":3041127455,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.052853+00:00\",\"event_loop.cycle_id\":\"beebe4f0-27a6-4aef-a0e3-09cda0c61a19\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.011757+00:00\",\"event_loop.parent_cycle_id\":\"23f63c39-82fe-4159-8035-ff8e6208b88f\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a1d0f9231c5db33d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"5aa1ead48a6ac0e1\",\"parentSpanId\":\"e1355f7dda87635b\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1786555392052368325,\"endTimeUnixNano\":1786555397053469078,\"durationNano\":5001100753,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":55860,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "5aa1ead48a6ac0e1" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a5e83c7e772efdf6\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392372920661,\"endTimeUnixNano\":1786555397053072269,\"durationNano\":4680151608,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2504,\"gen_ai.usage.output_tokens\":82,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2586,\"gen_ai.usage.completion_tokens\":82,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.372942+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.053049+00:00\",\"gen_ai.usage.input_tokens\":2504,\"aws.genai.token_count_total\":2586,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a5e83c7e772efdf6" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"42f4da466ab4c999\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411324466077,\"endTimeUnixNano\":1786555411324998705,\"durationNano\":532628,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "42f4da466ab4c999" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"6d003c695538a610\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411323051352,\"endTimeUnixNano\":1786555411324071465,\"durationNano\":1020113,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "6d003c695538a610" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"8a05cdd0a93d3a6a\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411325315616,\"endTimeUnixNano\":1786555411325825454,\"durationNano\":509838,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "8a05cdd0a93d3a6a" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"0745de30c628c8e5\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411431858312,\"endTimeUnixNano\":1786555411485160682,\"durationNano\":53302370,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "0745de30c628c8e5" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"cb5ae6a1e269e21b\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411487794771,\"endTimeUnixNano\":1786555411518235079,\"durationNano\":30440308,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "cb5ae6a1e269e21b" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"b043a295f3c1c4cf\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411519590941,\"endTimeUnixNano\":1786555411598192831,\"durationNano\":78601890,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "b043a295f3c1c4cf" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"b556c691098ede16\",\"parentSpanId\":\"627f6364bb0599fa\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411614408678,\"endTimeUnixNano\":1786555414476372175,\"durationNano\":2861963497,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"47288f71-5ae9-4b15-9720-d49cf3e180f2\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":48,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHYLSAHMVB\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "b556c691098ede16" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"627f6364bb0599fa\",\"parentSpanId\":\"e90c8e0cb4712af5\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411601581872,\"endTimeUnixNano\":1786555414476887232,\"durationNano\":2875305360,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":48,\"gen_ai.server.request.duration\":2842,\"gen_ai.usage.total_tokens\":1251,\"gen_ai.usage.completion_tokens\":48,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.601589+00:00\",\"gen_ai.server.time_to_first_token\":1966,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.476851+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "627f6364bb0599fa" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"e90c8e0cb4712af5\",\"parentSpanId\":\"00821fcc8a234519\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411601446390,\"endTimeUnixNano\":1786555414477214324,\"durationNano\":2875767934,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.477197+00:00\",\"event_loop.cycle_id\":\"cbaf6d7e-50f0-436e-8c06-6c5d9e6c5936\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.601458+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "e90c8e0cb4712af5" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA0YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"00821fcc8a234519\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411600856280,\"endTimeUnixNano\":1786555414477366577,\"durationNano\":2876510297,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":48,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1251,\"gen_ai.usage.completion_tokens\":48,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.600874+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.477345+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "00821fcc8a234519" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA4YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"82efc476a79eaee2\",\"parentSpanId\":\"1cc0af0f799f67f1\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1786555411282407024,\"endTimeUnixNano\":1786555414477671431,\"durationNano\":3195264407,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":34652,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "82efc476a79eaee2" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA8YAQ==" + } + ] + ], + "statistics": { + "recordsMatched": 26, + "recordsScanned": 26, + "estimatedRecordsSkipped": 0, + "bytesScanned": 47213, + "estimatedBytesSkipped": 0, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json new file mode 100644 index 000000000..a4401cbb3 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json @@ -0,0 +1,212 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394010008258,\"observedTimeUnixNano\":1786555395351842794,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"tool_use\",\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"db6cce231beea7f0\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "db6cce231beea7f0" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394011480146,\"observedTimeUnixNano\":1786555395352095155,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\",\"tool.result\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}},{\"role\":\"assistant\",\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7f719fd629b95bca\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7f719fd629b95bca" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394011315403,\"observedTimeUnixNano\":1786555395351997309,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"tool\",\"content\":{\"role\":\"tool\",\"content\":\"{\\\"a\\\": 2, \\\"b\\\": 2}\",\"id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"text\\\": \\\"4\\\"}]\",\"id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"26952cbd9a43fe35\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "26952cbd9a43fe35" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397052514047,\"observedTimeUnixNano\":1786555400454612062,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"2 + 2 = 4\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7576c2384130fece\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7576c2384130fece" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397052873967,\"observedTimeUnixNano\":1786555400454777045,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a1d0f9231c5db33d\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a1d0f9231c5db33d" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397053072269,\"observedTimeUnixNano\":1786555400454894614,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"2 + 2 = 4\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a5e83c7e772efdf6\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a5e83c7e772efdf6" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414476887232,\"observedTimeUnixNano\":1786555419396909949,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"A primary color is **red**.\\\\n\\\\nThe three primary colors are red, blue, and yellow (in traditional color theory) or red, green, and blue (in the RGB/additive color model used for light).\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"627f6364bb0599fa\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "627f6364bb0599fa" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414477366577,\"observedTimeUnixNano\":1786555419397161059,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"A primary color is **red**.\\n\\nThe three primary colors are red, blue, and yellow (in traditional color theory) or red, green, and blue (in the RGB/additive color model used for light).\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"00821fcc8a234519\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "00821fcc8a234519" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414477214324,\"observedTimeUnixNano\":1786555419397054030,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"e90c8e0cb4712af5\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "e90c8e0cb4712af5" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQAxgB" + } + ] + ], + "statistics": { + "recordsMatched": 9, + "recordsScanned": 72, + "estimatedRecordsSkipped": 3, + "bytesScanned": 68886, + "estimatedBytesSkipped": 349, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json new file mode 100644 index 000000000..4320f369e --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json @@ -0,0 +1,3 @@ +{ + "queryId": "3c802de9-ea3d-4268-ae53-944d121d2618" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json new file mode 100644 index 000000000..cd9387e26 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json @@ -0,0 +1,3 @@ +{ + "queryId": "cdc73753-869e-4222-b606-a7e5b2e7d9a5" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json b/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json new file mode 100644 index 000000000..458c13bd7 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json @@ -0,0 +1,44 @@ +{ + "sessionsRequested": 2, + "sessionsEvaluated": 2, + "results": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a", + "traceId": "6a7cabfa3bfe9a7348415c7b507648a8" + } + }, + "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 941, + "outputTokens": 118, + "totalTokens": 1059 + } + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", + "traceId": "6a7cac0e3fa42bfe5437eef070d1231c" + } + }, + "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.", + "value": 1, + "label": "Above And Beyond", + "tokenUsage": { + "inputTokens": 859, + "outputTokens": 154, + "totalTokens": 1013 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/evaluate/index.tsx b/src/handlers/eval/ondemand/evaluate/index.tsx new file mode 100644 index 000000000..b65416509 --- /dev/null +++ b/src/handlers/eval/ondemand/evaluate/index.tsx @@ -0,0 +1,135 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { coreOptsFromCtx, parseJsonArrayFlag } from "../../../utils"; +import type { SessionWindow } from "../../types"; + +export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) => + createHandler({ + name: "evaluate", + description: "evaluate existing sessions client-side (synchronous; prints scores)", + flags: [ + flag( + "agent", + "source: harness id or runtime id whose sessions to evaluate", + z.string().optional(), + ), + flag("endpoint", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag( + "lookback-days", + "time filter: evaluate sessions from the last N days", + z.number().optional(), + ), + flag( + "start-time", + "time filter: window start (ISO-8601, with --end-time)", + z.string().optional(), + ), + flag( + "end-time", + "time filter: window end (ISO-8601, with --start-time)", + z.string().optional(), + ), + flag("session-ids", "filter: specific session ids", z.array(z.string()).optional()), + flag( + "trace-id", + "filter: a single trace id (session id is read off the span)", + z.string().optional(), + ), + flag( + "ground-truth", + "ground truth (JSON EvaluationReferenceInput[]; inline, file://, or -)", + z.string().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["agent"]) { + throw new InputValidationError("on-demand requires '--agent'"); + } + if (!flags["evaluator"] || flags["evaluator"].length === 0) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + const window = resolveWindow(flags); + const sessionIds = flags["session-ids"]; + const traceId = flags["trace-id"]; + // On-demand reads sessions client-side, so it needs a concrete source — + // "evaluate everything" is not allowed. + if (!window && !sessionIds?.length && !traceId) { + throw new InputValidationError( + "specify a session source: --session-ids, --trace-id, --lookback-days, or --start-time/--end-time", + ); + } + + const opts = coreOptsFromCtx(ctx); + const traces = await core.eval.getTracesForAgent( + { agent: flags["agent"], endpoint: flags["endpoint"], window, sessionIds, traceId }, + opts, + ); + + // Ground truth is a typed SDK-shape passthrough (identical to batch's handler): + // resolve inline / file:// / -, then hand the array to core verbatim — core + // groups it by session. + const resolver = new SourceResolver({ stdin: io.stdin }); + const groundTruth = parseJsonArrayFlag( + "ground-truth", + await resolver.resolveText("ground-truth", flags["ground-truth"]), + ); + + const result = await core.eval.evaluate( + { traces, evaluatorIds: flags["evaluator"], groundTruth }, + opts, + ); + ctx.require(JsonRendererKey).renderJson(result); + }, + }); + +// resolveWindow validates on-demand's time filter: --lookback-days maps to +// [now - n days, now]; the explicit --start-time/--end-time pair must come together +// with start before end. On-demand owns this rather than reusing batch's resolver: +// batch has no --lookback-days and its window feeds a service-side data source, not +// a client-side Insights query. +function resolveWindow(flags: { + "lookback-days"?: number; + "start-time"?: string; + "end-time"?: string; +}): SessionWindow | undefined { + const lookback = flags["lookback-days"]; + const hasStart = flags["start-time"] !== undefined; + const hasEnd = flags["end-time"] !== undefined; + + if (lookback !== undefined) { + if (hasStart || hasEnd) { + throw new InputValidationError( + "--lookback-days cannot be combined with --start-time/--end-time", + ); + } + if (!Number.isFinite(lookback) || lookback <= 0) { + throw new InputValidationError("--lookback-days must be a positive number"); + } + const endTime = new Date(); + const startTime = new Date(+endTime - lookback * 24 * 60 * 60 * 1000); + return { startTime, endTime }; + } + + if (!hasStart && !hasEnd) return undefined; + if (!hasStart || !hasEnd) { + throw new InputValidationError("--start-time and --end-time must be provided together"); + } + const startTime = new Date(flags["start-time"]!); + const endTime = new Date(flags["end-time"]!); + if (Number.isNaN(+startTime) || Number.isNaN(+endTime)) { + throw new InputValidationError("--start-time and --end-time must be ISO-8601 timestamps"); + } + if (+startTime >= +endTime) { + throw new InputValidationError("--start-time must be before --end-time"); + } + return { startTime, endTime }; +} diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx new file mode 100644 index 000000000..d46cc38ea --- /dev/null +++ b/src/handlers/eval/ondemand/index.tsx @@ -0,0 +1,13 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createEvaluateOnDemandHandler } from "./evaluate"; + +// ondemand groups the synchronous, client-side evaluation commands. It has no TUI +// screen (unlike evaluator/online-eval), so a bare invocation prints help. +export function createOnDemandHandler(core: Core, io: AppIO): Router { + return new Router("ondemand", "evaluate existing sessions synchronously, client-side") + .default(createHelpDefault(io)) + .handler(createEvaluateOnDemandHandler(core, io)); +} diff --git a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx new file mode 100644 index 000000000..f34081c7d --- /dev/null +++ b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx @@ -0,0 +1,88 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with: RECORD=1 bun test src/handlers/eval/ondemand/ondemand.fixture.test.tsx +// +// This exercises the real seam end to end: parsing → handler → CoreClient → +// getTracesForAgent (GetAgentRuntime + CloudWatch Logs Insights StartQuery / +// GetQueryResults, read from aws/spans and the runtime group) → evaluate (the +// Evaluate data-plane API) → rendered scores. +// +// Determinism: the window is PINNED (not --lookback-days) so the StartQuery input — +// which embeds startTime/endTime epoch seconds — hashes to the same fixture on +// record and replay. --session-ids bounds the fetch to the two sessions recorded +// against the live agent below. +// +// Re-recording needs the agent to still exist AND those sessions' spans to still be +// within CloudWatch retention (they age out). If they've aged out, invoke the agent +// to create fresh sessions, then repoint FIXTURE_SESSION_IDS + the window at them. +const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; +const FIXTURE_SESSION_IDS = [ + "67ebf93b-65e3-4127-9e13-483b239f256a", + "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", +]; +const WINDOW_START = "2026-08-12T00:00:00Z"; +const WINDOW_END = "2026-08-13T00:00:00Z"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +describe("eval ondemand evaluate (fixture-backed)", () => { + test("evaluates the target sessions client-side and prints scores", async () => { + const stdout = await run([ + "eval", + "ondemand", + "evaluate", + "--agent", + FIXTURE_AGENT, + "--session-ids", + ...FIXTURE_SESSION_IDS, + "--start-time", + WINDOW_START, + "--end-time", + WINDOW_END, + "--evaluator", + "Builtin.Helpfulness", + ]); + + matchGolden(FIXTURES, "evaluate.golden.json", stdout); + const result = JSON.parse(stdout); + expect(result.sessionsEvaluated).toBeGreaterThan(0); + expect(result.results.length).toBeGreaterThan(0); + expect(result.results[0].evaluatorId).toBe("Builtin.Helpfulness"); + // Recording polls live CloudWatch Insights (1s between polls) and calls Evaluate + // per session, so it needs well over bun's 5s default; replay is instant. + }, 180_000); +}); diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx new file mode 100644 index 000000000..6d7173444 --- /dev/null +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -0,0 +1,178 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../index"; +import { + createSilentLogger, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../testing"; +import type { EvaluateResult, SessionTrace } from "../types"; + +// Command-flow tests for `eval ondemand evaluate`, driven through the real root +// handler against a TestCoreClient (no network). These cover the handler's +// orchestration (getTracesForAgent → evaluate), source-arm validation, and the +// local window resolution. The end-to-end SDK path (Insights + Evaluate) is proven +// by the golden fixture suite, which must be recorded against a live account. + +const TRACE: SessionTrace = { + sessionId: "s1", + spans: [{ traceId: "t1", spanId: "sp1" }], + traceIds: ["t1"], + toolCallSpanIds: [], +}; + +const RESULT: EvaluateResult = { + sessionsRequested: 1, + sessionsEvaluated: 1, + results: [{ evaluatorId: "Builtin.Helpfulness", value: 0.9 } as EvaluateResult["results"][number]], +}; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setGetTracesResponse([TRACE]).setEvaluateResponse(RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +const BASE = [ + "eval", + "ondemand", + "evaluate", + "--agent", + "a-1", + "--evaluator", + "Builtin.Helpfulness", +]; + +describe("eval ondemand command hierarchy", () => { + test("registers evaluate under eval → ondemand", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "ondemand"); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + }); +}); + +describe("eval ondemand evaluate validation", () => { + test("requires --agent", async () => { + await expect( + run([ + "eval", + "ondemand", + "evaluate", + "--evaluator", + "Builtin.Helpfulness", + "--session-ids", + "s1", + ]), + ).rejects.toThrow(/--agent/); + }); + + test("requires --evaluator", async () => { + await expect( + run(["eval", "ondemand", "evaluate", "--agent", "a-1", "--session-ids", "s1"]), + ).rejects.toThrow(/--evaluator/); + }); + + test("rejects an empty session source", async () => { + await expect(run(BASE)).rejects.toThrow(/session source/); + }); + + test("rejects --lookback-days combined with an explicit window", async () => { + await expect( + run([ + ...BASE, + "--lookback-days", + "7", + "--start-time", + "2026-01-01T00:00:00Z", + "--end-time", + "2026-01-02T00:00:00Z", + ]), + ).rejects.toThrow(/cannot be combined/); + }); + + test("rejects a half-open explicit window", async () => { + await expect(run([...BASE, "--start-time", "2026-01-01T00:00:00Z"])).rejects.toThrow( + /together/, + ); + }); + + test("rejects start-time not before end-time", async () => { + await expect( + run([...BASE, "--start-time", "2026-01-02T00:00:00Z", "--end-time", "2026-01-01T00:00:00Z"]), + ).rejects.toThrow(/before/); + }); +}); + +describe("eval ondemand evaluate orchestration", () => { + test("session-ids arm: fetches traces then evaluates them, rendering the result", async () => { + const { core, stdout } = await run([...BASE, "--session-ids", "s1", "s2"]); + + expect(JSON.parse(stdout)).toEqual(RESULT); + + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch?.args[0]).toMatchObject({ + agent: "a-1", + sessionIds: ["s1", "s2"], + window: undefined, + }); + + // evaluate receives exactly the traces getTracesForAgent returned. + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ + traces: [TRACE], + evaluatorIds: ["Builtin.Helpfulness"], + }); + // Order: fetch precedes evaluate. + expect(core.eval.calls.map((c) => c.method)).toEqual(["getTracesForAgent", "evaluate"]); + }); + + test("--trace-id alone is a valid source and is passed to the fetch", async () => { + const { core } = await run([...BASE, "--trace-id", "t1"]); + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch?.args[0]).toMatchObject({ traceId: "t1" }); + }); + + test("--lookback-days resolves to a now-N-days window (start before end)", async () => { + const { core } = await run([...BASE, "--lookback-days", "7"]); + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch).toBeDefined(); + const input = fetch!.args[0] as { window?: { startTime: Date; endTime: Date } }; + expect(input.window).toBeDefined(); + const window = input.window!; + expect(+window.startTime).toBeLessThan(+window.endTime); + const spanDays = (+window.endTime - +window.startTime) / (24 * 60 * 60 * 1000); + expect(spanDays).toBeCloseTo(7, 5); + }); + + test("ground-truth is parsed and passed to evaluate verbatim", async () => { + const groundTruth = [ + { context: { spanContext: { sessionId: "s1" } }, expectedResponse: { text: "hi" } }, + ]; + const { core } = await run([ + ...BASE, + "--session-ids", + "s1", + "--ground-truth", + JSON.stringify(groundTruth), + ]); + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ groundTruth }); + }); +}); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 4a12e554d..c7edd4970 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -25,6 +25,8 @@ import type { ListBatchEvaluationsResponse, StartBatchEvaluationResponse, SessionMetadataShape, + EvaluationReferenceInput, + EvaluationResultContent, DataSourceConfig as DataPlaneDataSourceConfig, } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; @@ -186,6 +188,54 @@ export type StartBatchEvaluationInput = { kmsKeyArn?: string; }; +// SpanRecord is one OTel span/log document — the parsed `@message` JSON of a +// CloudWatch Logs Insights result row. Left open (arbitrary JSON) because it is +// handed to the Evaluate API's `sessionSpans` verbatim; the CLI only reads a few +// well-known fields off it (traceId, spanId, attributes) to route evaluators. +export type SpanRecord = Record; + +// SessionTrace is one session's gathered telemetry, grouped client-side. Neutral +// by design — no Evaluate coupling — so getTracesForAgent stays reusable and +// EvalClient.evaluate owns the mapping into the Evaluate request shape. +export type SessionTrace = { + sessionId: string; // read from attributes.session.id + spans: SpanRecord[]; // full OTel span JSON (@message) + traceIds: string[]; // for TRACE-level evaluators (evaluationTarget.traceIds) + toolCallSpanIds: string[]; // for TOOL_CALL-level evaluators (evaluationTarget.spanIds) +}; + +// GetTracesInput selects which sessions' traces to read for one agent. `sessionIds` +// and `traceId` are independent, optional, AND-ed query filters; with neither, the +// `window` bounds discovery. `window` unset ⇒ the client defaults to now−7d. +export type GetTracesInput = { + agent: string; + endpoint?: string; + window?: SessionWindow; + sessionIds?: string[]; + traceId?: string; +}; + +// EvaluateInput is the CLI-facing shape for the synchronous Evaluate path. +// `groundTruth` is the SDK-native array, passed verbatim; EvalClient groups it by +// session (context.spanContext.sessionId) and attaches per session. +export type EvaluateInput = { + traces: SessionTrace[]; + evaluatorIds: string[]; + groundTruth?: EvaluationReferenceInput[]; +}; + +// EvaluateResult returns the raw Evaluate API results across all evaluators and +// sessions (each carries its own evaluatorId + span context). No aggregation — the +// caller renders the raw scores. The two counts are distinct on purpose: +// `sessionsRequested` is how many gathered sessions were handed to Evaluate; +// `sessionsEvaluated` is how many actually produced results (a TRACE/TOOL_CALL +// session with no matching ids is requested but not evaluated). +export type EvaluateResult = { + sessionsRequested: number; + sessionsEvaluated: number; + results: EvaluationResultContent[]; +}; + // CoreEvalClient is the evaluator, online evaluation, and dataset surface the eval // handlers depend on. It is declared here, next to the handlers that consume it, // and implemented by src/core/eval.tsx (dependency inversion: handlers own the @@ -237,6 +287,16 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; + // getTracesForAgent resolves the agent to its runtime log group and reads the + // target sessions' traces client-side (CloudWatch Logs Insights), grouped by + // session. Returns neutral SessionTrace records — the ondemand handler hands + // them to evaluate. Kept off the Evaluate path so the same fetch is reusable. + getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise; + // evaluate runs evaluators synchronously over already-gathered traces via the + // Evaluate API and returns per-session scores. No job, no CloudWatch — the + // trace read happened in getTracesForAgent. + evaluate(input: EvaluateInput, options: CoreOptions): Promise; + createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index df3414166..0f10e3790 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -110,8 +110,12 @@ import type { CoreEvalClient, CreateDatasetInput, CreateOnlineEvalInput, + EvaluateInput, + EvaluateResult, GetBatchEvaluationResult, + GetTracesInput, LlmAsAJudgeUpdate, + SessionTrace, StartBatchEvaluationInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -1222,6 +1226,12 @@ export class TestEvalClient implements CoreEvalClient { private batchEvalResults: BatchEvaluationResultEntry[] = []; private batchEvalResultsError?: unknown; private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE; + private getTracesResponse: SessionTrace[] = []; + private evaluateResponse: EvaluateResult = { + sessionsRequested: 0, + sessionsEvaluated: 0, + results: [], + }; private error?: Error; // setListResponse sets what listEvaluators resolves to (when not erroring). @@ -1468,6 +1478,31 @@ export class TestEvalClient implements CoreEvalClient { return this.startBatchEvalResponse; } + // setGetTracesResponse sets what getTracesForAgent resolves to (when not + // erroring). + setGetTracesResponse(traces: SessionTrace[]): this { + this.getTracesResponse = traces; + return this; + } + + // setEvaluateResponse sets what evaluate resolves to (when not erroring). + setEvaluateResponse(response: EvaluateResult): this { + this.evaluateResponse = response; + return this; + } + + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { + this.calls.push({ method: "getTracesForAgent", args: [input, options] }); + if (this.error) throw this.error; + return this.getTracesResponse; + } + + async evaluate(input: EvaluateInput, options: CoreOptions): Promise { + this.calls.push({ method: "evaluate", args: [input, options] }); + if (this.error) throw this.error; + return this.evaluateResponse; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, From e265d997563097163e90e23357c52cd4e358f102 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 13 Aug 2026 22:15:06 +0000 Subject: [PATCH 02/11] feat(eval): add batch-evaluation simulate (dataset replay) --- src/core/eval.tsx | 86 ++++++++- src/core/eval/simulate.ts | 94 ++++++++++ src/core/invokeRuntime.ts | 177 ++++++++++++++++++ src/core/runtime.tsx | 164 ++-------------- .../batch-evaluation.test.tsx | 2 +- src/handlers/eval/batch-evaluation/index.tsx | 2 + .../eval/batch-evaluation/simulate/index.tsx | 80 ++++++++ .../simulate/simulate.test.tsx | 130 +++++++++++++ src/handlers/eval/types.tsx | 33 ++++ src/io/index.ts | 1 + src/io/template.test.ts | 32 ++++ src/io/template.ts | 33 ++++ src/testing/TestCoreClient.tsx | 20 ++ 13 files changed, 699 insertions(+), 155 deletions(-) create mode 100644 src/core/eval/simulate.ts create mode 100644 src/core/invokeRuntime.ts create mode 100644 src/handlers/eval/batch-evaluation/simulate/index.tsx create mode 100644 src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx create mode 100644 src/io/template.test.ts create mode 100644 src/io/template.ts diff --git a/src/core/eval.tsx b/src/core/eval.tsx index ba9c0a2a7..7defe211c 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -79,6 +79,7 @@ import { } from "@aws-sdk/client-cloudwatch-logs"; import type { DocumentType } from "@smithy/types"; import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; import { basename, dirname, extname, join } from "node:path"; import { Transform } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; @@ -106,12 +107,17 @@ import type { LlmAsAJudgeUpdate, SessionSourceValue, SessionTrace, + SimulateInput, + SimulateResult, SpanRecord, StartBatchEvaluationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; -import { atomicWrite, atomicWriteStream, readTextFile } from "../io"; +import { atomicWrite, atomicWriteStream, readTextFile, renderJsonTemplate } from "../io"; +import { invokeRuntime } from "./invokeRuntime"; +import { loadDatasetFile, runScenarios, toSessionMetadata } from "./eval/simulate"; +import { normalizeRuntimeInvokeRequest } from "../handlers/runtime/invoke/request"; import { isTerminalStatus, readEvaluationResults } from "./batchEvaluationResults"; import { applyExampleIds, diffExamples, indexRemoteById, parseJsonl } from "./datasetDiff"; import type { Addition } from "./datasetDiff"; @@ -507,6 +513,84 @@ export class EvalClient implements CoreEvalClient { }; } + async simulate(input: SimulateInput, options: CoreOptions): Promise { + // Load scenarios: a local JSONL path directly, else download the dataset id. + const path = (await Bun.file(input.dataset).exists()) + ? input.dataset + : await this.downloadDatasetToTemp(input.dataset, input.datasetVersion, options); + const scenarios = await loadDatasetFile(path); + const byId = new Map(scenarios.map((s) => [s.scenarioId, s])); + + // Resolve the runtime once; normalizeRuntimeInvokeRequest validates auth + fills + // accountId. Invoke is the extracted free fn — no RuntimeClient dependency. + const runtime = await this.clients + .control(toClientConfig(options)) + .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId })); + const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; + + const { ok, failed } = await runScenarios(scenarios, async (scenario) => { + const request = normalizeRuntimeInvokeRequest(runtime, { + runtimeId: input.runtimeId, + qualifier: input.qualifier, + payload: renderJsonTemplate(input.payloadTemplate, { input: scenario.turns[0]?.input ?? "" }), + contentType: "application/json", + accept: "application/json", + applicationHeaders: input.headers, + bearerToken: input.bearerToken, + runtimeSessionId: input.sessionId, + runtimeUserId: input.userId, + }); + const response = await invokeRuntime(deps, request, options); + for await (const _chunk of response.body) { + // Drain the stream so the turn completes; only the session id is needed. + } + if (!response.runtimeSessionId) throw new Error("invoke returned no session id"); + return { scenarioId: scenario.scenarioId, sessionId: response.runtimeSessionId }; + }); + + if (ok.length === 0) { + throw new InputValidationError( + `no scenarios could be invoked (${failed} failed) — nothing to evaluate`, + ); + } + + const job = await this.startBatchEvaluation( + { + name: input.name, + description: input.description, + evaluatorIds: input.evaluatorIds, + source: { + origin: "agent", + agent: input.runtimeId, + endpoint: input.qualifier, + sessionIds: ok.map((r) => r.sessionId), + }, + groundTruth: ok.map((r) => toSessionMetadata(byId.get(r.scenarioId)!, r.sessionId)), + kmsKeyArn: input.kmsKeyArn, + }, + options, + ); + + return { + batchEvaluationId: job.batchEvaluationId, + status: job.status, + scenariosInvoked: ok.length, + scenariosFailed: failed, + }; + } + + // downloadDatasetToTemp streams a dataset version's JSONL to a temp file so + // loadDatasetFile can read it — reuses downloadDataset rather than re-fetching. + private async downloadDatasetToTemp( + id: string, + version: string | undefined, + options: CoreOptions, + ): Promise { + const path = join(tmpdir(), `agentcore-dataset-${randomUUID()}.jsonl`); + await this.downloadDataset(id, version, path, options); + return path; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, diff --git a/src/core/eval/simulate.ts b/src/core/eval/simulate.ts new file mode 100644 index 000000000..54849b0d3 --- /dev/null +++ b/src/core/eval/simulate.ts @@ -0,0 +1,94 @@ +import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; +import { InputValidationError } from "../../errors"; + +// Scenario is one dataset row for replay. Local to this module on purpose — it is +// not a handler contract, so it does not live in handlers/eval/types.tsx. Field +// names mirror the dataset JSONL (snake_case in, camelCase here). +export type Scenario = { + scenarioId: string; + turns: { input: string; expectedResponse?: string }[]; + assertions?: string[]; + expectedTrajectory?: string[]; +}; + +// parseScenarios reads dataset JSONL (one scenario per line) into Scenario records. +export function parseScenarios(text: string): Scenario[] { + const scenarios: Scenario[] = []; + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + let row: Record; + try { + row = JSON.parse(trimmed) as Record; + } catch { + throw new InputValidationError("dataset contains a line that is not valid JSON"); + } + scenarios.push(toScenario(row)); + } + if (scenarios.length === 0) throw new InputValidationError("dataset has no scenarios"); + return scenarios; +} + +function toScenario(row: Record): Scenario { + const turns = Array.isArray(row.turns) ? row.turns : []; + return { + scenarioId: String(row.scenario_id ?? ""), + turns: turns.map((t: Record) => ({ + input: String(t.input ?? ""), + expectedResponse: t.expected_response as string | undefined, + })), + assertions: row.assertions as string[] | undefined, + expectedTrajectory: row.expected_trajectory as string[] | undefined, + }; +} + +// loadDatasetFile reads scenarios from a local JSONL path. The dataset-id path is +// handled by the caller (downloadDataset to a temp file, then this). +export async function loadDatasetFile(path: string): Promise { + return parseScenarios(await Bun.file(path).text()); +} + +// runScenarios runs `worker` over every scenario with bounded concurrency, dropping +// failures (a single bad invocation must not sink the run). Returns the successful +// results plus a failure count so the caller can warn / error on all-failed. +export async function runScenarios( + scenarios: Scenario[], + worker: (scenario: Scenario) => Promise, + concurrency = 5, +): Promise<{ ok: T[]; failed: number }> { + const ok: T[] = []; + let failed = 0; + let next = 0; + const run = async (): Promise => { + while (next < scenarios.length) { + const scenario = scenarios[next++]!; + try { + ok.push(await worker(scenario)); + } catch { + failed++; + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, scenarios.length) }, run)); + return { ok, failed }; +} + +// toSessionMetadata maps a scenario's ground truth onto the session the replay +// created — the batch service's per-session ground-truth shape (inline arm). +export function toSessionMetadata(scenario: Scenario, sessionId: string): SessionMetadataShape { + return { + sessionId, + testScenarioId: scenario.scenarioId, + groundTruth: { + inline: { + assertions: scenario.assertions?.map((text) => ({ text })), + expectedTrajectory: scenario.expectedTrajectory + ? { toolNames: scenario.expectedTrajectory } + : undefined, + turns: scenario.turns + .filter((t) => t.expectedResponse !== undefined) + .map((t) => ({ expectedResponse: { text: t.expectedResponse! } })), + }, + }, + }; +} diff --git a/src/core/invokeRuntime.ts b/src/core/invokeRuntime.ts new file mode 100644 index 000000000..4f9f92e4a --- /dev/null +++ b/src/core/invokeRuntime.ts @@ -0,0 +1,177 @@ +import { randomUUID } from "node:crypto"; +import { InvokeAgentRuntimeCommand } from "@aws-sdk/client-bedrock-agentcore"; +import type { RuntimeInvokeRequest, RuntimeInvokeResponse } from "../handlers/runtime/types"; +import type { Logger } from "../logging"; +import type { AwsClients, CoreFetch, CoreOptions } from "./types"; +import { abortable } from "./abortable"; +import { toClientConfig } from "./utils"; + +// InvokeRuntimeDeps is the slice of a Core client an invoke needs. Passed in as a +// bag (not a sibling client) so both RuntimeClient and EvalClient.simulate can call +// these free functions off their own `this.clients`/`this.fetch`/`this.logger`. +export type InvokeRuntimeDeps = { + clients: AwsClients; + fetch: CoreFetch; + logger: Logger; +}; + +async function* emptyBody(): AsyncGenerator {} + +// invokeRuntime dispatches by auth mode: a bearer token routes to the CUSTOM_JWT +// (raw fetch) path, otherwise the SigV4 SDK path. +export async function invokeRuntime( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + options: CoreOptions, + signal?: AbortSignal, +): Promise { + const { runtimeId, bearerToken } = request; + if (bearerToken !== undefined) { + const logger = deps.logger.child({ + operation: "invokeRuntime", + authMode: "CUSTOM_JWT", + runtimeId, + qualifier: request.qualifier, + region: options.region, + }); + return invokeRuntimeWithCustomJwt(deps, request, bearerToken, options, logger, signal); + } + return invokeRuntimeWithIam(deps, request, options, signal); +} + +async function invokeRuntimeWithCustomJwt( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + bearerToken: string, + options: CoreOptions, + logger: Logger, + signal?: AbortSignal, +): Promise { + const client = deps.clients.data(toClientConfig(options)); + const endpoint = client.config.endpointProvider({ + Region: options.region, + Endpoint: options.endpointUrl, + }); + const url = new URL(endpoint.url); + if (url.protocol !== "https:") { + throw new TypeError("CUSTOM_JWT requires an HTTPS endpoint"); + } + url.pathname = `${url.pathname.replace(/\/?$/, "/")}runtimes/${encodeURIComponent(request.runtimeId)}/invocations`; + url.search = new URLSearchParams({ + accountId: request.accountId, + qualifier: request.qualifier, + }).toString(); + const headers = new Headers(request.applicationHeaders); + try { + headers.set("Authorization", `Bearer ${bearerToken}`); + } catch { + throw new TypeError("Invalid bearer token"); + } + try { + for (const [name, value] of [ + ["Content-Type", request.contentType], + ["Accept", request.accept], + ["Mcp-Session-Id", request.mcpSessionId], + ["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id", request.runtimeSessionId ?? randomUUID()], + ["Mcp-Protocol-Version", request.mcpProtocolVersion], + ["Mcp-Method", request.mcpMethod], + ["Mcp-Name", request.mcpName], + ["X-Amzn-Bedrock-AgentCore-Runtime-User-Id", request.runtimeUserId], + ["X-Amzn-Trace-Id", request.traceId], + ["traceparent", request.traceParent], + ["tracestate", request.traceState], + ["baggage", request.baggage], + ] as const) { + if (value !== undefined) headers.set(name, value); + } + } catch { + throw new TypeError("Invalid Runtime request header"); + } + let response: Response; + try { + response = await deps.fetch(url, { + method: "POST", + redirect: "error", + headers, + body: request.payload as RequestInit["body"], + signal, + }); + } catch (error) { + if (signal?.aborted) throw signal.reason ?? error; + logger + .child({ + errorName: + error instanceof TypeError + ? "TypeError" + : error instanceof Error + ? "Error" + : typeof error, + }) + .debug("Runtime invocation transport failed"); + throw new Error("Runtime invocation failed"); + } + if (!response.ok) { + logger + .child({ httpStatusCode: response.status }) + .debug("Runtime invocation returned a non-success response"); + await response.body?.cancel().catch(() => undefined); + throw new Error(`HTTP ${response.status}`); + } + const body = (response.body as AsyncIterable | null) ?? emptyBody(); + return { + statusCode: response.status, + contentType: response.headers.get("content-type") ?? "", + runtimeSessionId: + response.headers.get("x-amzn-bedrock-agentcore-runtime-session-id") ?? undefined, + mcpSessionId: response.headers.get("mcp-session-id") ?? undefined, + mcpProtocolVersion: response.headers.get("mcp-protocol-version") ?? undefined, + traceId: response.headers.get("x-amzn-trace-id") ?? undefined, + traceParent: response.headers.get("traceparent") ?? undefined, + traceState: response.headers.get("tracestate") ?? undefined, + baggage: response.headers.get("baggage") ?? undefined, + body: signal ? abortable(body, signal) : body, + }; +} + +async function invokeRuntimeWithIam( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + options: CoreOptions, + signal?: AbortSignal, +): Promise { + const { runtimeId, applicationHeaders, bearerToken: _bearerToken, ...input } = request; + const command = new InvokeAgentRuntimeCommand({ ...input, agentRuntimeArn: runtimeId }); + if (applicationHeaders?.length) { + command.middlewareStack.add( + (next) => async (args) => { + const sdkRequest = args.request as { headers: Record }; + for (const [name, value] of applicationHeaders) sdkRequest.headers[name] = value; + return next(args); + }, + { step: "build", name: "runtimeApplicationHeaders" }, + ); + } + let response; + try { + response = await deps.clients.data(toClientConfig(options)).send(command, { + abortSignal: signal, + }); + } catch (error) { + if (signal?.aborted) throw signal.reason ?? error; + throw error; + } + + const body = (response.response as AsyncIterable | undefined) ?? emptyBody(); + return { + statusCode: response.statusCode ?? 0, + contentType: response.contentType ?? "", + runtimeSessionId: response.runtimeSessionId, + mcpSessionId: response.mcpSessionId, + mcpProtocolVersion: response.mcpProtocolVersion, + traceId: response.traceId, + traceParent: response.traceParent, + traceState: response.traceState, + baggage: response.baggage, + body: signal ? abortable(body, signal) : body, + }; +} diff --git a/src/core/runtime.tsx b/src/core/runtime.tsx index 5de368453..1a3ed2027 100644 --- a/src/core/runtime.tsx +++ b/src/core/runtime.tsx @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { GetAgentRuntimeCommand, GetAgentRuntimeEndpointCommand, @@ -11,7 +10,6 @@ import { type ListAgentRuntimesResponse, type ListAgentRuntimeVersionsResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; -import { InvokeAgentRuntimeCommand } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreRuntimeClient, RuntimeInvokeRequest, @@ -19,11 +17,9 @@ import type { } from "../handlers/runtime/types"; import type { Logger } from "../logging"; import type { AwsClients, CoreFetch, CoreOptions } from "./types"; -import { abortable } from "./abortable"; +import { invokeRuntime } from "./invokeRuntime"; import { toClientConfig } from "./utils"; -async function* emptyBody(): AsyncGenerator {} - export class RuntimeClient implements CoreRuntimeClient { constructor( private readonly clients: AwsClients, @@ -31,158 +27,20 @@ export class RuntimeClient implements CoreRuntimeClient { private readonly logger: Logger, ) {} - async invokeRuntime( - request: RuntimeInvokeRequest, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const { runtimeId, bearerToken } = request; - if (bearerToken !== undefined) { - const logger = this.logger.child({ - operation: "invokeRuntime", - authMode: "CUSTOM_JWT", - runtimeId, - qualifier: request.qualifier, - region: options.region, - }); - return this.invokeRuntimeWithCustomJwt(request, bearerToken, options, logger, signal); - } - return this.invokeRuntimeWithIam(request, options, signal); - } - - private async invokeRuntimeWithCustomJwt( + // invokeRuntime delegates to the free function so EvalClient.simulate can reuse + // the same invoke logic without holding a RuntimeClient (both call it off their + // own clients/fetch/logger). + invokeRuntime( request: RuntimeInvokeRequest, - bearerToken: string, options: CoreOptions, - logger: Logger, signal?: AbortSignal, ): Promise { - const client = this.clients.data(toClientConfig(options)); - const endpoint = client.config.endpointProvider({ - Region: options.region, - Endpoint: options.endpointUrl, - }); - const url = new URL(endpoint.url); - if (url.protocol !== "https:") { - throw new TypeError("CUSTOM_JWT requires an HTTPS endpoint"); - } - url.pathname = `${url.pathname.replace(/\/?$/, "/")}runtimes/${encodeURIComponent(request.runtimeId)}/invocations`; - url.search = new URLSearchParams({ - accountId: request.accountId, - qualifier: request.qualifier, - }).toString(); - const headers = new Headers(request.applicationHeaders); - try { - headers.set("Authorization", `Bearer ${bearerToken}`); - } catch { - throw new TypeError("Invalid bearer token"); - } - try { - for (const [name, value] of [ - ["Content-Type", request.contentType], - ["Accept", request.accept], - ["Mcp-Session-Id", request.mcpSessionId], - ["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id", request.runtimeSessionId ?? randomUUID()], - ["Mcp-Protocol-Version", request.mcpProtocolVersion], - ["Mcp-Method", request.mcpMethod], - ["Mcp-Name", request.mcpName], - ["X-Amzn-Bedrock-AgentCore-Runtime-User-Id", request.runtimeUserId], - ["X-Amzn-Trace-Id", request.traceId], - ["traceparent", request.traceParent], - ["tracestate", request.traceState], - ["baggage", request.baggage], - ] as const) { - if (value !== undefined) headers.set(name, value); - } - } catch { - throw new TypeError("Invalid Runtime request header"); - } - let response: Response; - try { - response = await this.fetch(url, { - method: "POST", - redirect: "error", - headers, - body: request.payload as RequestInit["body"], - signal, - }); - } catch (error) { - if (signal?.aborted) throw signal.reason ?? error; - logger - .child({ - errorName: - error instanceof TypeError - ? "TypeError" - : error instanceof Error - ? "Error" - : typeof error, - }) - .debug("Runtime invocation transport failed"); - throw new Error("Runtime invocation failed"); - } - if (!response.ok) { - logger - .child({ httpStatusCode: response.status }) - .debug("Runtime invocation returned a non-success response"); - await response.body?.cancel().catch(() => undefined); - throw new Error(`HTTP ${response.status}`); - } - const body = (response.body as AsyncIterable | null) ?? emptyBody(); - return { - statusCode: response.status, - contentType: response.headers.get("content-type") ?? "", - runtimeSessionId: - response.headers.get("x-amzn-bedrock-agentcore-runtime-session-id") ?? undefined, - mcpSessionId: response.headers.get("mcp-session-id") ?? undefined, - mcpProtocolVersion: response.headers.get("mcp-protocol-version") ?? undefined, - traceId: response.headers.get("x-amzn-trace-id") ?? undefined, - traceParent: response.headers.get("traceparent") ?? undefined, - traceState: response.headers.get("tracestate") ?? undefined, - baggage: response.headers.get("baggage") ?? undefined, - body: signal ? abortable(body, signal) : body, - }; - } - - private async invokeRuntimeWithIam( - request: RuntimeInvokeRequest, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const { runtimeId, applicationHeaders, bearerToken: _bearerToken, ...input } = request; - const command = new InvokeAgentRuntimeCommand({ ...input, agentRuntimeArn: runtimeId }); - if (applicationHeaders?.length) { - command.middlewareStack.add( - (next) => async (args) => { - const sdkRequest = args.request as { headers: Record }; - for (const [name, value] of applicationHeaders) sdkRequest.headers[name] = value; - return next(args); - }, - { step: "build", name: "runtimeApplicationHeaders" }, - ); - } - let response; - try { - response = await this.clients.data(toClientConfig(options)).send(command, { - abortSignal: signal, - }); - } catch (error) { - if (signal?.aborted) throw signal.reason ?? error; - throw error; - } - - const body = (response.response as AsyncIterable | undefined) ?? emptyBody(); - return { - statusCode: response.statusCode ?? 0, - contentType: response.contentType ?? "", - runtimeSessionId: response.runtimeSessionId, - mcpSessionId: response.mcpSessionId, - mcpProtocolVersion: response.mcpProtocolVersion, - traceId: response.traceId, - traceParent: response.traceParent, - traceState: response.traceState, - baggage: response.baggage, - body: signal ? abortable(body, signal) : body, - }; + return invokeRuntime( + { clients: this.clients, fetch: this.fetch, logger: this.logger }, + request, + options, + signal, + ); } async getRuntime( diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index 9eb348b21..b710f529c 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -60,7 +60,7 @@ describe("eval batch-evaluation command hierarchy", () => { .find((c) => c.name() === "eval") ?.children() .find((c) => c.name() === "batch-evaluation"); - expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "get", "list"]); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate", "get", "list"]); }); test("prints help for `eval batch-evaluation --json` without an SDK call", async () => { diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx index da96049ad..3d865cb7c 100644 --- a/src/handlers/eval/batch-evaluation/index.tsx +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -6,6 +6,7 @@ import type { Core } from "../../types"; import { createGetBatchEvaluationHandler } from "./get"; import { createListBatchEvaluationsHandler } from "./list"; import { createEvaluateBatchEvaluationHandler } from "./evaluate"; +import { createSimulateBatchEvaluationHandler } from "./simulate"; // batch-evaluation supports evaluate (start an async job) plus get + list. A bare // invocation opens the interactive TUI (list → get), matching evaluator and @@ -15,6 +16,7 @@ export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { .use(withTuiOnEmptyFlagsAndArgs(core, io)) .default(renderTui(core, io)) .handler(createEvaluateBatchEvaluationHandler(core, io)) + .handler(createSimulateBatchEvaluationHandler(core, io)) .handler(createGetBatchEvaluationHandler(core, io)) .handler(createListBatchEvaluationsHandler(core)); } diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx new file mode 100644 index 000000000..e294ba867 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -0,0 +1,80 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; + +// batch-evaluation simulate replays a dataset against a runtime (invoke per scenario) +// then submits a batch evaluation over the sessions it created. Invoke flags mirror +// `runtime invoke`; content-type/accept are fixed to application/json. +export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "simulate", + description: "replay a dataset against a runtime, then batch-evaluate the resulting sessions", + flags: [ + flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()), + flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag( + "payload-template", + 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + z.string().optional(), + ), + flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()), + flag( + "bearer-token", + "CUSTOM_JWT bearer token (for JWT-auth runtimes)", + z.string().optional(), + ), + flag( + "session-id", + "runtime session id to reuse (default: fresh per scenario)", + z.string().optional(), + ), + flag("user-id", "runtime user id", z.string().optional()), + flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), + flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag("name", "batch evaluation name (unique in the account)", z.string().optional()), + flag("description", "optional description", z.string().optional()), + flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["runtime-id"]) + throw new InputValidationError("required option '--runtime-id' not specified"); + if (!flags["payload-template"]) { + throw new InputValidationError("required option '--payload-template' not specified"); + } + if (!flags["dataset"]) + throw new InputValidationError("required option '--dataset' not specified"); + if (!flags["evaluator"]?.length) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + + const result = await core.eval.simulate( + { + runtimeId: flags["runtime-id"], + qualifier: flags["qualifier"], + payloadTemplate: flags["payload-template"], + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + sessionId: flags["session-id"], + userId: flags["user-id"], + dataset: flags["dataset"], + datasetVersion: flags["dataset-version"], + evaluatorIds: flags["evaluator"], + name: flags["name"], + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(result); + }, + }); diff --git a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx new file mode 100644 index 000000000..5cb726eb3 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx @@ -0,0 +1,130 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../../index"; +import { + createSilentLogger, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../../testing"; +import type { SimulateResult } from "../../types"; + +const RESULT: SimulateResult = { + batchEvaluationId: "batch-eval-sim", + status: "RUNNING", + scenariosInvoked: 3, + scenariosFailed: 0, +}; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setSimulateResponse(RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +const BASE = [ + "eval", + "batch-evaluation", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", + "--name", + "sim-1", +]; + +describe("eval batch-evaluation simulate", () => { + test("registered under batch-evaluation", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "batch-evaluation"); + expect(group?.children().map((c) => c.name())).toContain("simulate"); + }); + + test.each([ + [ + [ + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "E", + "--name", + "n", + ], + /--runtime-id/, + ], + [ + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"], + /--payload-template/, + ], + [ + ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E", "--name", "n"], + /--dataset/, + ], + [ + [ + "--runtime-id", + "r-1", + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--name", + "n", + ], + /--evaluator/, + ], + [ + [ + "--runtime-id", + "r-1", + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "E", + ], + /--name/, + ], + ])("rejects missing required flag", async (args, expected) => { + await expect(run(["eval", "batch-evaluation", "simulate", ...args])).rejects.toThrow(expected); + }); + + test("maps flags to the simulate input and renders the result", async () => { + const { core, stdout } = await run([...BASE, "--qualifier", "PROD", "--header", "x-a:1"]); + expect(JSON.parse(stdout)).toEqual(RESULT); + const call = core.eval.calls.find((c) => c.method === "simulate"); + expect(call?.args[0]).toMatchObject({ + runtimeId: "r-1", + qualifier: "PROD", + payloadTemplate: '{"prompt":"{input}"}', + headers: [["x-a", "1"]], + dataset: "/tmp/ds.jsonl", + evaluatorIds: ["Builtin.Helpfulness"], + name: "sim-1", + }); + }); +}); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 6e1875b18..4e04295f4 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -205,6 +205,35 @@ export type StartBatchEvaluationInput = { kmsKeyArn?: string; }; +// SimulateInput is the CLI-facing shape for `batch-evaluation simulate` — dataset +// replay + batch grade. SDK-native fields only (no RuntimeInvokeRequest/EvaluateInput +// leak): Core invokes each scenario against the runtime, then submits a batch +// evaluation scoped to the sessions it created. Invoke fields mirror `runtime invoke`. +export type SimulateInput = { + runtimeId: string; + qualifier?: string; + payloadTemplate: string; // e.g. {"prompt":"{input}"} — {input} is the scenario's turn input + headers?: [string, string][]; + bearerToken?: string; + sessionId?: string; + userId?: string; + dataset: string; // local JSONL path or a dataset id + datasetVersion?: string; + evaluatorIds: string[]; + name: string; + description?: string; + kmsKeyArn?: string; +}; + +// SimulateResult reports the submitted job plus how many scenarios were actually +// invoked vs dropped (a failed invoke is skipped, not fatal, unless all fail). +export type SimulateResult = { + batchEvaluationId?: string; + status?: string; + scenariosInvoked: number; + scenariosFailed: number; +}; + // SpanRecord is one OTel span/log document — the parsed `@message` JSON of a // CloudWatch Logs Insights result row. Left open (arbitrary JSON) because it is // handed to the Evaluate API's `sessionSpans` verbatim; the CLI only reads a few @@ -329,6 +358,10 @@ export interface CoreEvalClient { // Evaluate API and returns per-session scores. No job, no CloudWatch — the // trace read happened in getTracesForAgent. evaluate(input: EvaluateInput, options: CoreOptions): Promise; + // simulate replays a dataset against the runtime (invoke per scenario, client-side) + // then submits a batch evaluation over the sessions it created. No dataset API + // exists service-side, so the CLI creates the sessions and the service grades them. + simulate(input: SimulateInput, options: CoreOptions): Promise; createOnlineEvaluationConfig( input: CreateOnlineEvalInput, diff --git a/src/io/index.ts b/src/io/index.ts index e93169768..7fb34175b 100644 --- a/src/io/index.ts +++ b/src/io/index.ts @@ -26,6 +26,7 @@ export { type JsonValue, } from "./jsonl"; export { SourceResolver, type SourceResolverConfig } from "./source"; +export { renderJsonTemplate } from "./template"; export { classifyStreamingResponse, writeStreamingResponse, diff --git a/src/io/template.test.ts b/src/io/template.test.ts new file mode 100644 index 000000000..6be6da2de --- /dev/null +++ b/src/io/template.test.ts @@ -0,0 +1,32 @@ +import { test, expect, describe } from "bun:test"; +import { renderJsonTemplate } from "./template"; + +const decode = (bytes: Uint8Array) => new TextDecoder().decode(bytes); + +describe("renderJsonTemplate", () => { + test("substitutes {input} inside a string value", () => { + const out = decode(renderJsonTemplate('{"prompt":"{input}"}', { input: "hello" })); + expect(JSON.parse(out)).toEqual({ prompt: "hello" }); + }); + + test("JSON-escapes substituted values (quotes, newlines don't break the payload)", () => { + const out = renderJsonTemplate('{"prompt":"{input}"}', { input: 'a "quote"\nline' }); + expect(JSON.parse(decode(out))).toEqual({ prompt: 'a "quote"\nline' }); + }); + + test("substitutes nested + array positions", () => { + const out = renderJsonTemplate('{"messages":[{"role":"user","content":"{input}"}]}', { + input: "hi", + }); + expect(JSON.parse(decode(out))).toEqual({ messages: [{ role: "user", content: "hi" }] }); + }); + + test("supports arbitrary keys, leaves unknown placeholders intact", () => { + const out = renderJsonTemplate('{"m":"{model}","p":"{input}"}', { input: "x" }); + expect(JSON.parse(decode(out))).toEqual({ m: "{model}", p: "x" }); + }); + + test("rejects invalid JSON template", () => { + expect(() => renderJsonTemplate("{not json", { input: "x" })).toThrow(/valid JSON/); + }); +}); diff --git a/src/io/template.ts b/src/io/template.ts new file mode 100644 index 000000000..e7d12cca8 --- /dev/null +++ b/src/io/template.ts @@ -0,0 +1,33 @@ +import { InputValidationError } from "../errors"; + +// renderJsonTemplate substitutes `{key}` placeholders inside the string values of a +// JSON template and returns the encoded bytes. General on purpose: `simulate` uses +// `{ input }`, but any `{model}`/`{sessionId}` a future caller adds works the same. +// Parsing the template first (rather than string-replacing raw) keeps the result +// valid JSON regardless of quotes/newlines in the substituted values. +export function renderJsonTemplate( + template: string, + values: Record, + flagName = "payload-template", +): Uint8Array { + let parsed: unknown; + try { + parsed = JSON.parse(template); + } catch { + throw new InputValidationError(`--${flagName} must be valid JSON`); + } + return new TextEncoder().encode(JSON.stringify(substitute(parsed, values))); +} + +function substitute(value: unknown, values: Record): unknown { + if (typeof value === "string") { + return value.replace(/\{(\w+)\}/g, (match, key) => values[key] ?? match); + } + if (Array.isArray(value)) return value.map((item) => substitute(item, values)); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, substitute(item, values)]), + ); + } + return value; +} diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 15ad2b854..0c5df6b26 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -133,6 +133,8 @@ import type { DatasetUpdateProgressEvent, EvaluateInput, EvaluateResult, + SimulateInput, + SimulateResult, GetBatchEvaluationResult, GetTracesInput, LlmAsAJudgeUpdate, @@ -1403,6 +1405,12 @@ export class TestEvalClient implements CoreEvalClient { sessionsEvaluated: 0, results: [], }; + private simulateResponse: SimulateResult = { + batchEvaluationId: "batch-eval-test", + status: "RUNNING", + scenariosInvoked: 0, + scenariosFailed: 0, + }; private error?: Error; // setListResponse sets what listEvaluators resolves to (when not erroring). @@ -1722,6 +1730,18 @@ export class TestEvalClient implements CoreEvalClient { return this.evaluateResponse; } + // setSimulateResponse sets what simulate resolves to (when not erroring). + setSimulateResponse(response: SimulateResult): this { + this.simulateResponse = response; + return this; + } + + async simulate(input: SimulateInput, options: CoreOptions): Promise { + this.calls.push({ method: "simulate", args: [input, options] }); + if (this.error) throw this.error; + return this.simulateResponse; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, From 95d341a9688e1b2155fa7970a3c8f0959031ce48 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 13 Aug 2026 22:19:03 +0000 Subject: [PATCH 03/11] fix(eval): omit empty ground-truth arrays (service rejects length 0) --- src/core/eval/simulate.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/core/eval/simulate.ts b/src/core/eval/simulate.ts index 54849b0d3..873383cd6 100644 --- a/src/core/eval/simulate.ts +++ b/src/core/eval/simulate.ts @@ -75,19 +75,24 @@ export async function runScenarios( // toSessionMetadata maps a scenario's ground truth onto the session the replay // created — the batch service's per-session ground-truth shape (inline arm). +// Omit array fields when empty: the service rejects zero-length `turns` / +// `assertions` (`length >= 1`) rather than treating an empty array as "no data". export function toSessionMetadata(scenario: Scenario, sessionId: string): SessionMetadataShape { + const turns = scenario.turns + .filter((t) => t.expectedResponse !== undefined) + .map((t) => ({ expectedResponse: { text: t.expectedResponse! } })); + const assertions = scenario.assertions?.map((text) => ({ text })); return { sessionId, testScenarioId: scenario.scenarioId, groundTruth: { inline: { - assertions: scenario.assertions?.map((text) => ({ text })), - expectedTrajectory: scenario.expectedTrajectory - ? { toolNames: scenario.expectedTrajectory } - : undefined, - turns: scenario.turns - .filter((t) => t.expectedResponse !== undefined) - .map((t) => ({ expectedResponse: { text: t.expectedResponse! } })), + ...(assertions && assertions.length > 0 && { assertions }), + ...(scenario.expectedTrajectory && + scenario.expectedTrajectory.length > 0 && { + expectedTrajectory: { toolNames: scenario.expectedTrajectory }, + }), + ...(turns.length > 0 && { turns }), }, }, }; From 80291f9a5751364087400d6d24a4691c819da4ec Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 13 Aug 2026 22:23:15 +0000 Subject: [PATCH 04/11] fix(eval): drop --session-id from simulate, add ingestion wait + reject multi-turn --- src/core/eval.tsx | 43 +++++++++++++++---- src/core/eval/simulate.ts | 38 +++++++++++++--- .../eval/batch-evaluation/simulate/index.tsx | 6 --- src/handlers/eval/types.tsx | 3 +- 4 files changed, 67 insertions(+), 23 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 7defe211c..88d155428 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -528,31 +528,56 @@ export class EvalClient implements CoreEvalClient { .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId })); const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; - const { ok, failed } = await runScenarios(scenarios, async (scenario) => { + const { ok, failed, firstError } = await runScenarios(scenarios, async (scenario) => { + // Generate the session id client-side. The CUSTOM_JWT invoke path relies on + // the server echoing back x-amzn-bedrock-agentcore-runtime-session-id, which + // is not guaranteed; picking the id here means we know it regardless of the + // auth mode. Also keeps IAM-path ids deterministic per scenario. + const runtimeSessionId = randomUUID(); const request = normalizeRuntimeInvokeRequest(runtime, { runtimeId: input.runtimeId, qualifier: input.qualifier, - payload: renderJsonTemplate(input.payloadTemplate, { input: scenario.turns[0]?.input ?? "" }), + payload: renderJsonTemplate(input.payloadTemplate, { input: scenario.turns[0]!.input }), contentType: "application/json", accept: "application/json", applicationHeaders: input.headers, bearerToken: input.bearerToken, - runtimeSessionId: input.sessionId, + runtimeSessionId, runtimeUserId: input.userId, }); - const response = await invokeRuntime(deps, request, options); - for await (const _chunk of response.body) { - // Drain the stream so the turn completes; only the session id is needed. + try { + const response = await invokeRuntime(deps, request, options); + for await (const _chunk of response.body) { + // Drain the stream so the turn completes; only the session id is needed. + } + } catch (error) { + this.logger.debug( + `simulate: invoke failed for scenario "${scenario.scenarioId}": ${(error as Error).message}`, + ); + throw error; } - if (!response.runtimeSessionId) throw new Error("invoke returned no session id"); - return { scenarioId: scenario.scenarioId, sessionId: response.runtimeSessionId }; + return { scenarioId: scenario.scenarioId, sessionId: runtimeSessionId }; }); if (ok.length === 0) { + const detail = firstError ? `; first error: ${firstError.message}` : ""; throw new InputValidationError( - `no scenarios could be invoked (${failed} failed) — nothing to evaluate`, + `no scenarios could be invoked (${failed} failed) — nothing to evaluate${detail}`, ); } + if (failed > 0) { + this.logger.warn(`simulate: ${failed} scenario(s) failed to invoke and were dropped`); + } + + // AgentCore takes ~30s-3min to emit spans for a freshly invoked session; submit + // too early and the service reads an empty log group and marks every session + // failed. Wait once for the batch to be safely ingestible (matches the old CLI's + // 180s wait). Skipped when disabled via `SIMULATE_INGESTION_WAIT_MS=0` (tests). + const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); + if (waitMs > 0) { + this.logger.info(`waiting ${Math.round(waitMs / 1000)}s for span ingestion before submitting`); + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } const job = await this.startBatchEvaluation( { diff --git a/src/core/eval/simulate.ts b/src/core/eval/simulate.ts index 873383cd6..204be251a 100644 --- a/src/core/eval/simulate.ts +++ b/src/core/eval/simulate.ts @@ -12,8 +12,12 @@ export type Scenario = { }; // parseScenarios reads dataset JSONL (one scenario per line) into Scenario records. +// Each scenario needs a non-empty, unique `scenario_id` — the id is the join key +// between the session created for it and its ground truth, so a missing/duplicate +// id silently misassigns ground truth to the wrong session. export function parseScenarios(text: string): Scenario[] { const scenarios: Scenario[] = []; + const seen = new Set(); for (const line of text.split("\n")) { const trimmed = line.trim(); if (!trimmed) continue; @@ -23,7 +27,23 @@ export function parseScenarios(text: string): Scenario[] { } catch { throw new InputValidationError("dataset contains a line that is not valid JSON"); } - scenarios.push(toScenario(row)); + const scenario = toScenario(row); + if (!scenario.scenarioId) { + throw new InputValidationError("dataset scenario is missing 'scenario_id'"); + } + if (seen.has(scenario.scenarioId)) { + throw new InputValidationError(`dataset has a duplicate scenario_id: "${scenario.scenarioId}"`); + } + if (scenario.turns.length !== 1) { + // Multi-turn replay isn't implemented yet; the ground-truth mapper emits + // per-turn entries but simulate only invokes turn[0]. Reject rather than + // silently misalign turns[i] to a single-turn session. + throw new InputValidationError( + `scenario "${scenario.scenarioId}" has ${scenario.turns.length} turns; simulate v1 supports single-turn scenarios only`, + ); + } + seen.add(scenario.scenarioId); + scenarios.push(scenario); } if (scenarios.length === 0) throw new InputValidationError("dataset has no scenarios"); return scenarios; @@ -48,29 +68,33 @@ export async function loadDatasetFile(path: string): Promise { return parseScenarios(await Bun.file(path).text()); } -// runScenarios runs `worker` over every scenario with bounded concurrency, dropping -// failures (a single bad invocation must not sink the run). Returns the successful -// results plus a failure count so the caller can warn / error on all-failed. +// runScenarios runs `worker` over every scenario with bounded concurrency. A failed +// worker doesn't sink the run — the failure is captured (so the caller can report +// on all-failed) but drops that scenario. Returns ok results + the first error we +// saw, which the caller can surface to explain a total failure. +export type ScenarioRun = { ok: T[]; failed: number; firstError?: Error }; export async function runScenarios( scenarios: Scenario[], worker: (scenario: Scenario) => Promise, concurrency = 5, -): Promise<{ ok: T[]; failed: number }> { +): Promise> { const ok: T[] = []; let failed = 0; + let firstError: Error | undefined; let next = 0; const run = async (): Promise => { while (next < scenarios.length) { const scenario = scenarios[next++]!; try { ok.push(await worker(scenario)); - } catch { + } catch (error) { failed++; + if (!firstError) firstError = error instanceof Error ? error : new Error(String(error)); } } }; await Promise.all(Array.from({ length: Math.min(concurrency, scenarios.length) }, run)); - return { ok, failed }; + return { ok, failed, firstError }; } // toSessionMetadata maps a scenario's ground truth onto the session the replay diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index e294ba867..accc73016 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -28,11 +28,6 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => "CUSTOM_JWT bearer token (for JWT-auth runtimes)", z.string().optional(), ), - flag( - "session-id", - "runtime session id to reuse (default: fresh per scenario)", - z.string().optional(), - ), flag("user-id", "runtime user id", z.string().optional()), flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), @@ -64,7 +59,6 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => payloadTemplate: flags["payload-template"], headers: parseRuntimeInvokeHeaders(flags["header"]), bearerToken: flags["bearer-token"], - sessionId: flags["session-id"], userId: flags["user-id"], dataset: flags["dataset"], datasetVersion: flags["dataset-version"], diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 4e04295f4..d2d3f5043 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -215,7 +215,8 @@ export type SimulateInput = { payloadTemplate: string; // e.g. {"prompt":"{input}"} — {input} is the scenario's turn input headers?: [string, string][]; bearerToken?: string; - sessionId?: string; + // simulate always creates a fresh session per scenario. Reusing one session + // across scenarios interleaves unrelated turns and collides ground-truth keys. userId?: string; dataset: string; // local JSONL path or a dataset id datasetVersion?: string; From 028bd0c6cebcb85cd63bbf5ce9bd2ff3fe450f7a Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 14 Aug 2026 19:46:55 +0000 Subject: [PATCH 05/11] feat(eval): support multi-turn scenarios (invoke each turn on one session, in order) --- src/core/eval.tsx | 43 ++++++++++++++++++++++----------------- src/core/eval/simulate.ts | 11 ++++------ 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 88d155428..a90217ffc 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -529,26 +529,29 @@ export class EvalClient implements CoreEvalClient { const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; const { ok, failed, firstError } = await runScenarios(scenarios, async (scenario) => { - // Generate the session id client-side. The CUSTOM_JWT invoke path relies on - // the server echoing back x-amzn-bedrock-agentcore-runtime-session-id, which - // is not guaranteed; picking the id here means we know it regardless of the - // auth mode. Also keeps IAM-path ids deterministic per scenario. + // One session per scenario; the id is generated client-side (session id is a + // client-owned input per the AgentCore docs, needed on the request before any + // response and reused across turns). Turns run sequentially against the SAME + // session so the conversation — and its per-turn traces — accumulate in order, + // matching the per-turn ground truth in toSessionMetadata. const runtimeSessionId = randomUUID(); - const request = normalizeRuntimeInvokeRequest(runtime, { - runtimeId: input.runtimeId, - qualifier: input.qualifier, - payload: renderJsonTemplate(input.payloadTemplate, { input: scenario.turns[0]!.input }), - contentType: "application/json", - accept: "application/json", - applicationHeaders: input.headers, - bearerToken: input.bearerToken, - runtimeSessionId, - runtimeUserId: input.userId, - }); try { - const response = await invokeRuntime(deps, request, options); - for await (const _chunk of response.body) { - // Drain the stream so the turn completes; only the session id is needed. + for (const turn of scenario.turns) { + const request = normalizeRuntimeInvokeRequest(runtime, { + runtimeId: input.runtimeId, + qualifier: input.qualifier, + payload: renderJsonTemplate(input.payloadTemplate, { input: turn.input }), + contentType: "application/json", + accept: "application/json", + applicationHeaders: input.headers, + bearerToken: input.bearerToken, + runtimeSessionId, + runtimeUserId: input.userId, + }); + const response = await invokeRuntime(deps, request, options); + for await (const _chunk of response.body) { + // Drain each turn's stream so it completes before the next turn. + } } } catch (error) { this.logger.debug( @@ -575,7 +578,9 @@ export class EvalClient implements CoreEvalClient { // 180s wait). Skipped when disabled via `SIMULATE_INGESTION_WAIT_MS=0` (tests). const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); if (waitMs > 0) { - this.logger.info(`waiting ${Math.round(waitMs / 1000)}s for span ingestion before submitting`); + this.logger.info( + `waiting ${Math.round(waitMs / 1000)}s for span ingestion before submitting`, + ); await new Promise((resolve) => setTimeout(resolve, waitMs)); } diff --git a/src/core/eval/simulate.ts b/src/core/eval/simulate.ts index 204be251a..81fc3f843 100644 --- a/src/core/eval/simulate.ts +++ b/src/core/eval/simulate.ts @@ -32,16 +32,13 @@ export function parseScenarios(text: string): Scenario[] { throw new InputValidationError("dataset scenario is missing 'scenario_id'"); } if (seen.has(scenario.scenarioId)) { - throw new InputValidationError(`dataset has a duplicate scenario_id: "${scenario.scenarioId}"`); - } - if (scenario.turns.length !== 1) { - // Multi-turn replay isn't implemented yet; the ground-truth mapper emits - // per-turn entries but simulate only invokes turn[0]. Reject rather than - // silently misalign turns[i] to a single-turn session. throw new InputValidationError( - `scenario "${scenario.scenarioId}" has ${scenario.turns.length} turns; simulate v1 supports single-turn scenarios only`, + `dataset has a duplicate scenario_id: "${scenario.scenarioId}"`, ); } + if (scenario.turns.length === 0) { + throw new InputValidationError(`scenario "${scenario.scenarioId}" has no turns`); + } seen.add(scenario.scenarioId); scenarios.push(scenario); } From 6ba19943adce77549687500741ea6b4525267482 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 14 Aug 2026 20:04:01 +0000 Subject: [PATCH 06/11] feat(eval): thread AbortSignal through simulate (Ctrl-C cancels invokes + ingestion wait) --- src/core/eval.tsx | 15 ++++-- .../eval/batch-evaluation/simulate/index.tsx | 49 ++++++++++++------- .../simulate/simulate.test.tsx | 2 + src/handlers/eval/types.tsx | 6 ++- src/testing/TestCoreClient.tsx | 8 ++- 5 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index a90217ffc..52ec0f7b5 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -513,11 +513,15 @@ export class EvalClient implements CoreEvalClient { }; } - async simulate(input: SimulateInput, options: CoreOptions): Promise { + async simulate( + input: SimulateInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { // Load scenarios: a local JSONL path directly, else download the dataset id. const path = (await Bun.file(input.dataset).exists()) ? input.dataset - : await this.downloadDatasetToTemp(input.dataset, input.datasetVersion, options); + : await this.downloadDatasetToTemp(input.dataset, input.datasetVersion, options, signal); const scenarios = await loadDatasetFile(path); const byId = new Map(scenarios.map((s) => [s.scenarioId, s])); @@ -548,7 +552,7 @@ export class EvalClient implements CoreEvalClient { runtimeSessionId, runtimeUserId: input.userId, }); - const response = await invokeRuntime(deps, request, options); + const response = await invokeRuntime(deps, request, options, signal); for await (const _chunk of response.body) { // Drain each turn's stream so it completes before the next turn. } @@ -581,7 +585,7 @@ export class EvalClient implements CoreEvalClient { this.logger.info( `waiting ${Math.round(waitMs / 1000)}s for span ingestion before submitting`, ); - await new Promise((resolve) => setTimeout(resolve, waitMs)); + await sleep(waitMs, undefined, { signal }); } const job = await this.startBatchEvaluation( @@ -615,9 +619,10 @@ export class EvalClient implements CoreEvalClient { id: string, version: string | undefined, options: CoreOptions, + signal?: AbortSignal, ): Promise { const path = join(tmpdir(), `agentcore-dataset-${randomUUID()}.jsonl`); - await this.downloadDataset(id, version, path, options); + await this.downloadDataset(id, version, path, options, signal); return path; } diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index accc73016..276e8f1b8 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -52,23 +52,36 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => if (!flags["name"]) throw new InputValidationError("required option '--name ' not specified"); - const result = await core.eval.simulate( - { - runtimeId: flags["runtime-id"], - qualifier: flags["qualifier"], - payloadTemplate: flags["payload-template"], - headers: parseRuntimeInvokeHeaders(flags["header"]), - bearerToken: flags["bearer-token"], - userId: flags["user-id"], - dataset: flags["dataset"], - datasetVersion: flags["dataset-version"], - evaluatorIds: flags["evaluator"], - name: flags["name"], - description: flags["description"], - kmsKeyArn: flags["kms-key-arn"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(result); + // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once("SIGINT", interrupt); + try { + const result = await core.eval.simulate( + { + runtimeId: flags["runtime-id"], + qualifier: flags["qualifier"], + payloadTemplate: flags["payload-template"], + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + userId: flags["user-id"], + dataset: flags["dataset"], + datasetVersion: flags["dataset-version"], + evaluatorIds: flags["evaluator"], + name: flags["name"], + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + }, + coreOptsFromCtx(ctx), + controller.signal, + ); + ctx.require(JsonRendererKey).renderJson(result); + } catch (error) { + // A Ctrl-C exits quietly; the half-created sessions grade nothing. + if (controller.signal.aborted) return; + throw error; + } finally { + process.off("SIGINT", interrupt); + } }, }); diff --git a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx index 5cb726eb3..fb0c6149d 100644 --- a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx @@ -126,5 +126,7 @@ describe("eval batch-evaluation simulate", () => { evaluatorIds: ["Builtin.Helpfulness"], name: "sim-1", }); + // Handler wires an AbortSignal (Ctrl-C) through to Core. + expect(call?.args[2]).toBeInstanceOf(AbortSignal); }); }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index d2d3f5043..07687048e 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -362,7 +362,11 @@ export interface CoreEvalClient { // simulate replays a dataset against the runtime (invoke per scenario, client-side) // then submits a batch evaluation over the sessions it created. No dataset API // exists service-side, so the CLI creates the sessions and the service grades them. - simulate(input: SimulateInput, options: CoreOptions): Promise; + simulate( + input: SimulateInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; createOnlineEvaluationConfig( input: CreateOnlineEvalInput, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 0c5df6b26..63392c571 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -1736,8 +1736,12 @@ export class TestEvalClient implements CoreEvalClient { return this; } - async simulate(input: SimulateInput, options: CoreOptions): Promise { - this.calls.push({ method: "simulate", args: [input, options] }); + async simulate( + input: SimulateInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + this.calls.push({ method: "simulate", args: [input, options, signal] }); if (this.error) throw this.error; return this.simulateResponse; } From a8675a200c1f1a73757de790f80b7ba037525ac9 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 14 Aug 2026 20:09:43 +0000 Subject: [PATCH 07/11] fix(eval): unlink temp dataset file after simulate (dataset-id path) --- src/core/eval.tsx | 175 ++++++++++++++++++++++++---------------------- 1 file changed, 93 insertions(+), 82 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 52ec0f7b5..816ed8463 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -79,6 +79,7 @@ import { } from "@aws-sdk/client-cloudwatch-logs"; import type { DocumentType } from "@smithy/types"; import { randomUUID } from "node:crypto"; +import { unlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, dirname, extname, join } from "node:path"; import { Transform } from "node:stream"; @@ -519,98 +520,108 @@ export class EvalClient implements CoreEvalClient { signal?: AbortSignal, ): Promise { // Load scenarios: a local JSONL path directly, else download the dataset id. + let tempDatasetPath: string | undefined; const path = (await Bun.file(input.dataset).exists()) ? input.dataset - : await this.downloadDatasetToTemp(input.dataset, input.datasetVersion, options, signal); - const scenarios = await loadDatasetFile(path); - const byId = new Map(scenarios.map((s) => [s.scenarioId, s])); - - // Resolve the runtime once; normalizeRuntimeInvokeRequest validates auth + fills - // accountId. Invoke is the extracted free fn — no RuntimeClient dependency. - const runtime = await this.clients - .control(toClientConfig(options)) - .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId })); - const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; - - const { ok, failed, firstError } = await runScenarios(scenarios, async (scenario) => { - // One session per scenario; the id is generated client-side (session id is a - // client-owned input per the AgentCore docs, needed on the request before any - // response and reused across turns). Turns run sequentially against the SAME - // session so the conversation — and its per-turn traces — accumulate in order, - // matching the per-turn ground truth in toSessionMetadata. - const runtimeSessionId = randomUUID(); - try { - for (const turn of scenario.turns) { - const request = normalizeRuntimeInvokeRequest(runtime, { - runtimeId: input.runtimeId, - qualifier: input.qualifier, - payload: renderJsonTemplate(input.payloadTemplate, { input: turn.input }), - contentType: "application/json", - accept: "application/json", - applicationHeaders: input.headers, - bearerToken: input.bearerToken, - runtimeSessionId, - runtimeUserId: input.userId, - }); - const response = await invokeRuntime(deps, request, options, signal); - for await (const _chunk of response.body) { - // Drain each turn's stream so it completes before the next turn. + : (tempDatasetPath = await this.downloadDatasetToTemp( + input.dataset, + input.datasetVersion, + options, + signal, + )); + try { + const scenarios = await loadDatasetFile(path); + const byId = new Map(scenarios.map((s) => [s.scenarioId, s])); + + // Resolve the runtime once; normalizeRuntimeInvokeRequest validates auth + fills + // accountId. Invoke is the extracted free fn — no RuntimeClient dependency. + const runtime = await this.clients + .control(toClientConfig(options)) + .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId })); + const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; + + const { ok, failed, firstError } = await runScenarios(scenarios, async (scenario) => { + // One session per scenario; the id is generated client-side (session id is a + // client-owned input per the AgentCore docs, needed on the request before any + // response and reused across turns). Turns run sequentially against the SAME + // session so the conversation — and its per-turn traces — accumulate in order, + // matching the per-turn ground truth in toSessionMetadata. + const runtimeSessionId = randomUUID(); + try { + for (const turn of scenario.turns) { + const request = normalizeRuntimeInvokeRequest(runtime, { + runtimeId: input.runtimeId, + qualifier: input.qualifier, + payload: renderJsonTemplate(input.payloadTemplate, { input: turn.input }), + contentType: "application/json", + accept: "application/json", + applicationHeaders: input.headers, + bearerToken: input.bearerToken, + runtimeSessionId, + runtimeUserId: input.userId, + }); + const response = await invokeRuntime(deps, request, options, signal); + for await (const _chunk of response.body) { + // Drain each turn's stream so it completes before the next turn. + } } + } catch (error) { + this.logger.debug( + `simulate: invoke failed for scenario "${scenario.scenarioId}": ${(error as Error).message}`, + ); + throw error; } - } catch (error) { - this.logger.debug( - `simulate: invoke failed for scenario "${scenario.scenarioId}": ${(error as Error).message}`, + return { scenarioId: scenario.scenarioId, sessionId: runtimeSessionId }; + }); + + if (ok.length === 0) { + const detail = firstError ? `; first error: ${firstError.message}` : ""; + throw new InputValidationError( + `no scenarios could be invoked (${failed} failed) — nothing to evaluate${detail}`, ); - throw error; } - return { scenarioId: scenario.scenarioId, sessionId: runtimeSessionId }; - }); - - if (ok.length === 0) { - const detail = firstError ? `; first error: ${firstError.message}` : ""; - throw new InputValidationError( - `no scenarios could be invoked (${failed} failed) — nothing to evaluate${detail}`, - ); - } - if (failed > 0) { - this.logger.warn(`simulate: ${failed} scenario(s) failed to invoke and were dropped`); - } + if (failed > 0) { + this.logger.warn(`simulate: ${failed} scenario(s) failed to invoke and were dropped`); + } - // AgentCore takes ~30s-3min to emit spans for a freshly invoked session; submit - // too early and the service reads an empty log group and marks every session - // failed. Wait once for the batch to be safely ingestible (matches the old CLI's - // 180s wait). Skipped when disabled via `SIMULATE_INGESTION_WAIT_MS=0` (tests). - const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); - if (waitMs > 0) { - this.logger.info( - `waiting ${Math.round(waitMs / 1000)}s for span ingestion before submitting`, - ); - await sleep(waitMs, undefined, { signal }); - } + // AgentCore takes ~30s-3min to emit spans for a freshly invoked session; submit + // too early and the service reads an empty log group and marks every session + // failed. Wait once for the batch to be safely ingestible (matches the old CLI's + // 180s wait). Skipped when disabled via `SIMULATE_INGESTION_WAIT_MS=0` (tests). + const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); + if (waitMs > 0) { + this.logger.info( + `waiting ${Math.round(waitMs / 1000)}s for span ingestion before submitting`, + ); + await sleep(waitMs, undefined, { signal }); + } - const job = await this.startBatchEvaluation( - { - name: input.name, - description: input.description, - evaluatorIds: input.evaluatorIds, - source: { - origin: "agent", - agent: input.runtimeId, - endpoint: input.qualifier, - sessionIds: ok.map((r) => r.sessionId), + const job = await this.startBatchEvaluation( + { + name: input.name, + description: input.description, + evaluatorIds: input.evaluatorIds, + source: { + origin: "agent", + agent: input.runtimeId, + endpoint: input.qualifier, + sessionIds: ok.map((r) => r.sessionId), + }, + groundTruth: ok.map((r) => toSessionMetadata(byId.get(r.scenarioId)!, r.sessionId)), + kmsKeyArn: input.kmsKeyArn, }, - groundTruth: ok.map((r) => toSessionMetadata(byId.get(r.scenarioId)!, r.sessionId)), - kmsKeyArn: input.kmsKeyArn, - }, - options, - ); + options, + ); - return { - batchEvaluationId: job.batchEvaluationId, - status: job.status, - scenariosInvoked: ok.length, - scenariosFailed: failed, - }; + return { + batchEvaluationId: job.batchEvaluationId, + status: job.status, + scenariosInvoked: ok.length, + scenariosFailed: failed, + }; + } finally { + if (tempDatasetPath) await unlink(tempDatasetPath).catch(() => {}); + } } // downloadDatasetToTemp streams a dataset version's JSONL to a temp file so From 889ea9baa3ff2a8fd44dfb27b4cb99c546850d11 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 18 Aug 2026 20:20:46 +0000 Subject: [PATCH 08/11] refactor(eval): replace core.eval.simulate with invokeDataset + per-type example classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the monolithic core.eval.simulate into two composable EvalClient calls the handler orchestrates — invokeDataset (replay) then startBatchEvaluation (grade) — and model each dataset type as a class that owns its parse + run + ground truth. - src/core/eval/dataset/: types (Example interface, RunContext), predefined + simulated example classes, DatasetLoader (shape-classify → switch → new), and a generic runExamples pool. Replaces src/core/eval/simulate.ts. - core.eval.invokeDataset returns { sessions, invoked, failed } with neutral inline ground truth; the batch-evaluation simulate handler wraps it as sessionMetadata and submits startBatchEvaluation. No Core method calls a sibling. - readDatasetText reuses readLocalDatasetFile + downloadDatasetToTemp (drops the combined loadDatasetFile); dispatch is a build() switch whose non-exhaustiveness on a new DatasetSchemaType fails the build (no map, no assertNever). - Carries the simulate fixes: sparse multi-turn ground truth keeps turn position (input.prompt per turn), both-row refusal, NotImplementedError for simulated, per-example failure isolation, 180s ingestion wait, AbortSignal, JSON-only. - Tests: dataset unit tests + a golden snapshot of the inline ground-truth shape; handler test asserts the invokeDataset→startBatchEvaluation composition. --- src/core/eval.tsx | 189 ++++++++---------- .../__snapshots__/dataset.test.ts.snap | 35 ++++ src/core/eval/dataset/dataset.test.ts | 137 +++++++++++++ src/core/eval/dataset/load.ts | 73 +++++++ src/core/eval/dataset/predefined.ts | 67 +++++++ src/core/eval/dataset/run.ts | 29 +++ src/core/eval/dataset/simulated.ts | 27 +++ src/core/eval/dataset/types.ts | 20 ++ src/core/eval/simulate.ts | 120 ----------- .../eval/batch-evaluation/simulate/index.tsx | 44 +++- .../simulate/simulate.test.tsx | 66 ++++-- src/handlers/eval/types.tsx | 54 ++--- src/testing/TestCoreClient.tsx | 29 ++- 13 files changed, 611 insertions(+), 279 deletions(-) create mode 100644 src/core/eval/dataset/__snapshots__/dataset.test.ts.snap create mode 100644 src/core/eval/dataset/dataset.test.ts create mode 100644 src/core/eval/dataset/load.ts create mode 100644 src/core/eval/dataset/predefined.ts create mode 100644 src/core/eval/dataset/run.ts create mode 100644 src/core/eval/dataset/simulated.ts create mode 100644 src/core/eval/dataset/types.ts delete mode 100644 src/core/eval/simulate.ts diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 816ed8463..e53c89cf5 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -108,8 +108,8 @@ import type { LlmAsAJudgeUpdate, SessionSourceValue, SessionTrace, - SimulateInput, - SimulateResult, + InvokeDatasetInput, + InvokeDatasetResult, SpanRecord, StartBatchEvaluationInput, UpdateConfigurationBundleInput, @@ -117,7 +117,9 @@ import type { } from "../handlers/eval/types"; import { atomicWrite, atomicWriteStream, readTextFile, renderJsonTemplate } from "../io"; import { invokeRuntime } from "./invokeRuntime"; -import { loadDatasetFile, runScenarios, toSessionMetadata } from "./eval/simulate"; +import { DatasetLoader } from "./eval/dataset/load"; +import { runExamples } from "./eval/dataset/run"; +import type { RunContext } from "./eval/dataset/types"; import { normalizeRuntimeInvokeRequest } from "../handlers/runtime/invoke/request"; import { isTerminalStatus, readEvaluationResults } from "./batchEvaluationResults"; import { applyExampleIds, diffExamples, indexRemoteById, parseJsonl } from "./datasetDiff"; @@ -514,118 +516,103 @@ export class EvalClient implements CoreEvalClient { }; } - async simulate( - input: SimulateInput, + async invokeDataset( + input: InvokeDatasetInput, options: CoreOptions, signal?: AbortSignal, - ): Promise { - // Load scenarios: a local JSONL path directly, else download the dataset id. - let tempDatasetPath: string | undefined; - const path = (await Bun.file(input.dataset).exists()) - ? input.dataset - : (tempDatasetPath = await this.downloadDatasetToTemp( - input.dataset, - input.datasetVersion, - options, - signal, - )); - try { - const scenarios = await loadDatasetFile(path); - const byId = new Map(scenarios.map((s) => [s.scenarioId, s])); - - // Resolve the runtime once; normalizeRuntimeInvokeRequest validates auth + fills - // accountId. Invoke is the extracted free fn — no RuntimeClient dependency. - const runtime = await this.clients - .control(toClientConfig(options)) - .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId })); - const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; - - const { ok, failed, firstError } = await runScenarios(scenarios, async (scenario) => { - // One session per scenario; the id is generated client-side (session id is a - // client-owned input per the AgentCore docs, needed on the request before any - // response and reused across turns). Turns run sequentially against the SAME - // session so the conversation — and its per-turn traces — accumulate in order, - // matching the per-turn ground truth in toSessionMetadata. - const runtimeSessionId = randomUUID(); - try { - for (const turn of scenario.turns) { - const request = normalizeRuntimeInvokeRequest(runtime, { - runtimeId: input.runtimeId, - qualifier: input.qualifier, - payload: renderJsonTemplate(input.payloadTemplate, { input: turn.input }), - contentType: "application/json", - accept: "application/json", - applicationHeaders: input.headers, - bearerToken: input.bearerToken, - runtimeSessionId, - runtimeUserId: input.userId, - }); - const response = await invokeRuntime(deps, request, options, signal); - for await (const _chunk of response.body) { - // Drain each turn's stream so it completes before the next turn. - } - } - } catch (error) { - this.logger.debug( - `simulate: invoke failed for scenario "${scenario.scenarioId}": ${(error as Error).message}`, - ); - throw error; - } - return { scenarioId: scenario.scenarioId, sessionId: runtimeSessionId }; - }); + ): Promise { + // readDatasetText owns the local-vs-id fetch + temp cleanup; DatasetLoader is the + // pure parse into Example instances (each dispatches its own replay via run()). + const examples = DatasetLoader.load( + await this.readDatasetText(input.dataset, input.datasetVersion, options, signal), + ); - if (ok.length === 0) { - const detail = firstError ? `; first error: ${firstError.message}` : ""; - throw new InputValidationError( - `no scenarios could be invoked (${failed} failed) — nothing to evaluate${detail}`, + // Resolve the runtime once; normalizeRuntimeInvokeRequest validates auth + fills + // accountId. Invoke is the extracted free fn — no RuntimeClient dependency. + const runtime = await this.clients + .control(toClientConfig(options)) + .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId }), { + abortSignal: signal, + }); + const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; + + const { ok, failed, firstError } = await runExamples(examples, async (example) => { + // One session per example; the id is client-generated (a client-owned input per + // the AgentCore docs, needed before any response) and reused across turns so the + // conversation and its per-turn traces accumulate in order. + const sessionId = randomUUID(); + const ctx: RunContext = { + invokeOnce: async (payload) => { + const request = normalizeRuntimeInvokeRequest(runtime, { + runtimeId: input.runtimeId, + qualifier: input.qualifier, + payload: renderJsonTemplate(input.payloadTemplate, { input: payload }), + contentType: "application/json", + accept: "application/json", + applicationHeaders: input.headers, + bearerToken: input.bearerToken, + runtimeSessionId: sessionId, + runtimeUserId: input.userId, + }); + const response = await invokeRuntime(deps, request, options, signal); + // Read the body to completion (frees the socket, feeds an actor loop later); + // a scripted example ignores the returned text. + let text = ""; + const decoder = new TextDecoder(); + for await (const chunk of response.body) text += decoder.decode(chunk, { stream: true }); + text += decoder.decode(); + return { text }; + }, + }; + try { + const groundTruth = await example.run(ctx); + return { exampleId: example.exampleId, sessionId, groundTruth }; + } catch (error) { + this.logger.debug( + `invokeDataset: invoke failed for example "${example.exampleId}" (${example.schemaType}): ${(error as Error).message}`, ); + throw error; } - if (failed > 0) { - this.logger.warn(`simulate: ${failed} scenario(s) failed to invoke and were dropped`); - } + }); - // AgentCore takes ~30s-3min to emit spans for a freshly invoked session; submit - // too early and the service reads an empty log group and marks every session - // failed. Wait once for the batch to be safely ingestible (matches the old CLI's - // 180s wait). Skipped when disabled via `SIMULATE_INGESTION_WAIT_MS=0` (tests). - const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); - if (waitMs > 0) { - this.logger.info( - `waiting ${Math.round(waitMs / 1000)}s for span ingestion before submitting`, - ); - await sleep(waitMs, undefined, { signal }); - } + if (failed > 0) { + this.logger.warn(`invokeDataset: ${failed} example(s) failed to invoke and were dropped`); + } - const job = await this.startBatchEvaluation( - { - name: input.name, - description: input.description, - evaluatorIds: input.evaluatorIds, - source: { - origin: "agent", - agent: input.runtimeId, - endpoint: input.qualifier, - sessionIds: ok.map((r) => r.sessionId), - }, - groundTruth: ok.map((r) => toSessionMetadata(byId.get(r.scenarioId)!, r.sessionId)), - kmsKeyArn: input.kmsKeyArn, - }, - options, + // AgentCore takes ~30s-3min to emit spans for a freshly invoked session; grade too + // early and the service reads an empty log group and marks every session failed. Wait + // once. Skipped when disabled via `SIMULATE_INGESTION_WAIT_MS=0` (tests). + const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); + if (ok.length > 0 && waitMs > 0) { + this.logger.info( + `waiting ${Math.round(waitMs / 1000)}s for span ingestion before evaluating`, ); + await sleep(waitMs, undefined, { signal }); + } - return { - batchEvaluationId: job.batchEvaluationId, - status: job.status, - scenariosInvoked: ok.length, - scenariosFailed: failed, - }; + return { sessions: ok, invoked: ok.length, failed, firstError }; + } + + // readDatasetText resolves a dataset ref to its JSONL text: a local path directly, else + // download the dataset id to a temp file (cleaned up here). Both funnel through the + // shared readLocalDatasetFile so replay and updateDatasetExamples read files the same way. + private async readDatasetText( + ref: string, + version: string | undefined, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + if (await Bun.file(ref).exists()) return readLocalDatasetFile(ref, signal); + const path = await this.downloadDatasetToTemp(ref, version, options, signal); + try { + return await readLocalDatasetFile(path, signal); } finally { - if (tempDatasetPath) await unlink(tempDatasetPath).catch(() => {}); + await unlink(path).catch(() => {}); } } // downloadDatasetToTemp streams a dataset version's JSONL to a temp file so - // loadDatasetFile can read it — reuses downloadDataset rather than re-fetching. + // readDatasetText can read it — reuses downloadDataset rather than re-fetching. private async downloadDatasetToTemp( id: string, version: string | undefined, diff --git a/src/core/eval/dataset/__snapshots__/dataset.test.ts.snap b/src/core/eval/dataset/__snapshots__/dataset.test.ts.snap new file mode 100644 index 000000000..079d038fc --- /dev/null +++ b/src/core/eval/dataset/__snapshots__/dataset.test.ts.snap @@ -0,0 +1,35 @@ +// Bun Snapshot v1, https://bun.sh/docs/test/snapshots + +exports[`PredefinedExample ground truth maps to the expected inline shape [golden] 1`] = ` +{ + "assertions": [ + { + "text": "stays polite", + }, + { + "text": "does not promise a date", + }, + ], + "expectedTrajectory": { + "toolNames": [ + "refund_lookup", + "refund_create", + ], + }, + "turns": [ + { + "input": { + "prompt": "I want a refund", + }, + }, + { + "expectedResponse": { + "text": "Refund started", + }, + "input": { + "prompt": "order 123", + }, + }, + ], +} +`; diff --git a/src/core/eval/dataset/dataset.test.ts b/src/core/eval/dataset/dataset.test.ts new file mode 100644 index 000000000..26c615c8e --- /dev/null +++ b/src/core/eval/dataset/dataset.test.ts @@ -0,0 +1,137 @@ +import { test, expect, describe } from "bun:test"; +import { DatasetLoader } from "./load"; +import { PredefinedExample } from "./predefined"; +import { SimulatedExample } from "./simulated"; +import type { RunContext, TurnResult } from "./types"; + +const row = (o: object) => JSON.stringify(o); + +// A fake transport that records what was said and returns a canned reply. No AWS. +function recordingCtx(reply = ""): { ctx: RunContext; calls: string[] } { + const calls: string[] = []; + const ctx: RunContext = { + invokeOnce: async (payload): Promise => { + calls.push(payload); + return { text: reply }; + }, + }; + return { ctx, calls }; +} + +describe("DatasetLoader.load", () => { + test("builds a PredefinedExample from a turns row", () => { + const [e] = DatasetLoader.load(row({ example_id: "x", turns: [{ input: "hi" }] })); + expect(e).toBeInstanceOf(PredefinedExample); + expect(e!.schemaType).toBe("AGENTCORE_EVALUATION_PREDEFINED_V1"); + expect(e!.exampleId).toBe("x"); + }); + + test("accepts the legacy scenario_id as the example id", () => { + const [e] = DatasetLoader.load(row({ scenario_id: "legacy", turns: [{ input: "hi" }] })); + expect(e!.exampleId).toBe("legacy"); + }); + + test("refuses a row that is both predefined and simulated", () => { + expect(() => + DatasetLoader.load(row({ example_id: "x", turns: [{ input: "a" }], actor_profile: {} })), + ).toThrow(/both 'turns' and 'actor_profile'/); + }); + + test("refuses a row that is neither", () => { + expect(() => DatasetLoader.load(row({ example_id: "x" }))).toThrow( + /neither 'turns' nor 'actor_profile'/, + ); + }); + + test("names a simulated row instead of blaming the data", () => { + expect(() => + DatasetLoader.load(row({ example_id: "x", actor_profile: { goal: "g" } })), + ).toThrow(/simulated example/); + }); + + test("rejects duplicate example ids", () => { + const two = [ + row({ example_id: "a", turns: [{ input: "1" }] }), + row({ example_id: "a", turns: [{ input: "2" }] }), + ].join("\n"); + expect(() => DatasetLoader.load(two)).toThrow(/duplicate example_id: "a"/); + }); + + test("rejects a missing example id", () => { + expect(() => DatasetLoader.load(row({ turns: [{ input: "hi" }] }))).toThrow( + /missing 'example_id'/, + ); + }); + + test("rejects an invalid JSON line", () => { + expect(() => DatasetLoader.load("{not json")).toThrow(/not valid JSON/); + }); + + test("rejects an empty dataset", () => { + expect(() => DatasetLoader.load("\n \n")).toThrow(/no examples/); + }); + + test("ignores blank lines between rows", () => { + const examples = DatasetLoader.load( + [ + row({ example_id: "a", turns: [{ input: "1" }] }), + "", + row({ example_id: "b", turns: [{ input: "2" }] }), + ].join("\n"), + ); + expect(examples.map((e) => e.exampleId)).toEqual(["a", "b"]); + }); +}); + +describe("SimulatedExample", () => { + test("construction throws NotImplementedError (never replayed)", () => { + expect(() => new SimulatedExample("x", { actor_profile: {} })).toThrow(/cannot replay yet/); + }); +}); + +describe("PredefinedExample", () => { + test("constructor rejects a row with no turns", () => { + expect(() => new PredefinedExample("x", { turns: [] })).toThrow(/has no turns/); + }); + + test("run replays every turn in order, on one session", async () => { + const { ctx, calls } = recordingCtx(); + await new PredefinedExample("x", { turns: [{ input: "a" }, { input: "b" }] }).run(ctx); + expect(calls).toEqual(["a", "b"]); + }); + + test("sparse expectations keep their turn position", async () => { + const { ctx } = recordingCtx(); + const gt = await new PredefinedExample("x", { + turns: [{ input: "t1" }, { input: "t2" }, { input: "t3", expected_response: "42" }], + }).run(ctx); + // Not 1 — filtering the two expectation-less turns would renumber the rest and score + // turn 3's "42" against turn 1. + expect(gt!.turns).toHaveLength(3); + expect(gt!.turns![2]!.expectedResponse).toEqual({ text: "42" }); + expect(gt!.turns![0]!.input).toEqual({ prompt: "t1" }); + expect(gt!.turns![0]!.expectedResponse).toBeUndefined(); + }); + + test("an example with no ground truth returns undefined", async () => { + const { ctx } = recordingCtx(); + const gt = await new PredefinedExample("x", { turns: [{ input: "t1" }] }).run(ctx); + expect(gt).toBeUndefined(); + }); + + // Golden: the full inline ground-truth shape for a representative example (assertions + + // trajectory + sparse turns). Locks the exact wire shape handed to the grader so a + // regression in the mapping is caught, not just its parts. + test("ground truth maps to the expected inline shape [golden]", async () => { + const { ctx } = recordingCtx(); + const gt = await new PredefinedExample("orders-1", { + turns: [ + { input: "I want a refund" }, + { input: "order 123", expected_response: "Refund started" }, + ], + assertions: ["stays polite", "does not promise a date"], + expected_trajectory: ["refund_lookup", "refund_create"], + }).run(ctx); + expect(gt).toMatchSnapshot(); + }); +}); diff --git a/src/core/eval/dataset/load.ts b/src/core/eval/dataset/load.ts new file mode 100644 index 000000000..516e94191 --- /dev/null +++ b/src/core/eval/dataset/load.ts @@ -0,0 +1,73 @@ +import type { DatasetSchemaType } from "@aws-sdk/client-bedrock-agentcore-control"; +import { InputValidationError } from "../../../errors"; +import type { Example } from "./types"; +import { PredefinedExample } from "./predefined"; +import { SimulatedExample } from "./simulated"; + +// DatasetLoader parses dataset JSONL into Example instances. Pure — no I/O, no AWS — so +// it's unit-testable with a plain string; fetching the text (local file or dataset id) is +// the caller's job. +export class DatasetLoader { + static load(text: string): Example[] { + const examples: Example[] = []; + const seen = new Set(); + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + let row: Record; + try { + row = JSON.parse(trimmed) as Record; + } catch { + throw new InputValidationError("dataset contains a line that is not valid JSON"); + } + + // The id is the join key between a session and its ground truth, so a missing or + // duplicate id silently misassigns ground truth to the wrong session. + const exampleId = String(row.example_id ?? row.scenario_id ?? ""); + if (!exampleId) { + throw new InputValidationError("dataset example is missing 'example_id'"); + } + if (seen.has(exampleId)) { + throw new InputValidationError(`dataset has a duplicate example_id: "${exampleId}"`); + } + seen.add(exampleId); + + examples.push(DatasetLoader.build(row, exampleId)); + } + if (examples.length === 0) throw new InputValidationError("dataset has no examples"); + return examples; + } + + // Classify by row shape — a local JSONL carries no schemaType, and AWS's own SDK + // dispatches this way, so a file the SDK accepts this CLI must accept too. Refuse a + // both-row: AWS's `if "turns" in raw` silently drops the actor profile, which reads as + // a passing run of the wrong test. + private static build(row: Record, exampleId: string): Example { + const hasTurns = Array.isArray(row.turns); + const hasActor = row.actor_profile != null; + if (hasTurns && hasActor) { + throw new InputValidationError( + `example "${exampleId}" has both 'turns' and 'actor_profile' — one row cannot be both`, + ); + } + if (!hasTurns && !hasActor) { + throw new InputValidationError( + `example "${exampleId}" has neither 'turns' nor 'actor_profile'`, + ); + } + + // `: DatasetSchemaType` types the scrutinee as the full SDK enum, so when a member is + // added the switch stops being exhaustive, build can reach its end without returning, + // and the compiler flags it (TS2366). The `new X(exampleId, row)` sites enforce the + // constructor shape — no map, no assertNever. + const schemaType: DatasetSchemaType = hasTurns + ? "AGENTCORE_EVALUATION_PREDEFINED_V1" + : "AGENTCORE_EVALUATION_SIMULATED_V1"; + switch (schemaType) { + case "AGENTCORE_EVALUATION_PREDEFINED_V1": + return new PredefinedExample(exampleId, row); + case "AGENTCORE_EVALUATION_SIMULATED_V1": + return new SimulatedExample(exampleId, row); + } + } +} diff --git a/src/core/eval/dataset/predefined.ts b/src/core/eval/dataset/predefined.ts new file mode 100644 index 000000000..5695f242c --- /dev/null +++ b/src/core/eval/dataset/predefined.ts @@ -0,0 +1,67 @@ +import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore"; +import { InputValidationError } from "../../../errors"; +import type { Example, RunContext } from "./types"; + +type Turn = { input: string; expectedResponse?: string }; + +// A predefined example has scripted turns: replay each verbatim, ignore the reply. Owns +// its parse (from the raw row) and its ground-truth mapping — everything predefined in +// one place. +export class PredefinedExample implements Example { + readonly schemaType = "AGENTCORE_EVALUATION_PREDEFINED_V1" as const; + readonly turns: Turn[]; + readonly assertions?: string[]; + readonly expectedTrajectory?: string[]; + + // Parsing happens in the constructor (validation-at-boundary). Fields are assigned in + // the body, not field initializers, so there's no "used before init" hazard. + constructor( + readonly exampleId: string, + row: Record, + ) { + const turns = (Array.isArray(row.turns) ? row.turns : []).map((t: Record) => ({ + input: String(t.input ?? ""), + expectedResponse: t.expected_response as string | undefined, + })); + if (turns.length === 0) { + throw new InputValidationError(`example "${exampleId}" has no turns`); + } + this.turns = turns; + this.assertions = row.assertions as string[] | undefined; + this.expectedTrajectory = row.expected_trajectory as string[] | undefined; + } + + // Turns share one session, so they run sequentially and awaited: racing them would + // interleave the conversation and misalign the per-turn traces with the ground truth. + async run(ctx: RunContext): Promise { + for (const turn of this.turns) await ctx.invokeOnce(turn.input); + return this.groundTruth(); + } + + // One entry per turn, each carrying its prompt in `input`, so a turn with no expected + // response still occupies its slot. Filtering the sparse turns out renumbers the rest, + // scoring turn 3's expectation against turn 1; the service's alignment rule is + // undocumented, so carrying the prompt is correct whether it aligns by index or content. + private groundTruth(): InlineGroundTruth | undefined { + const turns = this.turns.some((t) => t.expectedResponse !== undefined) + ? this.turns.map((t) => ({ + input: { prompt: t.input }, + ...(t.expectedResponse !== undefined && { + expectedResponse: { text: t.expectedResponse }, + }), + })) + : []; + const assertions = this.assertions?.map((text) => ({ text })); + const inline: InlineGroundTruth = { + // Omit empty arrays: the service rejects zero-length `assertions`/`turns` + // (documented min-1) rather than reading them as "no data". + ...(assertions && assertions.length > 0 && { assertions }), + ...(this.expectedTrajectory?.length && { + expectedTrajectory: { toolNames: this.expectedTrajectory }, + }), + ...(turns.length > 0 && { turns }), + }; + // An all-empty inline is not "no ground truth" — return undefined so the caller omits it. + return Object.keys(inline).length > 0 ? inline : undefined; + } +} diff --git a/src/core/eval/dataset/run.ts b/src/core/eval/dataset/run.ts new file mode 100644 index 000000000..1c9aeabc1 --- /dev/null +++ b/src/core/eval/dataset/run.ts @@ -0,0 +1,29 @@ +// runExamples runs `worker` over every item with bounded concurrency. A failed worker +// doesn't sink the run — the failure is counted (so the caller can report on all-failed) +// and its item dropped. Returns ok results + the first error, which the caller surfaces +// to explain a total failure. Generic in the item type; not eval-specific. +export type ExampleRun = { ok: Result[]; failed: number; firstError?: Error }; + +export async function runExamples( + items: Item[], + worker: (item: Item) => Promise, + concurrency = 5, +): Promise> { + const ok: Result[] = []; + let failed = 0; + let firstError: Error | undefined; + let next = 0; + const run = async (): Promise => { + while (next < items.length) { + const item = items[next++]!; + try { + ok.push(await worker(item)); + } catch (error) { + failed++; + if (!firstError) firstError = error instanceof Error ? error : new Error(String(error)); + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run)); + return { ok, failed, firstError }; +} diff --git a/src/core/eval/dataset/simulated.ts b/src/core/eval/dataset/simulated.ts new file mode 100644 index 000000000..7428fef84 --- /dev/null +++ b/src/core/eval/dataset/simulated.ts @@ -0,0 +1,27 @@ +import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore"; +import { NotImplementedError } from "../../../errors"; +import type { Example, RunContext } from "./types"; + +// A simulated example carries an actor profile instead of scripted turns: replaying it +// needs an LLM "user" to generate each next message from the agent's reply, which this +// command does not run yet. Throw at construction (= load time) so the user fails early +// with a clear message rather than a per-row "has no turns" misdiagnosis mid-run. +export class SimulatedExample implements Example { + readonly schemaType = "AGENTCORE_EVALUATION_SIMULATED_V1" as const; + + constructor( + readonly exampleId: string, + _row: Record, + ) { + throw new NotImplementedError( + `example "${exampleId}" is a simulated example (actor_profile), which this ` + + `command cannot replay yet — it has no scripted turns`, + ); + } + + // Unreachable today (the constructor throws). Implements the interface; the actor loop + // lands here when simulated ships. + run(_ctx: RunContext): Promise { + throw new NotImplementedError("simulated example replay is not implemented"); + } +} diff --git a/src/core/eval/dataset/types.ts b/src/core/eval/dataset/types.ts new file mode 100644 index 000000000..d6a9faf72 --- /dev/null +++ b/src/core/eval/dataset/types.ts @@ -0,0 +1,20 @@ +import type { DatasetSchemaType } from "@aws-sdk/client-bedrock-agentcore-control"; +import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore"; + +// TurnResult is the agent's reply to one turn. A record (not a bare string) so a future +// tool-branching dataset type can widen it by a field without touching every example. +export type TurnResult = { text: string }; + +// RunContext is the per-session transport handed to run(): one call = one turn. The +// session id, auth, and payload templating are bound by the caller (the machine), so an +// example only decides what to say next, never how the request is built. +export type RunContext = { invokeOnce: (payload: string) => Promise }; + +// Example is the contract each dataset type implements: identity plus a self-describing +// run. An interface, not a base class — there is no shared state or behaviour to inherit, +// and the machine dispatches by calling run(), so nothing needs a common superclass. +export interface Example { + readonly schemaType: DatasetSchemaType; + readonly exampleId: string; + run(ctx: RunContext): Promise; +} diff --git a/src/core/eval/simulate.ts b/src/core/eval/simulate.ts deleted file mode 100644 index 81fc3f843..000000000 --- a/src/core/eval/simulate.ts +++ /dev/null @@ -1,120 +0,0 @@ -import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; -import { InputValidationError } from "../../errors"; - -// Scenario is one dataset row for replay. Local to this module on purpose — it is -// not a handler contract, so it does not live in handlers/eval/types.tsx. Field -// names mirror the dataset JSONL (snake_case in, camelCase here). -export type Scenario = { - scenarioId: string; - turns: { input: string; expectedResponse?: string }[]; - assertions?: string[]; - expectedTrajectory?: string[]; -}; - -// parseScenarios reads dataset JSONL (one scenario per line) into Scenario records. -// Each scenario needs a non-empty, unique `scenario_id` — the id is the join key -// between the session created for it and its ground truth, so a missing/duplicate -// id silently misassigns ground truth to the wrong session. -export function parseScenarios(text: string): Scenario[] { - const scenarios: Scenario[] = []; - const seen = new Set(); - for (const line of text.split("\n")) { - const trimmed = line.trim(); - if (!trimmed) continue; - let row: Record; - try { - row = JSON.parse(trimmed) as Record; - } catch { - throw new InputValidationError("dataset contains a line that is not valid JSON"); - } - const scenario = toScenario(row); - if (!scenario.scenarioId) { - throw new InputValidationError("dataset scenario is missing 'scenario_id'"); - } - if (seen.has(scenario.scenarioId)) { - throw new InputValidationError( - `dataset has a duplicate scenario_id: "${scenario.scenarioId}"`, - ); - } - if (scenario.turns.length === 0) { - throw new InputValidationError(`scenario "${scenario.scenarioId}" has no turns`); - } - seen.add(scenario.scenarioId); - scenarios.push(scenario); - } - if (scenarios.length === 0) throw new InputValidationError("dataset has no scenarios"); - return scenarios; -} - -function toScenario(row: Record): Scenario { - const turns = Array.isArray(row.turns) ? row.turns : []; - return { - scenarioId: String(row.scenario_id ?? ""), - turns: turns.map((t: Record) => ({ - input: String(t.input ?? ""), - expectedResponse: t.expected_response as string | undefined, - })), - assertions: row.assertions as string[] | undefined, - expectedTrajectory: row.expected_trajectory as string[] | undefined, - }; -} - -// loadDatasetFile reads scenarios from a local JSONL path. The dataset-id path is -// handled by the caller (downloadDataset to a temp file, then this). -export async function loadDatasetFile(path: string): Promise { - return parseScenarios(await Bun.file(path).text()); -} - -// runScenarios runs `worker` over every scenario with bounded concurrency. A failed -// worker doesn't sink the run — the failure is captured (so the caller can report -// on all-failed) but drops that scenario. Returns ok results + the first error we -// saw, which the caller can surface to explain a total failure. -export type ScenarioRun = { ok: T[]; failed: number; firstError?: Error }; -export async function runScenarios( - scenarios: Scenario[], - worker: (scenario: Scenario) => Promise, - concurrency = 5, -): Promise> { - const ok: T[] = []; - let failed = 0; - let firstError: Error | undefined; - let next = 0; - const run = async (): Promise => { - while (next < scenarios.length) { - const scenario = scenarios[next++]!; - try { - ok.push(await worker(scenario)); - } catch (error) { - failed++; - if (!firstError) firstError = error instanceof Error ? error : new Error(String(error)); - } - } - }; - await Promise.all(Array.from({ length: Math.min(concurrency, scenarios.length) }, run)); - return { ok, failed, firstError }; -} - -// toSessionMetadata maps a scenario's ground truth onto the session the replay -// created — the batch service's per-session ground-truth shape (inline arm). -// Omit array fields when empty: the service rejects zero-length `turns` / -// `assertions` (`length >= 1`) rather than treating an empty array as "no data". -export function toSessionMetadata(scenario: Scenario, sessionId: string): SessionMetadataShape { - const turns = scenario.turns - .filter((t) => t.expectedResponse !== undefined) - .map((t) => ({ expectedResponse: { text: t.expectedResponse! } })); - const assertions = scenario.assertions?.map((text) => ({ text })); - return { - sessionId, - testScenarioId: scenario.scenarioId, - groundTruth: { - inline: { - ...(assertions && assertions.length > 0 && { assertions }), - ...(scenario.expectedTrajectory && - scenario.expectedTrajectory.length > 0 && { - expectedTrajectory: { toolNames: scenario.expectedTrajectory }, - }), - ...(turns.length > 0 && { turns }), - }, - }, - }; -} diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index 276e8f1b8..56fc0bbce 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -57,7 +57,10 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => const interrupt = () => controller.abort(); process.once("SIGINT", interrupt); try { - const result = await core.eval.simulate( + const opts = coreOptsFromCtx(ctx); + + // (1) Replay the dataset → one graded-ready session per example. + const r = await core.eval.invokeDataset( { runtimeId: flags["runtime-id"], qualifier: flags["qualifier"], @@ -67,15 +70,46 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => userId: flags["user-id"], dataset: flags["dataset"], datasetVersion: flags["dataset-version"], - evaluatorIds: flags["evaluator"], + }, + opts, + controller.signal, + ); + if (r.invoked === 0) { + const detail = r.firstError ? `; first error: ${r.firstError.message}` : ""; + throw new InputValidationError( + `no examples could be invoked (${r.failed} failed) — nothing to evaluate${detail}`, + ); + } + + // (2) Grade via the batch service — the example's neutral ground truth crosses + // over as sessionMetadata (inline arm). + const job = await core.eval.startBatchEvaluation( + { name: flags["name"], description: flags["description"], + evaluatorIds: flags["evaluator"], + source: { + origin: "agent", + agent: flags["runtime-id"], + endpoint: flags["qualifier"], + sessionIds: r.sessions.map((s) => s.sessionId), + }, + groundTruth: r.sessions.map((s) => ({ + sessionId: s.sessionId, + testScenarioId: s.exampleId, + ...(s.groundTruth && { groundTruth: { inline: s.groundTruth } }), + })), kmsKeyArn: flags["kms-key-arn"], }, - coreOptsFromCtx(ctx), - controller.signal, + opts, ); - ctx.require(JsonRendererKey).renderJson(result); + + ctx.require(JsonRendererKey).renderJson({ + batchEvaluationId: job.batchEvaluationId, + status: job.status, + examplesInvoked: r.invoked, + examplesFailed: r.failed, + }); } catch (error) { // A Ctrl-C exits quietly; the half-created sessions grade nothing. if (controller.signal.aborted) return; diff --git a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx index fb0c6149d..d840ba204 100644 --- a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx @@ -6,18 +6,22 @@ import { testIO, TestGlobalConfigAccessor, } from "../../../../testing"; -import type { SimulateResult } from "../../types"; +import type { InvokeDatasetResult } from "../../types"; -const RESULT: SimulateResult = { - batchEvaluationId: "batch-eval-sim", - status: "RUNNING", - scenariosInvoked: 3, - scenariosFailed: 0, +// Two invoked sessions; the handler feeds these into startBatchEvaluation and renders +// the job it returns (DEFAULT_START_BATCH_EVAL_RESPONSE: batch-eval-test / RUNNING). +const INVOKE_RESULT: InvokeDatasetResult = { + sessions: [ + { exampleId: "e1", sessionId: "s1", groundTruth: { assertions: [{ text: "polite" }] } }, + { exampleId: "e2", sessionId: "s2" }, + ], + invoked: 2, + failed: 0, }; async function run(args: string[], configure?: (core: TestCoreClient) => void) { const core = new TestCoreClient(); - core.eval.setSimulateResponse(RESULT); + core.eval.setInvokeDatasetResponse(INVOKE_RESULT); configure?.(core); const io = testIO(); const root = createRootHandler(core, { @@ -113,20 +117,56 @@ describe("eval batch-evaluation simulate", () => { await expect(run(["eval", "batch-evaluation", "simulate", ...args])).rejects.toThrow(expected); }); - test("maps flags to the simulate input and renders the result", async () => { - const { core, stdout } = await run([...BASE, "--qualifier", "PROD", "--header", "x-a:1"]); - expect(JSON.parse(stdout)).toEqual(RESULT); - const call = core.eval.calls.find((c) => c.method === "simulate"); + test("passes runtime-level flags to invokeDataset (no evaluator/name leak)", async () => { + const { core } = await run([...BASE, "--qualifier", "PROD", "--header", "x-a:1"]); + const call = core.eval.calls.find((c) => c.method === "invokeDataset"); expect(call?.args[0]).toMatchObject({ runtimeId: "r-1", qualifier: "PROD", payloadTemplate: '{"prompt":"{input}"}', headers: [["x-a", "1"]], dataset: "/tmp/ds.jsonl", - evaluatorIds: ["Builtin.Helpfulness"], - name: "sim-1", }); + // Grader-only flags are NOT part of the invokeDataset (runtime-level) input. + expect(call?.args[0]).not.toHaveProperty("evaluatorIds"); + expect(call?.args[0]).not.toHaveProperty("name"); // Handler wires an AbortSignal (Ctrl-C) through to Core. expect(call?.args[2]).toBeInstanceOf(AbortSignal); }); + + test("composes startBatchEvaluation over the created sessions + wrapped ground truth", async () => { + const { core, stdout } = await run(BASE); + + // Rendered output is the batch job + invoked/failed counts. + expect(JSON.parse(stdout)).toEqual({ + batchEvaluationId: "batch-eval-test", + status: "RUNNING", + examplesInvoked: 2, + examplesFailed: 0, + }); + + const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + expect(start?.args[0]).toMatchObject({ + name: "sim-1", + evaluatorIds: ["Builtin.Helpfulness"], + source: { origin: "agent", agent: "r-1", sessionIds: ["s1", "s2"] }, + // e1's inline GT is wrapped; e2 (no GT) omits the member. + groundTruth: [ + { + sessionId: "s1", + testScenarioId: "e1", + groundTruth: { inline: { assertions: [{ text: "polite" }] } }, + }, + { sessionId: "s2", testScenarioId: "e2" }, + ], + }); + }); + + test("refuses to grade when nothing was invoked", async () => { + await expect( + run(BASE, (core) => + core.eval.setInvokeDatasetResponse({ sessions: [], invoked: 0, failed: 3 }), + ), + ).rejects.toThrow(/no examples could be invoked \(3 failed\)/); + }); }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 07687048e..e5a407cab 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -34,6 +34,7 @@ import type { ListBatchEvaluationsResponse, StartBatchEvaluationResponse, SessionMetadataShape, + InlineGroundTruth, EvaluationReferenceInput, EvaluationResultContent, DataSourceConfig as DataPlaneDataSourceConfig, @@ -205,34 +206,37 @@ export type StartBatchEvaluationInput = { kmsKeyArn?: string; }; -// SimulateInput is the CLI-facing shape for `batch-evaluation simulate` — dataset -// replay + batch grade. SDK-native fields only (no RuntimeInvokeRequest/EvaluateInput -// leak): Core invokes each scenario against the runtime, then submits a batch -// evaluation scoped to the sessions it created. Invoke fields mirror `runtime invoke`. -export type SimulateInput = { +// InvokeDatasetInput is the runtime-level shape for replaying a dataset: invoke each +// example against the runtime, one client-generated session per example. Runtime fields +// only — no evaluator/name/kms (those belong to the grader the handler composes on top, +// e.g. startBatchEvaluation). Invoke fields mirror `runtime invoke`. +export type InvokeDatasetInput = { runtimeId: string; qualifier?: string; - payloadTemplate: string; // e.g. {"prompt":"{input}"} — {input} is the scenario's turn input + payloadTemplate: string; // e.g. {"prompt":"{input}"} — {input} is the example's turn input headers?: [string, string][]; bearerToken?: string; - // simulate always creates a fresh session per scenario. Reusing one session - // across scenarios interleaves unrelated turns and collides ground-truth keys. userId?: string; dataset: string; // local JSONL path or a dataset id datasetVersion?: string; - evaluatorIds: string[]; - name: string; - description?: string; - kmsKeyArn?: string; }; -// SimulateResult reports the submitted job plus how many scenarios were actually -// invoked vs dropped (a failed invoke is skipped, not fatal, unless all fail). -export type SimulateResult = { - batchEvaluationId?: string; - status?: string; - scenariosInvoked: number; - scenariosFailed: number; +// InvokedSession is one replayed example: the session created for it plus its neutral +// ground truth. Grader-agnostic — the batch handler wraps `groundTruth` as +// SessionMetadataShape; a future ondemand handler adapts it to EvaluationReferenceInput. +export type InvokedSession = { + exampleId: string; + sessionId: string; + groundTruth?: InlineGroundTruth; +}; + +// InvokeDatasetResult reports the created sessions plus how many examples were invoked +// vs dropped (a failed invoke is skipped, not fatal). firstError explains a total failure. +export type InvokeDatasetResult = { + sessions: InvokedSession[]; + invoked: number; + failed: number; + firstError?: Error; }; // SpanRecord is one OTel span/log document — the parsed `@message` JSON of a @@ -359,14 +363,14 @@ export interface CoreEvalClient { // Evaluate API and returns per-session scores. No job, no CloudWatch — the // trace read happened in getTracesForAgent. evaluate(input: EvaluateInput, options: CoreOptions): Promise; - // simulate replays a dataset against the runtime (invoke per scenario, client-side) - // then submits a batch evaluation over the sessions it created. No dataset API - // exists service-side, so the CLI creates the sessions and the service grades them. - simulate( - input: SimulateInput, + // invokeDataset replays a dataset against the runtime (invoke per example, client-side, + // one session each) and returns the created sessions + neutral ground truth. Grader- + // agnostic: the handler composes it with startBatchEvaluation (or, later, evaluate). + invokeDataset( + input: InvokeDatasetInput, options: CoreOptions, signal?: AbortSignal, - ): Promise; + ): Promise; createOnlineEvaluationConfig( input: CreateOnlineEvalInput, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 63392c571..f53012ee1 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -133,8 +133,8 @@ import type { DatasetUpdateProgressEvent, EvaluateInput, EvaluateResult, - SimulateInput, - SimulateResult, + InvokeDatasetInput, + InvokeDatasetResult, GetBatchEvaluationResult, GetTracesInput, LlmAsAJudgeUpdate, @@ -1405,11 +1405,10 @@ export class TestEvalClient implements CoreEvalClient { sessionsEvaluated: 0, results: [], }; - private simulateResponse: SimulateResult = { - batchEvaluationId: "batch-eval-test", - status: "RUNNING", - scenariosInvoked: 0, - scenariosFailed: 0, + private invokeDatasetResponse: InvokeDatasetResult = { + sessions: [], + invoked: 0, + failed: 0, }; private error?: Error; @@ -1730,20 +1729,20 @@ export class TestEvalClient implements CoreEvalClient { return this.evaluateResponse; } - // setSimulateResponse sets what simulate resolves to (when not erroring). - setSimulateResponse(response: SimulateResult): this { - this.simulateResponse = response; + // setInvokeDatasetResponse sets what invokeDataset resolves to (when not erroring). + setInvokeDatasetResponse(response: InvokeDatasetResult): this { + this.invokeDatasetResponse = response; return this; } - async simulate( - input: SimulateInput, + async invokeDataset( + input: InvokeDatasetInput, options: CoreOptions, signal?: AbortSignal, - ): Promise { - this.calls.push({ method: "simulate", args: [input, options, signal] }); + ): Promise { + this.calls.push({ method: "invokeDataset", args: [input, options, signal] }); if (this.error) throw this.error; - return this.simulateResponse; + return this.invokeDatasetResponse; } async createOnlineEvaluationConfig( From cb7b78b937a1934dc743fb5f87d4eb15837aaae9 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 18 Aug 2026 20:32:58 +0000 Subject: [PATCH 09/11] fix(eval): reject non-object dataset rows and turn entries instead of crashing A JSONL line that is valid JSON but not an object (e.g. bare `null`) threw a raw TypeError (`null is not an object`) when DatasetLoader dereferenced `row.example_id`; the same happened in PredefinedExample for a non-object turn entry (`turns: [null]`). Both now surface a clear InputValidationError at the parse boundary. Adds guard tests plus behavior locks for CRLF endings, unicode ids, empty assertions/trajectory omission, and `expected_response: ""`. --- src/core/eval/dataset/dataset.test.ts | 54 +++++++++++++++++++++++++++ src/core/eval/dataset/load.ts | 10 ++++- src/core/eval/dataset/predefined.ts | 16 ++++++-- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/src/core/eval/dataset/dataset.test.ts b/src/core/eval/dataset/dataset.test.ts index 26c615c8e..9c5efb5a6 100644 --- a/src/core/eval/dataset/dataset.test.ts +++ b/src/core/eval/dataset/dataset.test.ts @@ -81,6 +81,29 @@ describe("DatasetLoader.load", () => { ); expect(examples.map((e) => e.exampleId)).toEqual(["a", "b"]); }); + + // `null` is valid JSON but has no fields; dereferencing it once threw a raw TypeError + // instead of a clean validation error. + test.each([["null"], ["[1,2,3]"], ["42"], ['"hi"'], ["true"]])( + "rejects a non-object row (%s) with a clear error", + (line) => { + expect(() => DatasetLoader.load(line)).toThrow(/not a JSON object/); + }, + ); + + test("handles CRLF line endings", () => { + const crlf = + row({ example_id: "a", turns: [{ input: "1" }] }) + + "\r\n" + + row({ example_id: "b", turns: [{ input: "2" }] }) + + "\r\n"; + expect(DatasetLoader.load(crlf).map((e) => e.exampleId)).toEqual(["a", "b"]); + }); + + test("preserves unicode example ids", () => { + const [e] = DatasetLoader.load(row({ example_id: "café-日本-🎉", turns: [{ input: "1" }] })); + expect(e!.exampleId).toBe("café-日本-🎉"); + }); }); describe("SimulatedExample", () => { @@ -94,6 +117,37 @@ describe("PredefinedExample", () => { expect(() => new PredefinedExample("x", { turns: [] })).toThrow(/has no turns/); }); + test("constructor rejects a non-object turn entry", () => { + expect(() => new PredefinedExample("x", { turns: [null] })).toThrow(/turn 1 is not an object/); + }); + + test("omits empty assertions and expected_trajectory arrays", async () => { + const { ctx } = recordingCtx(); + // Only the expectation-bearing turn should survive to ground truth; the empty + // assertions/trajectory arrays are dropped (the service rejects zero-length ones). + const gt = await new PredefinedExample("x", { + turns: [{ input: "t1", expected_response: "r1" }], + assertions: [], + expected_trajectory: [], + }).run(ctx); + expect(gt).toBeDefined(); + expect(gt!.assertions).toBeUndefined(); + expect(gt!.expectedTrajectory).toBeUndefined(); + expect(gt!.turns).toHaveLength(1); + }); + + // A deliberate `expected_response: ""` means "expect an empty reply" — distinct from + // omitting the field. The `!== undefined` guard honors it: the turn carries + // expectedResponse { text: "" } rather than being treated as expectation-less. + test('treats expected_response "" as a real expectation', async () => { + const { ctx } = recordingCtx(); + const gt = await new PredefinedExample("x", { + turns: [{ input: "t1", expected_response: "" }], + }).run(ctx); + expect(gt!.turns).toHaveLength(1); + expect(gt!.turns![0]!.expectedResponse).toEqual({ text: "" }); + }); + test("run replays every turn in order, on one session", async () => { const { ctx, calls } = recordingCtx(); await new PredefinedExample("x", { turns: [{ input: "a" }, { input: "b" }] }).run(ctx); diff --git a/src/core/eval/dataset/load.ts b/src/core/eval/dataset/load.ts index 516e94191..8aba42160 100644 --- a/src/core/eval/dataset/load.ts +++ b/src/core/eval/dataset/load.ts @@ -14,12 +14,18 @@ export class DatasetLoader { for (const line of text.split("\n")) { const trimmed = line.trim(); if (!trimmed) continue; - let row: Record; + let parsed: unknown; try { - row = JSON.parse(trimmed) as Record; + parsed = JSON.parse(trimmed); } catch { throw new InputValidationError("dataset contains a line that is not valid JSON"); } + // Reject non-object rows before dereferencing: `null` (valid JSON) would throw a + // raw TypeError, and an array/primitive can never carry the fields a row needs. + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new InputValidationError("dataset contains a line that is not a JSON object"); + } + const row = parsed as Record; // The id is the join key between a session and its ground truth, so a missing or // duplicate id silently misassigns ground truth to the wrong session. diff --git a/src/core/eval/dataset/predefined.ts b/src/core/eval/dataset/predefined.ts index 5695f242c..fcc626251 100644 --- a/src/core/eval/dataset/predefined.ts +++ b/src/core/eval/dataset/predefined.ts @@ -19,10 +19,18 @@ export class PredefinedExample implements Example { readonly exampleId: string, row: Record, ) { - const turns = (Array.isArray(row.turns) ? row.turns : []).map((t: Record) => ({ - input: String(t.input ?? ""), - expectedResponse: t.expected_response as string | undefined, - })); + const turns = (Array.isArray(row.turns) ? row.turns : []).map((t: unknown, i: number) => { + // A non-object entry (e.g. `null`) has no fields to read; dereferencing it would + // throw a raw TypeError, so reject it here with the same boundary-validation intent. + if (typeof t !== "object" || t === null) { + throw new InputValidationError(`example "${exampleId}" turn ${i + 1} is not an object`); + } + const turn = t as Record; + return { + input: String(turn.input ?? ""), + expectedResponse: turn.expected_response as string | undefined, + }; + }); if (turns.length === 0) { throw new InputValidationError(`example "${exampleId}" has no turns`); } From 347534754152a5d1edd5e386132f46b8acc796d1 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 18 Aug 2026 20:33:04 +0000 Subject: [PATCH 10/11] test(eval): cover runExamples failure isolation + handler sessionMetadata golden runExamples: lock per-example failure isolation (one worker throws, the rest run and are returned; firstError captured and non-Error throws wrapped), exactly-once processing, the concurrency bound, and the empty-input case. Adds a golden snapshot of the sessionMetadata the simulate handler builds, pinning the `{ inline: gt }` wrapping and the omitted-member case for a session with no GT. --- src/core/eval/dataset/run.test.ts | 62 +++++++++++++++++++ .../__snapshots__/simulate.test.tsx.snap | 23 +++++++ .../simulate/simulate.test.tsx | 10 +++ 3 files changed, 95 insertions(+) create mode 100644 src/core/eval/dataset/run.test.ts create mode 100644 src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap diff --git a/src/core/eval/dataset/run.test.ts b/src/core/eval/dataset/run.test.ts new file mode 100644 index 000000000..7ebe17629 --- /dev/null +++ b/src/core/eval/dataset/run.test.ts @@ -0,0 +1,62 @@ +import { test, expect, describe } from "bun:test"; +import { runExamples } from "./run"; + +describe("runExamples", () => { + test("isolates a failing worker: it is counted, the rest still run", async () => { + const { ok, failed, firstError } = await runExamples([1, 2, 3, 4], async (n) => { + if (n === 2) throw new Error("boom 2"); + return n * 10; + }); + // The three survivors return; order among them is completion order, so compare as a set. + expect(new Set(ok)).toEqual(new Set([10, 30, 40])); + expect(failed).toBe(1); + expect(firstError?.message).toBe("boom 2"); + }); + + test("wraps a non-Error throw as an Error for firstError", async () => { + const { failed, firstError } = await runExamples([1], async () => { + throw "just a string"; + }); + expect(failed).toBe(1); + expect(firstError).toBeInstanceOf(Error); + expect(firstError?.message).toBe("just a string"); + }); + + test("processes every item exactly once", async () => { + const seen: number[] = []; + const items = Array.from({ length: 23 }, (_, i) => i); + const { ok } = await runExamples(items, async (n) => { + seen.push(n); + return n; + }); + expect(ok).toHaveLength(23); + expect([...seen].sort((a, b) => a - b)).toEqual(items); + }); + + test("never exceeds the concurrency bound", async () => { + let inFlight = 0; + let peak = 0; + await runExamples( + Array.from({ length: 20 }, (_, i) => i), + async () => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise((r) => setTimeout(r, 1)); + inFlight--; + }, + 3, + ); + expect(peak).toBeLessThanOrEqual(3); + }); + + test("empty input runs no workers and reports nothing invoked", async () => { + let called = false; + const { ok, failed, firstError } = await runExamples([], async () => { + called = true; + }); + expect(called).toBe(false); + expect(ok).toEqual([]); + expect(failed).toBe(0); + expect(firstError).toBeUndefined(); + }); +}); diff --git a/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap b/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap new file mode 100644 index 000000000..ddc632555 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap @@ -0,0 +1,23 @@ +// Bun Snapshot v1, https://bun.sh/docs/test/snapshots + +exports[`eval batch-evaluation simulate builds the sessionMetadata ground-truth shape [golden] 1`] = ` +[ + { + "groundTruth": { + "inline": { + "assertions": [ + { + "text": "polite", + }, + ], + }, + }, + "sessionId": "s1", + "testScenarioId": "e1", + }, + { + "sessionId": "s2", + "testScenarioId": "e2", + }, +] +`; diff --git a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx index d840ba204..2375a3b3f 100644 --- a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx @@ -162,6 +162,16 @@ describe("eval batch-evaluation simulate", () => { }); }); + // Golden: the exact evaluationMetadata (sessionMetadata) the handler builds from the + // invoked sessions. Locks the `{ inline: gt }` wrapping and the omitted-member case for + // a session with no ground truth — the wire shape the batch service reads. + test("builds the sessionMetadata ground-truth shape [golden]", async () => { + const { core } = await run(BASE); + const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + const input = start!.args[0] as { groundTruth: unknown }; + expect(input.groundTruth).toMatchSnapshot(); + }); + test("refuses to grade when nothing was invoked", async () => { await expect( run(BASE, (core) => From 98f4b1abab7d43101c748fd8a9bf6077f27b199e Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 18 Aug 2026 20:45:57 +0000 Subject: [PATCH 11/11] style(eval): trim code comments to why-not-how, drop restatements --- src/core/eval.tsx | 23 +++++++------------ src/core/eval/dataset/load.ts | 20 +++++----------- src/core/eval/dataset/predefined.ts | 23 ++++++------------- src/core/eval/dataset/run.ts | 6 ++--- src/core/eval/dataset/simulated.ts | 8 ++----- src/core/eval/dataset/types.ts | 13 ++++------- .../eval/batch-evaluation/simulate/index.tsx | 9 +++----- 7 files changed, 33 insertions(+), 69 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index e53c89cf5..1ed650d71 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -521,14 +521,11 @@ export class EvalClient implements CoreEvalClient { options: CoreOptions, signal?: AbortSignal, ): Promise { - // readDatasetText owns the local-vs-id fetch + temp cleanup; DatasetLoader is the - // pure parse into Example instances (each dispatches its own replay via run()). const examples = DatasetLoader.load( await this.readDatasetText(input.dataset, input.datasetVersion, options, signal), ); - // Resolve the runtime once; normalizeRuntimeInvokeRequest validates auth + fills - // accountId. Invoke is the extracted free fn — no RuntimeClient dependency. + // Resolve the runtime once, reused for every session. const runtime = await this.clients .control(toClientConfig(options)) .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId }), { @@ -537,9 +534,8 @@ export class EvalClient implements CoreEvalClient { const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; const { ok, failed, firstError } = await runExamples(examples, async (example) => { - // One session per example; the id is client-generated (a client-owned input per - // the AgentCore docs, needed before any response) and reused across turns so the - // conversation and its per-turn traces accumulate in order. + // One session per example; the id is a client-owned input per the AgentCore docs, + // reused across turns so the conversation and its per-turn traces stay in order. const sessionId = randomUUID(); const ctx: RunContext = { invokeOnce: async (payload) => { @@ -555,8 +551,7 @@ export class EvalClient implements CoreEvalClient { runtimeUserId: input.userId, }); const response = await invokeRuntime(deps, request, options, signal); - // Read the body to completion (frees the socket, feeds an actor loop later); - // a scripted example ignores the returned text. + // Read to completion to free the socket; a scripted example ignores the text. let text = ""; const decoder = new TextDecoder(); for await (const chunk of response.body) text += decoder.decode(chunk, { stream: true }); @@ -579,9 +574,8 @@ export class EvalClient implements CoreEvalClient { this.logger.warn(`invokeDataset: ${failed} example(s) failed to invoke and were dropped`); } - // AgentCore takes ~30s-3min to emit spans for a freshly invoked session; grade too - // early and the service reads an empty log group and marks every session failed. Wait - // once. Skipped when disabled via `SIMULATE_INGESTION_WAIT_MS=0` (tests). + // AgentCore emits spans ~30s-3min after invoke; grade too early and it reads an empty + // log group and fails every session. Disabled via SIMULATE_INGESTION_WAIT_MS=0 (tests). const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); if (ok.length > 0 && waitMs > 0) { this.logger.info( @@ -593,9 +587,8 @@ export class EvalClient implements CoreEvalClient { return { sessions: ok, invoked: ok.length, failed, firstError }; } - // readDatasetText resolves a dataset ref to its JSONL text: a local path directly, else - // download the dataset id to a temp file (cleaned up here). Both funnel through the - // shared readLocalDatasetFile so replay and updateDatasetExamples read files the same way. + // Resolve a dataset ref to JSONL text: a local path directly, else download the id to a + // temp file (cleaned up here). Reuses readLocalDatasetFile so replay reads like update. private async readDatasetText( ref: string, version: string | undefined, diff --git a/src/core/eval/dataset/load.ts b/src/core/eval/dataset/load.ts index 8aba42160..86c948d35 100644 --- a/src/core/eval/dataset/load.ts +++ b/src/core/eval/dataset/load.ts @@ -4,9 +4,7 @@ import type { Example } from "./types"; import { PredefinedExample } from "./predefined"; import { SimulatedExample } from "./simulated"; -// DatasetLoader parses dataset JSONL into Example instances. Pure — no I/O, no AWS — so -// it's unit-testable with a plain string; fetching the text (local file or dataset id) is -// the caller's job. +// Pure parse (no I/O) so it's testable with a plain string; the caller fetches the text. export class DatasetLoader { static load(text: string): Example[] { const examples: Example[] = []; @@ -20,15 +18,13 @@ export class DatasetLoader { } catch { throw new InputValidationError("dataset contains a line that is not valid JSON"); } - // Reject non-object rows before dereferencing: `null` (valid JSON) would throw a - // raw TypeError, and an array/primitive can never carry the fields a row needs. + // Reject non-object rows before dereferencing — `null` is valid JSON and would throw. if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { throw new InputValidationError("dataset contains a line that is not a JSON object"); } const row = parsed as Record; - // The id is the join key between a session and its ground truth, so a missing or - // duplicate id silently misassigns ground truth to the wrong session. + // The id joins a session to its ground truth — a missing/duplicate one misassigns it. const exampleId = String(row.example_id ?? row.scenario_id ?? ""); if (!exampleId) { throw new InputValidationError("dataset example is missing 'example_id'"); @@ -45,9 +41,7 @@ export class DatasetLoader { } // Classify by row shape — a local JSONL carries no schemaType, and AWS's own SDK - // dispatches this way, so a file the SDK accepts this CLI must accept too. Refuse a - // both-row: AWS's `if "turns" in raw` silently drops the actor profile, which reads as - // a passing run of the wrong test. + // dispatches this way. Refuse a both-row rather than silently dropping the actor profile. private static build(row: Record, exampleId: string): Example { const hasTurns = Array.isArray(row.turns); const hasActor = row.actor_profile != null; @@ -62,10 +56,8 @@ export class DatasetLoader { ); } - // `: DatasetSchemaType` types the scrutinee as the full SDK enum, so when a member is - // added the switch stops being exhaustive, build can reach its end without returning, - // and the compiler flags it (TS2366). The `new X(exampleId, row)` sites enforce the - // constructor shape — no map, no assertNever. + // Typed as the full SDK enum so a new member makes the switch non-exhaustive and + // build fails to compile (TS2366) — the build guard, no map or assertNever needed. const schemaType: DatasetSchemaType = hasTurns ? "AGENTCORE_EVALUATION_PREDEFINED_V1" : "AGENTCORE_EVALUATION_SIMULATED_V1"; diff --git a/src/core/eval/dataset/predefined.ts b/src/core/eval/dataset/predefined.ts index fcc626251..cd6c61d71 100644 --- a/src/core/eval/dataset/predefined.ts +++ b/src/core/eval/dataset/predefined.ts @@ -4,24 +4,18 @@ import type { Example, RunContext } from "./types"; type Turn = { input: string; expectedResponse?: string }; -// A predefined example has scripted turns: replay each verbatim, ignore the reply. Owns -// its parse (from the raw row) and its ground-truth mapping — everything predefined in -// one place. export class PredefinedExample implements Example { readonly schemaType = "AGENTCORE_EVALUATION_PREDEFINED_V1" as const; readonly turns: Turn[]; readonly assertions?: string[]; readonly expectedTrajectory?: string[]; - // Parsing happens in the constructor (validation-at-boundary). Fields are assigned in - // the body, not field initializers, so there's no "used before init" hazard. constructor( readonly exampleId: string, row: Record, ) { const turns = (Array.isArray(row.turns) ? row.turns : []).map((t: unknown, i: number) => { - // A non-object entry (e.g. `null`) has no fields to read; dereferencing it would - // throw a raw TypeError, so reject it here with the same boundary-validation intent. + // Reject a non-object entry here; dereferencing it below would throw a raw TypeError. if (typeof t !== "object" || t === null) { throw new InputValidationError(`example "${exampleId}" turn ${i + 1} is not an object`); } @@ -39,17 +33,16 @@ export class PredefinedExample implements Example { this.expectedTrajectory = row.expected_trajectory as string[] | undefined; } - // Turns share one session, so they run sequentially and awaited: racing them would - // interleave the conversation and misalign the per-turn traces with the ground truth. + // Sequential and awaited: the turns share one session, so racing them would interleave + // the conversation and misalign per-turn traces with the ground truth. async run(ctx: RunContext): Promise { for (const turn of this.turns) await ctx.invokeOnce(turn.input); return this.groundTruth(); } - // One entry per turn, each carrying its prompt in `input`, so a turn with no expected - // response still occupies its slot. Filtering the sparse turns out renumbers the rest, - // scoring turn 3's expectation against turn 1; the service's alignment rule is - // undocumented, so carrying the prompt is correct whether it aligns by index or content. + // Emit every turn (carrying its prompt), not just those with an expectation: filtering + // renumbers the rest, scoring turn 3's expectation against turn 1. The service's + // alignment rule is undocumented, so the prompt keeps index and content matching both valid. private groundTruth(): InlineGroundTruth | undefined { const turns = this.turns.some((t) => t.expectedResponse !== undefined) ? this.turns.map((t) => ({ @@ -61,15 +54,13 @@ export class PredefinedExample implements Example { : []; const assertions = this.assertions?.map((text) => ({ text })); const inline: InlineGroundTruth = { - // Omit empty arrays: the service rejects zero-length `assertions`/`turns` - // (documented min-1) rather than reading them as "no data". + // Omit empty arrays — the service rejects zero-length assertions/turns (min-1). ...(assertions && assertions.length > 0 && { assertions }), ...(this.expectedTrajectory?.length && { expectedTrajectory: { toolNames: this.expectedTrajectory }, }), ...(turns.length > 0 && { turns }), }; - // An all-empty inline is not "no ground truth" — return undefined so the caller omits it. return Object.keys(inline).length > 0 ? inline : undefined; } } diff --git a/src/core/eval/dataset/run.ts b/src/core/eval/dataset/run.ts index 1c9aeabc1..6f82a98b9 100644 --- a/src/core/eval/dataset/run.ts +++ b/src/core/eval/dataset/run.ts @@ -1,7 +1,5 @@ -// runExamples runs `worker` over every item with bounded concurrency. A failed worker -// doesn't sink the run — the failure is counted (so the caller can report on all-failed) -// and its item dropped. Returns ok results + the first error, which the caller surfaces -// to explain a total failure. Generic in the item type; not eval-specific. +// A failed worker is counted and dropped, not thrown — the caller reports all-failed via +// firstError. Bounded concurrency because each item invokes a live runtime. export type ExampleRun = { ok: Result[]; failed: number; firstError?: Error }; export async function runExamples( diff --git a/src/core/eval/dataset/simulated.ts b/src/core/eval/dataset/simulated.ts index 7428fef84..e5f9d7076 100644 --- a/src/core/eval/dataset/simulated.ts +++ b/src/core/eval/dataset/simulated.ts @@ -2,10 +2,8 @@ import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore"; import { NotImplementedError } from "../../../errors"; import type { Example, RunContext } from "./types"; -// A simulated example carries an actor profile instead of scripted turns: replaying it -// needs an LLM "user" to generate each next message from the agent's reply, which this -// command does not run yet. Throw at construction (= load time) so the user fails early -// with a clear message rather than a per-row "has no turns" misdiagnosis mid-run. +// Not shipped: replaying a simulated example needs an LLM actor we don't run yet. Throw +// at construction (= load time) so the user fails early, not with a per-row misdiagnosis. export class SimulatedExample implements Example { readonly schemaType = "AGENTCORE_EVALUATION_SIMULATED_V1" as const; @@ -19,8 +17,6 @@ export class SimulatedExample implements Example { ); } - // Unreachable today (the constructor throws). Implements the interface; the actor loop - // lands here when simulated ships. run(_ctx: RunContext): Promise { throw new NotImplementedError("simulated example replay is not implemented"); } diff --git a/src/core/eval/dataset/types.ts b/src/core/eval/dataset/types.ts index d6a9faf72..17ffce7b4 100644 --- a/src/core/eval/dataset/types.ts +++ b/src/core/eval/dataset/types.ts @@ -1,18 +1,15 @@ import type { DatasetSchemaType } from "@aws-sdk/client-bedrock-agentcore-control"; import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore"; -// TurnResult is the agent's reply to one turn. A record (not a bare string) so a future -// tool-branching dataset type can widen it by a field without touching every example. +// A record, not a bare string, so a future tool-branching type can widen it by a field. export type TurnResult = { text: string }; -// RunContext is the per-session transport handed to run(): one call = one turn. The -// session id, auth, and payload templating are bound by the caller (the machine), so an -// example only decides what to say next, never how the request is built. +// The per-session transport handed to run(): one call = one turn. Session id, auth, and +// templating are bound by the caller, so an example only decides what to say next. export type RunContext = { invokeOnce: (payload: string) => Promise }; -// Example is the contract each dataset type implements: identity plus a self-describing -// run. An interface, not a base class — there is no shared state or behaviour to inherit, -// and the machine dispatches by calling run(), so nothing needs a common superclass. +// An interface, not a base class: no shared state to inherit, and the machine dispatches +// by calling run(). export interface Example { readonly schemaType: DatasetSchemaType; readonly exampleId: string; diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index 56fc0bbce..ad6f5c4d0 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -7,9 +7,8 @@ import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; -// batch-evaluation simulate replays a dataset against a runtime (invoke per scenario) -// then submits a batch evaluation over the sessions it created. Invoke flags mirror -// `runtime invoke`; content-type/accept are fixed to application/json. +// Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror +// `runtime invoke`. export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => createHandler({ name: "simulate", @@ -59,7 +58,6 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => try { const opts = coreOptsFromCtx(ctx); - // (1) Replay the dataset → one graded-ready session per example. const r = await core.eval.invokeDataset( { runtimeId: flags["runtime-id"], @@ -81,8 +79,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => ); } - // (2) Grade via the batch service — the example's neutral ground truth crosses - // over as sessionMetadata (inline arm). + // The example's neutral ground truth crosses over as sessionMetadata (inline arm). const job = await core.eval.startBatchEvaluation( { name: flags["name"],