From 2a6caaf5d30b89688f44dd8a30407990fcc06269 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 29 Apr 2026 20:35:08 +0000 Subject: [PATCH 01/18] feat: wire telemetry withCommandRun into all add.* commands --- src/cli/primitives/AgentPrimitive.tsx | 157 +++++---- src/cli/primitives/CredentialPrimitive.tsx | 94 +++--- src/cli/primitives/EvaluatorPrimitive.ts | 195 +++++------ src/cli/primitives/GatewayPrimitive.ts | 89 ++--- src/cli/primitives/GatewayTargetPrimitive.ts | 314 ++++++++++-------- src/cli/primitives/MemoryPrimitive.tsx | 79 +++-- .../primitives/OnlineEvalConfigPrimitive.ts | 68 ++-- src/cli/primitives/PolicyEnginePrimitive.ts | 69 ++-- src/cli/primitives/PolicyPrimitive.ts | 71 ++-- .../primitives/RuntimeEndpointPrimitive.ts | 37 ++- src/cli/telemetry/schemas/command-run.ts | 1 + src/cli/telemetry/schemas/common-shapes.ts | 6 + 12 files changed, 658 insertions(+), 522 deletions(-) diff --git a/src/cli/primitives/AgentPrimitive.tsx b/src/cli/primitives/AgentPrimitive.tsx index 4702633ed..77518dc58 100644 --- a/src/cli/primitives/AgentPrimitive.tsx +++ b/src/cli/primitives/AgentPrimitive.tsx @@ -34,6 +34,19 @@ import { import { executeImportAgent } from '../operations/agent/import'; import { setupPythonProject } from '../operations/python'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { + AgentType, + AuthorizerType, + Build, + Framework, + Language, + Memory, + ModelProvider as ModelProviderEnum, + NetworkMode as NetworkModeEnum, + Protocol, + standardize, +} from '../telemetry/schemas/common-shapes.js'; import { createRenderer } from '../templates'; import { requireTTY } from '../tui/guards/tty'; import type { GenerateConfig, MemoryOption } from '../tui/screens/generate/types'; @@ -264,73 +277,93 @@ export class AgentPrimitive extends BasePrimitive { + const validation = validateAddAgentOptions(cliOptions); + if (!validation.valid) { + throw new Error(validation.error); + } + + // Parse custom claims JSON if provided (already validated by validateAddAgentOptions) + const customClaims = cliOptions.customClaims + ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) + : undefined; + + // Parse request header allowlist if provided + const requestHeaderAllowlist = cliOptions.requestHeaderAllowlist + ? parseAndNormalizeHeaders(cliOptions.requestHeaderAllowlist) + : undefined; + + const result = await this.add({ + name: cliOptions.name!, + type: cliOptions.type ?? 'create', + buildType: (cliOptions.build as BuildType) ?? 'CodeZip', + language: cliOptions.language!, + framework: cliOptions.framework!, + modelProvider: cliOptions.modelProvider!, + apiKey: cliOptions.apiKey, + memory: cliOptions.memory, + protocol: cliOptions.protocol, + networkMode: cliOptions.networkMode, + subnets: cliOptions.subnets, + securityGroups: cliOptions.securityGroups, + requestHeaderAllowlist, + codeLocation: cliOptions.codeLocation, + entrypoint: cliOptions.entrypoint, + bedrockAgentId: cliOptions.agentId, + bedrockAliasId: cliOptions.agentAliasId, + bedrockRegion: cliOptions.region, + authorizerType: cliOptions.authorizerType, + discoveryUrl: cliOptions.discoveryUrl, + allowedAudience: cliOptions.allowedAudience, + allowedClients: cliOptions.allowedClients, + allowedScopes: cliOptions.allowedScopes, + customClaims, + clientId: cliOptions.clientId, + clientSecret: cliOptions.clientSecret, + idleTimeout: cliOptions.idleTimeout ? Number(cliOptions.idleTimeout) : undefined, + maxLifetime: cliOptions.maxLifetime ? Number(cliOptions.maxLifetime) : undefined, + sessionStorageMountPath: cliOptions.sessionStorageMountPath, + }); + + if (!result.success) { + throw new Error(result.error); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added agent '${result.agentName}'`); + if (result.agentPath) { + console.log(`Agent code: ${result.agentPath}`); + } + if (cliOptions.networkMode === 'VPC') { + console.log(`\x1b[33mNote: ${VPC_ENDPOINT_WARNING}\x1b[0m`); + } + } + + return { + language: standardize(Language, cliOptions.language!), + framework: standardize(Framework, cliOptions.framework!), + model_provider: standardize(ModelProviderEnum, cliOptions.modelProvider!), + agent_type: standardize(AgentType, cliOptions.type ?? 'create'), + build: standardize(Build, cliOptions.build ?? 'CodeZip'), + protocol: standardize(Protocol, cliOptions.protocol ?? 'HTTP'), + network_mode: standardize(NetworkModeEnum, cliOptions.networkMode ?? 'PUBLIC'), + authorizer_type: standardize(AuthorizerType, cliOptions.authorizerType ?? 'NONE'), + memory: standardize(Memory, cliOptions.memory ?? 'none'), + }; + }); + process.exit(0); + } catch (error) { if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); + console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); } else { - console.error(validation.error); + console.error(getErrorMessage(error)); } process.exit(1); } - - // Parse custom claims JSON if provided (already validated by validateAddAgentOptions) - const customClaims = cliOptions.customClaims - ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) - : undefined; - - // Parse request header allowlist if provided - const requestHeaderAllowlist = cliOptions.requestHeaderAllowlist - ? parseAndNormalizeHeaders(cliOptions.requestHeaderAllowlist) - : undefined; - - const result = await this.add({ - name: cliOptions.name!, - type: cliOptions.type ?? 'create', - buildType: (cliOptions.build as BuildType) ?? 'CodeZip', - language: cliOptions.language!, - framework: cliOptions.framework!, - modelProvider: cliOptions.modelProvider!, - apiKey: cliOptions.apiKey, - memory: cliOptions.memory, - protocol: cliOptions.protocol, - networkMode: cliOptions.networkMode, - subnets: cliOptions.subnets, - securityGroups: cliOptions.securityGroups, - requestHeaderAllowlist, - codeLocation: cliOptions.codeLocation, - entrypoint: cliOptions.entrypoint, - bedrockAgentId: cliOptions.agentId, - bedrockAliasId: cliOptions.agentAliasId, - bedrockRegion: cliOptions.region, - authorizerType: cliOptions.authorizerType, - discoveryUrl: cliOptions.discoveryUrl, - allowedAudience: cliOptions.allowedAudience, - allowedClients: cliOptions.allowedClients, - allowedScopes: cliOptions.allowedScopes, - customClaims, - clientId: cliOptions.clientId, - clientSecret: cliOptions.clientSecret, - idleTimeout: cliOptions.idleTimeout ? Number(cliOptions.idleTimeout) : undefined, - maxLifetime: cliOptions.maxLifetime ? Number(cliOptions.maxLifetime) : undefined, - sessionStorageMountPath: cliOptions.sessionStorageMountPath, - }); - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added agent '${result.agentName}'`); - if (result.agentPath) { - console.log(`Agent code: ${result.agentPath}`); - } - if (cliOptions.networkMode === 'VPC') { - console.log(`\x1b[33mNote: ${VPC_ENDPOINT_WARNING}\x1b[0m`); - } - } else { - console.error(result.error); - } - - process.exit(result.success ? 0 : 1); } else { // TUI fallback — dynamic imports to avoid pulling ink (async) into registry requireTTY(); diff --git a/src/cli/primitives/CredentialPrimitive.tsx b/src/cli/primitives/CredentialPrimitive.tsx index 52f578235..ba9ddfc2d 100644 --- a/src/cli/primitives/CredentialPrimitive.tsx +++ b/src/cli/primitives/CredentialPrimitive.tsx @@ -4,6 +4,8 @@ import { CredentialSchema } from '../../schema'; import { validateAddCredentialOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { CredentialType, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; import { computeDefaultCredentialEnvVarName } from './credential-utils'; @@ -290,54 +292,58 @@ export class CredentialPrimitive extends BasePrimitive { + const validation = validateAddCredentialOptions({ + name: cliOptions.name, + type: cliOptions.type as 'api-key' | 'oauth' | undefined, + apiKey: cliOptions.apiKey, + discoveryUrl: cliOptions.discoveryUrl, + clientId: cliOptions.clientId, + clientSecret: cliOptions.clientSecret, + scopes: cliOptions.scopes, + }); + + if (!validation.valid) { + throw new Error(validation.error); + } + + const addOptions = + cliOptions.type === 'oauth' + ? { + authorizerType: 'OAuthCredentialProvider' as const, + name: cliOptions.name!, + discoveryUrl: cliOptions.discoveryUrl!, + clientId: cliOptions.clientId!, + clientSecret: cliOptions.clientSecret!, + scopes: cliOptions.scopes + ?.split(',') + .map(s => s.trim()) + .filter(Boolean), + } + : { + authorizerType: 'ApiKeyCredentialProvider' as const, + name: cliOptions.name!, + apiKey: cliOptions.apiKey!, + }; + + const result = await this.add(addOptions); + + if (!result.success) { + throw new Error(result.error); + } - if (!validation.valid) { if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); + console.log(JSON.stringify(result)); } else { - console.error(validation.error); + console.log(`Added credential '${result.credentialName}'`); } - process.exit(1); - } - - const addOptions = - cliOptions.type === 'oauth' - ? { - authorizerType: 'OAuthCredentialProvider' as const, - name: cliOptions.name!, - discoveryUrl: cliOptions.discoveryUrl!, - clientId: cliOptions.clientId!, - clientSecret: cliOptions.clientSecret!, - scopes: cliOptions.scopes - ?.split(',') - .map(s => s.trim()) - .filter(Boolean), - } - : { - authorizerType: 'ApiKeyCredentialProvider' as const, - name: cliOptions.name!, - apiKey: cliOptions.apiKey!, - }; - - const result = await this.add(addOptions); - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added credential '${result.credentialName}'`); - } else { - console.error(result.error); - } - process.exit(result.success ? 0 : 1); + + return { + credential_type: standardize(CredentialType, cliOptions.type ?? 'api-key'), + }; + }); + process.exit(0); } else { // TUI fallback — dynamic imports to avoid pulling ink (async) into registry requireTTY(); diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 73aaf8073..355bd0ce9 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -3,6 +3,8 @@ import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema'; import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { EvaluatorType, Level, standardize } from '../telemetry/schemas/common-shapes.js'; import { renderCodeBasedEvaluatorTemplate } from '../templates/EvaluatorRenderer'; import { requireTTY } from '../tui/guards/tty'; import { @@ -203,118 +205,123 @@ export class EvaluatorPrimitive extends BasePrimitive { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error })); - } else { - console.error(error); + const client = await TelemetryClientAccessor.get(); + await client.withCommandRun('add.evaluator', async () => { + const fail = (error: string): never => { + throw new Error(error); + }; + + if (!cliOptions.name || !cliOptions.level) { + fail('--name and --level are required in non-interactive mode'); } - process.exit(1); - }; - - if (!cliOptions.name || !cliOptions.level) { - fail('--name and --level are required in non-interactive mode'); - } - - const levelResult = EvaluationLevelSchema.safeParse(cliOptions.level); - if (!levelResult.success) { - fail(`Invalid --level "${cliOptions.level}". Must be one of: SESSION, TRACE, TOOL_CALL`); - } - - const evalType = cliOptions.type ?? 'llm-as-a-judge'; - if (evalType !== 'llm-as-a-judge' && evalType !== 'code-based') { - fail(`Invalid --type "${evalType}". Must be one of: llm-as-a-judge, code-based`); - } - - // Cross-validate flags against evaluator type - if (evalType !== 'code-based') { - if (cliOptions.lambdaArn) fail('--lambda-arn requires --type code-based'); - if (cliOptions.timeout) fail('--timeout requires --type code-based'); - } - if (evalType === 'code-based') { - if (cliOptions.model) fail('--model cannot be used with --type code-based'); - if (cliOptions.instructions) fail('--instructions cannot be used with --type code-based'); - if (cliOptions.ratingScale) fail('--rating-scale cannot be used with --type code-based'); - } - - let configJson: EvaluatorConfig; - - if (cliOptions.config) { - const { readFileSync } = await import('fs'); - configJson = JSON.parse(readFileSync(cliOptions.config, 'utf-8')) as EvaluatorConfig; - } else if (evalType === 'code-based') { - configJson = this.buildCodeBasedConfig(cliOptions.name!, cliOptions.lambdaArn, cliOptions.timeout); - } else { - // LLM-as-a-Judge flow - if (!cliOptions.model) { - fail('Either --config or --model is required for LLM-as-a-Judge evaluators'); + + const levelResult = EvaluationLevelSchema.safeParse(cliOptions.level); + if (!levelResult.success) { + fail(`Invalid --level "${cliOptions.level}". Must be one of: SESSION, TRACE, TOOL_CALL`); } - if (!cliOptions.instructions) { - const level = levelResult.data!; - const placeholders = LEVEL_PLACEHOLDERS[level].map(p => `{${p}}`).join(', '); - fail( - `--instructions is required in non-interactive mode (or use --config). ` + - `Must include at least one placeholder for ${level}: ${placeholders}` - ); + const evalType = cliOptions.type ?? 'llm-as-a-judge'; + if (evalType !== 'llm-as-a-judge' && evalType !== 'code-based') { + fail(`Invalid --type "${evalType}". Must be one of: llm-as-a-judge, code-based`); } - const placeholderCheck = validateInstructionPlaceholders(cliOptions.instructions!, levelResult.data!); - if (placeholderCheck !== true) { - fail(placeholderCheck); + // Cross-validate flags against evaluator type + if (evalType !== 'code-based') { + if (cliOptions.lambdaArn) fail('--lambda-arn requires --type code-based'); + if (cliOptions.timeout) fail('--timeout requires --type code-based'); + } + if (evalType === 'code-based') { + if (cliOptions.model) fail('--model cannot be used with --type code-based'); + if (cliOptions.instructions) fail('--instructions cannot be used with --type code-based'); + if (cliOptions.ratingScale) fail('--rating-scale cannot be used with --type code-based'); } - let ratingScale: NonNullable['ratingScale']; - const scaleInput = cliOptions.ratingScale ?? '1-5-quality'; + let configJson: EvaluatorConfig; - const preset = RATING_SCALE_PRESETS.find(p => p.id === scaleInput); - if (preset) { - ratingScale = preset.ratingScale; + if (cliOptions.config) { + const { readFileSync } = await import('fs'); + configJson = JSON.parse(readFileSync(cliOptions.config, 'utf-8')) as EvaluatorConfig; + } else if (evalType === 'code-based') { + configJson = this.buildCodeBasedConfig(cliOptions.name!, cliOptions.lambdaArn, cliOptions.timeout); } else { - const isNumerical = /^\d/.test(scaleInput.trim()); - const parsed = parseCustomRatingScale(scaleInput, isNumerical ? 'numerical' : 'categorical'); - if (!parsed.success) { + // LLM-as-a-Judge flow + if (!cliOptions.model) { + fail('Either --config or --model is required for LLM-as-a-Judge evaluators'); + } + + if (!cliOptions.instructions) { + const level = levelResult.data!; + const placeholders = LEVEL_PLACEHOLDERS[level].map(p => `{${p}}`).join(', '); fail( - `Invalid --rating-scale "${scaleInput}". Use a preset (${presetIds.join(', ')}) ` + - `or custom format: "1:Label:Definition, 2:Label:Definition" (numerical) ` + - `or "Label:Definition, Label:Definition" (categorical)` + `--instructions is required in non-interactive mode (or use --config). ` + + `Must include at least one placeholder for ${level}: ${placeholders}` ); } - ratingScale = parsed.success ? parsed.ratingScale : undefined!; + + const placeholderCheck = validateInstructionPlaceholders(cliOptions.instructions!, levelResult.data!); + if (placeholderCheck !== true) { + fail(placeholderCheck); + } + + let ratingScale: NonNullable['ratingScale']; + const scaleInput = cliOptions.ratingScale ?? '1-5-quality'; + + const preset = RATING_SCALE_PRESETS.find(p => p.id === scaleInput); + if (preset) { + ratingScale = preset.ratingScale; + } else { + const isNumerical = /^\d/.test(scaleInput.trim()); + const parsed = parseCustomRatingScale(scaleInput, isNumerical ? 'numerical' : 'categorical'); + if (!parsed.success) { + fail( + `Invalid --rating-scale "${scaleInput}". Use a preset (${presetIds.join(', ')}) ` + + `or custom format: "1:Label:Definition, 2:Label:Definition" (numerical) ` + + `or "Label:Definition, Label:Definition" (categorical)` + ); + } + ratingScale = parsed.success ? parsed.ratingScale : undefined!; + } + + configJson = { + llmAsAJudge: { + model: cliOptions.model!, + instructions: cliOptions.instructions!, + ratingScale, + }, + }; } - configJson = { - llmAsAJudge: { - model: cliOptions.model!, - instructions: cliOptions.instructions!, - ratingScale, - }, - }; - } + const result = await this.add({ + name: cliOptions.name!, + level: levelResult.data!, + config: configJson, + }); - const result = await this.add({ - name: cliOptions.name!, - level: levelResult.data!, - config: configJson, - }); + if (!result.success) { + throw new Error(result.error); + } - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - if (result.codePath) { - console.log(`Created evaluator '${result.evaluatorName}'`); - console.log(` Code: ${result.codePath}lambda_function.py`); - console.log(` IAM: ${result.codePath}execution-role-policy.json`); - console.log( - `\n Next: Edit lambda_function.py with your evaluation logic, then run \`agentcore deploy\`` - ); + if (cliOptions.json) { + console.log(JSON.stringify(result)); } else { - console.log(`Added evaluator '${result.evaluatorName}'`); + if (result.codePath) { + console.log(`Created evaluator '${result.evaluatorName}'`); + console.log(` Code: ${result.codePath}lambda_function.py`); + console.log(` IAM: ${result.codePath}execution-role-policy.json`); + console.log( + `\n Next: Edit lambda_function.py with your evaluation logic, then run \`agentcore deploy\`` + ); + } else { + console.log(`Added evaluator '${result.evaluatorName}'`); + } } - } else { - console.error(result.error); - } - process.exit(result.success ? 0 : 1); + + return { + evaluator_type: standardize(EvaluatorType, evalType), + level: standardize(Level, levelResult.data!), + }; + }); + process.exit(0); } else { // TUI fallback requireTTY(); diff --git a/src/cli/primitives/GatewayPrimitive.ts b/src/cli/primitives/GatewayPrimitive.ts index 427aa1533..04be5b473 100644 --- a/src/cli/primitives/GatewayPrimitive.ts +++ b/src/cli/primitives/GatewayPrimitive.ts @@ -12,6 +12,8 @@ import type { AddGatewayOptions as CLIAddGatewayOptions } from '../commands/add/ import { validateAddGatewayOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { AuthorizerType, PolicyEngineMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import type { AddGatewayConfig } from '../tui/screens/mcp/types'; import { BasePrimitive } from './BasePrimitive'; @@ -188,48 +190,61 @@ export class GatewayPrimitive extends BasePrimitive { + const validation = validateAddGatewayOptions(cliOptions); + if (!validation.valid) { + throw new Error(validation.error); + } + + // Parse custom claims JSON if provided (already validated) + const parsedCustomClaims = cliOptions.customClaims + ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) + : undefined; + + const result = await this.add({ + name: cliOptions.name!, + description: cliOptions.description, + authorizerType: cliOptions.authorizerType ?? 'NONE', + discoveryUrl: cliOptions.discoveryUrl, + allowedAudience: cliOptions.allowedAudience, + allowedClients: cliOptions.allowedClients, + allowedScopes: cliOptions.allowedScopes, + customClaims: parsedCustomClaims, + clientId: cliOptions.clientId, + clientSecret: cliOptions.clientSecret, + runtimes: cliOptions.runtimes, + enableSemanticSearch: cliOptions.semanticSearch !== false, + exceptionLevel: cliOptions.exceptionLevel, + policyEngine: cliOptions.policyEngine, + policyEngineMode: cliOptions.policyEngineMode, + }); + + if (!result.success) { + throw new Error(result.error); + } + if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); + console.log(JSON.stringify(result)); } else { - console.error(validation.error); + console.log(`Added gateway '${result.gatewayName}'`); } - process.exit(1); - } - // Parse custom claims JSON if provided (already validated) - const parsedCustomClaims = cliOptions.customClaims - ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) - : undefined; - - const result = await this.add({ - name: cliOptions.name!, - description: cliOptions.description, - authorizerType: cliOptions.authorizerType ?? 'NONE', - discoveryUrl: cliOptions.discoveryUrl, - allowedAudience: cliOptions.allowedAudience, - allowedClients: cliOptions.allowedClients, - allowedScopes: cliOptions.allowedScopes, - customClaims: parsedCustomClaims, - clientId: cliOptions.clientId, - clientSecret: cliOptions.clientSecret, - runtimes: cliOptions.runtimes, - enableSemanticSearch: cliOptions.semanticSearch !== false, - exceptionLevel: cliOptions.exceptionLevel, - policyEngine: cliOptions.policyEngine, - policyEngineMode: cliOptions.policyEngineMode, + const runtimeCount = cliOptions.runtimes + ? cliOptions.runtimes + .split(',') + .map(s => s.trim()) + .filter(Boolean).length + : 0; + return { + authorizer_type: standardize(AuthorizerType, cliOptions.authorizerType ?? 'NONE'), + has_policy_engine: !!cliOptions.policyEngine, + policy_engine_mode: standardize(PolicyEngineMode, cliOptions.policyEngineMode ?? 'log_only'), + semantic_search: cliOptions.semanticSearch !== false, + runtime_count: runtimeCount, + }; }); - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added gateway '${result.gatewayName}'`); - } else { - console.error(result.error); - } - - process.exit(result.success ? 0 : 1); + process.exit(0); } catch (error) { if (cliOptions.json) { console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); diff --git a/src/cli/primitives/GatewayTargetPrimitive.ts b/src/cli/primitives/GatewayTargetPrimitive.ts index 41a2e6a75..09c146fbb 100644 --- a/src/cli/primitives/GatewayTargetPrimitive.ts +++ b/src/cli/primitives/GatewayTargetPrimitive.ts @@ -14,6 +14,8 @@ import { validateAddGatewayTargetOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovableGatewayTarget } from '../operations/remove/remove-gateway-target'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { GatewayTargetHost, OutboundAuth, standardize } from '../telemetry/schemas/common-shapes.js'; import { getTemplateToolDefinitions, renderGatewayTargetTemplate } from '../templates/GatewayTargetRenderer'; import { requireTTY } from '../tui/guards/tty'; import type { @@ -303,170 +305,194 @@ export class GatewayTargetPrimitive extends BasePrimitive { + const validation = await validateAddGatewayTargetOptions(cliOptions); + if (!validation.valid) { + throw new Error(validation.error); } - process.exit(1); - } - // Map CLI flag values to internal types - const outboundAuthMap: Record = { - oauth: 'OAUTH', - 'api-key': 'API_KEY', - api_key: 'API_KEY', - none: 'NONE', - }; - - // Handle API Gateway targets (no code generation) - if (cliOptions.type === 'apiGateway') { - const config: ApiGatewayTargetConfig = { - targetType: 'apiGateway', - name: cliOptions.name!, - gateway: cliOptions.gateway!, - restApiId: cliOptions.restApiId!, - stage: cliOptions.stage!, - toolFilters: cliOptions.toolFilterPath - ? [ - { - filterPath: cliOptions.toolFilterPath, - methods: (cliOptions.toolFilterMethods?.split(',').map(m => m.trim()) ?? [ - 'GET', - ]) as ApiGatewayHttpMethod[], - }, - ] - : undefined, - ...(cliOptions.outboundAuthType - ? { - outboundAuth: { - type: (outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE') as - | 'API_KEY' - | 'NONE', - credentialName: cliOptions.credentialName, - }, - } - : {}), + // Map CLI flag values to internal types + const outboundAuthMap: Record = { + oauth: 'OAUTH', + 'api-key': 'API_KEY', + api_key: 'API_KEY', + none: 'NONE', }; - const result = await this.createApiGatewayTarget(config); - const output = { success: true, toolName: result.toolName }; - if (cliOptions.json) { - console.log(JSON.stringify(output)); - } else { - console.log(`Added gateway target '${result.toolName}'`); + + // Map target type from camelCase CLI to kebab-case telemetry + const targetTypeMap = { + apiGateway: 'api-gateway', + openApiSchema: 'open-api-schema', + smithyModel: 'smithy-model', + lambdaFunctionArn: 'lambda-function-arn', + mcpServer: 'mcp-server', + } as const; + type TargetTypeKey = keyof typeof targetTypeMap; + + const cliType = cliOptions.type ?? ''; + const telemetryTargetType = + cliType in targetTypeMap ? targetTypeMap[cliType as TargetTypeKey] : ('mcp-server' as const); + const telemetryOutboundAuth = standardize( + OutboundAuth, + (cliOptions.outboundAuthType ?? 'none').replace('_', '-') + ); + const telemetryHost = standardize(GatewayTargetHost, cliOptions.host ?? 'lambda'); + + // Handle API Gateway targets (no code generation) + if (cliOptions.type === 'apiGateway') { + const config: ApiGatewayTargetConfig = { + targetType: 'apiGateway', + name: cliOptions.name!, + gateway: cliOptions.gateway!, + restApiId: cliOptions.restApiId!, + stage: cliOptions.stage!, + toolFilters: cliOptions.toolFilterPath + ? [ + { + filterPath: cliOptions.toolFilterPath, + methods: (cliOptions.toolFilterMethods?.split(',').map(m => m.trim()) ?? [ + 'GET', + ]) as ApiGatewayHttpMethod[], + }, + ] + : undefined, + ...(cliOptions.outboundAuthType + ? { + outboundAuth: { + type: (outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE') as + | 'API_KEY' + | 'NONE', + credentialName: cliOptions.credentialName, + }, + } + : {}), + }; + const result = await this.createApiGatewayTarget(config); + const output = { success: true, toolName: result.toolName }; + if (cliOptions.json) { + console.log(JSON.stringify(output)); + } else { + console.log(`Added gateway target '${result.toolName}'`); + } + return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; } - process.exit(0); - } - // Handle schema-based targets (OpenAPI / Smithy) - if ((cliOptions.type === 'openApiSchema' || cliOptions.type === 'smithyModel') && cliOptions.schema) { - const isS3 = cliOptions.schema.startsWith('s3://'); - const schemaSource = isS3 - ? { - s3: { - uri: cliOptions.schema, - ...(cliOptions.schemaS3Account ? { bucketOwnerAccountId: cliOptions.schemaS3Account } : {}), - }, - } - : { inline: { path: cliOptions.schema } }; - - const config: SchemaBasedTargetConfig = { - name: cliOptions.name!, - targetType: cliOptions.type, - schemaSource, - gateway: cliOptions.gateway!, - ...(cliOptions.outboundAuthType + // Handle schema-based targets (OpenAPI / Smithy) + if ((cliOptions.type === 'openApiSchema' || cliOptions.type === 'smithyModel') && cliOptions.schema) { + const isS3 = cliOptions.schema.startsWith('s3://'); + const schemaSource = isS3 ? { - outboundAuth: { - type: outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE', - credentialName: cliOptions.credentialName, + s3: { + uri: cliOptions.schema, + ...(cliOptions.schemaS3Account ? { bucketOwnerAccountId: cliOptions.schemaS3Account } : {}), }, } - : {}), - }; - const result = await this.createSchemaBasedGatewayTarget(config); - const output = { success: true, toolName: result.toolName }; - if (cliOptions.json) { - console.log(JSON.stringify(output)); - } else { - console.log(`Added gateway target '${result.toolName}'`); + : { inline: { path: cliOptions.schema } }; + + const config: SchemaBasedTargetConfig = { + name: cliOptions.name!, + targetType: cliOptions.type, + schemaSource, + gateway: cliOptions.gateway!, + ...(cliOptions.outboundAuthType + ? { + outboundAuth: { + type: outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE', + credentialName: cliOptions.credentialName, + }, + } + : {}), + }; + const result = await this.createSchemaBasedGatewayTarget(config); + const output = { success: true, toolName: result.toolName }; + if (cliOptions.json) { + console.log(JSON.stringify(output)); + } else { + console.log(`Added gateway target '${result.toolName}'`); + } + return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; } - process.exit(0); - } - // Handle Lambda Function ARN targets (no code generation) - if (cliOptions.type === 'lambdaFunctionArn') { - const config = { - targetType: 'lambdaFunctionArn' as const, - name: cliOptions.name!, - gateway: cliOptions.gateway!, - lambdaArn: cliOptions.lambdaArn!, - toolSchemaFile: cliOptions.toolSchemaFile!, - }; - const result = await this.createLambdaFunctionArnTarget(config); - const output = { success: true, toolName: result.toolName }; - if (cliOptions.json) { - console.log(JSON.stringify(output)); - } else { - console.log(`Added gateway target '${result.toolName}'`); + // Handle Lambda Function ARN targets (no code generation) + if (cliOptions.type === 'lambdaFunctionArn') { + const config = { + targetType: 'lambdaFunctionArn' as const, + name: cliOptions.name!, + gateway: cliOptions.gateway!, + lambdaArn: cliOptions.lambdaArn!, + toolSchemaFile: cliOptions.toolSchemaFile!, + }; + const result = await this.createLambdaFunctionArnTarget(config); + const output = { success: true, toolName: result.toolName }; + if (cliOptions.json) { + console.log(JSON.stringify(output)); + } else { + console.log(`Added gateway target '${result.toolName}'`); + } + return { target_type: telemetryTargetType, host: 'lambda', outbound_auth: telemetryOutboundAuth }; } - process.exit(0); - } - // Handle MCP server targets (existing endpoint, no code generation) - if (cliOptions.type === 'mcpServer' && cliOptions.endpoint) { - const config: McpServerTargetConfig = { - targetType: 'mcpServer', - name: cliOptions.name!, - description: cliOptions.description ?? `Tool for ${cliOptions.name!}`, - endpoint: cliOptions.endpoint, - gateway: cliOptions.gateway!, - toolDefinition: { + // Handle MCP server targets (existing endpoint, no code generation) + if (cliOptions.type === 'mcpServer' && cliOptions.endpoint) { + const config: McpServerTargetConfig = { + targetType: 'mcpServer', name: cliOptions.name!, description: cliOptions.description ?? `Tool for ${cliOptions.name!}`, - inputSchema: { type: 'object' }, - }, - ...(cliOptions.outboundAuthType - ? { - outboundAuth: { - type: outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE', - credentialName: cliOptions.credentialName, - }, - } - : {}), - }; - const result = await this.createExternalGatewayTarget(config); - const output = { success: true, toolName: result.toolName, sourcePath: result.projectPath || undefined }; + endpoint: cliOptions.endpoint, + gateway: cliOptions.gateway!, + toolDefinition: { + name: cliOptions.name!, + description: cliOptions.description ?? `Tool for ${cliOptions.name!}`, + inputSchema: { type: 'object' }, + }, + ...(cliOptions.outboundAuthType + ? { + outboundAuth: { + type: outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE', + credentialName: cliOptions.credentialName, + }, + } + : {}), + }; + const result = await this.createExternalGatewayTarget(config); + const output = { + success: true, + toolName: result.toolName, + sourcePath: result.projectPath || undefined, + }; + if (cliOptions.json) { + console.log(JSON.stringify(output)); + } else { + console.log(`Added gateway target '${result.toolName}'`); + } + return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; + } + + const result = await this.add({ + name: cliOptions.name!, + description: cliOptions.description, + language: cliOptions.language ?? 'Python', + gateway: cliOptions.gateway, + host: cliOptions.host, + }); + + if (!result.success) { + throw new Error(result.error); + } + if (cliOptions.json) { - console.log(JSON.stringify(output)); + console.log(JSON.stringify(result)); } else { console.log(`Added gateway target '${result.toolName}'`); + if (result.sourcePath) { + console.log(`Tool code: ${result.sourcePath}`); + } } - process.exit(0); - } - const result = await this.add({ - name: cliOptions.name!, - description: cliOptions.description, - language: cliOptions.language ?? 'Python', - gateway: cliOptions.gateway, - host: cliOptions.host, + return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; }); - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added gateway target '${result.toolName}'`); - if (result.sourcePath) { - console.log(`Tool code: ${result.sourcePath}`); - } - } else { - console.error(result.error); - } - - process.exit(result.success ? 0 : 1); + process.exit(0); } catch (error) { if (cliOptions.json) { console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); diff --git a/src/cli/primitives/MemoryPrimitive.tsx b/src/cli/primitives/MemoryPrimitive.tsx index ad157c2cf..42dffd6a4 100644 --- a/src/cli/primitives/MemoryPrimitive.tsx +++ b/src/cli/primitives/MemoryPrimitive.tsx @@ -17,6 +17,7 @@ import { import { DEFAULT_DELIVERY_TYPE, validateAddMemoryOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; import { requireTTY } from '../tui/guards/tty'; import { DEFAULT_EVENT_EXPIRY } from '../tui/screens/memory/types'; import { BasePrimitive } from './BasePrimitive'; @@ -191,44 +192,56 @@ export class MemoryPrimitive extends BasePrimitive { + const expiry = cliOptions.expiry ? parseInt(cliOptions.expiry, 10) : undefined; + const validation = validateAddMemoryOptions({ + name: cliOptions.name, + strategies: cliOptions.strategies, + expiry, + deliveryType: cliOptions.deliveryType, + dataStreamArn: cliOptions.dataStreamArn, + contentLevel: cliOptions.streamContentLevel, + streamDeliveryResources: cliOptions.streamDeliveryResources, + }); + + if (!validation.valid) { + throw new Error(validation.error); + } + + const result = await this.add({ + name: cliOptions.name!, + strategies: cliOptions.strategies, + expiry, + deliveryType: cliOptions.deliveryType, + dataStreamArn: cliOptions.dataStreamArn, + contentLevel: cliOptions.streamContentLevel, + streamDeliveryResources: cliOptions.streamDeliveryResources, + }); + + if (!result.success) { + throw new Error(result.error); + } - if (!validation.valid) { if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); + console.log(JSON.stringify(result)); } else { - console.error(validation.error); + console.log(`Added memory '${result.memoryName}'`); } - process.exit(1); - } - - const result = await this.add({ - name: cliOptions.name!, - strategies: cliOptions.strategies, - expiry, - deliveryType: cliOptions.deliveryType, - dataStreamArn: cliOptions.dataStreamArn, - contentLevel: cliOptions.streamContentLevel, - streamDeliveryResources: cliOptions.streamDeliveryResources, - }); - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added memory '${result.memoryName}'`); - } else { - console.error(result.error); - } - process.exit(result.success ? 0 : 1); + const strategyList = (cliOptions.strategies ?? '') + .split(',') + .map(s => s.trim().toUpperCase()) + .filter(Boolean); + return { + strategy_count: strategyList.length, + strategy_semantic: strategyList.includes('SEMANTIC'), + strategy_summarization: strategyList.includes('SUMMARIZATION'), + strategy_user_preference: strategyList.includes('USER_PREFERENCE'), + strategy_episodic: strategyList.includes('EPISODIC'), + }; + }); + process.exit(0); } else { // TUI fallback — dynamic imports to avoid pulling ink (async) into registry requireTTY(); diff --git a/src/cli/primitives/OnlineEvalConfigPrimitive.ts b/src/cli/primitives/OnlineEvalConfigPrimitive.ts index c53bfb88f..44919aa71 100644 --- a/src/cli/primitives/OnlineEvalConfigPrimitive.ts +++ b/src/cli/primitives/OnlineEvalConfigPrimitive.ts @@ -3,6 +3,7 @@ import type { OnlineEvalConfig } from '../../schema'; import { OnlineEvalConfigSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; @@ -131,45 +132,46 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive { + if (!cliOptions.name || !cliOptions.runtime || allEvaluators.length === 0 || !cliOptions.samplingRate) { + throw new Error( + '--name, --runtime, --evaluator (and/or --evaluator-arn), and --sampling-rate are all required in non-interactive mode' + ); + } + + // Sampling rate as a percentage of requests to evaluate (0.01% to 100%) + const samplingRate = parseFloat(cliOptions.samplingRate); + if (isNaN(samplingRate) || samplingRate < 0.01 || samplingRate > 100) { + throw new Error( + `Invalid --sampling-rate "${cliOptions.samplingRate}". Must be a percentage between 0.01 and 100` + ); + } + + const result = await this.add({ + name: cliOptions.name, + agent: cliOptions.runtime, + evaluators: allEvaluators, + samplingRate, + enableOnCreate: cliOptions.enableOnCreate, + }); + + if (!result.success) { + throw new Error(result.error); } - process.exit(1); - } - // Sampling rate as a percentage of requests to evaluate (0.01% to 100%) - const samplingRate = parseFloat(cliOptions.samplingRate); - if (isNaN(samplingRate) || samplingRate < 0.01 || samplingRate > 100) { - const error = `Invalid --sampling-rate "${cliOptions.samplingRate}". Must be a percentage between 0.01 and 100`; if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error })); + console.log(JSON.stringify(result)); } else { - console.error(error); + console.log(`Added online eval config '${result.configName}'`); } - process.exit(1); - } - - const result = await this.add({ - name: cliOptions.name, - agent: cliOptions.runtime, - evaluators: allEvaluators, - samplingRate, - enableOnCreate: cliOptions.enableOnCreate, - }); - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added online eval config '${result.configName}'`); - } else { - console.error(result.error); - } - process.exit(result.success ? 0 : 1); + return { + evaluator_count: allEvaluators.length, + enable_on_create: cliOptions.enableOnCreate ?? false, + }; + }); + process.exit(0); } else { // TUI fallback requireTTY(); diff --git a/src/cli/primitives/PolicyEnginePrimitive.ts b/src/cli/primitives/PolicyEnginePrimitive.ts index a1f887547..6efd141cc 100644 --- a/src/cli/primitives/PolicyEnginePrimitive.ts +++ b/src/cli/primitives/PolicyEnginePrimitive.ts @@ -3,6 +3,8 @@ import type { AgentCoreProjectSpec, PolicyEngine } from '../../schema'; import { PolicyEngineModeSchema, PolicyEngineSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { AttachMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; import { SOURCE_CODE_NOTE } from './constants'; @@ -228,39 +230,50 @@ export class PolicyEnginePrimitive extends BasePrimitive { + if (!cliOptions.name) { + throw new Error('--name is required'); + } + + const result = await this.add({ + name: cliOptions.name, + description: cliOptions.description, + encryptionKeyArn: cliOptions.encryptionKeyArn, + }); + + // Attach to gateways if requested + if (result.success && cliOptions.attachToGateways) { + const mode = PolicyEngineModeSchema.parse(cliOptions.attachMode ?? 'LOG_ONLY'); + const gateways = cliOptions.attachToGateways + .split(',') + .map(s => s.trim()) + .filter(Boolean); + await this.attachToGateways(cliOptions.name, gateways, mode); + } + + if (!result.success) { + throw new Error(result.error); + } + if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: '--name is required' })); + console.log(JSON.stringify(result)); } else { - console.error('--name is required'); + console.log(`Added policy engine '${result.engineName}'`); } - process.exit(1); - } - const result = await this.add({ - name: cliOptions.name, - description: cliOptions.description, - encryptionKeyArn: cliOptions.encryptionKeyArn, + const gatewayCount = cliOptions.attachToGateways + ? cliOptions.attachToGateways + .split(',') + .map(s => s.trim()) + .filter(Boolean).length + : 0; + return { + attach_gateway_count: gatewayCount, + attach_mode: standardize(AttachMode, cliOptions.attachMode ?? 'log_only'), + }; }); - - // Attach to gateways if requested - if (result.success && cliOptions.attachToGateways) { - const mode = PolicyEngineModeSchema.parse(cliOptions.attachMode ?? 'LOG_ONLY'); - const gateways = cliOptions.attachToGateways - .split(',') - .map(s => s.trim()) - .filter(Boolean); - await this.attachToGateways(cliOptions.name, gateways, mode); - } - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added policy engine '${result.engineName}'`); - } else { - console.error(result.error); - } - process.exit(result.success ? 0 : 1); + process.exit(0); } else { requireTTY(); const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ diff --git a/src/cli/primitives/PolicyPrimitive.ts b/src/cli/primitives/PolicyPrimitive.ts index bc1d446cc..202585983 100644 --- a/src/cli/primitives/PolicyPrimitive.ts +++ b/src/cli/primitives/PolicyPrimitive.ts @@ -5,6 +5,8 @@ import { detectRegion } from '../aws'; import { getPolicyGeneration, startPolicyGeneration } from '../aws/policy-generation'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { ValidationMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; import { SOURCE_CODE_NOTE } from './constants'; @@ -305,44 +307,49 @@ export class PolicyPrimitive extends BasePrimitive { + if (!cliOptions.name) { + throw new Error('--name is required'); } - process.exit(1); - } - if (!cliOptions.engine) { + if (!cliOptions.engine) { + throw new Error('--engine is required'); + } + + const result = await this.add({ + name: cliOptions.name, + engine: cliOptions.engine, + description: cliOptions.description, + source: cliOptions.source, + statement: cliOptions.statement, + generate: cliOptions.generate, + gateway: cliOptions.gateway, + validationMode: cliOptions.validationMode + ? ValidationModeSchema.parse(cliOptions.validationMode) + : undefined, + }); + + if (!result.success) { + throw new Error(result.error); + } + if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: '--engine is required' })); + console.log(JSON.stringify(result)); } else { - console.error('--engine is required'); + console.log(`Added policy '${result.policyName}' to engine '${result.engineName}'`); } - process.exit(1); - } - const result = await this.add({ - name: cliOptions.name, - engine: cliOptions.engine, - description: cliOptions.description, - source: cliOptions.source, - statement: cliOptions.statement, - generate: cliOptions.generate, - gateway: cliOptions.gateway, - validationMode: cliOptions.validationMode - ? ValidationModeSchema.parse(cliOptions.validationMode) - : undefined, + const sourceType: 'file' | 'statement' | 'generate' = cliOptions.source + ? 'file' + : cliOptions.generate + ? 'generate' + : 'statement'; + return { + source_type: sourceType, + validation_mode: standardize(ValidationMode, cliOptions.validationMode ?? 'FAIL_ON_ANY_FINDINGS'), + }; }); - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added policy '${result.policyName}' to engine '${result.engineName}'`); - } else { - console.error(result.error); - } - process.exit(result.success ? 0 : 1); + process.exit(0); } else { requireTTY(); const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ diff --git a/src/cli/primitives/RuntimeEndpointPrimitive.ts b/src/cli/primitives/RuntimeEndpointPrimitive.ts index 358b5fa69..aef1aecb6 100644 --- a/src/cli/primitives/RuntimeEndpointPrimitive.ts +++ b/src/cli/primitives/RuntimeEndpointPrimitive.ts @@ -4,6 +4,7 @@ import { RuntimeEndpointSchema } from '../../schema'; import type { ResourceType } from '../commands/remove/types'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; import { BasePrimitive } from './BasePrimitive'; import { SOURCE_CODE_NOTE } from './constants'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; @@ -254,22 +255,28 @@ export class RuntimeEndpointPrimitive extends BasePrimitive { + const result = await this.add({ + runtime: cliOptions.runtime, + endpoint: cliOptions.endpoint, + version: cliOptions.version, + description: cliOptions.description, + }); + + if (!result.success) { + throw new Error(result.error); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added runtime endpoint '${cliOptions.endpoint}' to runtime '${cliOptions.runtime}'`); + } + + return {}; }); - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added runtime endpoint '${cliOptions.endpoint}' to runtime '${cliOptions.runtime}'`); - } else { - console.error(result.error); - } - - process.exit(result.success ? 0 : 1); + process.exit(0); } catch (error) { if (cliOptions.json) { console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index dfd127a98..0acfcaf1b 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -157,6 +157,7 @@ export const COMMAND_SCHEMAS = { 'add.gateway-target': AddGatewayTargetAttrs, 'add.policy-engine': AddPolicyEngineAttrs, 'add.policy': AddPolicyAttrs, + 'add.runtime-endpoint': NoAttrs, // deploy deploy: DeployAttrs, diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index 732bb3d61..3d52c28e3 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -26,6 +26,12 @@ export function resilientParse( return result; } +/** Lowercase a CLI value and parse it through a Zod enum, returning the narrowed type. */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function standardize>(schema: T, value: string): z.infer { + return schema.parse(value.toLowerCase()) as z.infer; +} + // Primitive types export const Count = z.number().int().nonnegative(); From a0eb4ecd6031232d992459f2202234491596e23c Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 29 Apr 2026 21:17:38 +0000 Subject: [PATCH 02/18] refactor: extract cliCommandRun helper, apply to all add.* primitives --- src/cli/primitives/AgentPrimitive.tsx | 153 ++++---- src/cli/primitives/CredentialPrimitive.tsx | 181 +++++---- src/cli/primitives/EvaluatorPrimitive.ts | 257 ++++++------- src/cli/primitives/GatewayPrimitive.ts | 114 +++--- src/cli/primitives/GatewayTargetPrimitive.ts | 349 +++++++++--------- src/cli/primitives/MemoryPrimitive.tsx | 159 ++++---- .../primitives/OnlineEvalConfigPrimitive.ts | 143 ++++--- src/cli/primitives/PolicyEnginePrimitive.ts | 137 ++++--- src/cli/primitives/PolicyPrimitive.ts | 153 ++++---- .../primitives/RuntimeEndpointPrimitive.ts | 53 ++- src/cli/telemetry/cli-command-run.ts | 26 ++ 11 files changed, 820 insertions(+), 905 deletions(-) create mode 100644 src/cli/telemetry/cli-command-run.ts diff --git a/src/cli/primitives/AgentPrimitive.tsx b/src/cli/primitives/AgentPrimitive.tsx index 77518dc58..7f94e1595 100644 --- a/src/cli/primitives/AgentPrimitive.tsx +++ b/src/cli/primitives/AgentPrimitive.tsx @@ -34,7 +34,7 @@ import { import { executeImportAgent } from '../operations/agent/import'; import { setupPythonProject } from '../operations/python'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { AgentType, AuthorizerType, @@ -277,93 +277,82 @@ export class AgentPrimitive extends BasePrimitive { - const validation = validateAddAgentOptions(cliOptions); - if (!validation.valid) { - throw new Error(validation.error); - } + await cliCommandRun('add.agent', !!cliOptions.json, async () => { + const validation = validateAddAgentOptions(cliOptions); + if (!validation.valid) { + throw new Error(validation.error); + } - // Parse custom claims JSON if provided (already validated by validateAddAgentOptions) - const customClaims = cliOptions.customClaims - ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) - : undefined; - - // Parse request header allowlist if provided - const requestHeaderAllowlist = cliOptions.requestHeaderAllowlist - ? parseAndNormalizeHeaders(cliOptions.requestHeaderAllowlist) - : undefined; - - const result = await this.add({ - name: cliOptions.name!, - type: cliOptions.type ?? 'create', - buildType: (cliOptions.build as BuildType) ?? 'CodeZip', - language: cliOptions.language!, - framework: cliOptions.framework!, - modelProvider: cliOptions.modelProvider!, - apiKey: cliOptions.apiKey, - memory: cliOptions.memory, - protocol: cliOptions.protocol, - networkMode: cliOptions.networkMode, - subnets: cliOptions.subnets, - securityGroups: cliOptions.securityGroups, - requestHeaderAllowlist, - codeLocation: cliOptions.codeLocation, - entrypoint: cliOptions.entrypoint, - bedrockAgentId: cliOptions.agentId, - bedrockAliasId: cliOptions.agentAliasId, - bedrockRegion: cliOptions.region, - authorizerType: cliOptions.authorizerType, - discoveryUrl: cliOptions.discoveryUrl, - allowedAudience: cliOptions.allowedAudience, - allowedClients: cliOptions.allowedClients, - allowedScopes: cliOptions.allowedScopes, - customClaims, - clientId: cliOptions.clientId, - clientSecret: cliOptions.clientSecret, - idleTimeout: cliOptions.idleTimeout ? Number(cliOptions.idleTimeout) : undefined, - maxLifetime: cliOptions.maxLifetime ? Number(cliOptions.maxLifetime) : undefined, - sessionStorageMountPath: cliOptions.sessionStorageMountPath, - }); - - if (!result.success) { - throw new Error(result.error); - } + // Parse custom claims JSON if provided (already validated by validateAddAgentOptions) + const customClaims = cliOptions.customClaims + ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) + : undefined; + + // Parse request header allowlist if provided + const requestHeaderAllowlist = cliOptions.requestHeaderAllowlist + ? parseAndNormalizeHeaders(cliOptions.requestHeaderAllowlist) + : undefined; + + const result = await this.add({ + name: cliOptions.name!, + type: cliOptions.type ?? 'create', + buildType: (cliOptions.build as BuildType) ?? 'CodeZip', + language: cliOptions.language!, + framework: cliOptions.framework!, + modelProvider: cliOptions.modelProvider!, + apiKey: cliOptions.apiKey, + memory: cliOptions.memory, + protocol: cliOptions.protocol, + networkMode: cliOptions.networkMode, + subnets: cliOptions.subnets, + securityGroups: cliOptions.securityGroups, + requestHeaderAllowlist, + codeLocation: cliOptions.codeLocation, + entrypoint: cliOptions.entrypoint, + bedrockAgentId: cliOptions.agentId, + bedrockAliasId: cliOptions.agentAliasId, + bedrockRegion: cliOptions.region, + authorizerType: cliOptions.authorizerType, + discoveryUrl: cliOptions.discoveryUrl, + allowedAudience: cliOptions.allowedAudience, + allowedClients: cliOptions.allowedClients, + allowedScopes: cliOptions.allowedScopes, + customClaims, + clientId: cliOptions.clientId, + clientSecret: cliOptions.clientSecret, + idleTimeout: cliOptions.idleTimeout ? Number(cliOptions.idleTimeout) : undefined, + maxLifetime: cliOptions.maxLifetime ? Number(cliOptions.maxLifetime) : undefined, + sessionStorageMountPath: cliOptions.sessionStorageMountPath, + }); - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else { - console.log(`Added agent '${result.agentName}'`); - if (result.agentPath) { - console.log(`Agent code: ${result.agentPath}`); - } - if (cliOptions.networkMode === 'VPC') { - console.log(`\x1b[33mNote: ${VPC_ENDPOINT_WARNING}\x1b[0m`); - } - } + if (!result.success) { + throw new Error(result.error); + } - return { - language: standardize(Language, cliOptions.language!), - framework: standardize(Framework, cliOptions.framework!), - model_provider: standardize(ModelProviderEnum, cliOptions.modelProvider!), - agent_type: standardize(AgentType, cliOptions.type ?? 'create'), - build: standardize(Build, cliOptions.build ?? 'CodeZip'), - protocol: standardize(Protocol, cliOptions.protocol ?? 'HTTP'), - network_mode: standardize(NetworkModeEnum, cliOptions.networkMode ?? 'PUBLIC'), - authorizer_type: standardize(AuthorizerType, cliOptions.authorizerType ?? 'NONE'), - memory: standardize(Memory, cliOptions.memory ?? 'none'), - }; - }); - process.exit(0); - } catch (error) { if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + console.log(JSON.stringify(result)); } else { - console.error(getErrorMessage(error)); + console.log(`Added agent '${result.agentName}'`); + if (result.agentPath) { + console.log(`Agent code: ${result.agentPath}`); + } + if (cliOptions.networkMode === 'VPC') { + console.log(`\x1b[33mNote: ${VPC_ENDPOINT_WARNING}\x1b[0m`); + } } - process.exit(1); - } + + return { + language: standardize(Language, cliOptions.language!), + framework: standardize(Framework, cliOptions.framework!), + model_provider: standardize(ModelProviderEnum, cliOptions.modelProvider!), + agent_type: standardize(AgentType, cliOptions.type ?? 'create'), + build: standardize(Build, cliOptions.build ?? 'CodeZip'), + protocol: standardize(Protocol, cliOptions.protocol ?? 'HTTP'), + network_mode: standardize(NetworkModeEnum, cliOptions.networkMode ?? 'PUBLIC'), + authorizer_type: standardize(AuthorizerType, cliOptions.authorizerType ?? 'NONE'), + memory: standardize(Memory, cliOptions.memory ?? 'none'), + }; + }); } else { // TUI fallback — dynamic imports to avoid pulling ink (async) into registry requireTTY(); diff --git a/src/cli/primitives/CredentialPrimitive.tsx b/src/cli/primitives/CredentialPrimitive.tsx index ba9ddfc2d..e9bf61e24 100644 --- a/src/cli/primitives/CredentialPrimitive.tsx +++ b/src/cli/primitives/CredentialPrimitive.tsx @@ -4,7 +4,7 @@ import { CredentialSchema } from '../../schema'; import { validateAddCredentialOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { CredentialType, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; @@ -275,103 +275,92 @@ export class CredentialPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } - - if ( - cliOptions.name || - cliOptions.apiKey || - cliOptions.json || - cliOptions.type || - cliOptions.discoveryUrl || - cliOptions.clientId || - cliOptions.clientSecret || - cliOptions.scopes - ) { - // CLI mode - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('add.credential', async () => { - const validation = validateAddCredentialOptions({ - name: cliOptions.name, - type: cliOptions.type as 'api-key' | 'oauth' | undefined, - apiKey: cliOptions.apiKey, - discoveryUrl: cliOptions.discoveryUrl, - clientId: cliOptions.clientId, - clientSecret: cliOptions.clientSecret, - scopes: cliOptions.scopes, - }); - - if (!validation.valid) { - throw new Error(validation.error); - } - - const addOptions = - cliOptions.type === 'oauth' - ? { - authorizerType: 'OAuthCredentialProvider' as const, - name: cliOptions.name!, - discoveryUrl: cliOptions.discoveryUrl!, - clientId: cliOptions.clientId!, - clientSecret: cliOptions.clientSecret!, - scopes: cliOptions.scopes - ?.split(',') - .map(s => s.trim()) - .filter(Boolean), - } - : { - authorizerType: 'ApiKeyCredentialProvider' as const, - name: cliOptions.name!, - apiKey: cliOptions.apiKey!, - }; - - const result = await this.add(addOptions); - - if (!result.success) { - throw new Error(result.error); - } - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else { - console.log(`Added credential '${result.credentialName}'`); - } - - return { - credential_type: standardize(CredentialType, cliOptions.type ?? 'api-key'), - }; - }); - process.exit(0); - } else { - // TUI fallback — dynamic imports to avoid pulling ink (async) into registry - requireTTY(); - const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ - import('ink'), - import('react'), - import('../tui/screens/add/AddFlow'), - ]); - const { clear, unmount } = render( - React.createElement(AddFlow, { - isInteractive: false, - initialResource: 'credential', - onExit: () => { - clear(); - unmount(); - process.exit(0); - }, - }) - ); - } - } catch (error) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); - } else { - console.error(getErrorMessage(error)); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); process.exit(1); } + + if ( + cliOptions.name || + cliOptions.apiKey || + cliOptions.json || + cliOptions.type || + cliOptions.discoveryUrl || + cliOptions.clientId || + cliOptions.clientSecret || + cliOptions.scopes + ) { + // CLI mode + await cliCommandRun('add.credential', !!cliOptions.json, async () => { + const validation = validateAddCredentialOptions({ + name: cliOptions.name, + type: cliOptions.type as 'api-key' | 'oauth' | undefined, + apiKey: cliOptions.apiKey, + discoveryUrl: cliOptions.discoveryUrl, + clientId: cliOptions.clientId, + clientSecret: cliOptions.clientSecret, + scopes: cliOptions.scopes, + }); + + if (!validation.valid) { + throw new Error(validation.error); + } + + const addOptions = + cliOptions.type === 'oauth' + ? { + authorizerType: 'OAuthCredentialProvider' as const, + name: cliOptions.name!, + discoveryUrl: cliOptions.discoveryUrl!, + clientId: cliOptions.clientId!, + clientSecret: cliOptions.clientSecret!, + scopes: cliOptions.scopes + ?.split(',') + .map(s => s.trim()) + .filter(Boolean), + } + : { + authorizerType: 'ApiKeyCredentialProvider' as const, + name: cliOptions.name!, + apiKey: cliOptions.apiKey!, + }; + + const result = await this.add(addOptions); + + if (!result.success) { + throw new Error(result.error); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added credential '${result.credentialName}'`); + } + + return { + credential_type: standardize(CredentialType, cliOptions.type ?? 'api-key'), + }; + }); + } else { + // TUI fallback — dynamic imports to avoid pulling ink (async) into registry + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'credential', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } } ); diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 355bd0ce9..f8a7d3fa2 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -3,7 +3,7 @@ import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema'; import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { EvaluatorType, Level, standardize } from '../telemetry/schemas/common-shapes.js'; import { renderCodeBasedEvaluatorTemplate } from '../templates/EvaluatorRenderer'; import { requireTTY } from '../tui/guards/tty'; @@ -198,157 +198,146 @@ export class EvaluatorPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } - - if (cliOptions.name || cliOptions.json) { - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('add.evaluator', async () => { - const fail = (error: string): never => { - throw new Error(error); - }; - - if (!cliOptions.name || !cliOptions.level) { - fail('--name and --level are required in non-interactive mode'); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - const levelResult = EvaluationLevelSchema.safeParse(cliOptions.level); - if (!levelResult.success) { - fail(`Invalid --level "${cliOptions.level}". Must be one of: SESSION, TRACE, TOOL_CALL`); + if (cliOptions.name || cliOptions.json) { + await cliCommandRun('add.evaluator', !!cliOptions.json, async () => { + const fail = (error: string): never => { + throw new Error(error); + }; + + if (!cliOptions.name || !cliOptions.level) { + fail('--name and --level are required in non-interactive mode'); + } + + const levelResult = EvaluationLevelSchema.safeParse(cliOptions.level); + if (!levelResult.success) { + fail(`Invalid --level "${cliOptions.level}". Must be one of: SESSION, TRACE, TOOL_CALL`); + } + + const evalType = cliOptions.type ?? 'llm-as-a-judge'; + if (evalType !== 'llm-as-a-judge' && evalType !== 'code-based') { + fail(`Invalid --type "${evalType}". Must be one of: llm-as-a-judge, code-based`); + } + + // Cross-validate flags against evaluator type + if (evalType !== 'code-based') { + if (cliOptions.lambdaArn) fail('--lambda-arn requires --type code-based'); + if (cliOptions.timeout) fail('--timeout requires --type code-based'); + } + if (evalType === 'code-based') { + if (cliOptions.model) fail('--model cannot be used with --type code-based'); + if (cliOptions.instructions) fail('--instructions cannot be used with --type code-based'); + if (cliOptions.ratingScale) fail('--rating-scale cannot be used with --type code-based'); + } + + let configJson: EvaluatorConfig; + + if (cliOptions.config) { + const { readFileSync } = await import('fs'); + configJson = JSON.parse(readFileSync(cliOptions.config, 'utf-8')) as EvaluatorConfig; + } else if (evalType === 'code-based') { + configJson = this.buildCodeBasedConfig(cliOptions.name!, cliOptions.lambdaArn, cliOptions.timeout); + } else { + // LLM-as-a-Judge flow + if (!cliOptions.model) { + fail('Either --config or --model is required for LLM-as-a-Judge evaluators'); } - const evalType = cliOptions.type ?? 'llm-as-a-judge'; - if (evalType !== 'llm-as-a-judge' && evalType !== 'code-based') { - fail(`Invalid --type "${evalType}". Must be one of: llm-as-a-judge, code-based`); + if (!cliOptions.instructions) { + const level = levelResult.data!; + const placeholders = LEVEL_PLACEHOLDERS[level].map(p => `{${p}}`).join(', '); + fail( + `--instructions is required in non-interactive mode (or use --config). ` + + `Must include at least one placeholder for ${level}: ${placeholders}` + ); } - // Cross-validate flags against evaluator type - if (evalType !== 'code-based') { - if (cliOptions.lambdaArn) fail('--lambda-arn requires --type code-based'); - if (cliOptions.timeout) fail('--timeout requires --type code-based'); - } - if (evalType === 'code-based') { - if (cliOptions.model) fail('--model cannot be used with --type code-based'); - if (cliOptions.instructions) fail('--instructions cannot be used with --type code-based'); - if (cliOptions.ratingScale) fail('--rating-scale cannot be used with --type code-based'); + const placeholderCheck = validateInstructionPlaceholders(cliOptions.instructions!, levelResult.data!); + if (placeholderCheck !== true) { + fail(placeholderCheck); } - let configJson: EvaluatorConfig; + let ratingScale: NonNullable['ratingScale']; + const scaleInput = cliOptions.ratingScale ?? '1-5-quality'; - if (cliOptions.config) { - const { readFileSync } = await import('fs'); - configJson = JSON.parse(readFileSync(cliOptions.config, 'utf-8')) as EvaluatorConfig; - } else if (evalType === 'code-based') { - configJson = this.buildCodeBasedConfig(cliOptions.name!, cliOptions.lambdaArn, cliOptions.timeout); + const preset = RATING_SCALE_PRESETS.find(p => p.id === scaleInput); + if (preset) { + ratingScale = preset.ratingScale; } else { - // LLM-as-a-Judge flow - if (!cliOptions.model) { - fail('Either --config or --model is required for LLM-as-a-Judge evaluators'); - } - - if (!cliOptions.instructions) { - const level = levelResult.data!; - const placeholders = LEVEL_PLACEHOLDERS[level].map(p => `{${p}}`).join(', '); + const isNumerical = /^\d/.test(scaleInput.trim()); + const parsed = parseCustomRatingScale(scaleInput, isNumerical ? 'numerical' : 'categorical'); + if (!parsed.success) { fail( - `--instructions is required in non-interactive mode (or use --config). ` + - `Must include at least one placeholder for ${level}: ${placeholders}` + `Invalid --rating-scale "${scaleInput}". Use a preset (${presetIds.join(', ')}) ` + + `or custom format: "1:Label:Definition, 2:Label:Definition" (numerical) ` + + `or "Label:Definition, Label:Definition" (categorical)` ); } - - const placeholderCheck = validateInstructionPlaceholders(cliOptions.instructions!, levelResult.data!); - if (placeholderCheck !== true) { - fail(placeholderCheck); - } - - let ratingScale: NonNullable['ratingScale']; - const scaleInput = cliOptions.ratingScale ?? '1-5-quality'; - - const preset = RATING_SCALE_PRESETS.find(p => p.id === scaleInput); - if (preset) { - ratingScale = preset.ratingScale; - } else { - const isNumerical = /^\d/.test(scaleInput.trim()); - const parsed = parseCustomRatingScale(scaleInput, isNumerical ? 'numerical' : 'categorical'); - if (!parsed.success) { - fail( - `Invalid --rating-scale "${scaleInput}". Use a preset (${presetIds.join(', ')}) ` + - `or custom format: "1:Label:Definition, 2:Label:Definition" (numerical) ` + - `or "Label:Definition, Label:Definition" (categorical)` - ); - } - ratingScale = parsed.success ? parsed.ratingScale : undefined!; - } - - configJson = { - llmAsAJudge: { - model: cliOptions.model!, - instructions: cliOptions.instructions!, - ratingScale, - }, - }; + ratingScale = parsed.success ? parsed.ratingScale : undefined!; } - const result = await this.add({ - name: cliOptions.name!, - level: levelResult.data!, - config: configJson, - }); + configJson = { + llmAsAJudge: { + model: cliOptions.model!, + instructions: cliOptions.instructions!, + ratingScale, + }, + }; + } - if (!result.success) { - throw new Error(result.error); - } + const result = await this.add({ + name: cliOptions.name!, + level: levelResult.data!, + config: configJson, + }); - if (cliOptions.json) { - console.log(JSON.stringify(result)); + if (!result.success) { + throw new Error(result.error); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + if (result.codePath) { + console.log(`Created evaluator '${result.evaluatorName}'`); + console.log(` Code: ${result.codePath}lambda_function.py`); + console.log(` IAM: ${result.codePath}execution-role-policy.json`); + console.log( + `\n Next: Edit lambda_function.py with your evaluation logic, then run \`agentcore deploy\`` + ); } else { - if (result.codePath) { - console.log(`Created evaluator '${result.evaluatorName}'`); - console.log(` Code: ${result.codePath}lambda_function.py`); - console.log(` IAM: ${result.codePath}execution-role-policy.json`); - console.log( - `\n Next: Edit lambda_function.py with your evaluation logic, then run \`agentcore deploy\`` - ); - } else { - console.log(`Added evaluator '${result.evaluatorName}'`); - } + console.log(`Added evaluator '${result.evaluatorName}'`); } - - return { - evaluator_type: standardize(EvaluatorType, evalType), - level: standardize(Level, levelResult.data!), - }; - }); - process.exit(0); - } else { - // TUI fallback - requireTTY(); - const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ - import('ink'), - import('react'), - import('../tui/screens/add/AddFlow'), - ]); - const { clear, unmount } = render( - React.createElement(AddFlow, { - isInteractive: false, - initialResource: 'evaluator', - onExit: () => { - clear(); - unmount(); - process.exit(0); - }, - }) - ); - } - } catch (error) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); - } else { - console.error(getErrorMessage(error)); - } - process.exit(1); + } + + return { + evaluator_type: standardize(EvaluatorType, evalType), + level: standardize(Level, levelResult.data!), + }; + }); + } else { + // TUI fallback + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'evaluator', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); } } ); diff --git a/src/cli/primitives/GatewayPrimitive.ts b/src/cli/primitives/GatewayPrimitive.ts index 04be5b473..625a3c4a5 100644 --- a/src/cli/primitives/GatewayPrimitive.ts +++ b/src/cli/primitives/GatewayPrimitive.ts @@ -12,7 +12,7 @@ import type { AddGatewayOptions as CLIAddGatewayOptions } from '../commands/add/ import { validateAddGatewayOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { AuthorizerType, PolicyEngineMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import type { AddGatewayConfig } from '../tui/screens/mcp/types'; @@ -184,75 +184,63 @@ export class GatewayPrimitive extends BasePrimitive) => { const cliOptions = rawOptions as unknown as CLIAddGatewayOptions; - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } + await cliCommandRun('add.gateway', !!cliOptions.json, async () => { + const validation = validateAddGatewayOptions(cliOptions); + if (!validation.valid) { + throw new Error(validation.error); } - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('add.gateway', async () => { - const validation = validateAddGatewayOptions(cliOptions); - if (!validation.valid) { - throw new Error(validation.error); - } - - // Parse custom claims JSON if provided (already validated) - const parsedCustomClaims = cliOptions.customClaims - ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) - : undefined; - - const result = await this.add({ - name: cliOptions.name!, - description: cliOptions.description, - authorizerType: cliOptions.authorizerType ?? 'NONE', - discoveryUrl: cliOptions.discoveryUrl, - allowedAudience: cliOptions.allowedAudience, - allowedClients: cliOptions.allowedClients, - allowedScopes: cliOptions.allowedScopes, - customClaims: parsedCustomClaims, - clientId: cliOptions.clientId, - clientSecret: cliOptions.clientSecret, - runtimes: cliOptions.runtimes, - enableSemanticSearch: cliOptions.semanticSearch !== false, - exceptionLevel: cliOptions.exceptionLevel, - policyEngine: cliOptions.policyEngine, - policyEngineMode: cliOptions.policyEngineMode, - }); - - if (!result.success) { - throw new Error(result.error); - } + // Parse custom claims JSON if provided (already validated) + const parsedCustomClaims = cliOptions.customClaims + ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) + : undefined; + + const result = await this.add({ + name: cliOptions.name!, + description: cliOptions.description, + authorizerType: cliOptions.authorizerType ?? 'NONE', + discoveryUrl: cliOptions.discoveryUrl, + allowedAudience: cliOptions.allowedAudience, + allowedClients: cliOptions.allowedClients, + allowedScopes: cliOptions.allowedScopes, + customClaims: parsedCustomClaims, + clientId: cliOptions.clientId, + clientSecret: cliOptions.clientSecret, + runtimes: cliOptions.runtimes, + enableSemanticSearch: cliOptions.semanticSearch !== false, + exceptionLevel: cliOptions.exceptionLevel, + policyEngine: cliOptions.policyEngine, + policyEngineMode: cliOptions.policyEngineMode, + }); - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else { - console.log(`Added gateway '${result.gatewayName}'`); - } + if (!result.success) { + throw new Error(result.error); + } - const runtimeCount = cliOptions.runtimes - ? cliOptions.runtimes - .split(',') - .map(s => s.trim()) - .filter(Boolean).length - : 0; - return { - authorizer_type: standardize(AuthorizerType, cliOptions.authorizerType ?? 'NONE'), - has_policy_engine: !!cliOptions.policyEngine, - policy_engine_mode: standardize(PolicyEngineMode, cliOptions.policyEngineMode ?? 'log_only'), - semantic_search: cliOptions.semanticSearch !== false, - runtime_count: runtimeCount, - }; - }); - process.exit(0); - } catch (error) { if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + console.log(JSON.stringify(result)); } else { - console.error(`Error: ${getErrorMessage(error)}`); + console.log(`Added gateway '${result.gatewayName}'`); } - process.exit(1); - } + + const runtimeCount = cliOptions.runtimes + ? cliOptions.runtimes + .split(',') + .map(s => s.trim()) + .filter(Boolean).length + : 0; + return { + authorizer_type: standardize(AuthorizerType, cliOptions.authorizerType ?? 'NONE'), + has_policy_engine: !!cliOptions.policyEngine, + policy_engine_mode: standardize(PolicyEngineMode, cliOptions.policyEngineMode ?? 'log_only'), + semantic_search: cliOptions.semanticSearch !== false, + runtime_count: runtimeCount, + }; + }); }); removeCmd diff --git a/src/cli/primitives/GatewayTargetPrimitive.ts b/src/cli/primitives/GatewayTargetPrimitive.ts index 09c146fbb..2323a37a0 100644 --- a/src/cli/primitives/GatewayTargetPrimitive.ts +++ b/src/cli/primitives/GatewayTargetPrimitive.ts @@ -14,7 +14,7 @@ import { validateAddGatewayTargetOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovableGatewayTarget } from '../operations/remove/remove-gateway-target'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { GatewayTargetHost, OutboundAuth, standardize } from '../telemetry/schemas/common-shapes.js'; import { getTemplateToolDefinitions, renderGatewayTargetTemplate } from '../templates/GatewayTargetRenderer'; import { requireTTY } from '../tui/guards/tty'; @@ -299,208 +299,197 @@ export class GatewayTargetPrimitive extends BasePrimitive { - const validation = await validateAddGatewayTargetOptions(cliOptions); - if (!validation.valid) { - throw new Error(validation.error); - } + await cliCommandRun('add.gateway-target', !!cliOptions.json, async () => { + const validation = await validateAddGatewayTargetOptions(cliOptions); + if (!validation.valid) { + throw new Error(validation.error); + } - // Map CLI flag values to internal types - const outboundAuthMap: Record = { - oauth: 'OAUTH', - 'api-key': 'API_KEY', - api_key: 'API_KEY', - none: 'NONE', + // Map CLI flag values to internal types + const outboundAuthMap: Record = { + oauth: 'OAUTH', + 'api-key': 'API_KEY', + api_key: 'API_KEY', + none: 'NONE', + }; + + // Map target type from camelCase CLI to kebab-case telemetry + const targetTypeMap = { + apiGateway: 'api-gateway', + openApiSchema: 'open-api-schema', + smithyModel: 'smithy-model', + lambdaFunctionArn: 'lambda-function-arn', + mcpServer: 'mcp-server', + } as const; + type TargetTypeKey = keyof typeof targetTypeMap; + + const cliType = cliOptions.type ?? ''; + const telemetryTargetType = + cliType in targetTypeMap ? targetTypeMap[cliType as TargetTypeKey] : ('mcp-server' as const); + const telemetryOutboundAuth = standardize( + OutboundAuth, + (cliOptions.outboundAuthType ?? 'none').replace('_', '-') + ); + const telemetryHost = standardize(GatewayTargetHost, cliOptions.host ?? 'lambda'); + + // Handle API Gateway targets (no code generation) + if (cliOptions.type === 'apiGateway') { + const config: ApiGatewayTargetConfig = { + targetType: 'apiGateway', + name: cliOptions.name!, + gateway: cliOptions.gateway!, + restApiId: cliOptions.restApiId!, + stage: cliOptions.stage!, + toolFilters: cliOptions.toolFilterPath + ? [ + { + filterPath: cliOptions.toolFilterPath, + methods: (cliOptions.toolFilterMethods?.split(',').map(m => m.trim()) ?? [ + 'GET', + ]) as ApiGatewayHttpMethod[], + }, + ] + : undefined, + ...(cliOptions.outboundAuthType + ? { + outboundAuth: { + type: (outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE') as + | 'API_KEY' + | 'NONE', + credentialName: cliOptions.credentialName, + }, + } + : {}), }; - - // Map target type from camelCase CLI to kebab-case telemetry - const targetTypeMap = { - apiGateway: 'api-gateway', - openApiSchema: 'open-api-schema', - smithyModel: 'smithy-model', - lambdaFunctionArn: 'lambda-function-arn', - mcpServer: 'mcp-server', - } as const; - type TargetTypeKey = keyof typeof targetTypeMap; - - const cliType = cliOptions.type ?? ''; - const telemetryTargetType = - cliType in targetTypeMap ? targetTypeMap[cliType as TargetTypeKey] : ('mcp-server' as const); - const telemetryOutboundAuth = standardize( - OutboundAuth, - (cliOptions.outboundAuthType ?? 'none').replace('_', '-') - ); - const telemetryHost = standardize(GatewayTargetHost, cliOptions.host ?? 'lambda'); - - // Handle API Gateway targets (no code generation) - if (cliOptions.type === 'apiGateway') { - const config: ApiGatewayTargetConfig = { - targetType: 'apiGateway', - name: cliOptions.name!, - gateway: cliOptions.gateway!, - restApiId: cliOptions.restApiId!, - stage: cliOptions.stage!, - toolFilters: cliOptions.toolFilterPath - ? [ - { - filterPath: cliOptions.toolFilterPath, - methods: (cliOptions.toolFilterMethods?.split(',').map(m => m.trim()) ?? [ - 'GET', - ]) as ApiGatewayHttpMethod[], - }, - ] - : undefined, - ...(cliOptions.outboundAuthType - ? { - outboundAuth: { - type: (outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE') as - | 'API_KEY' - | 'NONE', - credentialName: cliOptions.credentialName, - }, - } - : {}), - }; - const result = await this.createApiGatewayTarget(config); - const output = { success: true, toolName: result.toolName }; - if (cliOptions.json) { - console.log(JSON.stringify(output)); - } else { - console.log(`Added gateway target '${result.toolName}'`); - } - return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; + const result = await this.createApiGatewayTarget(config); + const output = { success: true, toolName: result.toolName }; + if (cliOptions.json) { + console.log(JSON.stringify(output)); + } else { + console.log(`Added gateway target '${result.toolName}'`); } + return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; + } - // Handle schema-based targets (OpenAPI / Smithy) - if ((cliOptions.type === 'openApiSchema' || cliOptions.type === 'smithyModel') && cliOptions.schema) { - const isS3 = cliOptions.schema.startsWith('s3://'); - const schemaSource = isS3 + // Handle schema-based targets (OpenAPI / Smithy) + if ((cliOptions.type === 'openApiSchema' || cliOptions.type === 'smithyModel') && cliOptions.schema) { + const isS3 = cliOptions.schema.startsWith('s3://'); + const schemaSource = isS3 + ? { + s3: { + uri: cliOptions.schema, + ...(cliOptions.schemaS3Account ? { bucketOwnerAccountId: cliOptions.schemaS3Account } : {}), + }, + } + : { inline: { path: cliOptions.schema } }; + + const config: SchemaBasedTargetConfig = { + name: cliOptions.name!, + targetType: cliOptions.type, + schemaSource, + gateway: cliOptions.gateway!, + ...(cliOptions.outboundAuthType ? { - s3: { - uri: cliOptions.schema, - ...(cliOptions.schemaS3Account ? { bucketOwnerAccountId: cliOptions.schemaS3Account } : {}), + outboundAuth: { + type: outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE', + credentialName: cliOptions.credentialName, }, } - : { inline: { path: cliOptions.schema } }; - - const config: SchemaBasedTargetConfig = { - name: cliOptions.name!, - targetType: cliOptions.type, - schemaSource, - gateway: cliOptions.gateway!, - ...(cliOptions.outboundAuthType - ? { - outboundAuth: { - type: outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE', - credentialName: cliOptions.credentialName, - }, - } - : {}), - }; - const result = await this.createSchemaBasedGatewayTarget(config); - const output = { success: true, toolName: result.toolName }; - if (cliOptions.json) { - console.log(JSON.stringify(output)); - } else { - console.log(`Added gateway target '${result.toolName}'`); - } - return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; + : {}), + }; + const result = await this.createSchemaBasedGatewayTarget(config); + const output = { success: true, toolName: result.toolName }; + if (cliOptions.json) { + console.log(JSON.stringify(output)); + } else { + console.log(`Added gateway target '${result.toolName}'`); } + return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; + } - // Handle Lambda Function ARN targets (no code generation) - if (cliOptions.type === 'lambdaFunctionArn') { - const config = { - targetType: 'lambdaFunctionArn' as const, - name: cliOptions.name!, - gateway: cliOptions.gateway!, - lambdaArn: cliOptions.lambdaArn!, - toolSchemaFile: cliOptions.toolSchemaFile!, - }; - const result = await this.createLambdaFunctionArnTarget(config); - const output = { success: true, toolName: result.toolName }; - if (cliOptions.json) { - console.log(JSON.stringify(output)); - } else { - console.log(`Added gateway target '${result.toolName}'`); - } - return { target_type: telemetryTargetType, host: 'lambda', outbound_auth: telemetryOutboundAuth }; + // Handle Lambda Function ARN targets (no code generation) + if (cliOptions.type === 'lambdaFunctionArn') { + const config = { + targetType: 'lambdaFunctionArn' as const, + name: cliOptions.name!, + gateway: cliOptions.gateway!, + lambdaArn: cliOptions.lambdaArn!, + toolSchemaFile: cliOptions.toolSchemaFile!, + }; + const result = await this.createLambdaFunctionArnTarget(config); + const output = { success: true, toolName: result.toolName }; + if (cliOptions.json) { + console.log(JSON.stringify(output)); + } else { + console.log(`Added gateway target '${result.toolName}'`); } + return { target_type: telemetryTargetType, host: 'lambda', outbound_auth: telemetryOutboundAuth }; + } - // Handle MCP server targets (existing endpoint, no code generation) - if (cliOptions.type === 'mcpServer' && cliOptions.endpoint) { - const config: McpServerTargetConfig = { - targetType: 'mcpServer', + // Handle MCP server targets (existing endpoint, no code generation) + if (cliOptions.type === 'mcpServer' && cliOptions.endpoint) { + const config: McpServerTargetConfig = { + targetType: 'mcpServer', + name: cliOptions.name!, + description: cliOptions.description ?? `Tool for ${cliOptions.name!}`, + endpoint: cliOptions.endpoint, + gateway: cliOptions.gateway!, + toolDefinition: { name: cliOptions.name!, description: cliOptions.description ?? `Tool for ${cliOptions.name!}`, - endpoint: cliOptions.endpoint, - gateway: cliOptions.gateway!, - toolDefinition: { - name: cliOptions.name!, - description: cliOptions.description ?? `Tool for ${cliOptions.name!}`, - inputSchema: { type: 'object' }, - }, - ...(cliOptions.outboundAuthType - ? { - outboundAuth: { - type: outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE', - credentialName: cliOptions.credentialName, - }, - } - : {}), - }; - const result = await this.createExternalGatewayTarget(config); - const output = { - success: true, - toolName: result.toolName, - sourcePath: result.projectPath || undefined, - }; - if (cliOptions.json) { - console.log(JSON.stringify(output)); - } else { - console.log(`Added gateway target '${result.toolName}'`); - } - return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; - } - - const result = await this.add({ - name: cliOptions.name!, - description: cliOptions.description, - language: cliOptions.language ?? 'Python', - gateway: cliOptions.gateway, - host: cliOptions.host, - }); - - if (!result.success) { - throw new Error(result.error); - } - + inputSchema: { type: 'object' }, + }, + ...(cliOptions.outboundAuthType + ? { + outboundAuth: { + type: outboundAuthMap[cliOptions.outboundAuthType.toLowerCase()] ?? 'NONE', + credentialName: cliOptions.credentialName, + }, + } + : {}), + }; + const result = await this.createExternalGatewayTarget(config); + const output = { + success: true, + toolName: result.toolName, + sourcePath: result.projectPath || undefined, + }; if (cliOptions.json) { - console.log(JSON.stringify(result)); + console.log(JSON.stringify(output)); } else { console.log(`Added gateway target '${result.toolName}'`); - if (result.sourcePath) { - console.log(`Tool code: ${result.sourcePath}`); - } } - return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; + } + + const result = await this.add({ + name: cliOptions.name!, + description: cliOptions.description, + language: cliOptions.language ?? 'Python', + gateway: cliOptions.gateway, + host: cliOptions.host, }); - process.exit(0); - } catch (error) { + + if (!result.success) { + throw new Error(result.error); + } + if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + console.log(JSON.stringify(result)); } else { - console.error(`Error: ${getErrorMessage(error)}`); + console.log(`Added gateway target '${result.toolName}'`); + if (result.sourcePath) { + console.log(`Tool code: ${result.sourcePath}`); + } } - process.exit(1); - } + + return { target_type: telemetryTargetType, host: telemetryHost, outbound_auth: telemetryOutboundAuth }; + }); }); removeCmd diff --git a/src/cli/primitives/MemoryPrimitive.tsx b/src/cli/primitives/MemoryPrimitive.tsx index 42dffd6a4..8f425d9eb 100644 --- a/src/cli/primitives/MemoryPrimitive.tsx +++ b/src/cli/primitives/MemoryPrimitive.tsx @@ -17,7 +17,7 @@ import { import { DEFAULT_DELIVERY_TYPE, validateAddMemoryOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { requireTTY } from '../tui/guards/tty'; import { DEFAULT_EVENT_EXPIRY } from '../tui/screens/memory/types'; import { BasePrimitive } from './BasePrimitive'; @@ -184,92 +184,81 @@ export class MemoryPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } - - if (cliOptions.name || cliOptions.json) { - // CLI mode - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('add.memory', async () => { - const expiry = cliOptions.expiry ? parseInt(cliOptions.expiry, 10) : undefined; - const validation = validateAddMemoryOptions({ - name: cliOptions.name, - strategies: cliOptions.strategies, - expiry, - deliveryType: cliOptions.deliveryType, - dataStreamArn: cliOptions.dataStreamArn, - contentLevel: cliOptions.streamContentLevel, - streamDeliveryResources: cliOptions.streamDeliveryResources, - }); - - if (!validation.valid) { - throw new Error(validation.error); - } - - const result = await this.add({ - name: cliOptions.name!, - strategies: cliOptions.strategies, - expiry, - deliveryType: cliOptions.deliveryType, - dataStreamArn: cliOptions.dataStreamArn, - contentLevel: cliOptions.streamContentLevel, - streamDeliveryResources: cliOptions.streamDeliveryResources, - }); - - if (!result.success) { - throw new Error(result.error); - } - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else { - console.log(`Added memory '${result.memoryName}'`); - } - - const strategyList = (cliOptions.strategies ?? '') - .split(',') - .map(s => s.trim().toUpperCase()) - .filter(Boolean); - return { - strategy_count: strategyList.length, - strategy_semantic: strategyList.includes('SEMANTIC'), - strategy_summarization: strategyList.includes('SUMMARIZATION'), - strategy_user_preference: strategyList.includes('USER_PREFERENCE'), - strategy_episodic: strategyList.includes('EPISODIC'), - }; - }); - process.exit(0); - } else { - // TUI fallback — dynamic imports to avoid pulling ink (async) into registry - requireTTY(); - const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ - import('ink'), - import('react'), - import('../tui/screens/add/AddFlow'), - ]); - const { clear, unmount } = render( - React.createElement(AddFlow, { - isInteractive: false, - initialResource: 'memory', - onExit: () => { - clear(); - unmount(); - process.exit(0); - }, - }) - ); - } - } catch (error) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); - } else { - console.error(getErrorMessage(error)); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); process.exit(1); } + + if (cliOptions.name || cliOptions.json) { + // CLI mode + await cliCommandRun('add.memory', !!cliOptions.json, async () => { + const expiry = cliOptions.expiry ? parseInt(cliOptions.expiry, 10) : undefined; + const validation = validateAddMemoryOptions({ + name: cliOptions.name, + strategies: cliOptions.strategies, + expiry, + deliveryType: cliOptions.deliveryType, + dataStreamArn: cliOptions.dataStreamArn, + contentLevel: cliOptions.streamContentLevel, + streamDeliveryResources: cliOptions.streamDeliveryResources, + }); + + if (!validation.valid) { + throw new Error(validation.error); + } + + const result = await this.add({ + name: cliOptions.name!, + strategies: cliOptions.strategies, + expiry, + deliveryType: cliOptions.deliveryType, + dataStreamArn: cliOptions.dataStreamArn, + contentLevel: cliOptions.streamContentLevel, + streamDeliveryResources: cliOptions.streamDeliveryResources, + }); + + if (!result.success) { + throw new Error(result.error); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added memory '${result.memoryName}'`); + } + + const strategyList = (cliOptions.strategies ?? '') + .split(',') + .map(s => s.trim().toUpperCase()) + .filter(Boolean); + return { + strategy_count: strategyList.length, + strategy_semantic: strategyList.includes('SEMANTIC'), + strategy_summarization: strategyList.includes('SUMMARIZATION'), + strategy_user_preference: strategyList.includes('USER_PREFERENCE'), + strategy_episodic: strategyList.includes('EPISODIC'), + }; + }); + } else { + // TUI fallback — dynamic imports to avoid pulling ink (async) into registry + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'memory', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } } ); diff --git a/src/cli/primitives/OnlineEvalConfigPrimitive.ts b/src/cli/primitives/OnlineEvalConfigPrimitive.ts index 44919aa71..3af51ff84 100644 --- a/src/cli/primitives/OnlineEvalConfigPrimitive.ts +++ b/src/cli/primitives/OnlineEvalConfigPrimitive.ts @@ -3,7 +3,7 @@ import type { OnlineEvalConfig } from '../../schema'; import { OnlineEvalConfigSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; @@ -122,84 +122,73 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } - - if (cliOptions.name || cliOptions.json) { - // Merge --evaluator and --evaluator-arn into a single list - const allEvaluators = [...(cliOptions.evaluator ?? []), ...(cliOptions.evaluatorArn ?? [])]; - - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('add.online-eval', async () => { - if (!cliOptions.name || !cliOptions.runtime || allEvaluators.length === 0 || !cliOptions.samplingRate) { - throw new Error( - '--name, --runtime, --evaluator (and/or --evaluator-arn), and --sampling-rate are all required in non-interactive mode' - ); - } - - // Sampling rate as a percentage of requests to evaluate (0.01% to 100%) - const samplingRate = parseFloat(cliOptions.samplingRate); - if (isNaN(samplingRate) || samplingRate < 0.01 || samplingRate > 100) { - throw new Error( - `Invalid --sampling-rate "${cliOptions.samplingRate}". Must be a percentage between 0.01 and 100` - ); - } - - const result = await this.add({ - name: cliOptions.name, - agent: cliOptions.runtime, - evaluators: allEvaluators, - samplingRate, - enableOnCreate: cliOptions.enableOnCreate, - }); - - if (!result.success) { - throw new Error(result.error); - } - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else { - console.log(`Added online eval config '${result.configName}'`); - } - - return { - evaluator_count: allEvaluators.length, - enable_on_create: cliOptions.enableOnCreate ?? false, - }; - }); - process.exit(0); - } else { - // TUI fallback - requireTTY(); - const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ - import('ink'), - import('react'), - import('../tui/screens/add/AddFlow'), - ]); - const { clear, unmount } = render( - React.createElement(AddFlow, { - isInteractive: false, - initialResource: 'online-eval', - onExit: () => { - clear(); - unmount(); - process.exit(0); - }, - }) - ); - } - } catch (error) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); - } else { - console.error(getErrorMessage(error)); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); process.exit(1); } + + if (cliOptions.name || cliOptions.json) { + // Merge --evaluator and --evaluator-arn into a single list + const allEvaluators = [...(cliOptions.evaluator ?? []), ...(cliOptions.evaluatorArn ?? [])]; + + await cliCommandRun('add.online-eval', !!cliOptions.json, async () => { + if (!cliOptions.name || !cliOptions.runtime || allEvaluators.length === 0 || !cliOptions.samplingRate) { + throw new Error( + '--name, --runtime, --evaluator (and/or --evaluator-arn), and --sampling-rate are all required in non-interactive mode' + ); + } + + // Sampling rate as a percentage of requests to evaluate (0.01% to 100%) + const samplingRate = parseFloat(cliOptions.samplingRate); + if (isNaN(samplingRate) || samplingRate < 0.01 || samplingRate > 100) { + throw new Error( + `Invalid --sampling-rate "${cliOptions.samplingRate}". Must be a percentage between 0.01 and 100` + ); + } + + const result = await this.add({ + name: cliOptions.name, + agent: cliOptions.runtime, + evaluators: allEvaluators, + samplingRate, + enableOnCreate: cliOptions.enableOnCreate, + }); + + if (!result.success) { + throw new Error(result.error); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added online eval config '${result.configName}'`); + } + + return { + evaluator_count: allEvaluators.length, + enable_on_create: cliOptions.enableOnCreate ?? false, + }; + }); + } else { + // TUI fallback + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'online-eval', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } } ); diff --git a/src/cli/primitives/PolicyEnginePrimitive.ts b/src/cli/primitives/PolicyEnginePrimitive.ts index 6efd141cc..a8d870b7c 100644 --- a/src/cli/primitives/PolicyEnginePrimitive.ts +++ b/src/cli/primitives/PolicyEnginePrimitive.ts @@ -3,7 +3,7 @@ import type { AgentCoreProjectSpec, PolicyEngine } from '../../schema'; import { PolicyEngineModeSchema, PolicyEngineSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { AttachMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; @@ -223,82 +223,71 @@ export class PolicyEnginePrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } + + if (cliOptions.name || cliOptions.description || cliOptions.encryptionKeyArn || cliOptions.json) { + await cliCommandRun('add.policy-engine', !!cliOptions.json, async () => { + if (!cliOptions.name) { + throw new Error('--name is required'); + } + + const result = await this.add({ + name: cliOptions.name, + description: cliOptions.description, + encryptionKeyArn: cliOptions.encryptionKeyArn, + }); - if (cliOptions.name || cliOptions.description || cliOptions.encryptionKeyArn || cliOptions.json) { - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('add.policy-engine', async () => { - if (!cliOptions.name) { - throw new Error('--name is required'); - } - - const result = await this.add({ - name: cliOptions.name, - description: cliOptions.description, - encryptionKeyArn: cliOptions.encryptionKeyArn, - }); - - // Attach to gateways if requested - if (result.success && cliOptions.attachToGateways) { - const mode = PolicyEngineModeSchema.parse(cliOptions.attachMode ?? 'LOG_ONLY'); - const gateways = cliOptions.attachToGateways + // Attach to gateways if requested + if (result.success && cliOptions.attachToGateways) { + const mode = PolicyEngineModeSchema.parse(cliOptions.attachMode ?? 'LOG_ONLY'); + const gateways = cliOptions.attachToGateways + .split(',') + .map(s => s.trim()) + .filter(Boolean); + await this.attachToGateways(cliOptions.name, gateways, mode); + } + + if (!result.success) { + throw new Error(result.error); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added policy engine '${result.engineName}'`); + } + + const gatewayCount = cliOptions.attachToGateways + ? cliOptions.attachToGateways .split(',') .map(s => s.trim()) - .filter(Boolean); - await this.attachToGateways(cliOptions.name, gateways, mode); - } - - if (!result.success) { - throw new Error(result.error); - } - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else { - console.log(`Added policy engine '${result.engineName}'`); - } - - const gatewayCount = cliOptions.attachToGateways - ? cliOptions.attachToGateways - .split(',') - .map(s => s.trim()) - .filter(Boolean).length - : 0; - return { - attach_gateway_count: gatewayCount, - attach_mode: standardize(AttachMode, cliOptions.attachMode ?? 'log_only'), - }; - }); - process.exit(0); - } else { - requireTTY(); - const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ - import('ink'), - import('react'), - import('../tui/screens/add/AddFlow'), - ]); - const { clear, unmount } = render( - React.createElement(AddFlow, { - isInteractive: false, - onExit: () => { - clear(); - unmount(); - process.exit(0); - }, - }) - ); - } - } catch (error) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); - } else { - console.error(`Error: ${getErrorMessage(error)}`); - } - process.exit(1); + .filter(Boolean).length + : 0; + return { + attach_gateway_count: gatewayCount, + attach_mode: standardize(AttachMode, cliOptions.attachMode ?? 'log_only'), + }; + }); + } else { + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); } } ); diff --git a/src/cli/primitives/PolicyPrimitive.ts b/src/cli/primitives/PolicyPrimitive.ts index 202585983..e129306bf 100644 --- a/src/cli/primitives/PolicyPrimitive.ts +++ b/src/cli/primitives/PolicyPrimitive.ts @@ -5,7 +5,7 @@ import { detectRegion } from '../aws'; import { getPolicyGeneration, startPolicyGeneration } from '../aws/policy-generation'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { ValidationMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; @@ -293,89 +293,78 @@ export class PolicyPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - if ( - cliOptions.name || - cliOptions.engine || - cliOptions.source || - cliOptions.statement || - cliOptions.generate || - cliOptions.json - ) { - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('add.policy', async () => { - if (!cliOptions.name) { - throw new Error('--name is required'); - } - if (!cliOptions.engine) { - throw new Error('--engine is required'); - } - - const result = await this.add({ - name: cliOptions.name, - engine: cliOptions.engine, - description: cliOptions.description, - source: cliOptions.source, - statement: cliOptions.statement, - generate: cliOptions.generate, - gateway: cliOptions.gateway, - validationMode: cliOptions.validationMode - ? ValidationModeSchema.parse(cliOptions.validationMode) - : undefined, - }); - - if (!result.success) { - throw new Error(result.error); - } - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else { - console.log(`Added policy '${result.policyName}' to engine '${result.engineName}'`); - } - - const sourceType: 'file' | 'statement' | 'generate' = cliOptions.source - ? 'file' - : cliOptions.generate - ? 'generate' - : 'statement'; - return { - source_type: sourceType, - validation_mode: standardize(ValidationMode, cliOptions.validationMode ?? 'FAIL_ON_ANY_FINDINGS'), - }; + if ( + cliOptions.name || + cliOptions.engine || + cliOptions.source || + cliOptions.statement || + cliOptions.generate || + cliOptions.json + ) { + await cliCommandRun('add.policy', !!cliOptions.json, async () => { + if (!cliOptions.name) { + throw new Error('--name is required'); + } + if (!cliOptions.engine) { + throw new Error('--engine is required'); + } + + const result = await this.add({ + name: cliOptions.name, + engine: cliOptions.engine, + description: cliOptions.description, + source: cliOptions.source, + statement: cliOptions.statement, + generate: cliOptions.generate, + gateway: cliOptions.gateway, + validationMode: cliOptions.validationMode + ? ValidationModeSchema.parse(cliOptions.validationMode) + : undefined, }); - process.exit(0); - } else { - requireTTY(); - const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ - import('ink'), - import('react'), - import('../tui/screens/add/AddFlow'), - ]); - const { clear, unmount } = render( - React.createElement(AddFlow, { - isInteractive: false, - initialResource: 'policy', - onExit: () => { - clear(); - unmount(); - process.exit(0); - }, - }) - ); - } - } catch (error) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); - } else { - console.error(`Error: ${getErrorMessage(error)}`); - } - process.exit(1); + + if (!result.success) { + throw new Error(result.error); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added policy '${result.policyName}' to engine '${result.engineName}'`); + } + + const sourceType: 'file' | 'statement' | 'generate' = cliOptions.source + ? 'file' + : cliOptions.generate + ? 'generate' + : 'statement'; + return { + source_type: sourceType, + validation_mode: standardize(ValidationMode, cliOptions.validationMode ?? 'FAIL_ON_ANY_FINDINGS'), + }; + }); + } else { + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'policy', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); } } ); diff --git a/src/cli/primitives/RuntimeEndpointPrimitive.ts b/src/cli/primitives/RuntimeEndpointPrimitive.ts index aef1aecb6..e055fd17a 100644 --- a/src/cli/primitives/RuntimeEndpointPrimitive.ts +++ b/src/cli/primitives/RuntimeEndpointPrimitive.ts @@ -4,7 +4,7 @@ import { RuntimeEndpointSchema } from '../../schema'; import type { ResourceType } from '../commands/remove/types'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { TelemetryClientAccessor } from '../telemetry/client-accessor.js'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { BasePrimitive } from './BasePrimitive'; import { SOURCE_CODE_NOTE } from './constants'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; @@ -249,42 +249,31 @@ export class RuntimeEndpointPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - const client = await TelemetryClientAccessor.get(); - await client.withCommandRun('add.runtime-endpoint', async () => { - const result = await this.add({ - runtime: cliOptions.runtime, - endpoint: cliOptions.endpoint, - version: cliOptions.version, - description: cliOptions.description, - }); - - if (!result.success) { - throw new Error(result.error); - } - - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else { - console.log(`Added runtime endpoint '${cliOptions.endpoint}' to runtime '${cliOptions.runtime}'`); - } - - return {}; + await cliCommandRun('add.runtime-endpoint', !!cliOptions.json, async () => { + const result = await this.add({ + runtime: cliOptions.runtime, + endpoint: cliOptions.endpoint, + version: cliOptions.version, + description: cliOptions.description, }); - process.exit(0); - } catch (error) { + + if (!result.success) { + throw new Error(result.error); + } + if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + console.log(JSON.stringify(result)); } else { - console.error(`Error: ${getErrorMessage(error)}`); + console.log(`Added runtime endpoint '${cliOptions.endpoint}' to runtime '${cliOptions.runtime}'`); } - process.exit(1); - } + + return {}; + }); } ); diff --git a/src/cli/telemetry/cli-command-run.ts b/src/cli/telemetry/cli-command-run.ts new file mode 100644 index 000000000..1e3ec1959 --- /dev/null +++ b/src/cli/telemetry/cli-command-run.ts @@ -0,0 +1,26 @@ +import { getErrorMessage } from '../errors'; +import { TelemetryClientAccessor } from './client-accessor.js'; +import type { Command, CommandAttrs } from './schemas/command-run.js'; + +/** + * Run a CLI command with telemetry, standardized error output, and process.exit. + * The callback should throw on failure and return telemetry attrs on success. + */ +export async function cliCommandRun( + command: C, + json: boolean, + fn: () => Promise> +): Promise { + try { + const client = await TelemetryClientAccessor.get(); + await client.withCommandRun(command, fn); + process.exit(0); + } catch (error) { + if (json) { + console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + } else { + console.error(getErrorMessage(error)); + } + process.exit(1); + } +} From b102b94168b22cbd4cea30bf4d4f01147aca155d Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 29 Apr 2026 21:51:53 +0000 Subject: [PATCH 03/18] test: add audit file assertions for all add.* telemetry --- integ-tests/telemetry.test.ts | 147 +++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 2 deletions(-) diff --git a/integ-tests/telemetry.test.ts b/integ-tests/telemetry.test.ts index 5347af0c8..2746b0680 100644 --- a/integ-tests/telemetry.test.ts +++ b/integ-tests/telemetry.test.ts @@ -1,9 +1,12 @@ import { spawnAndCollect } from '../src/test-utils/cli-runner.js'; -import { mkdtempSync } from 'node:fs'; +import { createTestProject } from '../src/test-utils/index.js'; +import type { TestProject } from '../src/test-utils/index.js'; +import { globSync } from 'glob'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; const testConfigDir = mkdtempSync(join(tmpdir(), 'agentcore-integ-')); const cliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); @@ -30,3 +33,143 @@ describe('telemetry e2e', () => { expect(status.stdout).toContain('global config'); }); }); + +// ── Audit file tests ─────────────────────────────────────────────────────── + +interface TelemetryEntry { + value: number; + attrs: Record; +} + +function readAuditEntries(configDir: string): TelemetryEntry[] { + const files = globSync(join(configDir, 'telemetry', '*.json')); + return files.flatMap(f => + readFileSync(f, 'utf-8') + .trim() + .split('\n') + .map(line => JSON.parse(line) as TelemetryEntry) + ); +} + +function clearAudit(configDir: string) { + try { + rmSync(join(configDir, 'telemetry'), { recursive: true, force: true }); + } catch { + /* ignore */ + } +} + +describe('telemetry audit: add.* commands', () => { + let project: TestProject; + const auditConfigDir = mkdtempSync(join(tmpdir(), 'agentcore-audit-')); + + function runWithAudit(args: string[]) { + return spawnAndCollect('node', [cliPath, ...args], project.projectPath, { + AGENTCORE_SKIP_INSTALL: '1', + AGENTCORE_TELEMETRY_AUDIT: '1', + AGENTCORE_CONFIG_DIR: auditConfigDir, + }); + } + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + await rm(auditConfigDir, { recursive: true, force: true }); + }); + + beforeEach(() => clearAudit(auditConfigDir)); + + it('add.gateway emits correct telemetry', async () => { + const result = await runWithAudit(['add', 'gateway', '--name', 'testgw', '--authorizer-type', 'NONE', '--json']); + expect(result.exitCode).toBe(0); + + const entries = readAuditEntries(auditConfigDir); + expect(entries).toHaveLength(1); + expect(entries[0]!.attrs).toMatchObject({ + command: 'add.gateway', + command_group: 'add', + exit_reason: 'success', + authorizer_type: 'none', + has_policy_engine: 'false', + semantic_search: 'true', + runtime_count: 0, + }); + }); + + it('add.credential emits correct telemetry', async () => { + const result = await runWithAudit([ + 'add', + 'credential', + '--name', + 'testcred', + '--type', + 'api-key', + '--api-key', + 'secret123', + '--json', + ]); + expect(result.exitCode).toBe(0); + + const entries = readAuditEntries(auditConfigDir); + expect(entries).toHaveLength(1); + expect(entries[0]!.attrs).toMatchObject({ + command: 'add.credential', + exit_reason: 'success', + credential_type: 'api-key', + }); + }); + + it('add.memory emits correct telemetry', async () => { + const result = await runWithAudit([ + 'add', + 'memory', + '--name', + 'testmem', + '--strategies', + 'SEMANTIC,SUMMARIZATION', + '--expiry', + '30', + '--json', + ]); + expect(result.exitCode).toBe(0); + + const entries = readAuditEntries(auditConfigDir); + expect(entries).toHaveLength(1); + expect(entries[0]!.attrs).toMatchObject({ + command: 'add.memory', + exit_reason: 'success', + strategy_count: 2, + strategy_semantic: 'true', + strategy_summarization: 'true', + }); + }); + + it('add.policy-engine emits correct telemetry', async () => { + const result = await runWithAudit(['add', 'policy-engine', '--name', 'testengine', '--json']); + expect(result.exitCode).toBe(0); + + const entries = readAuditEntries(auditConfigDir); + expect(entries).toHaveLength(1); + expect(entries[0]!.attrs).toMatchObject({ + command: 'add.policy-engine', + exit_reason: 'success', + attach_gateway_count: 0, + attach_mode: 'log_only', + }); + }); + + it('records failure telemetry when validation fails', async () => { + const result = await runWithAudit(['add', 'gateway', '--json']); + expect(result.exitCode).toBe(1); + + const entries = readAuditEntries(auditConfigDir); + expect(entries).toHaveLength(1); + expect(entries[0]!.attrs).toMatchObject({ + command: 'add.gateway', + exit_reason: 'failure', + }); + }); +}); From 72ab9d3bcca84d554c8c85f9105796ac82686583 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 29 Apr 2026 21:58:01 +0000 Subject: [PATCH 04/18] test: add telemetry audit assertions to existing add integ tests --- e2e-tests/byo-custom-jwt.test.ts | 2 +- integ-tests/add-remove-resources.test.ts | 41 ++++++- integ-tests/create-no-agent.test.ts | 2 +- integ-tests/create-with-agent.test.ts | 2 +- integ-tests/dev-server.test.ts | 2 +- integ-tests/telemetry.test.ts | 147 +---------------------- src/test-utils/cli-runner.ts | 12 +- src/test-utils/project-factory.ts | 2 +- 8 files changed, 56 insertions(+), 154 deletions(-) diff --git a/e2e-tests/byo-custom-jwt.test.ts b/e2e-tests/byo-custom-jwt.test.ts index b7391a522..64e534e20 100644 --- a/e2e-tests/byo-custom-jwt.test.ts +++ b/e2e-tests/byo-custom-jwt.test.ts @@ -48,7 +48,7 @@ const region = process.env.AWS_REGION ?? 'us-east-1'; * Run the local CLI build without skipping install (needed for deploy). */ function runLocalCLI(args: string[], cwd: string): Promise { - return runCLI(args, cwd, /* skipInstall */ false); + return runCLI(args, cwd, { skipInstall: false }); } describe.sequential('e2e: BYO agent with CUSTOM_JWT auth', () => { diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index 57dd48483..154e00de1 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -1,7 +1,28 @@ import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; +import { globSync } from 'glob'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +const auditDir = mkdtempSync(join(tmpdir(), 'agentcore-audit-')); +const auditEnv = { AGENTCORE_TELEMETRY_AUDIT: '1', AGENTCORE_CONFIG_DIR: auditDir }; + +interface TelemetryEntry { + value: number; + attrs: Record; +} + +function readAuditEntries(): TelemetryEntry[] { + return globSync(join(auditDir, 'telemetry', '*.json')).flatMap(f => + readFileSync(f, 'utf-8') + .trim() + .split('\n') + .map(line => JSON.parse(line) as TelemetryEntry) + ); +} + describe('integration: add and remove resources', () => { let project: TestProject; @@ -16,13 +37,16 @@ describe('integration: add and remove resources', () => { afterAll(async () => { await project.cleanup(); + rmSync(auditDir, { recursive: true, force: true }); }); describe('memory lifecycle', () => { const memoryName = `IntegMem${Date.now().toString().slice(-6)}`; it('adds a memory resource', async () => { - const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath); + const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath, { + env: auditEnv, + }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); const json = JSON.parse(result.stdout); @@ -34,6 +58,12 @@ describe('integration: add and remove resources', () => { expect(memories, 'memories should exist').toBeDefined(); const found = memories!.some((m: Record) => m.name === memoryName); expect(found, `Memory "${memoryName}" should be in config`).toBe(true); + + // Verify telemetry + const entries = readAuditEntries(); + expect(entries.length).toBeGreaterThan(0); + const last = entries[entries.length - 1]!.attrs; + expect(last).toMatchObject({ command: 'add.memory', exit_reason: 'success' }); }); it('adds a memory with EPISODIC strategy and verifies reflectionNamespaces', async () => { @@ -86,7 +116,8 @@ describe('integration: add and remove resources', () => { it('adds a credential resource', async () => { const result = await runCLI( ['add', 'credential', '--name', credentialName, '--api-key', 'test-key-integ-123', '--json'], - project.projectPath + project.projectPath, + { env: auditEnv } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -99,6 +130,12 @@ describe('integration: add and remove resources', () => { expect(credentials, 'credentials should exist').toBeDefined(); const found = credentials!.some((c: Record) => c.name === credentialName); expect(found, `Credential "${credentialName}" should be in config`).toBe(true); + + // Verify telemetry + const entries = readAuditEntries(); + expect(entries.length).toBeGreaterThan(0); + const last = entries[entries.length - 1]!.attrs; + expect(last).toMatchObject({ command: 'add.credential', exit_reason: 'success', credential_type: 'api-key' }); }); it('removes the credential resource', async () => { diff --git a/integ-tests/create-no-agent.test.ts b/integ-tests/create-no-agent.test.ts index 4bcca2690..bcdf80eaa 100644 --- a/integ-tests/create-no-agent.test.ts +++ b/integ-tests/create-no-agent.test.ts @@ -32,7 +32,7 @@ describe('integration: create without agent', () => { it.skipIf(!hasNpm || !hasGit)('creates project with real npm install and git init', async () => { const name = `NoAgent${Date.now().toString().slice(-6)}`; - const result = await runCLI(['create', '--name', name, '--no-agent', '--json'], testDir, false); + const result = await runCLI(['create', '--name', name, '--no-agent', '--json'], testDir, { skipInstall: false }); expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0); diff --git a/integ-tests/create-with-agent.test.ts b/integ-tests/create-with-agent.test.ts index 7fb20bdbf..69f0594b8 100644 --- a/integ-tests/create-with-agent.test.ts +++ b/integ-tests/create-with-agent.test.ts @@ -49,7 +49,7 @@ describe('integration: create with Python agent', () => { '--json', ], testDir, - false + { skipInstall: false } ); expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0); diff --git a/integ-tests/dev-server.test.ts b/integ-tests/dev-server.test.ts index 5f60976e7..4b07b7284 100644 --- a/integ-tests/dev-server.test.ts +++ b/integ-tests/dev-server.test.ts @@ -60,7 +60,7 @@ describe('integration: dev server', () => { '--json', ], testDir, - false + { skipInstall: false } ); if (result.exitCode === 0) { diff --git a/integ-tests/telemetry.test.ts b/integ-tests/telemetry.test.ts index 2746b0680..5347af0c8 100644 --- a/integ-tests/telemetry.test.ts +++ b/integ-tests/telemetry.test.ts @@ -1,12 +1,9 @@ import { spawnAndCollect } from '../src/test-utils/cli-runner.js'; -import { createTestProject } from '../src/test-utils/index.js'; -import type { TestProject } from '../src/test-utils/index.js'; -import { globSync } from 'glob'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { mkdtempSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { afterAll, describe, expect, it } from 'vitest'; const testConfigDir = mkdtempSync(join(tmpdir(), 'agentcore-integ-')); const cliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); @@ -33,143 +30,3 @@ describe('telemetry e2e', () => { expect(status.stdout).toContain('global config'); }); }); - -// ── Audit file tests ─────────────────────────────────────────────────────── - -interface TelemetryEntry { - value: number; - attrs: Record; -} - -function readAuditEntries(configDir: string): TelemetryEntry[] { - const files = globSync(join(configDir, 'telemetry', '*.json')); - return files.flatMap(f => - readFileSync(f, 'utf-8') - .trim() - .split('\n') - .map(line => JSON.parse(line) as TelemetryEntry) - ); -} - -function clearAudit(configDir: string) { - try { - rmSync(join(configDir, 'telemetry'), { recursive: true, force: true }); - } catch { - /* ignore */ - } -} - -describe('telemetry audit: add.* commands', () => { - let project: TestProject; - const auditConfigDir = mkdtempSync(join(tmpdir(), 'agentcore-audit-')); - - function runWithAudit(args: string[]) { - return spawnAndCollect('node', [cliPath, ...args], project.projectPath, { - AGENTCORE_SKIP_INSTALL: '1', - AGENTCORE_TELEMETRY_AUDIT: '1', - AGENTCORE_CONFIG_DIR: auditConfigDir, - }); - } - - beforeAll(async () => { - project = await createTestProject({ noAgent: true }); - }); - - afterAll(async () => { - await project.cleanup(); - await rm(auditConfigDir, { recursive: true, force: true }); - }); - - beforeEach(() => clearAudit(auditConfigDir)); - - it('add.gateway emits correct telemetry', async () => { - const result = await runWithAudit(['add', 'gateway', '--name', 'testgw', '--authorizer-type', 'NONE', '--json']); - expect(result.exitCode).toBe(0); - - const entries = readAuditEntries(auditConfigDir); - expect(entries).toHaveLength(1); - expect(entries[0]!.attrs).toMatchObject({ - command: 'add.gateway', - command_group: 'add', - exit_reason: 'success', - authorizer_type: 'none', - has_policy_engine: 'false', - semantic_search: 'true', - runtime_count: 0, - }); - }); - - it('add.credential emits correct telemetry', async () => { - const result = await runWithAudit([ - 'add', - 'credential', - '--name', - 'testcred', - '--type', - 'api-key', - '--api-key', - 'secret123', - '--json', - ]); - expect(result.exitCode).toBe(0); - - const entries = readAuditEntries(auditConfigDir); - expect(entries).toHaveLength(1); - expect(entries[0]!.attrs).toMatchObject({ - command: 'add.credential', - exit_reason: 'success', - credential_type: 'api-key', - }); - }); - - it('add.memory emits correct telemetry', async () => { - const result = await runWithAudit([ - 'add', - 'memory', - '--name', - 'testmem', - '--strategies', - 'SEMANTIC,SUMMARIZATION', - '--expiry', - '30', - '--json', - ]); - expect(result.exitCode).toBe(0); - - const entries = readAuditEntries(auditConfigDir); - expect(entries).toHaveLength(1); - expect(entries[0]!.attrs).toMatchObject({ - command: 'add.memory', - exit_reason: 'success', - strategy_count: 2, - strategy_semantic: 'true', - strategy_summarization: 'true', - }); - }); - - it('add.policy-engine emits correct telemetry', async () => { - const result = await runWithAudit(['add', 'policy-engine', '--name', 'testengine', '--json']); - expect(result.exitCode).toBe(0); - - const entries = readAuditEntries(auditConfigDir); - expect(entries).toHaveLength(1); - expect(entries[0]!.attrs).toMatchObject({ - command: 'add.policy-engine', - exit_reason: 'success', - attach_gateway_count: 0, - attach_mode: 'log_only', - }); - }); - - it('records failure telemetry when validation fails', async () => { - const result = await runWithAudit(['add', 'gateway', '--json']); - expect(result.exitCode).toBe(1); - - const entries = readAuditEntries(auditConfigDir); - expect(entries).toHaveLength(1); - expect(entries[0]!.attrs).toMatchObject({ - command: 'add.gateway', - exit_reason: 'failure', - }); - }); -}); diff --git a/src/test-utils/cli-runner.ts b/src/test-utils/cli-runner.ts index 789624364..4526546c5 100644 --- a/src/test-utils/cli-runner.ts +++ b/src/test-utils/cli-runner.ts @@ -72,6 +72,14 @@ function getCLIPath(): string { * Run the AgentCore CLI via the local build (unit/integ tests). * Skips dependency installation by default for speed. */ -export async function runCLI(args: string[], cwd: string, skipInstall = true): Promise { - return spawnAndCollect('node', [getCLIPath(), ...args], cwd, skipInstall ? { AGENTCORE_SKIP_INSTALL: '1' } : {}); +export async function runCLI( + args: string[], + cwd: string, + options: { skipInstall?: boolean; env?: Record } = {} +): Promise { + const { skipInstall = true, env } = options; + return spawnAndCollect('node', [getCLIPath(), ...args], cwd, { + ...(skipInstall ? { AGENTCORE_SKIP_INSTALL: '1' } : {}), + ...env, + }); } diff --git a/src/test-utils/project-factory.ts b/src/test-utils/project-factory.ts index 77c77e1c6..61525f6c5 100644 --- a/src/test-utils/project-factory.ts +++ b/src/test-utils/project-factory.ts @@ -65,7 +65,7 @@ export async function createTestProject(options: CreateTestProjectOptions = {}): args.push('--json'); - const result = await runCLI(args, testDir, skipInstall); + const result = await runCLI(args, testDir, { skipInstall }); if (result.exitCode !== 0) { // Clean up on failure From e521dbccd94f8756bb10398852018960fecabad5 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 29 Apr 2026 22:05:47 +0000 Subject: [PATCH 05/18] refactor: extract shared audit test utils into src/test-utils/audit.ts --- integ-tests/add-remove-resources.test.ts | 48 +++++++++--------------- integ-tests/help.test.ts | 45 ++++++++++------------ src/test-utils/audit.ts | 44 ++++++++++++++++++++++ src/test-utils/index.ts | 1 + 4 files changed, 82 insertions(+), 56 deletions(-) create mode 100644 src/test-utils/audit.ts diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index 154e00de1..844708741 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -1,27 +1,9 @@ +import { createAuditContext } from '../src/test-utils/audit.js'; import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; -import { globSync } from 'glob'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -const auditDir = mkdtempSync(join(tmpdir(), 'agentcore-audit-')); -const auditEnv = { AGENTCORE_TELEMETRY_AUDIT: '1', AGENTCORE_CONFIG_DIR: auditDir }; - -interface TelemetryEntry { - value: number; - attrs: Record; -} - -function readAuditEntries(): TelemetryEntry[] { - return globSync(join(auditDir, 'telemetry', '*.json')).flatMap(f => - readFileSync(f, 'utf-8') - .trim() - .split('\n') - .map(line => JSON.parse(line) as TelemetryEntry) - ); -} +const audit = createAuditContext(); describe('integration: add and remove resources', () => { let project: TestProject; @@ -37,7 +19,7 @@ describe('integration: add and remove resources', () => { afterAll(async () => { await project.cleanup(); - rmSync(auditDir, { recursive: true, force: true }); + audit.cleanup(); }); describe('memory lifecycle', () => { @@ -45,7 +27,7 @@ describe('integration: add and remove resources', () => { it('adds a memory resource', async () => { const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath, { - env: auditEnv, + env: audit.env, }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -60,10 +42,10 @@ describe('integration: add and remove resources', () => { expect(found, `Memory "${memoryName}" should be in config`).toBe(true); // Verify telemetry - const entries = readAuditEntries(); - expect(entries.length).toBeGreaterThan(0); - const last = entries[entries.length - 1]!.attrs; - expect(last).toMatchObject({ command: 'add.memory', exit_reason: 'success' }); + const entries = audit.readEntries(); + const memEntry = entries.find(e => e.attrs.command === 'add.memory'); + expect(memEntry).toBeDefined(); + expect(memEntry!.attrs).toMatchObject({ command: 'add.memory', exit_reason: 'success' }); }); it('adds a memory with EPISODIC strategy and verifies reflectionNamespaces', async () => { @@ -117,7 +99,7 @@ describe('integration: add and remove resources', () => { const result = await runCLI( ['add', 'credential', '--name', credentialName, '--api-key', 'test-key-integ-123', '--json'], project.projectPath, - { env: auditEnv } + { env: audit.env } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -132,10 +114,14 @@ describe('integration: add and remove resources', () => { expect(found, `Credential "${credentialName}" should be in config`).toBe(true); // Verify telemetry - const entries = readAuditEntries(); - expect(entries.length).toBeGreaterThan(0); - const last = entries[entries.length - 1]!.attrs; - expect(last).toMatchObject({ command: 'add.credential', exit_reason: 'success', credential_type: 'api-key' }); + const entries = audit.readEntries(); + const credEntry = entries.find(e => e.attrs.command === 'add.credential'); + expect(credEntry).toBeDefined(); + expect(credEntry!.attrs).toMatchObject({ + command: 'add.credential', + exit_reason: 'success', + credential_type: 'api-key', + }); }); it('removes the credential resource', async () => { diff --git a/integ-tests/help.test.ts b/integ-tests/help.test.ts index 052605c7a..93aa447d3 100644 --- a/integ-tests/help.test.ts +++ b/integ-tests/help.test.ts @@ -1,10 +1,9 @@ +import { createAuditContext } from '../src/test-utils/audit.js'; import { spawnAndCollect } from '../src/test-utils/cli-runner.js'; import { runCLI } from '../src/test-utils/index.js'; import { readdirSync } from 'node:fs'; -import { mkdir, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, describe, expect, it } from 'vitest'; const COMMANDS = [ 'create', @@ -45,52 +44,48 @@ describe('CLI help', () => { }); describe('help modes telemetry', () => { - let testConfigDir: string; + const audit = createAuditContext(); const cliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); - beforeAll(async () => { - testConfigDir = join(tmpdir(), `agentcore-help-telemetry-${Date.now()}`); - await mkdir(testConfigDir, { recursive: true }); - }); - afterAll(() => rm(testConfigDir, { recursive: true, force: true })); + afterAll(() => audit.cleanup()); function run(args: string[], extraEnv: Record = {}) { - return spawnAndCollect('node', [cliPath, ...args], tmpdir(), { + return spawnAndCollect('node', [cliPath, ...args], process.cwd(), { AGENTCORE_SKIP_INSTALL: '1', - AGENTCORE_CONFIG_DIR: testConfigDir, + ...audit.env, ...extraEnv, }); } it('writes JSONL audit file when audit is enabled via env var', async () => { - const result = await run(['help', 'modes'], { AGENTCORE_TELEMETRY_AUDIT: '1' }); + const result = await run(['help', 'modes']); expect(result.exitCode).toBe(0); - const telemetryDir = join(testConfigDir, 'telemetry'); - const files = readdirSync(telemetryDir).filter(f => f.startsWith('help-')); - expect(files).toHaveLength(1); - - const content = await readFile(join(telemetryDir, files[0]!), 'utf-8'); - const entry = JSON.parse(content.trim()); - expect(entry.attrs).toMatchObject({ + const entries = audit.readEntries(); + expect(entries).toHaveLength(1); + expect(entries[0]!.attrs).toMatchObject({ 'service.name': 'agentcore-cli', 'agentcore-cli.mode': 'cli', command_group: 'help', command: 'help.modes', exit_reason: 'success', }); - expect(entry.attrs['agentcore-cli.session_id']).toBeDefined(); - expect(entry.attrs['os.type']).toBeDefined(); - expect(entry.value).toBeGreaterThanOrEqual(0); + expect(entries[0]!.attrs['agentcore-cli.session_id']).toBeDefined(); + expect(entries[0]!.attrs['os.type']).toBeDefined(); + expect(entries[0]!.value).toBeGreaterThanOrEqual(0); }); it('does not write audit file when audit is not enabled', async () => { - const telemetryDir = join(testConfigDir, 'telemetry'); - await rm(telemetryDir, { recursive: true, force: true }); + audit.clear(); - const result = await run(['help', 'modes']); + const noAuditCliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); + const result = await spawnAndCollect('node', [noAuditCliPath, 'help', 'modes'], process.cwd(), { + AGENTCORE_SKIP_INSTALL: '1', + AGENTCORE_CONFIG_DIR: audit.dir, + }); expect(result.exitCode).toBe(0); + const telemetryDir = join(audit.dir, 'telemetry'); try { const files = readdirSync(telemetryDir); expect(files).toHaveLength(0); diff --git a/src/test-utils/audit.ts b/src/test-utils/audit.ts new file mode 100644 index 000000000..857a00e05 --- /dev/null +++ b/src/test-utils/audit.ts @@ -0,0 +1,44 @@ +import { globSync } from 'glob'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +export interface TelemetryEntry { + value: number; + attrs: Record; +} + +export interface AuditContext { + /** Temp directory used as AGENTCORE_CONFIG_DIR */ + dir: string; + /** Env vars to pass to runCLI */ + env: { AGENTCORE_TELEMETRY_AUDIT: '1'; AGENTCORE_CONFIG_DIR: string }; + /** Read all JSONL entries from the audit telemetry directory */ + readEntries: () => TelemetryEntry[]; + /** Delete the telemetry subdirectory */ + clear: () => void; + /** Delete the entire config directory */ + cleanup: () => void; +} + +export function createAuditContext(): AuditContext { + const dir = mkdtempSync(join(tmpdir(), 'agentcore-audit-')); + return { + dir, + env: { AGENTCORE_TELEMETRY_AUDIT: '1', AGENTCORE_CONFIG_DIR: dir }, + readEntries() { + return globSync(join(dir, 'telemetry', '*.json')).flatMap(f => + readFileSync(f, 'utf-8') + .trim() + .split('\n') + .map(line => JSON.parse(line) as TelemetryEntry) + ); + }, + clear() { + rmSync(join(dir, 'telemetry'), { recursive: true, force: true }); + }, + cleanup() { + rmSync(dir, { recursive: true, force: true }); + }, + }; +} diff --git a/src/test-utils/index.ts b/src/test-utils/index.ts index ff127a35e..a9aad665d 100644 --- a/src/test-utils/index.ts +++ b/src/test-utils/index.ts @@ -4,6 +4,7 @@ */ export { runCLI, spawnAndCollect, cleanSpawnEnv, type RunResult } from './cli-runner.js'; +export { createAuditContext, type AuditContext, type TelemetryEntry } from './audit.js'; export { exists } from './fs-helpers.js'; export { hasCommand, hasAwsCredentials, prereqs } from './prereqs.js'; export { createTestProject, type TestProject, type CreateTestProjectOptions } from './project-factory.js'; From f152b0c67ac37676dd8eb718f2c66686e70d45a3 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 29 Apr 2026 22:25:26 +0000 Subject: [PATCH 06/18] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20guard=20telemetry=20init,=20replaceAll,=20unknown?= =?UTF-8?q?=20fallback,=20TUI=20try/catch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cli/primitives/CredentialPrimitive.tsx | 41 +++++++++++-------- src/cli/primitives/EvaluatorPrimitive.ts | 41 +++++++++++-------- src/cli/primitives/GatewayTargetPrimitive.ts | 4 +- src/cli/primitives/MemoryPrimitive.tsx | 41 +++++++++++-------- .../primitives/OnlineEvalConfigPrimitive.ts | 41 +++++++++++-------- src/cli/primitives/PolicyEnginePrimitive.ts | 37 +++++++++-------- src/cli/primitives/PolicyPrimitive.ts | 39 ++++++++++-------- src/cli/telemetry/cli-command-run.ts | 13 +++++- src/cli/telemetry/schemas/common-shapes.ts | 1 + 9 files changed, 150 insertions(+), 108 deletions(-) diff --git a/src/cli/primitives/CredentialPrimitive.tsx b/src/cli/primitives/CredentialPrimitive.tsx index e9bf61e24..9607094f8 100644 --- a/src/cli/primitives/CredentialPrimitive.tsx +++ b/src/cli/primitives/CredentialPrimitive.tsx @@ -342,24 +342,29 @@ export class CredentialPrimitive extends BasePrimitive { - clear(); - unmount(); - process.exit(0); - }, - }) - ); + try { + // TUI fallback — dynamic imports to avoid pulling ink (async) into registry + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'credential', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } catch (error) { + console.error(getErrorMessage(error)); + process.exit(1); + } } } ); diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index f8a7d3fa2..45459fec7 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -320,24 +320,29 @@ export class EvaluatorPrimitive extends BasePrimitive { - clear(); - unmount(); - process.exit(0); - }, - }) - ); + try { + // TUI fallback + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'evaluator', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } catch (error) { + console.error(getErrorMessage(error)); + process.exit(1); + } } } ); diff --git a/src/cli/primitives/GatewayTargetPrimitive.ts b/src/cli/primitives/GatewayTargetPrimitive.ts index 2323a37a0..95d237e65 100644 --- a/src/cli/primitives/GatewayTargetPrimitive.ts +++ b/src/cli/primitives/GatewayTargetPrimitive.ts @@ -330,10 +330,10 @@ export class GatewayTargetPrimitive extends BasePrimitive { - clear(); - unmount(); - process.exit(0); - }, - }) - ); + try { + // TUI fallback — dynamic imports to avoid pulling ink (async) into registry + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'memory', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } catch (error) { + console.error(getErrorMessage(error)); + process.exit(1); + } } } ); diff --git a/src/cli/primitives/OnlineEvalConfigPrimitive.ts b/src/cli/primitives/OnlineEvalConfigPrimitive.ts index 3af51ff84..5a85e950a 100644 --- a/src/cli/primitives/OnlineEvalConfigPrimitive.ts +++ b/src/cli/primitives/OnlineEvalConfigPrimitive.ts @@ -170,24 +170,29 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive { - clear(); - unmount(); - process.exit(0); - }, - }) - ); + try { + // TUI fallback + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'online-eval', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } catch (error) { + console.error(getErrorMessage(error)); + process.exit(1); + } } } ); diff --git a/src/cli/primitives/PolicyEnginePrimitive.ts b/src/cli/primitives/PolicyEnginePrimitive.ts index a8d870b7c..bb9b314d8 100644 --- a/src/cli/primitives/PolicyEnginePrimitive.ts +++ b/src/cli/primitives/PolicyEnginePrimitive.ts @@ -272,22 +272,27 @@ export class PolicyEnginePrimitive extends BasePrimitive { - clear(); - unmount(); - process.exit(0); - }, - }) - ); + try { + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } catch (error) { + console.error(getErrorMessage(error)); + process.exit(1); + } } } ); diff --git a/src/cli/primitives/PolicyPrimitive.ts b/src/cli/primitives/PolicyPrimitive.ts index e129306bf..beefe3008 100644 --- a/src/cli/primitives/PolicyPrimitive.ts +++ b/src/cli/primitives/PolicyPrimitive.ts @@ -348,23 +348,28 @@ export class PolicyPrimitive extends BasePrimitive { - clear(); - unmount(); - process.exit(0); - }, - }) - ); + try { + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'policy', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } catch (error) { + console.error(getErrorMessage(error)); + process.exit(1); + } } } ); diff --git a/src/cli/telemetry/cli-command-run.ts b/src/cli/telemetry/cli-command-run.ts index 1e3ec1959..4d3613e6e 100644 --- a/src/cli/telemetry/cli-command-run.ts +++ b/src/cli/telemetry/cli-command-run.ts @@ -5,6 +5,9 @@ import type { Command, CommandAttrs } from './schemas/command-run.js'; /** * Run a CLI command with telemetry, standardized error output, and process.exit. * The callback should throw on failure and return telemetry attrs on success. + * + * If telemetry initialization fails, the command still runs without telemetry — + * telemetry must never block CLI behavior. */ export async function cliCommandRun( command: C, @@ -12,7 +15,15 @@ export async function cliCommandRun( fn: () => Promise> ): Promise { try { - const client = await TelemetryClientAccessor.get(); + let client; + try { + client = await TelemetryClientAccessor.get(); + } catch { + // Telemetry init failed — run without it + await fn(); + process.exit(0); + } + // withCommandRun records success/failure telemetry, then re-throws on failure await client.withCommandRun(command, fn); process.exit(0); } catch (error) { diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index 3d52c28e3..80a3a06c7 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -65,6 +65,7 @@ export const GatewayTargetType = z.enum([ 'open-api-schema', 'smithy-model', 'lambda-function-arn', + 'unknown', ]); export const Language = z.enum(['python', 'typescript', 'other']); export const Level = z.enum(['session', 'trace', 'tool_call']); From f7003ca74dbab5cda583328256ee290266717e92 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Wed, 29 Apr 2026 22:54:07 +0000 Subject: [PATCH 07/18] fix: AgentPrimitive TUI try/catch, standardize uses safeParse --- src/cli/primitives/AgentPrimitive.tsx | 41 ++++++++++++---------- src/cli/telemetry/schemas/common-shapes.ts | 6 +++- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/cli/primitives/AgentPrimitive.tsx b/src/cli/primitives/AgentPrimitive.tsx index 7f94e1595..3bd04d6ee 100644 --- a/src/cli/primitives/AgentPrimitive.tsx +++ b/src/cli/primitives/AgentPrimitive.tsx @@ -354,24 +354,29 @@ export class AgentPrimitive extends BasePrimitive { - clear(); - unmount(); - process.exit(0); - }, - }) - ); + try { + // TUI fallback — dynamic imports to avoid pulling ink (async) into registry + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'agent', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } catch (error) { + console.error(getErrorMessage(error)); + process.exit(1); + } } }); diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index 80a3a06c7..6ee451a10 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -29,7 +29,11 @@ export function resilientParse( /** Lowercase a CLI value and parse it through a Zod enum, returning the narrowed type. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export function standardize>(schema: T, value: string): z.infer { - return schema.parse(value.toLowerCase()) as z.infer; + const lower = value.toLowerCase(); + const result = schema.safeParse(lower); + // If the value doesn't match the enum, return the lowercased value anyway — + // recordCommandRun's try/catch will silently drop the invalid metric. + return (result.success ? result.data : lower) as z.infer; } // Primitive types From 091264c3483f9743c76045bab5a23039821874cd Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 00:25:49 +0000 Subject: [PATCH 08/18] refactor: extract standalone assertTelemetry helper --- integ-tests/add-remove-resources.test.ts | 12 +++--------- integ-tests/help.test.ts | 6 ++---- src/test-utils/audit.ts | 7 +++++++ src/test-utils/index.ts | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index 844708741..7d340252e 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -1,4 +1,4 @@ -import { createAuditContext } from '../src/test-utils/audit.js'; +import { assertTelemetry, createAuditContext } from '../src/test-utils/audit.js'; import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; @@ -42,10 +42,7 @@ describe('integration: add and remove resources', () => { expect(found, `Memory "${memoryName}" should be in config`).toBe(true); // Verify telemetry - const entries = audit.readEntries(); - const memEntry = entries.find(e => e.attrs.command === 'add.memory'); - expect(memEntry).toBeDefined(); - expect(memEntry!.attrs).toMatchObject({ command: 'add.memory', exit_reason: 'success' }); + assertTelemetry(audit.readEntries(), { command: 'add.memory', exit_reason: 'success' }); }); it('adds a memory with EPISODIC strategy and verifies reflectionNamespaces', async () => { @@ -114,10 +111,7 @@ describe('integration: add and remove resources', () => { expect(found, `Credential "${credentialName}" should be in config`).toBe(true); // Verify telemetry - const entries = audit.readEntries(); - const credEntry = entries.find(e => e.attrs.command === 'add.credential'); - expect(credEntry).toBeDefined(); - expect(credEntry!.attrs).toMatchObject({ + assertTelemetry(audit.readEntries(), { command: 'add.credential', exit_reason: 'success', credential_type: 'api-key', diff --git a/integ-tests/help.test.ts b/integ-tests/help.test.ts index 93aa447d3..4ed6706fd 100644 --- a/integ-tests/help.test.ts +++ b/integ-tests/help.test.ts @@ -1,4 +1,4 @@ -import { createAuditContext } from '../src/test-utils/audit.js'; +import { assertTelemetry, createAuditContext } from '../src/test-utils/audit.js'; import { spawnAndCollect } from '../src/test-utils/cli-runner.js'; import { runCLI } from '../src/test-utils/index.js'; import { readdirSync } from 'node:fs'; @@ -63,9 +63,7 @@ describe('help modes telemetry', () => { const entries = audit.readEntries(); expect(entries).toHaveLength(1); - expect(entries[0]!.attrs).toMatchObject({ - 'service.name': 'agentcore-cli', - 'agentcore-cli.mode': 'cli', + assertTelemetry(entries, { command_group: 'help', command: 'help.modes', exit_reason: 'success', diff --git a/src/test-utils/audit.ts b/src/test-utils/audit.ts index 857a00e05..79a6d1c27 100644 --- a/src/test-utils/audit.ts +++ b/src/test-utils/audit.ts @@ -2,6 +2,7 @@ import { globSync } from 'glob'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { expect } from 'vitest'; export interface TelemetryEntry { value: number; @@ -42,3 +43,9 @@ export function createAuditContext(): AuditContext { }, }; } + +/** Assert that at least one telemetry entry was emitted matching the given attrs. */ +export function assertTelemetry(entries: TelemetryEntry[], expected: Record): void { + const match = entries.find(e => Object.entries(expected).every(([k, v]) => String(e.attrs[k]) === String(v))); + expect(match, `No telemetry entry matching ${JSON.stringify(expected)}`).toBeDefined(); +} diff --git a/src/test-utils/index.ts b/src/test-utils/index.ts index a9aad665d..a77a22d70 100644 --- a/src/test-utils/index.ts +++ b/src/test-utils/index.ts @@ -4,7 +4,7 @@ */ export { runCLI, spawnAndCollect, cleanSpawnEnv, type RunResult } from './cli-runner.js'; -export { createAuditContext, type AuditContext, type TelemetryEntry } from './audit.js'; +export { createAuditContext, assertTelemetry, type AuditContext, type TelemetryEntry } from './audit.js'; export { exists } from './fs-helpers.js'; export { hasCommand, hasAwsCredentials, prereqs } from './prereqs.js'; export { createTestProject, type TestProject, type CreateTestProjectOptions } from './project-factory.js'; From 8aecebfd103b79f5fc67000fd12276aff74dd359 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 00:30:17 +0000 Subject: [PATCH 09/18] refactor: rename audit.ts to telemetry-helper.ts, clarify method names --- integ-tests/add-remove-resources.test.ts | 14 +++++++------- integ-tests/help.test.ts | 16 ++++++++-------- src/test-utils/index.ts | 7 ++++++- .../{audit.ts => telemetry-helper.ts} | 18 +++++++++--------- 4 files changed, 30 insertions(+), 25 deletions(-) rename src/test-utils/{audit.ts => telemetry-helper.ts} (80%) diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index 7d340252e..7a43494ce 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -1,9 +1,9 @@ -import { assertTelemetry, createAuditContext } from '../src/test-utils/audit.js'; import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; +import { assertTelemetry, createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -const audit = createAuditContext(); +const telemetry = createTelemetryHelper(); describe('integration: add and remove resources', () => { let project: TestProject; @@ -19,7 +19,7 @@ describe('integration: add and remove resources', () => { afterAll(async () => { await project.cleanup(); - audit.cleanup(); + telemetry.destroy(); }); describe('memory lifecycle', () => { @@ -27,7 +27,7 @@ describe('integration: add and remove resources', () => { it('adds a memory resource', async () => { const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath, { - env: audit.env, + env: telemetry.env, }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -42,7 +42,7 @@ describe('integration: add and remove resources', () => { expect(found, `Memory "${memoryName}" should be in config`).toBe(true); // Verify telemetry - assertTelemetry(audit.readEntries(), { command: 'add.memory', exit_reason: 'success' }); + assertTelemetry(telemetry.readEntries(), { command: 'add.memory', exit_reason: 'success' }); }); it('adds a memory with EPISODIC strategy and verifies reflectionNamespaces', async () => { @@ -96,7 +96,7 @@ describe('integration: add and remove resources', () => { const result = await runCLI( ['add', 'credential', '--name', credentialName, '--api-key', 'test-key-integ-123', '--json'], project.projectPath, - { env: audit.env } + { env: telemetry.env } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -111,7 +111,7 @@ describe('integration: add and remove resources', () => { expect(found, `Credential "${credentialName}" should be in config`).toBe(true); // Verify telemetry - assertTelemetry(audit.readEntries(), { + assertTelemetry(telemetry.readEntries(), { command: 'add.credential', exit_reason: 'success', credential_type: 'api-key', diff --git a/integ-tests/help.test.ts b/integ-tests/help.test.ts index 4ed6706fd..fe13fdf17 100644 --- a/integ-tests/help.test.ts +++ b/integ-tests/help.test.ts @@ -1,6 +1,6 @@ -import { assertTelemetry, createAuditContext } from '../src/test-utils/audit.js'; import { spawnAndCollect } from '../src/test-utils/cli-runner.js'; import { runCLI } from '../src/test-utils/index.js'; +import { assertTelemetry, createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { readdirSync } from 'node:fs'; import { join } from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; @@ -44,15 +44,15 @@ describe('CLI help', () => { }); describe('help modes telemetry', () => { - const audit = createAuditContext(); + const telemetry = createTelemetryHelper(); const cliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); - afterAll(() => audit.cleanup()); + afterAll(() => telemetry.destroy()); function run(args: string[], extraEnv: Record = {}) { return spawnAndCollect('node', [cliPath, ...args], process.cwd(), { AGENTCORE_SKIP_INSTALL: '1', - ...audit.env, + ...telemetry.env, ...extraEnv, }); } @@ -61,7 +61,7 @@ describe('help modes telemetry', () => { const result = await run(['help', 'modes']); expect(result.exitCode).toBe(0); - const entries = audit.readEntries(); + const entries = telemetry.readEntries(); expect(entries).toHaveLength(1); assertTelemetry(entries, { command_group: 'help', @@ -74,16 +74,16 @@ describe('help modes telemetry', () => { }); it('does not write audit file when audit is not enabled', async () => { - audit.clear(); + telemetry.clearEntries(); const noAuditCliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); const result = await spawnAndCollect('node', [noAuditCliPath, 'help', 'modes'], process.cwd(), { AGENTCORE_SKIP_INSTALL: '1', - AGENTCORE_CONFIG_DIR: audit.dir, + AGENTCORE_CONFIG_DIR: telemetry.dir, }); expect(result.exitCode).toBe(0); - const telemetryDir = join(audit.dir, 'telemetry'); + const telemetryDir = join(telemetry.dir, 'telemetry'); try { const files = readdirSync(telemetryDir); expect(files).toHaveLength(0); diff --git a/src/test-utils/index.ts b/src/test-utils/index.ts index a77a22d70..8802ea803 100644 --- a/src/test-utils/index.ts +++ b/src/test-utils/index.ts @@ -4,7 +4,12 @@ */ export { runCLI, spawnAndCollect, cleanSpawnEnv, type RunResult } from './cli-runner.js'; -export { createAuditContext, assertTelemetry, type AuditContext, type TelemetryEntry } from './audit.js'; +export { + createTelemetryHelper, + assertTelemetry, + type TelemetryHelper, + type TelemetryEntry, +} from './telemetry-helper.js'; export { exists } from './fs-helpers.js'; export { hasCommand, hasAwsCredentials, prereqs } from './prereqs.js'; export { createTestProject, type TestProject, type CreateTestProjectOptions } from './project-factory.js'; diff --git a/src/test-utils/audit.ts b/src/test-utils/telemetry-helper.ts similarity index 80% rename from src/test-utils/audit.ts rename to src/test-utils/telemetry-helper.ts index 79a6d1c27..e49c74d9c 100644 --- a/src/test-utils/audit.ts +++ b/src/test-utils/telemetry-helper.ts @@ -9,20 +9,20 @@ export interface TelemetryEntry { attrs: Record; } -export interface AuditContext { +export interface TelemetryHelper { /** Temp directory used as AGENTCORE_CONFIG_DIR */ dir: string; - /** Env vars to pass to runCLI */ + /** Env vars to pass to runCLI to enable audit mode */ env: { AGENTCORE_TELEMETRY_AUDIT: '1'; AGENTCORE_CONFIG_DIR: string }; /** Read all JSONL entries from the audit telemetry directory */ readEntries: () => TelemetryEntry[]; - /** Delete the telemetry subdirectory */ - clear: () => void; - /** Delete the entire config directory */ - cleanup: () => void; + /** Delete telemetry entries only (keeps the config dir) */ + clearEntries: () => void; + /** Delete the entire config directory — call in afterAll */ + destroy: () => void; } -export function createAuditContext(): AuditContext { +export function createTelemetryHelper(): TelemetryHelper { const dir = mkdtempSync(join(tmpdir(), 'agentcore-audit-')); return { dir, @@ -35,10 +35,10 @@ export function createAuditContext(): AuditContext { .map(line => JSON.parse(line) as TelemetryEntry) ); }, - clear() { + clearEntries() { rmSync(join(dir, 'telemetry'), { recursive: true, force: true }); }, - cleanup() { + destroy() { rmSync(dir, { recursive: true, force: true }); }, }; From 8e4a22942eda50d9d4ad8ba0166f2b4e31f73014 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 00:38:06 +0000 Subject: [PATCH 10/18] refactor: move assertTelemetry into TelemetryHelper as assertMetricEmitted --- integ-tests/add-remove-resources.test.ts | 6 +++--- integ-tests/help.test.ts | 4 ++-- src/test-utils/index.ts | 7 +------ src/test-utils/telemetry-helper.ts | 16 +++++++++------- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index 7a43494ce..67ff28637 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -1,6 +1,6 @@ import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; -import { assertTelemetry, createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; const telemetry = createTelemetryHelper(); @@ -42,7 +42,7 @@ describe('integration: add and remove resources', () => { expect(found, `Memory "${memoryName}" should be in config`).toBe(true); // Verify telemetry - assertTelemetry(telemetry.readEntries(), { command: 'add.memory', exit_reason: 'success' }); + telemetry.assertMetricEmitted({ command: 'add.memory', exit_reason: 'success' }); }); it('adds a memory with EPISODIC strategy and verifies reflectionNamespaces', async () => { @@ -111,7 +111,7 @@ describe('integration: add and remove resources', () => { expect(found, `Credential "${credentialName}" should be in config`).toBe(true); // Verify telemetry - assertTelemetry(telemetry.readEntries(), { + telemetry.assertMetricEmitted({ command: 'add.credential', exit_reason: 'success', credential_type: 'api-key', diff --git a/integ-tests/help.test.ts b/integ-tests/help.test.ts index fe13fdf17..7e2176e2f 100644 --- a/integ-tests/help.test.ts +++ b/integ-tests/help.test.ts @@ -1,6 +1,6 @@ import { spawnAndCollect } from '../src/test-utils/cli-runner.js'; import { runCLI } from '../src/test-utils/index.js'; -import { assertTelemetry, createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { readdirSync } from 'node:fs'; import { join } from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; @@ -63,7 +63,7 @@ describe('help modes telemetry', () => { const entries = telemetry.readEntries(); expect(entries).toHaveLength(1); - assertTelemetry(entries, { + telemetry.assertMetricEmitted({ command_group: 'help', command: 'help.modes', exit_reason: 'success', diff --git a/src/test-utils/index.ts b/src/test-utils/index.ts index 8802ea803..1c032519c 100644 --- a/src/test-utils/index.ts +++ b/src/test-utils/index.ts @@ -4,12 +4,7 @@ */ export { runCLI, spawnAndCollect, cleanSpawnEnv, type RunResult } from './cli-runner.js'; -export { - createTelemetryHelper, - assertTelemetry, - type TelemetryHelper, - type TelemetryEntry, -} from './telemetry-helper.js'; +export { createTelemetryHelper, type TelemetryHelper, type TelemetryEntry } from './telemetry-helper.js'; export { exists } from './fs-helpers.js'; export { hasCommand, hasAwsCredentials, prereqs } from './prereqs.js'; export { createTestProject, type TestProject, type CreateTestProjectOptions } from './project-factory.js'; diff --git a/src/test-utils/telemetry-helper.ts b/src/test-utils/telemetry-helper.ts index e49c74d9c..9d7f70163 100644 --- a/src/test-utils/telemetry-helper.ts +++ b/src/test-utils/telemetry-helper.ts @@ -16,6 +16,8 @@ export interface TelemetryHelper { env: { AGENTCORE_TELEMETRY_AUDIT: '1'; AGENTCORE_CONFIG_DIR: string }; /** Read all JSONL entries from the audit telemetry directory */ readEntries: () => TelemetryEntry[]; + /** Assert a metric was emitted with attrs matching the given subset */ + assertMetricEmitted: (expected: Record) => void; /** Delete telemetry entries only (keeps the config dir) */ clearEntries: () => void; /** Delete the entire config directory — call in afterAll */ @@ -24,7 +26,7 @@ export interface TelemetryHelper { export function createTelemetryHelper(): TelemetryHelper { const dir = mkdtempSync(join(tmpdir(), 'agentcore-audit-')); - return { + const helper: TelemetryHelper = { dir, env: { AGENTCORE_TELEMETRY_AUDIT: '1', AGENTCORE_CONFIG_DIR: dir }, readEntries() { @@ -35,6 +37,11 @@ export function createTelemetryHelper(): TelemetryHelper { .map(line => JSON.parse(line) as TelemetryEntry) ); }, + assertMetricEmitted(expected) { + const entries = helper.readEntries(); + const match = entries.find(e => Object.entries(expected).every(([k, v]) => String(e.attrs[k]) === String(v))); + expect(match, `No telemetry entry matching ${JSON.stringify(expected)}`).toBeDefined(); + }, clearEntries() { rmSync(join(dir, 'telemetry'), { recursive: true, force: true }); }, @@ -42,10 +49,5 @@ export function createTelemetryHelper(): TelemetryHelper { rmSync(dir, { recursive: true, force: true }); }, }; -} - -/** Assert that at least one telemetry entry was emitted matching the given attrs. */ -export function assertTelemetry(entries: TelemetryEntry[], expected: Record): void { - const match = entries.find(e => Object.entries(expected).every(([k, v]) => String(e.attrs[k]) === String(v))); - expect(match, `No telemetry entry matching ${JSON.stringify(expected)}`).toBeDefined(); + return helper; } From 445a44832ab3c82d92d7c560c38d1e82963b934c Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 13:04:11 +0000 Subject: [PATCH 11/18] feat: add telemetry to TUI add paths via withAddTelemetry --- src/cli/telemetry/cli-command-run.ts | 30 +++++++++++++ src/cli/tui/hooks/useCreateEvaluator.ts | 20 ++++++--- src/cli/tui/hooks/useCreateMcp.ts | 45 ++++++++++++------- src/cli/tui/hooks/useCreateMemory.ts | 27 ++++++++--- src/cli/tui/hooks/useCreateOnlineEval.ts | 23 +++++++--- .../tui/screens/identity/useCreateIdentity.ts | 9 +++- src/cli/tui/screens/policy/AddPolicyFlow.tsx | 33 ++++++++++---- .../AddRuntimeEndpointFlow.tsx | 27 +++++------ 8 files changed, 157 insertions(+), 57 deletions(-) diff --git a/src/cli/telemetry/cli-command-run.ts b/src/cli/telemetry/cli-command-run.ts index 4d3613e6e..02223b76e 100644 --- a/src/cli/telemetry/cli-command-run.ts +++ b/src/cli/telemetry/cli-command-run.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '../errors'; +import type { AddResult } from '../primitives/types.js'; import { TelemetryClientAccessor } from './client-accessor.js'; import type { Command, CommandAttrs } from './schemas/command-run.js'; @@ -35,3 +36,32 @@ export async function cliCommandRun( process.exit(1); } } + +/** + * Wrap a primitive .add() call with telemetry — used by TUI paths. + * CLI paths use {@link cliCommandRun} instead. + */ +export async function withAddTelemetry>( + command: C, + attrs: CommandAttrs, + fn: () => Promise> +): Promise> { + let client; + try { + client = await TelemetryClientAccessor.get(); + } catch { + return fn(); + } + + let result: AddResult | undefined; + try { + await client.withCommandRun(command, async () => { + result = await fn(); + if (!result.success) throw new Error(result.error); + return attrs; + }); + } catch { + if (!result) return fn(); + } + return result!; +} diff --git a/src/cli/tui/hooks/useCreateEvaluator.ts b/src/cli/tui/hooks/useCreateEvaluator.ts index 6e1d8f052..f1cad666f 100644 --- a/src/cli/tui/hooks/useCreateEvaluator.ts +++ b/src/cli/tui/hooks/useCreateEvaluator.ts @@ -1,5 +1,7 @@ import type { EvaluatorConfig } from '../../../schema'; import { evaluatorPrimitive } from '../../primitives/registry'; +import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; +import { Level, standardize } from '../../telemetry/schemas/common-shapes.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateEvaluatorConfig { @@ -16,11 +18,19 @@ export function useCreateEvaluator() { const create = useCallback(async (config: CreateEvaluatorConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await evaluatorPrimitive.add({ - name: config.name, - level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL', - config: config.config, - }); + const addResult = await withAddTelemetry( + 'add.evaluator', + { + evaluator_type: config.config.codeBased ? 'code-based' : 'llm-as-a-judge', + level: standardize(Level, config.level), + }, + () => + evaluatorPrimitive.add({ + name: config.name, + level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL', + config: config.config, + }) + ); if (!addResult.success) { throw new Error(addResult.error ?? 'Failed to create evaluator'); } diff --git a/src/cli/tui/hooks/useCreateMcp.ts b/src/cli/tui/hooks/useCreateMcp.ts index 2b3b3b25a..ec91666d0 100644 --- a/src/cli/tui/hooks/useCreateMcp.ts +++ b/src/cli/tui/hooks/useCreateMcp.ts @@ -4,6 +4,8 @@ import { gatewayTargetPrimitive, policyEnginePrimitive, } from '../../primitives/registry'; +import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; +import { AuthorizerType, PolicyEngineMode, standardize } from '../../telemetry/schemas/common-shapes.js'; import type { AddGatewayConfig } from '../screens/mcp/types'; import { useCallback, useEffect, useState } from 'react'; @@ -23,22 +25,33 @@ export function useCreateGateway() { const createGateway = useCallback(async (config: AddGatewayConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await gatewayPrimitive.add({ - name: config.name, - description: config.description, - authorizerType: config.authorizerType, - discoveryUrl: config.jwtConfig?.discoveryUrl, - allowedAudience: config.jwtConfig?.allowedAudience?.join(','), - allowedClients: config.jwtConfig?.allowedClients?.join(','), - allowedScopes: config.jwtConfig?.allowedScopes?.join(','), - customClaims: config.jwtConfig?.customClaims, - clientId: config.jwtConfig?.clientId, - clientSecret: config.jwtConfig?.clientSecret, - enableSemanticSearch: config.enableSemanticSearch, - exceptionLevel: config.exceptionLevel, - policyEngine: config.policyEngineConfiguration?.policyEngineName, - policyEngineMode: config.policyEngineConfiguration?.mode, - }); + const addResult = await withAddTelemetry( + 'add.gateway', + { + authorizer_type: standardize(AuthorizerType, config.authorizerType ?? 'NONE'), + has_policy_engine: !!config.policyEngineConfiguration?.policyEngineName, + policy_engine_mode: standardize(PolicyEngineMode, config.policyEngineConfiguration?.mode ?? 'log_only'), + semantic_search: config.enableSemanticSearch !== false, + runtime_count: 0, + }, + () => + gatewayPrimitive.add({ + name: config.name, + description: config.description, + authorizerType: config.authorizerType, + discoveryUrl: config.jwtConfig?.discoveryUrl, + allowedAudience: config.jwtConfig?.allowedAudience?.join(','), + allowedClients: config.jwtConfig?.allowedClients?.join(','), + allowedScopes: config.jwtConfig?.allowedScopes?.join(','), + customClaims: config.jwtConfig?.customClaims, + clientId: config.jwtConfig?.clientId, + clientSecret: config.jwtConfig?.clientSecret, + enableSemanticSearch: config.enableSemanticSearch, + exceptionLevel: config.exceptionLevel, + policyEngine: config.policyEngineConfiguration?.policyEngineName, + policyEngineMode: config.policyEngineConfiguration?.mode, + }) + ); if (!addResult.success) { throw new Error(addResult.error ?? 'Failed to create gateway'); } diff --git a/src/cli/tui/hooks/useCreateMemory.ts b/src/cli/tui/hooks/useCreateMemory.ts index 4345b4ead..d4196582f 100644 --- a/src/cli/tui/hooks/useCreateMemory.ts +++ b/src/cli/tui/hooks/useCreateMemory.ts @@ -2,6 +2,7 @@ import { ConfigIO } from '../../../lib'; import type { Memory } from '../../../schema'; import { getAvailableAgents } from '../../operations/attach'; import { memoryPrimitive } from '../../primitives/registry'; +import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateMemoryConfig { @@ -24,13 +25,25 @@ export function useCreateMemory() { setStatus({ state: 'loading' }); try { const strategiesStr = config.strategies.map(s => s.type).join(','); - const addResult = await memoryPrimitive.add({ - name: config.name, - expiry: config.eventExpiryDuration, - strategies: strategiesStr || undefined, - dataStreamArn: config.streaming?.dataStreamArn, - contentLevel: config.streaming?.contentLevel, - }); + const strategyList = strategiesStr ? strategiesStr.split(',').map(s => s.trim().toUpperCase()) : []; + const addResult = await withAddTelemetry( + 'add.memory', + { + strategy_count: strategyList.length, + strategy_semantic: strategyList.includes('SEMANTIC'), + strategy_summarization: strategyList.includes('SUMMARIZATION'), + strategy_user_preference: strategyList.includes('USER_PREFERENCE'), + strategy_episodic: strategyList.includes('EPISODIC'), + }, + () => + memoryPrimitive.add({ + name: config.name, + expiry: config.eventExpiryDuration, + strategies: strategiesStr || undefined, + dataStreamArn: config.streaming?.dataStreamArn, + contentLevel: config.streaming?.contentLevel, + }) + ); if (!addResult.success) { throw new Error(addResult.error ?? 'Failed to create memory'); } diff --git a/src/cli/tui/hooks/useCreateOnlineEval.ts b/src/cli/tui/hooks/useCreateOnlineEval.ts index 2d0190552..840b8ae5c 100644 --- a/src/cli/tui/hooks/useCreateOnlineEval.ts +++ b/src/cli/tui/hooks/useCreateOnlineEval.ts @@ -1,4 +1,5 @@ import { onlineEvalConfigPrimitive } from '../../primitives/registry'; +import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateOnlineEvalConfig { @@ -17,13 +18,21 @@ export function useCreateOnlineEval() { const create = useCallback(async (config: CreateOnlineEvalConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await onlineEvalConfigPrimitive.add({ - name: config.name, - agent: config.agent, - evaluators: config.evaluators, - samplingRate: config.samplingRate, - enableOnCreate: config.enableOnCreate, - }); + const addResult = await withAddTelemetry( + 'add.online-eval', + { + evaluator_count: config.evaluators.length, + enable_on_create: config.enableOnCreate ?? false, + }, + () => + onlineEvalConfigPrimitive.add({ + name: config.name, + agent: config.agent, + evaluators: config.evaluators, + samplingRate: config.samplingRate, + enableOnCreate: config.enableOnCreate, + }) + ); if (!addResult.success) { throw new Error(addResult.error ?? 'Failed to create online eval config'); } diff --git a/src/cli/tui/screens/identity/useCreateIdentity.ts b/src/cli/tui/screens/identity/useCreateIdentity.ts index 42aace21b..e214d1e67 100644 --- a/src/cli/tui/screens/identity/useCreateIdentity.ts +++ b/src/cli/tui/screens/identity/useCreateIdentity.ts @@ -2,6 +2,7 @@ import { ConfigIO } from '../../../../lib'; import type { Credential } from '../../../../schema'; import type { AddCredentialOptions } from '../../../primitives/CredentialPrimitive'; import { credentialPrimitive } from '../../../primitives/registry'; +import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateStatus { @@ -16,7 +17,13 @@ export function useCreateIdentity() { const create = useCallback(async (config: AddCredentialOptions) => { setStatus({ state: 'loading' }); try { - const result = await credentialPrimitive.add(config); + const result = await withAddTelemetry( + 'add.credential', + { + credential_type: config.authorizerType === 'OAuthCredentialProvider' ? 'oauth' : 'api-key', + }, + () => credentialPrimitive.add(config) + ); if (!result.success) { throw new Error(result.error ?? 'Failed to create credential'); } diff --git a/src/cli/tui/screens/policy/AddPolicyFlow.tsx b/src/cli/tui/screens/policy/AddPolicyFlow.tsx index 720984563..aa9110b08 100644 --- a/src/cli/tui/screens/policy/AddPolicyFlow.tsx +++ b/src/cli/tui/screens/policy/AddPolicyFlow.tsx @@ -1,4 +1,6 @@ import { policyEnginePrimitive, policyPrimitive } from '../../../primitives/registry'; +import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; +import { ValidationMode, standardize } from '../../../telemetry/schemas/common-shapes.js'; import { ErrorPrompt, Panel, @@ -128,7 +130,14 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD }, []); const commitEngine = useCallback(async (engineName: string, gateways?: string[], mode?: 'LOG_ONLY' | 'ENFORCE') => { - const result = await policyEnginePrimitive.add({ name: engineName }); + const result = await withAddTelemetry( + 'add.policy-engine', + { + attach_gateway_count: 0, + attach_mode: 'log_only', + }, + () => policyEnginePrimitive.add({ name: engineName }) + ); if (!result.success) { setFlow({ name: 'error', message: result.error }); return; @@ -155,13 +164,21 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD ); const handlePolicyComplete = useCallback(async (config: AddPolicyConfig) => { - const result = await policyPrimitive.add({ - name: config.name, - engine: config.engine, - statement: config.statement, - source: config.sourceFile || undefined, - validationMode: config.validationMode, - }); + const result = await withAddTelemetry( + 'add.policy', + { + source_type: config.sourceFile ? 'file' : 'statement', + validation_mode: standardize(ValidationMode, config.validationMode ?? 'FAIL_ON_ANY_FINDINGS'), + }, + () => + policyPrimitive.add({ + name: config.name, + engine: config.engine, + statement: config.statement, + source: config.sourceFile || undefined, + validationMode: config.validationMode, + }) + ); if (result.success) { setPolicyNames(prev => [...prev, config.name]); diff --git a/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx b/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx index 2404109fc..83bda78e5 100644 --- a/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx +++ b/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx @@ -1,5 +1,6 @@ import { ConfigIO } from '../../../../lib'; import { runtimeEndpointPrimitive } from '../../../primitives/registry'; +import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; import { ErrorPrompt } from '../../components'; import { AddSuccessScreen } from '../add/AddSuccessScreen'; import { AddRuntimeEndpointScreen } from './AddRuntimeEndpointScreen'; @@ -78,24 +79,24 @@ export function AddRuntimeEndpointFlow({ }, [isInteractive, flow.name, onExit]); const handleCreateComplete = useCallback((config: RuntimeEndpointWizardConfig) => { - void runtimeEndpointPrimitive - .add({ + void withAddTelemetry('add.runtime-endpoint', {}, () => + runtimeEndpointPrimitive.add({ runtime: config.runtimeName, endpoint: config.endpointName, version: config.version, description: config.description, }) - .then(result => { - if (result.success) { - setFlow({ - name: 'create-success', - endpointName: config.endpointName, - runtimeName: config.runtimeName, - }); - return; - } - setFlow({ name: 'error', message: result.error ?? 'Unknown error' }); - }); + ).then(result => { + if (result.success) { + setFlow({ + name: 'create-success', + endpointName: config.endpointName, + runtimeName: config.runtimeName, + }); + return; + } + setFlow({ name: 'error', message: result.error ?? 'Unknown error' }); + }); }, []); if (flow.name === 'loading') { From c0af1ea5f08216897761daa446c9597ccad64e72 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 13:50:50 +0000 Subject: [PATCH 12/18] =?UTF-8?q?fix:=20review=20feedback=20=E2=80=94=20wi?= =?UTF-8?q?thAddTelemetry=20safety,=20standardize=20handles=20undefined,?= =?UTF-8?q?=20MCP=20agent=20attrs,=20policy=20TUI=20attrs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cli/primitives/AgentPrimitive.tsx | 6 +++--- src/cli/telemetry/cli-command-run.ts | 5 +++-- src/cli/telemetry/schemas/common-shapes.ts | 4 ++-- src/cli/tui/screens/policy/AddPolicyFlow.tsx | 8 ++++---- src/test-utils/telemetry-helper.ts | 5 ++++- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/cli/primitives/AgentPrimitive.tsx b/src/cli/primitives/AgentPrimitive.tsx index 3bd04d6ee..8f354a43c 100644 --- a/src/cli/primitives/AgentPrimitive.tsx +++ b/src/cli/primitives/AgentPrimitive.tsx @@ -342,9 +342,9 @@ export class AgentPrimitive extends BasePrimitive); } diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index 6ee451a10..f3dac5cfc 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -28,8 +28,8 @@ export function resilientParse( /** Lowercase a CLI value and parse it through a Zod enum, returning the narrowed type. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any -export function standardize>(schema: T, value: string): z.infer { - const lower = value.toLowerCase(); +export function standardize>(schema: T, value: string | undefined): z.infer { + const lower = (value ?? '').toLowerCase(); const result = schema.safeParse(lower); // If the value doesn't match the enum, return the lowercased value anyway — // recordCommandRun's try/catch will silently drop the invalid metric. diff --git a/src/cli/tui/screens/policy/AddPolicyFlow.tsx b/src/cli/tui/screens/policy/AddPolicyFlow.tsx index aa9110b08..9b3542cb8 100644 --- a/src/cli/tui/screens/policy/AddPolicyFlow.tsx +++ b/src/cli/tui/screens/policy/AddPolicyFlow.tsx @@ -1,6 +1,6 @@ import { policyEnginePrimitive, policyPrimitive } from '../../../primitives/registry'; import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; -import { ValidationMode, standardize } from '../../../telemetry/schemas/common-shapes.js'; +import { AttachMode, ValidationMode, standardize } from '../../../telemetry/schemas/common-shapes.js'; import { ErrorPrompt, Panel, @@ -133,8 +133,8 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD const result = await withAddTelemetry( 'add.policy-engine', { - attach_gateway_count: 0, - attach_mode: 'log_only', + attach_gateway_count: gateways?.length ?? 0, + attach_mode: standardize(AttachMode, mode ?? 'log_only'), }, () => policyEnginePrimitive.add({ name: engineName }) ); @@ -167,7 +167,7 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD const result = await withAddTelemetry( 'add.policy', { - source_type: config.sourceFile ? 'file' : 'statement', + source_type: config.sourceFile ? 'file' : config.sourceMethod === 'generate' ? 'generate' : 'statement', validation_mode: standardize(ValidationMode, config.validationMode ?? 'FAIL_ON_ANY_FINDINGS'), }, () => diff --git a/src/test-utils/telemetry-helper.ts b/src/test-utils/telemetry-helper.ts index 9d7f70163..e7fa58949 100644 --- a/src/test-utils/telemetry-helper.ts +++ b/src/test-utils/telemetry-helper.ts @@ -40,7 +40,10 @@ export function createTelemetryHelper(): TelemetryHelper { assertMetricEmitted(expected) { const entries = helper.readEntries(); const match = entries.find(e => Object.entries(expected).every(([k, v]) => String(e.attrs[k]) === String(v))); - expect(match, `No telemetry entry matching ${JSON.stringify(expected)}`).toBeDefined(); + expect( + match, + `No telemetry entry matching ${JSON.stringify(expected)}\nFound ${entries.length} entries:\n${entries.map(e => JSON.stringify(e.attrs)).join('\n')}` + ).toBeDefined(); }, clearEntries() { rmSync(join(dir, 'telemetry'), { recursive: true, force: true }); From db7300bfe12b4b0e47dcbd72663b07c81ed681f8 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 15:48:50 +0000 Subject: [PATCH 13/18] fix: remove unnecessary type assertion --- src/cli/primitives/EvaluatorPrimitive.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 45459fec7..ec7372b08 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -316,7 +316,7 @@ export class EvaluatorPrimitive extends BasePrimitive Date: Thu, 30 Apr 2026 16:01:19 +0000 Subject: [PATCH 14/18] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20doc?= =?UTF-8?q?ument=20standardize=20cast,=20add=20policy-engine=20+=20episodi?= =?UTF-8?q?c=20telemetry=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- integ-tests/add-remove-resources.test.ts | 37 +++++++++++++++++++++- src/cli/telemetry/schemas/common-shapes.ts | 8 ++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index 67ff28637..a89c761dd 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -49,7 +49,8 @@ describe('integration: add and remove resources', () => { const episodicMemName = `EpiMem${Date.now().toString().slice(-6)}`; const result = await runCLI( ['add', 'memory', '--name', episodicMemName, '--strategies', 'EPISODIC', '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -70,6 +71,14 @@ describe('integration: add and remove resources', () => { expect(episodic!.reflectionNamespaces, 'Should have reflectionNamespaces').toBeDefined(); expect(episodic!.reflectionNamespaces!.length).toBeGreaterThan(0); + // Verify telemetry + telemetry.assertMetricEmitted({ + command: 'add.memory', + exit_reason: 'success', + strategy_count: '1', + strategy_episodic: 'true', + }); + // Clean up await runCLI(['remove', 'memory', '--name', episodicMemName, '--json'], project.projectPath); }); @@ -132,4 +141,30 @@ describe('integration: add and remove resources', () => { expect(found, `Credential "${credentialName}" should be removed from config`).toBe(false); }); }); + + describe('policy-engine', () => { + const engineName = `TestEngine${Date.now().toString().slice(-6)}`; + + it('adds a policy engine resource', async () => { + const result = await runCLI(['add', 'policy-engine', '--name', engineName, '--json'], project.projectPath, { + env: telemetry.env, + }); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + telemetry.assertMetricEmitted({ + command: 'add.policy-engine', + exit_reason: 'success', + attach_gateway_count: '0', + }); + }); + + it('removes the policy engine resource', async () => { + const result = await runCLI(['remove', 'policy-engine', '--name', engineName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + }); + }); }); diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index f3dac5cfc..73568f907 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -26,7 +26,13 @@ export function resilientParse( return result; } -/** Lowercase a CLI value and parse it through a Zod enum, returning the narrowed type. */ +/** + * Lowercase a CLI value and parse it through a Zod enum, returning the narrowed type. + * The `as` cast on the failure branch is intentional: invalid values pass through to + * recordCommandRun, where COMMAND_SCHEMAS[command].parse(attrs) validates the full + * attr object in a try/catch — silently dropping the metric if any field is invalid. + * This ensures telemetry never crashes the CLI while keeping the happy-path type-safe. + */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export function standardize>(schema: T, value: string | undefined): z.infer { const lower = (value ?? '').toLowerCase(); From 9087af05824c4f5e34390a02c5fcd8b2574f52a1 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 16:10:12 +0000 Subject: [PATCH 15/18] refactor: centralize gateway target type mapping in common-shapes --- src/cli/primitives/GatewayTargetPrimitive.ts | 20 +++++++------------- src/cli/telemetry/schemas/common-shapes.ts | 9 +++++++++ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/cli/primitives/GatewayTargetPrimitive.ts b/src/cli/primitives/GatewayTargetPrimitive.ts index 95d237e65..56812f6e8 100644 --- a/src/cli/primitives/GatewayTargetPrimitive.ts +++ b/src/cli/primitives/GatewayTargetPrimitive.ts @@ -15,7 +15,12 @@ import { getErrorMessage } from '../errors'; import type { RemovableGatewayTarget } from '../operations/remove/remove-gateway-target'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; import { cliCommandRun } from '../telemetry/cli-command-run.js'; -import { GatewayTargetHost, OutboundAuth, standardize } from '../telemetry/schemas/common-shapes.js'; +import { + GATEWAY_TARGET_TYPE_MAP, + GatewayTargetHost, + OutboundAuth, + standardize, +} from '../telemetry/schemas/common-shapes.js'; import { getTemplateToolDefinitions, renderGatewayTargetTemplate } from '../templates/GatewayTargetRenderer'; import { requireTTY } from '../tui/guards/tty'; import type { @@ -318,19 +323,8 @@ export class GatewayTargetPrimitive extends BasePrimitive> = { + apiGateway: 'api-gateway', + openApiSchema: 'open-api-schema', + smithyModel: 'smithy-model', + lambdaFunctionArn: 'lambda-function-arn', + mcpServer: 'mcp-server', +}; export const Language = z.enum(['python', 'typescript', 'other']); export const Level = z.enum(['session', 'trace', 'tool_call']); export const Memory = z.enum(['none', 'shortterm', 'longandshortterm']); From 35a0531c38cad79e8f80c8767cebe15d63ad03db Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 17:05:07 +0000 Subject: [PATCH 16/18] fix: preserve original function error with telemetry wrapper --- src/cli/telemetry/cli-command-run.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cli/telemetry/cli-command-run.ts b/src/cli/telemetry/cli-command-run.ts index e5fe0a84c..987f05730 100644 --- a/src/cli/telemetry/cli-command-run.ts +++ b/src/cli/telemetry/cli-command-run.ts @@ -60,9 +60,12 @@ export async function withAddTelemetry); + return result!; } From 0d09e63c4b7e0ee66b9e12a82c3238c96c03ea50 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 17:12:00 +0000 Subject: [PATCH 17/18] refactor: extract telemetryAttrs into a single line --- src/cli/primitives/GatewayTargetPrimitive.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/cli/primitives/GatewayTargetPrimitive.ts b/src/cli/primitives/GatewayTargetPrimitive.ts index 56812f6e8..418455b33 100644 --- a/src/cli/primitives/GatewayTargetPrimitive.ts +++ b/src/cli/primitives/GatewayTargetPrimitive.ts @@ -330,6 +330,11 @@ export class GatewayTargetPrimitive extends BasePrimitive Date: Thu, 30 Apr 2026 17:23:26 +0000 Subject: [PATCH 18/18] feat: wire up telemetry for addAgent --- src/cli/primitives/GatewayTargetPrimitive.ts | 2 +- src/cli/tui/screens/agent/useAddAgent.ts | 96 ++++++++++++++------ 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/src/cli/primitives/GatewayTargetPrimitive.ts b/src/cli/primitives/GatewayTargetPrimitive.ts index 418455b33..e8a1da996 100644 --- a/src/cli/primitives/GatewayTargetPrimitive.ts +++ b/src/cli/primitives/GatewayTargetPrimitive.ts @@ -427,7 +427,7 @@ export class GatewayTargetPrimitive extends BasePrimitive => { setIsLoading(true); try { - const configBaseDir = findConfigRoot(); - if (!configBaseDir) { - return { ok: false, error: new NoProjectError().message }; - } - - const configIO = new ConfigIO({ baseDir: configBaseDir }); - - if (!configIO.configExists('project')) { - return { ok: false, error: new NoProjectError().message }; - } - - // Check for duplicate agent name - const project = await configIO.readProjectSpec(); - const existingAgent = project.runtimes.find(agent => agent.name === config.name); - if (existingAgent) { - return { ok: false, error: `Agent "${config.name}" already exists in this project.` }; - } - - // Branch based on agent type - if (config.agentType === 'import') { - return await handleImportPath(config, configBaseDir); - } else if (config.agentType === 'create') { - return await handleCreatePath(config, configBaseDir); - } else { - return await handleByoPath(config, configIO, configBaseDir); + const result = await withAddTelemetry( + 'add.agent', + { + language: standardize(Language, config.language), + framework: standardize(Framework, config.framework), + model_provider: standardize(ModelProvider, config.modelProvider), + agent_type: standardize(AgentTypeEnum, config.agentType), + build: standardize(Build, config.buildType), + protocol: standardize(Protocol, config.protocol ?? 'HTTP'), + network_mode: standardize(NetworkMode, config.networkMode ?? 'PUBLIC'), + authorizer_type: standardize(AuthorizerTypeEnum, config.authorizerType ?? 'NONE'), + memory: standardize(MemoryEnum, config.memory ?? 'none'), + }, + () => addAgentInner(config) + ); + if (!result.success) { + return { ok: false, error: result.error }; } - } catch (err) { - return { ok: false, error: getErrorMessage(err) }; + return result.outcome; } finally { setIsLoading(false); } @@ -175,6 +178,43 @@ export function useAddAgent() { return { addAgent, isLoading, reset }; } +type AddAgentInnerResult = + | { success: true; outcome: AddAgentCreateResult | AddAgentByoResult } + | { success: false; error: string }; + +async function addAgentInner(config: AddAgentConfig): Promise { + const configBaseDir = findConfigRoot(); + if (!configBaseDir) { + return { success: false, error: new NoProjectError().message }; + } + + const configIO = new ConfigIO({ baseDir: configBaseDir }); + + if (!configIO.configExists('project')) { + return { success: false, error: new NoProjectError().message }; + } + + const project = await configIO.readProjectSpec(); + const existingAgent = project.runtimes.find(agent => agent.name === config.name); + if (existingAgent) { + return { success: false, error: `Agent "${config.name}" already exists in this project.` }; + } + + let outcome: AddAgentCreateResult | AddAgentByoResult | AddAgentError; + if (config.agentType === 'import') { + outcome = await handleImportPath(config, configBaseDir); + } else if (config.agentType === 'create') { + outcome = await handleCreatePath(config, configBaseDir); + } else { + outcome = await handleByoPath(config, configIO, configBaseDir); + } + + if (!outcome.ok) { + return { success: false, error: outcome.error }; + } + return { success: true, outcome }; +} + /** * Handle the "create" path: generate agent from template and write to project. */