diff --git a/e2e-tests/byo-custom-jwt.test.ts b/e2e-tests/byo-custom-jwt.test.ts index b7391a522..64e534e20 100644 --- a/e2e-tests/byo-custom-jwt.test.ts +++ b/e2e-tests/byo-custom-jwt.test.ts @@ -48,7 +48,7 @@ const region = process.env.AWS_REGION ?? 'us-east-1'; * Run the local CLI build without skipping install (needed for deploy). */ function runLocalCLI(args: string[], cwd: string): Promise { - return runCLI(args, cwd, /* skipInstall */ false); + return runCLI(args, cwd, { skipInstall: false }); } describe.sequential('e2e: BYO agent with CUSTOM_JWT auth', () => { diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index 57dd48483..a89c761dd 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -1,7 +1,10 @@ import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +const telemetry = createTelemetryHelper(); + describe('integration: add and remove resources', () => { let project: TestProject; @@ -16,13 +19,16 @@ describe('integration: add and remove resources', () => { afterAll(async () => { await project.cleanup(); + telemetry.destroy(); }); describe('memory lifecycle', () => { const memoryName = `IntegMem${Date.now().toString().slice(-6)}`; it('adds a memory resource', async () => { - const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath); + const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath, { + env: telemetry.env, + }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); const json = JSON.parse(result.stdout); @@ -34,13 +40,17 @@ describe('integration: add and remove resources', () => { expect(memories, 'memories should exist').toBeDefined(); const found = memories!.some((m: Record) => m.name === memoryName); expect(found, `Memory "${memoryName}" should be in config`).toBe(true); + + // Verify telemetry + telemetry.assertMetricEmitted({ command: 'add.memory', exit_reason: 'success' }); }); it('adds a memory with EPISODIC strategy and verifies reflectionNamespaces', async () => { const episodicMemName = `EpiMem${Date.now().toString().slice(-6)}`; const result = await runCLI( ['add', 'memory', '--name', episodicMemName, '--strategies', 'EPISODIC', '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -61,6 +71,14 @@ describe('integration: add and remove resources', () => { expect(episodic!.reflectionNamespaces, 'Should have reflectionNamespaces').toBeDefined(); expect(episodic!.reflectionNamespaces!.length).toBeGreaterThan(0); + // Verify telemetry + telemetry.assertMetricEmitted({ + command: 'add.memory', + exit_reason: 'success', + strategy_count: '1', + strategy_episodic: 'true', + }); + // Clean up await runCLI(['remove', 'memory', '--name', episodicMemName, '--json'], project.projectPath); }); @@ -86,7 +104,8 @@ describe('integration: add and remove resources', () => { it('adds a credential resource', async () => { const result = await runCLI( ['add', 'credential', '--name', credentialName, '--api-key', 'test-key-integ-123', '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -99,6 +118,13 @@ describe('integration: add and remove resources', () => { expect(credentials, 'credentials should exist').toBeDefined(); const found = credentials!.some((c: Record) => c.name === credentialName); expect(found, `Credential "${credentialName}" should be in config`).toBe(true); + + // Verify telemetry + telemetry.assertMetricEmitted({ + command: 'add.credential', + exit_reason: 'success', + credential_type: 'api-key', + }); }); it('removes the credential resource', async () => { @@ -115,4 +141,30 @@ describe('integration: add and remove resources', () => { expect(found, `Credential "${credentialName}" should be removed from config`).toBe(false); }); }); + + describe('policy-engine', () => { + const engineName = `TestEngine${Date.now().toString().slice(-6)}`; + + it('adds a policy engine resource', async () => { + const result = await runCLI(['add', 'policy-engine', '--name', engineName, '--json'], project.projectPath, { + env: telemetry.env, + }); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + telemetry.assertMetricEmitted({ + command: 'add.policy-engine', + exit_reason: 'success', + attach_gateway_count: '0', + }); + }); + + it('removes the policy engine resource', async () => { + const result = await runCLI(['remove', 'policy-engine', '--name', engineName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + }); + }); }); diff --git a/integ-tests/create-no-agent.test.ts b/integ-tests/create-no-agent.test.ts index 4bcca2690..bcdf80eaa 100644 --- a/integ-tests/create-no-agent.test.ts +++ b/integ-tests/create-no-agent.test.ts @@ -32,7 +32,7 @@ describe('integration: create without agent', () => { it.skipIf(!hasNpm || !hasGit)('creates project with real npm install and git init', async () => { const name = `NoAgent${Date.now().toString().slice(-6)}`; - const result = await runCLI(['create', '--name', name, '--no-agent', '--json'], testDir, false); + const result = await runCLI(['create', '--name', name, '--no-agent', '--json'], testDir, { skipInstall: false }); expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0); diff --git a/integ-tests/create-with-agent.test.ts b/integ-tests/create-with-agent.test.ts index 7fb20bdbf..69f0594b8 100644 --- a/integ-tests/create-with-agent.test.ts +++ b/integ-tests/create-with-agent.test.ts @@ -49,7 +49,7 @@ describe('integration: create with Python agent', () => { '--json', ], testDir, - false + { skipInstall: false } ); expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0); diff --git a/integ-tests/dev-server.test.ts b/integ-tests/dev-server.test.ts index 5f60976e7..4b07b7284 100644 --- a/integ-tests/dev-server.test.ts +++ b/integ-tests/dev-server.test.ts @@ -60,7 +60,7 @@ describe('integration: dev server', () => { '--json', ], testDir, - false + { skipInstall: false } ); if (result.exitCode === 0) { diff --git a/integ-tests/help.test.ts b/integ-tests/help.test.ts index 052605c7a..7e2176e2f 100644 --- a/integ-tests/help.test.ts +++ b/integ-tests/help.test.ts @@ -1,10 +1,9 @@ import { spawnAndCollect } from '../src/test-utils/cli-runner.js'; import { runCLI } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { readdirSync } from 'node:fs'; -import { mkdir, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, describe, expect, it } from 'vitest'; const COMMANDS = [ 'create', @@ -45,52 +44,46 @@ describe('CLI help', () => { }); describe('help modes telemetry', () => { - let testConfigDir: string; + const telemetry = createTelemetryHelper(); const cliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); - beforeAll(async () => { - testConfigDir = join(tmpdir(), `agentcore-help-telemetry-${Date.now()}`); - await mkdir(testConfigDir, { recursive: true }); - }); - afterAll(() => rm(testConfigDir, { recursive: true, force: true })); + afterAll(() => telemetry.destroy()); function run(args: string[], extraEnv: Record = {}) { - return spawnAndCollect('node', [cliPath, ...args], tmpdir(), { + return spawnAndCollect('node', [cliPath, ...args], process.cwd(), { AGENTCORE_SKIP_INSTALL: '1', - AGENTCORE_CONFIG_DIR: testConfigDir, + ...telemetry.env, ...extraEnv, }); } it('writes JSONL audit file when audit is enabled via env var', async () => { - const result = await run(['help', 'modes'], { AGENTCORE_TELEMETRY_AUDIT: '1' }); + const result = await run(['help', 'modes']); expect(result.exitCode).toBe(0); - const telemetryDir = join(testConfigDir, 'telemetry'); - const files = readdirSync(telemetryDir).filter(f => f.startsWith('help-')); - expect(files).toHaveLength(1); - - const content = await readFile(join(telemetryDir, files[0]!), 'utf-8'); - const entry = JSON.parse(content.trim()); - expect(entry.attrs).toMatchObject({ - 'service.name': 'agentcore-cli', - 'agentcore-cli.mode': 'cli', + const entries = telemetry.readEntries(); + expect(entries).toHaveLength(1); + telemetry.assertMetricEmitted({ command_group: 'help', command: 'help.modes', exit_reason: 'success', }); - expect(entry.attrs['agentcore-cli.session_id']).toBeDefined(); - expect(entry.attrs['os.type']).toBeDefined(); - expect(entry.value).toBeGreaterThanOrEqual(0); + expect(entries[0]!.attrs['agentcore-cli.session_id']).toBeDefined(); + expect(entries[0]!.attrs['os.type']).toBeDefined(); + expect(entries[0]!.value).toBeGreaterThanOrEqual(0); }); it('does not write audit file when audit is not enabled', async () => { - const telemetryDir = join(testConfigDir, 'telemetry'); - await rm(telemetryDir, { recursive: true, force: true }); + telemetry.clearEntries(); - const result = await run(['help', 'modes']); + const noAuditCliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); + const result = await spawnAndCollect('node', [noAuditCliPath, 'help', 'modes'], process.cwd(), { + AGENTCORE_SKIP_INSTALL: '1', + AGENTCORE_CONFIG_DIR: telemetry.dir, + }); expect(result.exitCode).toBe(0); + const telemetryDir = join(telemetry.dir, 'telemetry'); try { const files = readdirSync(telemetryDir); expect(files).toHaveLength(0); diff --git a/src/cli/primitives/AgentPrimitive.tsx b/src/cli/primitives/AgentPrimitive.tsx index 4702633ed..8f354a43c 100644 --- a/src/cli/primitives/AgentPrimitive.tsx +++ b/src/cli/primitives/AgentPrimitive.tsx @@ -34,6 +34,19 @@ import { import { executeImportAgent } from '../operations/agent/import'; import { setupPythonProject } from '../operations/python'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { + AgentType, + AuthorizerType, + Build, + Framework, + Language, + Memory, + ModelProvider as ModelProviderEnum, + NetworkMode as NetworkModeEnum, + Protocol, + standardize, +} from '../telemetry/schemas/common-shapes.js'; import { createRenderer } from '../templates'; import { requireTTY } from '../tui/guards/tty'; import type { GenerateConfig, MemoryOption } from '../tui/screens/generate/types'; @@ -264,92 +277,106 @@ export class AgentPrimitive extends BasePrimitive { + const validation = validateAddAgentOptions(cliOptions); + if (!validation.valid) { + throw new Error(validation.error); } - process.exit(1); - } - - // Parse custom claims JSON if provided (already validated by validateAddAgentOptions) - const customClaims = cliOptions.customClaims - ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) - : undefined; - - // Parse request header allowlist if provided - const requestHeaderAllowlist = cliOptions.requestHeaderAllowlist - ? parseAndNormalizeHeaders(cliOptions.requestHeaderAllowlist) - : undefined; - - const result = await this.add({ - name: cliOptions.name!, - type: cliOptions.type ?? 'create', - buildType: (cliOptions.build as BuildType) ?? 'CodeZip', - language: cliOptions.language!, - framework: cliOptions.framework!, - modelProvider: cliOptions.modelProvider!, - apiKey: cliOptions.apiKey, - memory: cliOptions.memory, - protocol: cliOptions.protocol, - networkMode: cliOptions.networkMode, - subnets: cliOptions.subnets, - securityGroups: cliOptions.securityGroups, - requestHeaderAllowlist, - codeLocation: cliOptions.codeLocation, - entrypoint: cliOptions.entrypoint, - bedrockAgentId: cliOptions.agentId, - bedrockAliasId: cliOptions.agentAliasId, - bedrockRegion: cliOptions.region, - authorizerType: cliOptions.authorizerType, - discoveryUrl: cliOptions.discoveryUrl, - allowedAudience: cliOptions.allowedAudience, - allowedClients: cliOptions.allowedClients, - allowedScopes: cliOptions.allowedScopes, - customClaims, - clientId: cliOptions.clientId, - clientSecret: cliOptions.clientSecret, - idleTimeout: cliOptions.idleTimeout ? Number(cliOptions.idleTimeout) : undefined, - maxLifetime: cliOptions.maxLifetime ? Number(cliOptions.maxLifetime) : undefined, - sessionStorageMountPath: cliOptions.sessionStorageMountPath, - }); - if (cliOptions.json) { - console.log(JSON.stringify(result)); - } else if (result.success) { - console.log(`Added agent '${result.agentName}'`); - if (result.agentPath) { - console.log(`Agent code: ${result.agentPath}`); + // Parse custom claims JSON if provided (already validated by validateAddAgentOptions) + const customClaims = cliOptions.customClaims + ? (JSON.parse(cliOptions.customClaims) as CustomClaimValidation[]) + : undefined; + + // Parse request header allowlist if provided + const requestHeaderAllowlist = cliOptions.requestHeaderAllowlist + ? parseAndNormalizeHeaders(cliOptions.requestHeaderAllowlist) + : undefined; + + const result = await this.add({ + name: cliOptions.name!, + type: cliOptions.type ?? 'create', + buildType: (cliOptions.build as BuildType) ?? 'CodeZip', + language: cliOptions.language!, + framework: cliOptions.framework!, + modelProvider: cliOptions.modelProvider!, + apiKey: cliOptions.apiKey, + memory: cliOptions.memory, + protocol: cliOptions.protocol, + networkMode: cliOptions.networkMode, + subnets: cliOptions.subnets, + securityGroups: cliOptions.securityGroups, + requestHeaderAllowlist, + codeLocation: cliOptions.codeLocation, + entrypoint: cliOptions.entrypoint, + bedrockAgentId: cliOptions.agentId, + bedrockAliasId: cliOptions.agentAliasId, + bedrockRegion: cliOptions.region, + authorizerType: cliOptions.authorizerType, + discoveryUrl: cliOptions.discoveryUrl, + allowedAudience: cliOptions.allowedAudience, + allowedClients: cliOptions.allowedClients, + allowedScopes: cliOptions.allowedScopes, + customClaims, + clientId: cliOptions.clientId, + clientSecret: cliOptions.clientSecret, + idleTimeout: cliOptions.idleTimeout ? Number(cliOptions.idleTimeout) : undefined, + maxLifetime: cliOptions.maxLifetime ? Number(cliOptions.maxLifetime) : undefined, + sessionStorageMountPath: cliOptions.sessionStorageMountPath, + }); + + if (!result.success) { + throw new Error(result.error); } - if (cliOptions.networkMode === 'VPC') { - console.log(`\x1b[33mNote: ${VPC_ENDPOINT_WARNING}\x1b[0m`); + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added agent '${result.agentName}'`); + if (result.agentPath) { + console.log(`Agent code: ${result.agentPath}`); + } + if (cliOptions.networkMode === 'VPC') { + console.log(`\x1b[33mNote: ${VPC_ENDPOINT_WARNING}\x1b[0m`); + } } - } else { - console.error(result.error); - } - process.exit(result.success ? 0 : 1); + return { + language: standardize(Language, cliOptions.language), + framework: standardize(Framework, cliOptions.framework), + model_provider: standardize(ModelProviderEnum, cliOptions.modelProvider), + agent_type: standardize(AgentType, cliOptions.type ?? 'create'), + build: standardize(Build, cliOptions.build ?? 'CodeZip'), + protocol: standardize(Protocol, cliOptions.protocol ?? 'HTTP'), + network_mode: standardize(NetworkModeEnum, cliOptions.networkMode ?? 'PUBLIC'), + authorizer_type: standardize(AuthorizerType, cliOptions.authorizerType ?? 'NONE'), + memory: standardize(Memory, cliOptions.memory ?? 'none'), + }; + }); } else { - // TUI fallback — dynamic imports to avoid pulling ink (async) into registry - requireTTY(); - const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ - import('ink'), - import('react'), - import('../tui/screens/add/AddFlow'), - ]); - const { clear, unmount } = render( - React.createElement(AddFlow, { - isInteractive: false, - initialResource: 'agent', - onExit: () => { - clear(); - unmount(); - process.exit(0); - }, - }) - ); + try { + // TUI fallback — dynamic imports to avoid pulling ink (async) into registry + requireTTY(); + const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ + import('ink'), + import('react'), + import('../tui/screens/add/AddFlow'), + ]); + const { clear, unmount } = render( + React.createElement(AddFlow, { + isInteractive: false, + initialResource: 'agent', + onExit: () => { + clear(); + unmount(); + process.exit(0); + }, + }) + ); + } catch (error) { + console.error(getErrorMessage(error)); + process.exit(1); + } } }); diff --git a/src/cli/primitives/CredentialPrimitive.tsx b/src/cli/primitives/CredentialPrimitive.tsx index 52f578235..9607094f8 100644 --- a/src/cli/primitives/CredentialPrimitive.tsx +++ b/src/cli/primitives/CredentialPrimitive.tsx @@ -4,6 +4,8 @@ import { CredentialSchema } from '../../schema'; import { validateAddCredentialOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { CredentialType, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; import { computeDefaultCredentialEnvVarName } from './credential-utils'; @@ -273,23 +275,23 @@ export class CredentialPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - if ( - cliOptions.name || - cliOptions.apiKey || - cliOptions.json || - cliOptions.type || - cliOptions.discoveryUrl || - cliOptions.clientId || - cliOptions.clientSecret || - cliOptions.scopes - ) { - // CLI mode + if ( + cliOptions.name || + cliOptions.apiKey || + cliOptions.json || + cliOptions.type || + cliOptions.discoveryUrl || + cliOptions.clientId || + cliOptions.clientSecret || + cliOptions.scopes + ) { + // CLI mode + await cliCommandRun('add.credential', !!cliOptions.json, async () => { const validation = validateAddCredentialOptions({ name: cliOptions.name, type: cliOptions.type as 'api-key' | 'oauth' | undefined, @@ -301,12 +303,7 @@ export class CredentialPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - if (cliOptions.name || cliOptions.json) { - const fail = (error: string) => { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error })); - } else { - console.error(error); - } - process.exit(1); + if (cliOptions.name || cliOptions.json) { + await cliCommandRun('add.evaluator', !!cliOptions.json, async () => { + const fail = (error: string): never => { + throw new Error(error); }; if (!cliOptions.name || !cliOptions.level) { @@ -298,9 +295,13 @@ export class EvaluatorPrimitive extends BasePrimitive) => { const cliOptions = rawOptions as unknown as CLIAddGatewayOptions; - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } - + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } + await cliCommandRun('add.gateway', !!cliOptions.json, async () => { const validation = validateAddGatewayOptions(cliOptions); if (!validation.valid) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); - } else { - console.error(validation.error); - } - process.exit(1); + throw new Error(validation.error); } // Parse custom claims JSON if provided (already validated) @@ -221,23 +217,30 @@ export class GatewayPrimitive extends BasePrimitive s.trim()) + .filter(Boolean).length + : 0; + return { + authorizer_type: standardize(AuthorizerType, cliOptions.authorizerType ?? 'NONE'), + has_policy_engine: !!cliOptions.policyEngine, + policy_engine_mode: standardize(PolicyEngineMode, cliOptions.policyEngineMode ?? 'log_only'), + semantic_search: cliOptions.semanticSearch !== false, + runtime_count: runtimeCount, + }; + }); }); removeCmd diff --git a/src/cli/primitives/GatewayTargetPrimitive.ts b/src/cli/primitives/GatewayTargetPrimitive.ts index 41a2e6a75..e8a1da996 100644 --- a/src/cli/primitives/GatewayTargetPrimitive.ts +++ b/src/cli/primitives/GatewayTargetPrimitive.ts @@ -14,6 +14,13 @@ import { validateAddGatewayTargetOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovableGatewayTarget } from '../operations/remove/remove-gateway-target'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { + GATEWAY_TARGET_TYPE_MAP, + GatewayTargetHost, + OutboundAuth, + standardize, +} from '../telemetry/schemas/common-shapes.js'; import { getTemplateToolDefinitions, renderGatewayTargetTemplate } from '../templates/GatewayTargetRenderer'; import { requireTTY } from '../tui/guards/tty'; import type { @@ -297,20 +304,15 @@ export class GatewayTargetPrimitive extends BasePrimitive { const validation = await validateAddGatewayTargetOptions(cliOptions); if (!validation.valid) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); - } else { - console.error(validation.error); - } - process.exit(1); + throw new Error(validation.error); } // Map CLI flag values to internal types @@ -321,6 +323,19 @@ export class GatewayTargetPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - if (cliOptions.name || cliOptions.json) { - // CLI mode + if (cliOptions.name || cliOptions.json) { + // CLI mode + await cliCommandRun('add.memory', !!cliOptions.json, async () => { const expiry = cliOptions.expiry ? parseInt(cliOptions.expiry, 10) : undefined; const validation = validateAddMemoryOptions({ name: cliOptions.name, @@ -203,12 +204,7 @@ export class MemoryPrimitive extends BasePrimitive s.trim().toUpperCase()) + .filter(Boolean); + return { + strategy_count: strategyList.length, + strategy_semantic: strategyList.includes('SEMANTIC'), + strategy_summarization: strategyList.includes('SUMMARIZATION'), + strategy_user_preference: strategyList.includes('USER_PREFERENCE'), + strategy_episodic: strategyList.includes('EPISODIC'), + }; + }); + } else { + try { // TUI fallback — dynamic imports to avoid pulling ink (async) into registry requireTTY(); const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ @@ -248,14 +259,10 @@ export class MemoryPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - if (cliOptions.name || cliOptions.json) { - // Merge --evaluator and --evaluator-arn into a single list - const allEvaluators = [...(cliOptions.evaluator ?? []), ...(cliOptions.evaluatorArn ?? [])]; + if (cliOptions.name || cliOptions.json) { + // Merge --evaluator and --evaluator-arn into a single list + const allEvaluators = [...(cliOptions.evaluator ?? []), ...(cliOptions.evaluatorArn ?? [])]; + await cliCommandRun('add.online-eval', !!cliOptions.json, async () => { if (!cliOptions.name || !cliOptions.runtime || allEvaluators.length === 0 || !cliOptions.samplingRate) { - const error = - '--name, --runtime, --evaluator (and/or --evaluator-arn), and --sampling-rate are all required in non-interactive mode'; - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error })); - } else { - console.error(error); - } - process.exit(1); + throw new Error( + '--name, --runtime, --evaluator (and/or --evaluator-arn), and --sampling-rate are all required in non-interactive mode' + ); } // Sampling rate as a percentage of requests to evaluate (0.01% to 100%) const samplingRate = parseFloat(cliOptions.samplingRate); if (isNaN(samplingRate) || samplingRate < 0.01 || samplingRate > 100) { - const error = `Invalid --sampling-rate "${cliOptions.samplingRate}". Must be a percentage between 0.01 and 100`; - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error })); - } else { - console.error(error); - } - process.exit(1); + throw new Error( + `Invalid --sampling-rate "${cliOptions.samplingRate}". Must be a percentage between 0.01 and 100` + ); } const result = await this.add({ @@ -162,15 +154,23 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - if (cliOptions.name || cliOptions.description || cliOptions.encryptionKeyArn || cliOptions.json) { + if (cliOptions.name || cliOptions.description || cliOptions.encryptionKeyArn || cliOptions.json) { + await cliCommandRun('add.policy-engine', !!cliOptions.json, async () => { if (!cliOptions.name) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: '--name is required' })); - } else { - console.error('--name is required'); - } - process.exit(1); + throw new Error('--name is required'); } const result = await this.add({ @@ -253,15 +250,29 @@ export class PolicyEnginePrimitive extends BasePrimitive s.trim()) + .filter(Boolean).length + : 0; + return { + attach_gateway_count: gatewayCount, + attach_mode: standardize(AttachMode, cliOptions.attachMode ?? 'log_only'), + }; + }); + } else { + try { requireTTY(); const [{ render }, { default: React }, { AddFlow }] = await Promise.all([ import('ink'), @@ -278,14 +289,10 @@ export class PolicyEnginePrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } - if ( - cliOptions.name || - cliOptions.engine || - cliOptions.source || - cliOptions.statement || - cliOptions.generate || - cliOptions.json - ) { + if ( + cliOptions.name || + cliOptions.engine || + cliOptions.source || + cliOptions.statement || + cliOptions.generate || + cliOptions.json + ) { + await cliCommandRun('add.policy', !!cliOptions.json, async () => { if (!cliOptions.name) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: '--name is required' })); - } else { - console.error('--name is required'); - } - process.exit(1); + throw new Error('--name is required'); } if (!cliOptions.engine) { - if (cliOptions.json) { - console.log(JSON.stringify({ success: false, error: '--engine is required' })); - } else { - console.error('--engine is required'); - } - process.exit(1); + throw new Error('--engine is required'); } const result = await this.add({ @@ -335,15 +327,28 @@ export class PolicyPrimitive extends BasePrimitive { - try { - if (!findConfigRoot()) { - console.error('No agentcore project found. Run `agentcore create` first.'); - process.exit(1); - } + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } + await cliCommandRun('add.runtime-endpoint', !!cliOptions.json, async () => { const result = await this.add({ runtime: cliOptions.runtime, endpoint: cliOptions.endpoint, @@ -261,23 +262,18 @@ export class RuntimeEndpointPrimitive extends BasePrimitive( + command: C, + json: boolean, + fn: () => Promise> +): Promise { + try { + let client; + try { + client = await TelemetryClientAccessor.get(); + } catch { + // Telemetry init failed — run without it + await fn(); + process.exit(0); + } + // withCommandRun records success/failure telemetry, then re-throws on failure + await client.withCommandRun(command, fn); + process.exit(0); + } catch (error) { + if (json) { + console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + } else { + console.error(getErrorMessage(error)); + } + process.exit(1); + } +} + +/** + * Wrap a primitive .add() call with telemetry — used by TUI paths. + * CLI paths use {@link cliCommandRun} instead. + */ +export async function withAddTelemetry>( + command: C, + attrs: CommandAttrs, + fn: () => Promise> +): Promise> { + let client; + try { + client = await TelemetryClientAccessor.get(); + } catch { + return fn(); + } + + let result: AddResult | undefined; + try { + await client.withCommandRun(command, async () => { + result = await fn(); + if (!result.success) throw new Error(result.error); + return attrs; + }); + } catch (err) { + // withCommandRun re-throws after recording failure telemetry. + // result is set if fn() ran; if not, fn() itself threw. + if (!result) { + return { success: false, error: getErrorMessage(err) }; + } + } + return result!; +} diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index dfd127a98..0acfcaf1b 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -157,6 +157,7 @@ export const COMMAND_SCHEMAS = { 'add.gateway-target': AddGatewayTargetAttrs, 'add.policy-engine': AddPolicyEngineAttrs, 'add.policy': AddPolicyAttrs, + 'add.runtime-endpoint': NoAttrs, // deploy deploy: DeployAttrs, diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index 732bb3d61..4624883cd 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -26,6 +26,22 @@ export function resilientParse( return result; } +/** + * Lowercase a CLI value and parse it through a Zod enum, returning the narrowed type. + * The `as` cast on the failure branch is intentional: invalid values pass through to + * recordCommandRun, where COMMAND_SCHEMAS[command].parse(attrs) validates the full + * attr object in a try/catch — silently dropping the metric if any field is invalid. + * This ensures telemetry never crashes the CLI while keeping the happy-path type-safe. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function standardize>(schema: T, value: string | undefined): z.infer { + const lower = (value ?? '').toLowerCase(); + const result = schema.safeParse(lower); + // If the value doesn't match the enum, return the lowercased value anyway — + // recordCommandRun's try/catch will silently drop the invalid metric. + return (result.success ? result.data : lower) as z.infer; +} + // Primitive types export const Count = z.number().int().nonnegative(); @@ -59,7 +75,17 @@ export const GatewayTargetType = z.enum([ 'open-api-schema', 'smithy-model', 'lambda-function-arn', + 'unknown', ]); + +/** Map camelCase CLI target type to kebab-case telemetry enum value. */ +export const GATEWAY_TARGET_TYPE_MAP: Record> = { + apiGateway: 'api-gateway', + openApiSchema: 'open-api-schema', + smithyModel: 'smithy-model', + lambdaFunctionArn: 'lambda-function-arn', + mcpServer: 'mcp-server', +}; export const Language = z.enum(['python', 'typescript', 'other']); export const Level = z.enum(['session', 'trace', 'tool_call']); export const Memory = z.enum(['none', 'shortterm', 'longandshortterm']); diff --git a/src/cli/tui/hooks/useCreateEvaluator.ts b/src/cli/tui/hooks/useCreateEvaluator.ts index 6e1d8f052..f1cad666f 100644 --- a/src/cli/tui/hooks/useCreateEvaluator.ts +++ b/src/cli/tui/hooks/useCreateEvaluator.ts @@ -1,5 +1,7 @@ import type { EvaluatorConfig } from '../../../schema'; import { evaluatorPrimitive } from '../../primitives/registry'; +import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; +import { Level, standardize } from '../../telemetry/schemas/common-shapes.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateEvaluatorConfig { @@ -16,11 +18,19 @@ export function useCreateEvaluator() { const create = useCallback(async (config: CreateEvaluatorConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await evaluatorPrimitive.add({ - name: config.name, - level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL', - config: config.config, - }); + const addResult = await withAddTelemetry( + 'add.evaluator', + { + evaluator_type: config.config.codeBased ? 'code-based' : 'llm-as-a-judge', + level: standardize(Level, config.level), + }, + () => + evaluatorPrimitive.add({ + name: config.name, + level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL', + config: config.config, + }) + ); if (!addResult.success) { throw new Error(addResult.error ?? 'Failed to create evaluator'); } diff --git a/src/cli/tui/hooks/useCreateMcp.ts b/src/cli/tui/hooks/useCreateMcp.ts index 2b3b3b25a..ec91666d0 100644 --- a/src/cli/tui/hooks/useCreateMcp.ts +++ b/src/cli/tui/hooks/useCreateMcp.ts @@ -4,6 +4,8 @@ import { gatewayTargetPrimitive, policyEnginePrimitive, } from '../../primitives/registry'; +import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; +import { AuthorizerType, PolicyEngineMode, standardize } from '../../telemetry/schemas/common-shapes.js'; import type { AddGatewayConfig } from '../screens/mcp/types'; import { useCallback, useEffect, useState } from 'react'; @@ -23,22 +25,33 @@ export function useCreateGateway() { const createGateway = useCallback(async (config: AddGatewayConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await gatewayPrimitive.add({ - name: config.name, - description: config.description, - authorizerType: config.authorizerType, - discoveryUrl: config.jwtConfig?.discoveryUrl, - allowedAudience: config.jwtConfig?.allowedAudience?.join(','), - allowedClients: config.jwtConfig?.allowedClients?.join(','), - allowedScopes: config.jwtConfig?.allowedScopes?.join(','), - customClaims: config.jwtConfig?.customClaims, - clientId: config.jwtConfig?.clientId, - clientSecret: config.jwtConfig?.clientSecret, - enableSemanticSearch: config.enableSemanticSearch, - exceptionLevel: config.exceptionLevel, - policyEngine: config.policyEngineConfiguration?.policyEngineName, - policyEngineMode: config.policyEngineConfiguration?.mode, - }); + const addResult = await withAddTelemetry( + 'add.gateway', + { + authorizer_type: standardize(AuthorizerType, config.authorizerType ?? 'NONE'), + has_policy_engine: !!config.policyEngineConfiguration?.policyEngineName, + policy_engine_mode: standardize(PolicyEngineMode, config.policyEngineConfiguration?.mode ?? 'log_only'), + semantic_search: config.enableSemanticSearch !== false, + runtime_count: 0, + }, + () => + gatewayPrimitive.add({ + name: config.name, + description: config.description, + authorizerType: config.authorizerType, + discoveryUrl: config.jwtConfig?.discoveryUrl, + allowedAudience: config.jwtConfig?.allowedAudience?.join(','), + allowedClients: config.jwtConfig?.allowedClients?.join(','), + allowedScopes: config.jwtConfig?.allowedScopes?.join(','), + customClaims: config.jwtConfig?.customClaims, + clientId: config.jwtConfig?.clientId, + clientSecret: config.jwtConfig?.clientSecret, + enableSemanticSearch: config.enableSemanticSearch, + exceptionLevel: config.exceptionLevel, + policyEngine: config.policyEngineConfiguration?.policyEngineName, + policyEngineMode: config.policyEngineConfiguration?.mode, + }) + ); if (!addResult.success) { throw new Error(addResult.error ?? 'Failed to create gateway'); } diff --git a/src/cli/tui/hooks/useCreateMemory.ts b/src/cli/tui/hooks/useCreateMemory.ts index 4345b4ead..d4196582f 100644 --- a/src/cli/tui/hooks/useCreateMemory.ts +++ b/src/cli/tui/hooks/useCreateMemory.ts @@ -2,6 +2,7 @@ import { ConfigIO } from '../../../lib'; import type { Memory } from '../../../schema'; import { getAvailableAgents } from '../../operations/attach'; import { memoryPrimitive } from '../../primitives/registry'; +import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateMemoryConfig { @@ -24,13 +25,25 @@ export function useCreateMemory() { setStatus({ state: 'loading' }); try { const strategiesStr = config.strategies.map(s => s.type).join(','); - const addResult = await memoryPrimitive.add({ - name: config.name, - expiry: config.eventExpiryDuration, - strategies: strategiesStr || undefined, - dataStreamArn: config.streaming?.dataStreamArn, - contentLevel: config.streaming?.contentLevel, - }); + const strategyList = strategiesStr ? strategiesStr.split(',').map(s => s.trim().toUpperCase()) : []; + const addResult = await withAddTelemetry( + 'add.memory', + { + strategy_count: strategyList.length, + strategy_semantic: strategyList.includes('SEMANTIC'), + strategy_summarization: strategyList.includes('SUMMARIZATION'), + strategy_user_preference: strategyList.includes('USER_PREFERENCE'), + strategy_episodic: strategyList.includes('EPISODIC'), + }, + () => + memoryPrimitive.add({ + name: config.name, + expiry: config.eventExpiryDuration, + strategies: strategiesStr || undefined, + dataStreamArn: config.streaming?.dataStreamArn, + contentLevel: config.streaming?.contentLevel, + }) + ); if (!addResult.success) { throw new Error(addResult.error ?? 'Failed to create memory'); } diff --git a/src/cli/tui/hooks/useCreateOnlineEval.ts b/src/cli/tui/hooks/useCreateOnlineEval.ts index 2d0190552..840b8ae5c 100644 --- a/src/cli/tui/hooks/useCreateOnlineEval.ts +++ b/src/cli/tui/hooks/useCreateOnlineEval.ts @@ -1,4 +1,5 @@ import { onlineEvalConfigPrimitive } from '../../primitives/registry'; +import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateOnlineEvalConfig { @@ -17,13 +18,21 @@ export function useCreateOnlineEval() { const create = useCallback(async (config: CreateOnlineEvalConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await onlineEvalConfigPrimitive.add({ - name: config.name, - agent: config.agent, - evaluators: config.evaluators, - samplingRate: config.samplingRate, - enableOnCreate: config.enableOnCreate, - }); + const addResult = await withAddTelemetry( + 'add.online-eval', + { + evaluator_count: config.evaluators.length, + enable_on_create: config.enableOnCreate ?? false, + }, + () => + onlineEvalConfigPrimitive.add({ + name: config.name, + agent: config.agent, + evaluators: config.evaluators, + samplingRate: config.samplingRate, + enableOnCreate: config.enableOnCreate, + }) + ); if (!addResult.success) { throw new Error(addResult.error ?? 'Failed to create online eval config'); } diff --git a/src/cli/tui/screens/agent/useAddAgent.ts b/src/cli/tui/screens/agent/useAddAgent.ts index 3160e8712..273eff4aa 100644 --- a/src/cli/tui/screens/agent/useAddAgent.ts +++ b/src/cli/tui/screens/agent/useAddAgent.ts @@ -1,6 +1,5 @@ import { APP_DIR, ConfigIO, NoProjectError, findConfigRoot, setEnvVar } from '../../../../lib'; import type { AgentEnvSpec, DirectoryPath, FilePath } from '../../../../schema'; -import { getErrorMessage } from '../../../errors'; import { type PythonSetupResult, setupPythonProject } from '../../../operations'; import { mapGenerateConfigToRenderConfig, @@ -12,6 +11,19 @@ import { executeImportAgent } from '../../../operations/agent/import'; import { buildAuthorizerConfigFromJwtConfig, createManagedOAuthCredential } from '../../../primitives/auth-utils'; import { computeDefaultCredentialEnvVarName } from '../../../primitives/credential-utils'; import { credentialPrimitive } from '../../../primitives/registry'; +import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; +import { + AgentType as AgentTypeEnum, + AuthorizerType as AuthorizerTypeEnum, + Build, + Framework, + Language, + Memory as MemoryEnum, + ModelProvider, + NetworkMode, + Protocol, + standardize, +} from '../../../telemetry/schemas/common-shapes.js'; import { createRenderer } from '../../../templates'; import type { GenerateConfig } from '../generate/types'; import type { AddAgentConfig } from './types'; @@ -135,34 +147,25 @@ export function useAddAgent() { const addAgent = useCallback(async (config: AddAgentConfig): Promise => { setIsLoading(true); try { - const configBaseDir = findConfigRoot(); - if (!configBaseDir) { - return { ok: false, error: new NoProjectError().message }; - } - - const configIO = new ConfigIO({ baseDir: configBaseDir }); - - if (!configIO.configExists('project')) { - return { ok: false, error: new NoProjectError().message }; - } - - // Check for duplicate agent name - const project = await configIO.readProjectSpec(); - const existingAgent = project.runtimes.find(agent => agent.name === config.name); - if (existingAgent) { - return { ok: false, error: `Agent "${config.name}" already exists in this project.` }; - } - - // Branch based on agent type - if (config.agentType === 'import') { - return await handleImportPath(config, configBaseDir); - } else if (config.agentType === 'create') { - return await handleCreatePath(config, configBaseDir); - } else { - return await handleByoPath(config, configIO, configBaseDir); + const result = await withAddTelemetry( + 'add.agent', + { + language: standardize(Language, config.language), + framework: standardize(Framework, config.framework), + model_provider: standardize(ModelProvider, config.modelProvider), + agent_type: standardize(AgentTypeEnum, config.agentType), + build: standardize(Build, config.buildType), + protocol: standardize(Protocol, config.protocol ?? 'HTTP'), + network_mode: standardize(NetworkMode, config.networkMode ?? 'PUBLIC'), + authorizer_type: standardize(AuthorizerTypeEnum, config.authorizerType ?? 'NONE'), + memory: standardize(MemoryEnum, config.memory ?? 'none'), + }, + () => addAgentInner(config) + ); + if (!result.success) { + return { ok: false, error: result.error }; } - } catch (err) { - return { ok: false, error: getErrorMessage(err) }; + return result.outcome; } finally { setIsLoading(false); } @@ -175,6 +178,43 @@ export function useAddAgent() { return { addAgent, isLoading, reset }; } +type AddAgentInnerResult = + | { success: true; outcome: AddAgentCreateResult | AddAgentByoResult } + | { success: false; error: string }; + +async function addAgentInner(config: AddAgentConfig): Promise { + const configBaseDir = findConfigRoot(); + if (!configBaseDir) { + return { success: false, error: new NoProjectError().message }; + } + + const configIO = new ConfigIO({ baseDir: configBaseDir }); + + if (!configIO.configExists('project')) { + return { success: false, error: new NoProjectError().message }; + } + + const project = await configIO.readProjectSpec(); + const existingAgent = project.runtimes.find(agent => agent.name === config.name); + if (existingAgent) { + return { success: false, error: `Agent "${config.name}" already exists in this project.` }; + } + + let outcome: AddAgentCreateResult | AddAgentByoResult | AddAgentError; + if (config.agentType === 'import') { + outcome = await handleImportPath(config, configBaseDir); + } else if (config.agentType === 'create') { + outcome = await handleCreatePath(config, configBaseDir); + } else { + outcome = await handleByoPath(config, configIO, configBaseDir); + } + + if (!outcome.ok) { + return { success: false, error: outcome.error }; + } + return { success: true, outcome }; +} + /** * Handle the "create" path: generate agent from template and write to project. */ diff --git a/src/cli/tui/screens/identity/useCreateIdentity.ts b/src/cli/tui/screens/identity/useCreateIdentity.ts index 42aace21b..e214d1e67 100644 --- a/src/cli/tui/screens/identity/useCreateIdentity.ts +++ b/src/cli/tui/screens/identity/useCreateIdentity.ts @@ -2,6 +2,7 @@ import { ConfigIO } from '../../../../lib'; import type { Credential } from '../../../../schema'; import type { AddCredentialOptions } from '../../../primitives/CredentialPrimitive'; import { credentialPrimitive } from '../../../primitives/registry'; +import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateStatus { @@ -16,7 +17,13 @@ export function useCreateIdentity() { const create = useCallback(async (config: AddCredentialOptions) => { setStatus({ state: 'loading' }); try { - const result = await credentialPrimitive.add(config); + const result = await withAddTelemetry( + 'add.credential', + { + credential_type: config.authorizerType === 'OAuthCredentialProvider' ? 'oauth' : 'api-key', + }, + () => credentialPrimitive.add(config) + ); if (!result.success) { throw new Error(result.error ?? 'Failed to create credential'); } diff --git a/src/cli/tui/screens/policy/AddPolicyFlow.tsx b/src/cli/tui/screens/policy/AddPolicyFlow.tsx index 720984563..9b3542cb8 100644 --- a/src/cli/tui/screens/policy/AddPolicyFlow.tsx +++ b/src/cli/tui/screens/policy/AddPolicyFlow.tsx @@ -1,4 +1,6 @@ import { policyEnginePrimitive, policyPrimitive } from '../../../primitives/registry'; +import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; +import { AttachMode, ValidationMode, standardize } from '../../../telemetry/schemas/common-shapes.js'; import { ErrorPrompt, Panel, @@ -128,7 +130,14 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD }, []); const commitEngine = useCallback(async (engineName: string, gateways?: string[], mode?: 'LOG_ONLY' | 'ENFORCE') => { - const result = await policyEnginePrimitive.add({ name: engineName }); + const result = await withAddTelemetry( + 'add.policy-engine', + { + attach_gateway_count: gateways?.length ?? 0, + attach_mode: standardize(AttachMode, mode ?? 'log_only'), + }, + () => policyEnginePrimitive.add({ name: engineName }) + ); if (!result.success) { setFlow({ name: 'error', message: result.error }); return; @@ -155,13 +164,21 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD ); const handlePolicyComplete = useCallback(async (config: AddPolicyConfig) => { - const result = await policyPrimitive.add({ - name: config.name, - engine: config.engine, - statement: config.statement, - source: config.sourceFile || undefined, - validationMode: config.validationMode, - }); + const result = await withAddTelemetry( + 'add.policy', + { + source_type: config.sourceFile ? 'file' : config.sourceMethod === 'generate' ? 'generate' : 'statement', + validation_mode: standardize(ValidationMode, config.validationMode ?? 'FAIL_ON_ANY_FINDINGS'), + }, + () => + policyPrimitive.add({ + name: config.name, + engine: config.engine, + statement: config.statement, + source: config.sourceFile || undefined, + validationMode: config.validationMode, + }) + ); if (result.success) { setPolicyNames(prev => [...prev, config.name]); diff --git a/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx b/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx index 2404109fc..83bda78e5 100644 --- a/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx +++ b/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx @@ -1,5 +1,6 @@ import { ConfigIO } from '../../../../lib'; import { runtimeEndpointPrimitive } from '../../../primitives/registry'; +import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; import { ErrorPrompt } from '../../components'; import { AddSuccessScreen } from '../add/AddSuccessScreen'; import { AddRuntimeEndpointScreen } from './AddRuntimeEndpointScreen'; @@ -78,24 +79,24 @@ export function AddRuntimeEndpointFlow({ }, [isInteractive, flow.name, onExit]); const handleCreateComplete = useCallback((config: RuntimeEndpointWizardConfig) => { - void runtimeEndpointPrimitive - .add({ + void withAddTelemetry('add.runtime-endpoint', {}, () => + runtimeEndpointPrimitive.add({ runtime: config.runtimeName, endpoint: config.endpointName, version: config.version, description: config.description, }) - .then(result => { - if (result.success) { - setFlow({ - name: 'create-success', - endpointName: config.endpointName, - runtimeName: config.runtimeName, - }); - return; - } - setFlow({ name: 'error', message: result.error ?? 'Unknown error' }); - }); + ).then(result => { + if (result.success) { + setFlow({ + name: 'create-success', + endpointName: config.endpointName, + runtimeName: config.runtimeName, + }); + return; + } + setFlow({ name: 'error', message: result.error ?? 'Unknown error' }); + }); }, []); if (flow.name === 'loading') { diff --git a/src/test-utils/cli-runner.ts b/src/test-utils/cli-runner.ts index 789624364..4526546c5 100644 --- a/src/test-utils/cli-runner.ts +++ b/src/test-utils/cli-runner.ts @@ -72,6 +72,14 @@ function getCLIPath(): string { * Run the AgentCore CLI via the local build (unit/integ tests). * Skips dependency installation by default for speed. */ -export async function runCLI(args: string[], cwd: string, skipInstall = true): Promise { - return spawnAndCollect('node', [getCLIPath(), ...args], cwd, skipInstall ? { AGENTCORE_SKIP_INSTALL: '1' } : {}); +export async function runCLI( + args: string[], + cwd: string, + options: { skipInstall?: boolean; env?: Record } = {} +): Promise { + const { skipInstall = true, env } = options; + return spawnAndCollect('node', [getCLIPath(), ...args], cwd, { + ...(skipInstall ? { AGENTCORE_SKIP_INSTALL: '1' } : {}), + ...env, + }); } diff --git a/src/test-utils/index.ts b/src/test-utils/index.ts index ff127a35e..1c032519c 100644 --- a/src/test-utils/index.ts +++ b/src/test-utils/index.ts @@ -4,6 +4,7 @@ */ export { runCLI, spawnAndCollect, cleanSpawnEnv, type RunResult } from './cli-runner.js'; +export { createTelemetryHelper, type TelemetryHelper, type TelemetryEntry } from './telemetry-helper.js'; export { exists } from './fs-helpers.js'; export { hasCommand, hasAwsCredentials, prereqs } from './prereqs.js'; export { createTestProject, type TestProject, type CreateTestProjectOptions } from './project-factory.js'; diff --git a/src/test-utils/project-factory.ts b/src/test-utils/project-factory.ts index 77c77e1c6..61525f6c5 100644 --- a/src/test-utils/project-factory.ts +++ b/src/test-utils/project-factory.ts @@ -65,7 +65,7 @@ export async function createTestProject(options: CreateTestProjectOptions = {}): args.push('--json'); - const result = await runCLI(args, testDir, skipInstall); + const result = await runCLI(args, testDir, { skipInstall }); if (result.exitCode !== 0) { // Clean up on failure diff --git a/src/test-utils/telemetry-helper.ts b/src/test-utils/telemetry-helper.ts new file mode 100644 index 000000000..e7fa58949 --- /dev/null +++ b/src/test-utils/telemetry-helper.ts @@ -0,0 +1,56 @@ +import { globSync } from 'glob'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { expect } from 'vitest'; + +export interface TelemetryEntry { + value: number; + attrs: Record; +} + +export interface TelemetryHelper { + /** Temp directory used as AGENTCORE_CONFIG_DIR */ + dir: string; + /** Env vars to pass to runCLI to enable audit mode */ + env: { AGENTCORE_TELEMETRY_AUDIT: '1'; AGENTCORE_CONFIG_DIR: string }; + /** Read all JSONL entries from the audit telemetry directory */ + readEntries: () => TelemetryEntry[]; + /** Assert a metric was emitted with attrs matching the given subset */ + assertMetricEmitted: (expected: Record) => void; + /** Delete telemetry entries only (keeps the config dir) */ + clearEntries: () => void; + /** Delete the entire config directory — call in afterAll */ + destroy: () => void; +} + +export function createTelemetryHelper(): TelemetryHelper { + const dir = mkdtempSync(join(tmpdir(), 'agentcore-audit-')); + const helper: TelemetryHelper = { + dir, + env: { AGENTCORE_TELEMETRY_AUDIT: '1', AGENTCORE_CONFIG_DIR: dir }, + readEntries() { + return globSync(join(dir, 'telemetry', '*.json')).flatMap(f => + readFileSync(f, 'utf-8') + .trim() + .split('\n') + .map(line => JSON.parse(line) as TelemetryEntry) + ); + }, + assertMetricEmitted(expected) { + const entries = helper.readEntries(); + const match = entries.find(e => Object.entries(expected).every(([k, v]) => String(e.attrs[k]) === String(v))); + expect( + match, + `No telemetry entry matching ${JSON.stringify(expected)}\nFound ${entries.length} entries:\n${entries.map(e => JSON.stringify(e.attrs)).join('\n')}` + ).toBeDefined(); + }, + clearEntries() { + rmSync(join(dir, 'telemetry'), { recursive: true, force: true }); + }, + destroy() { + rmSync(dir, { recursive: true, force: true }); + }, + }; + return helper; +}