diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e5f4aa9e2..6c143e375 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -101,6 +101,7 @@ jobs: BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }} CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \ | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \ + | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \ | tr '\n' ' ') echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT" echo "Changed e2e tests: ${CHANGED:-none}" @@ -113,5 +114,7 @@ jobs: OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} CDK_TARBALL: ${{ env.CDK_TARBALL }} - # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR - run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }} + # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR + run: + npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{ + steps.changed.outputs.extra_tests }} diff --git a/e2e-tests/harness-bedrock.test.ts b/e2e-tests/harness-bedrock.test.ts new file mode 100644 index 000000000..7b53e18bb --- /dev/null +++ b/e2e-tests/harness-bedrock.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'bedrock' }); diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts new file mode 100644 index 000000000..ca29ae4f3 --- /dev/null +++ b/e2e-tests/harness-e2e-helper.ts @@ -0,0 +1,163 @@ +import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js'; +import { + cleanupStaleCredentialProviders, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const hasAws = hasAwsCredentials(); +const baseCanRun = prereqs.npm && prereqs.git && hasAws; + +interface HarnessE2EConfig { + modelProvider: 'bedrock' | 'open_ai' | 'gemini'; + requiredEnvVar?: string; + skipMemory?: boolean; +} + +export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { + const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar]; + const canRun = baseCanRun && hasRequiredVar; + + const providerLabel = + cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock'; + + describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => { + let testDir: string; + let projectPath: string; + let harnessName: string; + + beforeAll(async () => { + if (!canRun) return; + + await cleanupStaleCredentialProviders(); + + testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4); + harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`; + + const createArgs = [ + 'create', + '--name', + harnessName, + '--model-provider', + cfg.modelProvider, + '--json', + '--skip-git', + ]; + + if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) { + createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!); + } + + if (cfg.skipMemory) { + createArgs.push('--no-harness-memory'); + } + + const result = await runAgentCoreCLI(createArgs, testDir); + + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { projectPath: string }; + projectPath = json.projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, harnessName, cfg.modelProvider); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + it.skipIf(!canRun)( + 'deploys to AWS successfully', + async () => { + expect(projectPath, 'Project should have been created').toBeTruthy(); + + await retry( + async () => { + const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath); + + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + + expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success, 'Deploy should report success').toBe(true); + }, + 1, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'invokes the deployed harness', + async () => { + expect(projectPath, 'Project should have been created').toBeTruthy(); + + await retry( + async () => { + const result = await runAgentCoreCLI( + ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'], + projectPath + ); + + if (result.exitCode !== 0) { + console.log('Invoke stdout:', result.stdout); + console.log('Invoke stderr:', result.stderr); + } + + expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success, 'Invoke should report success').toBe(true); + }, + 3, + 15000 + ); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'status shows the deployed harness', + async () => { + const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath); + + expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0); + + const json = parseJsonOutput(statusResult.stdout) as { + success: boolean; + resources: { + resourceType: string; + name: string; + deploymentState: string; + identifier?: string; + }[]; + }; + expect(json.success).toBe(true); + + const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName); + expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined(); + expect(harness!.deploymentState).toBe('deployed'); + expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy(); + }, + 120000 + ); + }); +} diff --git a/e2e-tests/harness-gemini.test.ts b/e2e-tests/harness-gemini.test.ts new file mode 100644 index 000000000..8fd024147 --- /dev/null +++ b/e2e-tests/harness-gemini.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true }); diff --git a/e2e-tests/harness-openai.test.ts b/e2e-tests/harness-openai.test.ts new file mode 100644 index 000000000..bdb9c3772 --- /dev/null +++ b/e2e-tests/harness-openai.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true }); diff --git a/integ-tests/add-remove-harness.test.ts b/integ-tests/add-remove-harness.test.ts new file mode 100644 index 000000000..2f69270db --- /dev/null +++ b/integ-tests/add-remove-harness.test.ts @@ -0,0 +1,204 @@ +import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js'; +import type { TestProject } from '../src/test-utils/index.js'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +async function readHarnessSpec(projectPath: string, harnessName: string) { + return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8')); +} + +describe('integration: harness add/remove lifecycle', () => { + let project: TestProject; + const harnessName = 'TestHarness'; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('adds a harness with defaults', async () => { + const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + const config = await readProjectConfig(project.projectPath); + const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy(); + expect(harness!.path).toBe(`app/${harnessName}`); + }); + + it('creates harness.json with correct model config', async () => { + const spec = await readHarnessSpec(project.projectPath, harnessName); + expect(spec.model).toBeDefined(); + expect(spec.model.provider).toBe('bedrock'); + expect(spec.model.modelId).toBeTruthy(); + }); + + it('creates system-prompt.md', async () => { + const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`); + expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true); + }); + + it('auto-creates memory resource', async () => { + const config = await readProjectConfig(project.projectPath); + const memories = config.memories ?? []; + expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0); + }); + + it('rejects duplicate harness name', async () => { + const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('removes the harness', async () => { + const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + const config = await readProjectConfig(project.projectPath); + const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy(); + }); +}); + +describe('integration: harness configuration options', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('adds harness with truncation strategy', async () => { + const name = 'TruncHarness'; + const result = await runCLI( + ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.truncation?.strategy).toBe('sliding_window'); + }); + + it('adds harness with lifecycle config', async () => { + const name = 'LifecycleHarness'; + const result = await runCLI( + ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300); + expect(spec.lifecycleConfig?.maxLifetime).toBe(3600); + }); + + it('adds harness without memory when --no-memory is set', async () => { + const name = 'NoMemHarness'; + const configBefore = await readProjectConfig(project.projectPath); + const memoriesBefore = (configBefore.memories ?? []).length; + + const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const configAfter = await readProjectConfig(project.projectPath); + const memoriesAfter = (configAfter.memories ?? []).length; + expect(memoriesAfter).toBe(memoriesBefore); + }); + + it('adds harness with non-bedrock model provider', async () => { + const name = 'OpenAIHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--model-provider', + 'open_ai', + '--model-id', + 'gpt-5', + '--api-key-arn', + 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.model.provider).toBe('open_ai'); + expect(spec.model.modelId).toBe('gpt-5'); + expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key'); + }); +}); + +describe('integration: harness validation errors', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('rejects invalid harness name with special characters', async () => { + const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('rejects harness name starting with a number', async () => { + const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('rejects add harness without --name when --json is passed', async () => { + const result = await runCLI(['add', 'harness', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); +}); + +describe('integration: create project with harness', () => { + let project: TestProject; + const harnessName = 'CreateHarness'; + + beforeAll(async () => { + project = await createTestProject({ name: harnessName, noAgent: true }); + await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('has correct project scaffolding', async () => { + expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true); + expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true); + expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true); + expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true); + }); + + it('has harness registered in project config', async () => { + const config = await readProjectConfig(project.projectPath); + const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(harness).toBeTruthy(); + }); +});