From 7335079570dc8ee25d8e81b89ef1951b75aea0ab Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Tue, 28 Apr 2026 18:46:57 -0400 Subject: [PATCH 1/4] feat(tests): add E2E and integ tests for Harness primitive E2E tests verify create/deploy/invoke/status against real AWS for Bedrock, OpenAI, and Gemini providers using a factory pattern that mirrors the existing agent E2E suite. Integ tests cover the add/remove lifecycle, configuration options (truncation, lifecycle, model provider), validation errors, and project scaffolding. Harness-bedrock is added to the PR E2E baseline alongside strands-bedrock so every PR exercises the harness deploy path. --- .github/workflows/e2e-tests.yml | 7 +- e2e-tests/harness-bedrock.test.ts | 3 + e2e-tests/harness-e2e-helper.ts | 158 +++++++++++++++++++ e2e-tests/harness-gemini.test.ts | 3 + e2e-tests/harness-openai.test.ts | 3 + integ-tests/add-remove-harness.test.ts | 208 +++++++++++++++++++++++++ 6 files changed, 380 insertions(+), 2 deletions(-) create mode 100644 e2e-tests/harness-bedrock.test.ts create mode 100644 e2e-tests/harness-e2e-helper.ts create mode 100644 e2e-tests/harness-gemini.test.ts create mode 100644 e2e-tests/harness-openai.test.ts create mode 100644 integ-tests/add-remove-harness.test.ts diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e5f4aa9e2..6c143e375 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -101,6 +101,7 @@ jobs: BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }} CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \ | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \ + | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \ | tr '\n' ' ') echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT" echo "Changed e2e tests: ${CHANGED:-none}" @@ -113,5 +114,7 @@ jobs: OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} CDK_TARBALL: ${{ env.CDK_TARBALL }} - # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR - run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }} + # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR + run: + npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{ + steps.changed.outputs.extra_tests }} diff --git a/e2e-tests/harness-bedrock.test.ts b/e2e-tests/harness-bedrock.test.ts new file mode 100644 index 000000000..7b53e18bb --- /dev/null +++ b/e2e-tests/harness-bedrock.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'bedrock' }); diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts new file mode 100644 index 000000000..3a12b91c2 --- /dev/null +++ b/e2e-tests/harness-e2e-helper.ts @@ -0,0 +1,158 @@ +import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js'; +import { + cleanupStaleCredentialProviders, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const hasAws = hasAwsCredentials(); +const baseCanRun = prereqs.npm && prereqs.git && hasAws; + +interface HarnessE2EConfig { + modelProvider: 'bedrock' | 'open_ai' | 'gemini'; + requiredEnvVar?: string; +} + +export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { + const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar]; + const canRun = baseCanRun && hasRequiredVar; + + const providerLabel = + cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock'; + + describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => { + let testDir: string; + let projectPath: string; + let harnessName: string; + + beforeAll(async () => { + if (!canRun) return; + + await cleanupStaleCredentialProviders(); + + testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4); + harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`; + + const createArgs = [ + 'create', + '--name', + harnessName, + '--model-provider', + cfg.modelProvider, + '--json', + '--skip-git', + ]; + + if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) { + createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!); + } + + const result = await runAgentCoreCLI(createArgs, testDir); + + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { projectPath: string }; + projectPath = json.projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, harnessName, cfg.modelProvider); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + it.skipIf(!canRun)( + 'deploys to AWS successfully', + async () => { + expect(projectPath, 'Project should have been created').toBeTruthy(); + + await retry( + async () => { + const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath); + + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + + expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success, 'Deploy should report success').toBe(true); + }, + 1, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'invokes the deployed harness', + async () => { + expect(projectPath, 'Project should have been created').toBeTruthy(); + + await retry( + async () => { + const result = await runAgentCoreCLI( + ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'], + projectPath + ); + + if (result.exitCode !== 0) { + console.log('Invoke stdout:', result.stdout); + console.log('Invoke stderr:', result.stderr); + } + + expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success, 'Invoke should report success').toBe(true); + }, + 3, + 15000 + ); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'status shows the deployed harness', + async () => { + const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath); + + expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0); + + const json = parseJsonOutput(statusResult.stdout) as { + success: boolean; + resources: { + resourceType: string; + name: string; + deploymentState: string; + identifier?: string; + }[]; + }; + expect(json.success).toBe(true); + + const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName); + expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined(); + expect(harness!.deploymentState).toBe('deployed'); + expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy(); + }, + 120000 + ); + }); +} diff --git a/e2e-tests/harness-gemini.test.ts b/e2e-tests/harness-gemini.test.ts new file mode 100644 index 000000000..adb5bbc24 --- /dev/null +++ b/e2e-tests/harness-gemini.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN' }); diff --git a/e2e-tests/harness-openai.test.ts b/e2e-tests/harness-openai.test.ts new file mode 100644 index 000000000..85076bfbd --- /dev/null +++ b/e2e-tests/harness-openai.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN' }); diff --git a/integ-tests/add-remove-harness.test.ts b/integ-tests/add-remove-harness.test.ts new file mode 100644 index 000000000..95c2ce90c --- /dev/null +++ b/integ-tests/add-remove-harness.test.ts @@ -0,0 +1,208 @@ +import { createTestProject, exists, runCLI } from '../src/test-utils/index.js'; +import type { TestProject } from '../src/test-utils/index.js'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +async function readProjectConfig(projectPath: string) { + return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8')); +} + +async function readHarnessSpec(projectPath: string, harnessName: string) { + return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8')); +} + +describe('integration: harness add/remove lifecycle', () => { + let project: TestProject; + const harnessName = 'TestHarness'; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('adds a harness with defaults', async () => { + const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + const config = await readProjectConfig(project.projectPath); + const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy(); + expect(harness.path).toBe(`app/${harnessName}`); + }); + + it('creates harness.json with correct model config', async () => { + const spec = await readHarnessSpec(project.projectPath, harnessName); + expect(spec.model).toBeDefined(); + expect(spec.model.provider).toBe('bedrock'); + expect(spec.model.modelId).toBeTruthy(); + }); + + it('creates system-prompt.md', async () => { + const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`); + expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true); + }); + + it('auto-creates memory resource', async () => { + const config = await readProjectConfig(project.projectPath); + const memories = config.memories ?? []; + expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0); + }); + + it('rejects duplicate harness name', async () => { + const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('removes the harness', async () => { + const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + const config = await readProjectConfig(project.projectPath); + const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy(); + }); +}); + +describe('integration: harness configuration options', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('adds harness with truncation strategy', async () => { + const name = 'TruncHarness'; + const result = await runCLI( + ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.truncation?.strategy).toBe('sliding_window'); + }); + + it('adds harness with lifecycle config', async () => { + const name = 'LifecycleHarness'; + const result = await runCLI( + ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300); + expect(spec.lifecycleConfig?.maxLifetime).toBe(3600); + }); + + it('adds harness without memory when --no-memory is set', async () => { + const name = 'NoMemHarness'; + const configBefore = await readProjectConfig(project.projectPath); + const memoriesBefore = (configBefore.memories ?? []).length; + + const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const configAfter = await readProjectConfig(project.projectPath); + const memoriesAfter = (configAfter.memories ?? []).length; + expect(memoriesAfter).toBe(memoriesBefore); + }); + + it('adds harness with non-bedrock model provider', async () => { + const name = 'OpenAIHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--model-provider', + 'open_ai', + '--model-id', + 'gpt-5', + '--api-key-arn', + 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.model.provider).toBe('open_ai'); + expect(spec.model.modelId).toBe('gpt-5'); + expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key'); + }); +}); + +describe('integration: harness validation errors', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('rejects invalid harness name with special characters', async () => { + const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('rejects harness name starting with a number', async () => { + const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('rejects add harness without --name when --json is passed', async () => { + const result = await runCLI(['add', 'harness', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); +}); + +describe('integration: create project with harness', () => { + let project: TestProject; + const harnessName = 'CreateHarness'; + + beforeAll(async () => { + project = await createTestProject({ name: harnessName, noAgent: true }); + await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('has correct project scaffolding', async () => { + expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true); + expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true); + expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true); + expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true); + }); + + it('has harness registered in project config', async () => { + const config = await readProjectConfig(project.projectPath); + const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(harness).toBeTruthy(); + }); +}); From 9a572eb272721c771a6f42b1d06d21399b242a91 Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Wed, 29 Apr 2026 13:28:06 -0400 Subject: [PATCH 2/4] skip memory for OpenAI/Gemini E2E harness tests to reduce deploy time --- e2e-tests/harness-e2e-helper.ts | 5 +++++ e2e-tests/harness-gemini.test.ts | 2 +- e2e-tests/harness-openai.test.ts | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts index 3a12b91c2..ca29ae4f3 100644 --- a/e2e-tests/harness-e2e-helper.ts +++ b/e2e-tests/harness-e2e-helper.ts @@ -18,6 +18,7 @@ const baseCanRun = prereqs.npm && prereqs.git && hasAws; interface HarnessE2EConfig { modelProvider: 'bedrock' | 'open_ai' | 'gemini'; requiredEnvVar?: string; + skipMemory?: boolean; } export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { @@ -57,6 +58,10 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!); } + if (cfg.skipMemory) { + createArgs.push('--no-harness-memory'); + } + const result = await runAgentCoreCLI(createArgs, testDir); expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); diff --git a/e2e-tests/harness-gemini.test.ts b/e2e-tests/harness-gemini.test.ts index adb5bbc24..8fd024147 100644 --- a/e2e-tests/harness-gemini.test.ts +++ b/e2e-tests/harness-gemini.test.ts @@ -1,3 +1,3 @@ import { createHarnessE2ESuite } from './harness-e2e-helper.js'; -createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN' }); +createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true }); diff --git a/e2e-tests/harness-openai.test.ts b/e2e-tests/harness-openai.test.ts index 85076bfbd..bdb9c3772 100644 --- a/e2e-tests/harness-openai.test.ts +++ b/e2e-tests/harness-openai.test.ts @@ -1,3 +1,3 @@ import { createHarnessE2ESuite } from './harness-e2e-helper.js'; -createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN' }); +createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true }); From 191f80a643345eaf3a2a97042c0761d703199af8 Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Wed, 29 Apr 2026 15:02:09 -0400 Subject: [PATCH 3/4] fix: use shared readProjectConfig from test-utils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review comment — reuse existing config-reader utility instead of inline helper. --- integ-tests/add-remove-harness.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/integ-tests/add-remove-harness.test.ts b/integ-tests/add-remove-harness.test.ts index 95c2ce90c..e5541accc 100644 --- a/integ-tests/add-remove-harness.test.ts +++ b/integ-tests/add-remove-harness.test.ts @@ -1,13 +1,9 @@ -import { createTestProject, exists, runCLI } from '../src/test-utils/index.js'; +import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -async function readProjectConfig(projectPath: string) { - return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8')); -} - async function readHarnessSpec(projectPath: string, harnessName: string) { return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8')); } From e8bc7bbd6dd62a4ae1ca4201e1c8f8fe38e342c5 Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Wed, 29 Apr 2026 15:13:16 -0400 Subject: [PATCH 4/4] fix: add non-null assertion to fix typecheck failure The Zod-validated return type from readProjectConfig makes the find() result possibly undefined. The preceding expect() guards against it at runtime. --- integ-tests/add-remove-harness.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-tests/add-remove-harness.test.ts b/integ-tests/add-remove-harness.test.ts index e5541accc..2f69270db 100644 --- a/integ-tests/add-remove-harness.test.ts +++ b/integ-tests/add-remove-harness.test.ts @@ -30,7 +30,7 @@ describe('integration: harness add/remove lifecycle', () => { const config = await readProjectConfig(project.projectPath); const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName); expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy(); - expect(harness.path).toBe(`app/${harnessName}`); + expect(harness!.path).toBe(`app/${harnessName}`); }); it('creates harness.json with correct model config', async () => {