feat(tests): add E2E and integration tests for Harness primitive

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
           BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
           CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
             | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+            | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
             | tr '\n' ' ')
           echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
           echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
           OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
           GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
           CDK_TARBALL: ${{ env.CDK_TARBALL }}
-        # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
-        run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+        # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+        run:
+          npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+          steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+  cleanupStaleCredentialProviders,
+  installCdkTarball,
+  runAgentCoreCLI,
+  teardownE2EProject,
+  writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+  modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+  requiredEnvVar?: string;
+  skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+  const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+  const canRun = baseCanRun && hasRequiredVar;
+
+  const providerLabel =
+    cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+  describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+    let testDir: string;
+    let projectPath: string;
+    let harnessName: string;
+
+    beforeAll(async () => {
+      if (!canRun) return;
+
+      await cleanupStaleCredentialProviders();
+
+      testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+      await mkdir(testDir, { recursive: true });
+
+      const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+      harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+      const createArgs = [
+        'create',
+        '--name',
+        harnessName,
+        '--model-provider',
+        cfg.modelProvider,
+        '--json',
+        '--skip-git',
+      ];
+
+      if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+        createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+      }
+
+      if (cfg.skipMemory) {
+        createArgs.push('--no-harness-memory');
+      }
+
+      const result = await runAgentCoreCLI(createArgs, testDir);
+
+      expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+      const json = parseJsonOutput(result.stdout) as { projectPath: string };
+      projectPath = json.projectPath;
+
+      await writeAwsTargets(projectPath);
+      installCdkTarball(projectPath);
+    }, 300000);
+
+    afterAll(async () => {
+      if (projectPath && hasAws) {
+        await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+      }
+      if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+    }, 600000);
+
+    it.skipIf(!canRun)(
+      'deploys to AWS successfully',
+      async () => {
+        expect(projectPath, 'Project should have been created').toBeTruthy();
+
+        await retry(
+          async () => {
+            const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+            if (result.exitCode !== 0) {
+              console.log('Deploy stdout:', result.stdout);
+              console.log('Deploy stderr:', result.stderr);
+            }
+
+            expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+            const json = parseJsonOutput(result.stdout) as { success: boolean };
+            expect(json.success, 'Deploy should report success').toBe(true);
+          },
+          1,
+          30000
+        );
+      },
+      600000
+    );
+
+    it.skipIf(!canRun)(
+      'invokes the deployed harness',
+      async () => {
+        expect(projectPath, 'Project should have been created').toBeTruthy();
+
+        await retry(
+          async () => {
+            const result = await runAgentCoreCLI(
+              ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+              projectPath
+            );
+
+            if (result.exitCode !== 0) {
+              console.log('Invoke stdout:', result.stdout);
+              console.log('Invoke stderr:', result.stderr);
+            }
+
+            expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+            const json = parseJsonOutput(result.stdout) as { success: boolean };
+            expect(json.success, 'Invoke should report success').toBe(true);
+          },
+          3,
+          15000
+        );
+      },
+      180000
+    );
+
+    it.skipIf(!canRun)(
+      'status shows the deployed harness',
+      async () => {
+        const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+        expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+        const json = parseJsonOutput(statusResult.stdout) as {
+          success: boolean;
+          resources: {
+            resourceType: string;
+            name: string;
+            deploymentState: string;
+            identifier?: string;
+          }[];
+        };
+        expect(json.success).toBe(true);
+
+        const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+        expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+        expect(harness!.deploymentState).toBe('deployed');
+        expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+      },
+      120000
+    );
+  });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+  return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+  let project: TestProject;
+  const harnessName = 'TestHarness';
+
+  beforeAll(async () => {
+    project = await createTestProject({ noAgent: true });
+  });
+
+  afterAll(async () => {
+    await project.cleanup();
+  });
+
+  it('adds a harness with defaults', async () => {
+    const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+    expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+    const json = JSON.parse(result.stdout);
+    expect(json.success).toBe(true);
+
+    const config = await readProjectConfig(project.projectPath);
+    const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+    expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+    expect(harness!.path).toBe(`app/${harnessName}`);
+  });
+
+  it('creates harness.json with correct model config', async () => {
+    const spec = await readHarnessSpec(project.projectPath, harnessName);
+    expect(spec.model).toBeDefined();
+    expect(spec.model.provider).toBe('bedrock');
+    expect(spec.model.modelId).toBeTruthy();
+  });
+
+  it('creates system-prompt.md', async () => {
+    const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+    expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+  });
+
+  it('auto-creates memory resource', async () => {
+    const config = await readProjectConfig(project.projectPath);
+    const memories = config.memories ?? [];
+    expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+  });
+
+  it('rejects duplicate harness name', async () => {
+    const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+    expect(result.exitCode).not.toBe(0);
+  });
+
+  it('removes the harness', async () => {
+    const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+    expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+    const json = JSON.parse(result.stdout);
+    expect(json.success).toBe(true);
+
+    const config = await readProjectConfig(project.projectPath);
+    const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+    expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+  });
+});
+
+describe('integration: harness configuration options', () => {
+  let project: TestProject;
+
+  beforeAll(async () => {
+    project = await createTestProject({ noAgent: true });
+  });
+
+  afterAll(async () => {
+    await project.cleanup();
+  });
+
+  it('adds harness with truncation strategy', async () => {
+    const name = 'TruncHarness';
+    const result = await runCLI(
+      ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+      project.projectPath
+    );
+
+    expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+    const spec = await readHarnessSpec(project.projectPath, name);
+    expect(spec.truncation?.strategy).toBe('sliding_window');
+  });
+
+  it('adds harness with lifecycle config', async () => {
+    const name = 'LifecycleHarness';
+    const result = await runCLI(
+      ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+      project.projectPath
+    );
+
+    expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+    const spec = await readHarnessSpec(project.projectPath, name);
+    expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+    expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+  });
+
+  it('adds harness without memory when --no-memory is set', async () => {
+    const name = 'NoMemHarness';
+    const configBefore = await readProjectConfig(project.projectPath);
+    const memoriesBefore = (configBefore.memories ?? []).length;
+
+    const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+    expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+    const configAfter = await readProjectConfig(project.projectPath);
+    const memoriesAfter = (configAfter.memories ?? []).length;
+    expect(memoriesAfter).toBe(memoriesBefore);
+  });
+
+  it('adds harness with non-bedrock model provider', async () => {
+    const name = 'OpenAIHarness';
+    const result = await runCLI(
+      [
+        'add',
+        'harness',
+        '--name',
+        name,
+        '--model-provider',
+        'open_ai',
+        '--model-id',
+        'gpt-5',
+        '--api-key-arn',
+        'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+        '--json',
+      ],
+      project.projectPath
+    );
+
+    expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+    const spec = await readHarnessSpec(project.projectPath, name);
+    expect(spec.model.provider).toBe('open_ai');
+    expect(spec.model.modelId).toBe('gpt-5');
+    expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+  });
+});
+
+describe('integration: harness validation errors', () => {
+  let project: TestProject;
+
+  beforeAll(async () => {
+    project = await createTestProject({ noAgent: true });
+  });
+
+  afterAll(async () => {
+    await project.cleanup();
+  });
+
+  it('rejects invalid harness name with special characters', async () => {
+    const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+    expect(result.exitCode).not.toBe(0);
+  });
+
+  it('rejects harness name starting with a number', async () => {
+    const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+    expect(result.exitCode).not.toBe(0);
+  });
+
+  it('rejects add harness without --name when --json is passed', async () => {
+    const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+    expect(result.exitCode).not.toBe(0);
+  });
+});
+
+describe('integration: create project with harness', () => {
+  let project: TestProject;
+  const harnessName = 'CreateHarness';
+
+  beforeAll(async () => {
+    project = await createTestProject({ name: harnessName, noAgent: true });
+    await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+  });
+
+  afterAll(async () => {
+    await project.cleanup();
+  });
+
+  it('has correct project scaffolding', async () => {
+    expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+    expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+    expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+    expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+  });
+
+  it('has harness registered in project config', async () => {
+    const config = await readProjectConfig(project.projectPath);
+    const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+    expect(harness).toBeTruthy();
+  });
+});