test: per-directory coverage + E2E cleanup hardening (Days 5-6)

.github/workflows/build-and-test.yml

@@ -134,16 +134,29 @@ jobs:
 
     steps:
       - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 20.x
+          cache: 'npm'
       - name: Download coverage artifact
         uses: actions/download-artifact@v8
         with:
           name: coverage-report
           path: coverage/
-      - name: Coverage Report
+      - name: Coverage Report (PR comment)
         uses: davelosert/vitest-coverage-report-action@v2
         with:
           json-summary-path: coverage/coverage-summary.json
           json-final-path: coverage/coverage-final.json
           vite-config-path: vitest.config.ts
-          file-coverage-mode: none
-          coverage-thresholds: '{ "lines": 50, "branches": 50, "functions": 50, "statements": 50 }'
+          file-coverage-mode: changes
+      - name: Per-directory coverage gate
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+        run: node scripts/check-coverage.mjs

e2e-tests/ab-test-config-bundle.test.ts

@@ -54,10 +54,13 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
   }, 300000);
 
   afterAll(async () => {
-    if (projectPath && hasAws) {
-      await teardownE2EProject(projectPath, agentName, 'Bedrock');
+    try {
+      if (projectPath && hasAws) {
+        await teardownE2EProject(projectPath, agentName, 'Bedrock');
+      }
+    } finally {
+      if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
     }
-    if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
   }, 600000);
 
   const run = (args: string[]) => runAgentCoreCLI(args, projectPath);

e2e-tests/ab-test-target-based.test.ts

@@ -55,10 +55,13 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
   }, 300000);
 
   afterAll(async () => {
-    if (projectPath && hasAws) {
-      await teardownE2EProject(projectPath, agentName, 'Bedrock');
+    try {
+      if (projectPath && hasAws) {
+        await teardownE2EProject(projectPath, agentName, 'Bedrock');
+      }
+    } finally {
+      if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
     }
-    if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
   }, 600000);
 
   const run = (args: string[]) => runAgentCoreCLI(args, projectPath);

e2e-tests/archive-lifecycle.test.ts

@@ -67,10 +67,13 @@ describe.sequential('e2e: archive command lifecycle', () => {
   }, 300000);
 
   afterAll(async () => {
-    if (projectPath && hasAws) {
-      await teardownE2EProject(projectPath, agentName, 'Bedrock');
+    try {
+      if (projectPath && hasAws) {
+        await teardownE2EProject(projectPath, agentName, 'Bedrock');
+      }
+    } finally {
+      if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
     }
-    if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
   }, 600000);
 
   const run = (args: string[]) => runAgentCoreCLI(args, projectPath);

e2e-tests/byo-custom-jwt.test.ts

@@ -48,7 +48,36 @@ const region = process.env.AWS_REGION ?? 'us-east-1';
  * Run the local CLI build without skipping install (needed for deploy).
  */
 function runLocalCLI(args: string[], cwd: string): Promise<RunResult> {
-  return runCLI(args, cwd, { skipInstall: false });
+  return runCLI(args, cwd, {
+    skipInstall: false,
+    env: {
+      AGENTCORE_E2E_TEST: '1',
+      AGENTCORE_E2E_CREATOR: process.env.AGENTCORE_E2E_CREATOR ?? 'github-actions',
+    },
+  });
+}
+
+async function deleteCognitoResourceWithRetry(
+  label: string,
+  op: () => Promise<unknown>,
+  attempts = 3,
+  delayMs = 5000
+): Promise<void> {
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      await op();
+      return;
+    } catch (err) {
+      const name = (err as { name?: string }).name ?? 'Unknown';
+      const msg = (err as { message?: string }).message ?? String(err);
+      if (attempt === attempts) {
+        console.error(`[cognito-cleanup] ${label} failed after ${attempts} attempts: [${name}] ${msg}`);
+        return;
+      }
+      console.warn(`[cognito-cleanup] ${label} attempt ${attempt}/${attempts} failed: [${name}] — retrying`);
+      await new Promise(resolve => setTimeout(resolve, delayMs));
+    }
+  }
 }
 
 describe.sequential('e2e: BYO agent with CUSTOM_JWT auth', () => {
@@ -201,21 +230,15 @@ describe.sequential('e2e: BYO agent with CUSTOM_JWT auth', () => {
 
     // ── Delete Cognito resources ──
     if (userPoolId) {
-      try {
-        await cognitoClient.send(new DeleteResourceServerCommand({ UserPoolId: userPoolId, Identifier: 'agentcore' }));
-      } catch {
-        /* best-effort */
-      }
-      try {
-        await cognitoClient.send(new DeleteUserPoolDomainCommand({ UserPoolId: userPoolId, Domain: domainPrefix }));
-      } catch {
-        /* best-effort */
-      }
-      try {
-        await cognitoClient.send(new DeleteUserPoolCommand({ UserPoolId: userPoolId }));
-      } catch {
-        /* best-effort */
-      }
+      await deleteCognitoResourceWithRetry('DeleteResourceServer', () =>
+        cognitoClient.send(new DeleteResourceServerCommand({ UserPoolId: userPoolId, Identifier: 'agentcore' }))
+      );
+      await deleteCognitoResourceWithRetry('DeleteUserPoolDomain', () =>
+        cognitoClient.send(new DeleteUserPoolDomainCommand({ UserPoolId: userPoolId, Domain: domainPrefix }))
+      );
+      await deleteCognitoResourceWithRetry('DeleteUserPool', () =>
+        cognitoClient.send(new DeleteUserPoolCommand({ UserPoolId: userPoolId }))
+      );
     }
 
     // ── Clean up temp directory ──

e2e-tests/config-bundle-eval-rec.test.ts

@@ -64,10 +64,13 @@ describe.sequential('e2e: config bundles, batch evaluation, and recommendations'
   }, 300000);
 
   afterAll(async () => {
-    if (projectPath && hasAws) {
-      await teardownE2EProject(projectPath, agentName, 'Bedrock');
+    try {
+      if (projectPath && hasAws) {
+        await teardownE2EProject(projectPath, agentName, 'Bedrock');
+      }
+    } finally {
+      if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
     }
-    if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
   }, 600000);
 
   const run = (args: string[]) => runAgentCoreCLI(args, projectPath);

e2e-tests/e2e-helper.ts

@@ -96,10 +96,13 @@ export function createE2ESuite(cfg: E2EConfig) {
     }, 300000);
 
     afterAll(async () => {
-      if (projectPath && hasAws) {
-        await teardownE2EProject(projectPath, agentName, cfg.modelProvider);
+      try {
+        if (projectPath && hasAws) {
+          await teardownE2EProject(projectPath, agentName, cfg.modelProvider);
+        }
+      } finally {
+        if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
       }
-      if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
     }, 600000);
 
     // Container builds go through CodeBuild which is slower and more prone to transient failures.
@@ -308,7 +311,10 @@ export function createE2ESuite(cfg: E2EConfig) {
 export { hasAws, baseCanRun };
 
 export function runAgentCoreCLI(args: string[], cwd: string): Promise<RunResult> {
-  return spawnAndCollect('agentcore', args, cwd);
+  return spawnAndCollect('agentcore', args, cwd, {
+    AGENTCORE_E2E_TEST: '1',
+    AGENTCORE_E2E_CREATOR: process.env.AGENTCORE_E2E_CREATOR ?? 'github-actions',
+  });
 }
 
 // TODO: Replace with `agentcore add target` once the CLI command is re-introduced
@@ -347,7 +353,9 @@ async function deleteCredentialProvider(client: BedrockAgentCoreControlClient, n
  * Runs in beforeAll to prevent accumulation from previous runs that
  * crashed or timed out before their afterAll teardown could execute.
  */
-export async function cleanupStaleCredentialProviders(maxAgeMs: number = 30 * 60 * 1000): Promise<void> {
+export async function cleanupStaleCredentialProviders(
+  maxAgeMs: number = parseInt(process.env.E2E_STALE_CRED_MAX_AGE_MS ?? '', 10) || 30 * 60 * 1000
+): Promise<void> {
   const region = process.env.AWS_REGION ?? 'us-east-1';
   const client = new BedrockAgentCoreControlClient({ region });
   const cutoff = new Date(Date.now() - maxAgeMs);
@@ -365,17 +373,45 @@ export async function cleanupStaleCredentialProviders(maxAgeMs: number = 30 * 60
 }
 
 export async function teardownE2EProject(projectPath: string, agentName: string, modelProvider: string): Promise<void> {
-  await spawnAndCollect('agentcore', ['remove', 'all', '--json'], projectPath);
-  const result = await spawnAndCollect('agentcore', ['deploy', '--yes', '--json'], projectPath);
-  if (result.exitCode !== 0) {
-    console.log('Teardown stdout:', result.stdout);
-    console.log('Teardown stderr:', result.stderr);
+  const failures: string[] = [];
+
+  const removeResult = await runAgentCoreCLI(['remove', 'all', '--json'], projectPath);
+  if (removeResult.exitCode !== 0) {
+    console.error(`[teardown] remove all failed (exit ${removeResult.exitCode})`);
+    console.error('[teardown] remove stdout:', removeResult.stdout);
+    console.error('[teardown] remove stderr:', removeResult.stderr);
+    failures.push(`remove all: exit ${removeResult.exitCode}`);
+  }
+
+  const MAX_DEPLOY_ATTEMPTS = 3;
+  let deploySucceeded = false;
+  for (let attempt = 1; attempt <= MAX_DEPLOY_ATTEMPTS; attempt++) {
+    const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+    if (result.exitCode === 0) {
+      deploySucceeded = true;
+      if (attempt > 1) console.error(`[teardown] deploy succeeded on attempt ${attempt}`);
+      break;
+    }
+    console.error(`[teardown] deploy attempt ${attempt}/${MAX_DEPLOY_ATTEMPTS} failed (exit ${result.exitCode})`);
+    console.error('[teardown] deploy stdout:', result.stdout);
+    console.error('[teardown] deploy stderr:', result.stderr);
+    if (attempt < MAX_DEPLOY_ATTEMPTS) {
+      await new Promise(resolve => setTimeout(resolve, 15000));
+    }
+  }
+  if (!deploySucceeded) {
+    failures.push(`deploy teardown failed after ${MAX_DEPLOY_ATTEMPTS} attempts`);
   }
+
   if (modelProvider !== 'Bedrock' && agentName) {
     const region = process.env.AWS_REGION ?? 'us-east-1';
     const client = new BedrockAgentCoreControlClient({ region });
     await deleteCredentialProvider(client, `${agentName}${modelProvider}`);
   }
+
+  if (failures.length > 0) {
+    throw new Error(`E2E teardown failed: ${failures.join('; ')}`);
+  }
 }
 
 export async function dumpImportDebugInfo(

e2e-tests/evals-lifecycle.test.ts

@@ -53,10 +53,13 @@ describe.sequential('e2e: evaluations lifecycle', () => {
   }, 300000);
 
   afterAll(async () => {
-    if (projectPath && hasAws) {
-      await teardownE2EProject(projectPath, agentName, 'Bedrock');
+    try {
+      if (projectPath && hasAws) {
+        await teardownE2EProject(projectPath, agentName, 'Bedrock');
+      }
+    } finally {
+      if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
     }
-    if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
   }, 600000);
 
   const run = (args: string[]) => runAgentCoreCLI(args, projectPath);