Fix Design Decision Gate false failures caused by the LLM invocation cap

.github/workflows/design-decision-gate.md

@@ -30,7 +30,7 @@ permissions:
   contents: read
   pull-requests: read
   issues: read
-max-turns: 20
+max-turns: 30
 model: claude-sonnet-4-6
 engine:
   id: claude

actions/setup/js/claude_harness.cjs

@@ -540,6 +540,10 @@ async function main() {
           log(`attempt ${attempt + 1}: AI credits budget enforced — exiting 0 (budget control, not an error)`);
           return { action: "stop", exitCode: 0 };
         }
+        if (nonRetryableGuard.maxRunsExceeded && safeOutputsPath && hasExpectedSafeOutputs(safeOutputsPath, { logger: log })) {
+          log(`attempt ${attempt + 1}: invocation cap saturated but safe-outputs already contain expected output — suppressing terminal verdict (false-red: core work succeeded)`);
+          return { action: "stop", exitCode: 0 };
+        }
         return { action: "stop" };
       }
 

actions/setup/js/claude_harness.test.cjs

@@ -691,6 +691,29 @@ process.exit(1);
       expect(result.stderr).toContain("maximum LLM invocations exceeded — not retrying");
     });
 
+    it("exits 0 without retrying when max invocations are exceeded but safe-outputs already contain the expected result", () => {
+      const tempDir = makeHarnessTempDir("claude-harness-");
+      const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl");
+      fs.writeFileSync(safeOutputsPath, '{"type":"add_comment","body":"ADR reviewed"}\n', "utf8");
+      const stubScript = `
+const fs = require("fs");
+const callsPath = process.env.CLAUDE_HARNESS_STUB_CALLS;
+const args = process.argv.slice(2);
+const priorCalls = fs.existsSync(callsPath) ? fs.readFileSync(callsPath, "utf8").trim().split("\\n").filter(Boolean).length : 0;
+fs.appendFileSync(callsPath, JSON.stringify({ args }) + "\\n", "utf8");
+if (priorCalls > 0) {
+  process.stderr.write("unexpected retry after max_runs_exceeded\\n");
+  process.exit(9);
+}
+process.stderr.write('{"error":{"type":"max_runs_exceeded","message":"Maximum LLM invocations exceeded (20 / 20)."}}\\n');
+process.exit(1);
+`;
+      const { result, calls } = runHarnessWithStub({ stubScript, extraEnv: { GH_AW_SAFE_OUTPUTS: safeOutputsPath } });
+      expect(result.status).toBe(0);
+      expect(calls.length).toBe(1);
+      expect(result.stderr).toContain("invocation cap saturated but safe-outputs already contain expected output");
+    });
+
     it("returns true for normal partial-execution retry", () => {
       const result = shouldRetryWithContinue({
         attempt: 0,

actions/setup/js/codex_harness.cjs

@@ -654,6 +654,10 @@ async function main() {
           log(`attempt ${attempt + 1}: AI credits budget enforced — exiting 0 (budget control, not an error)`);
           return { action: "stop", exitCode: 0 };
         }
+        if (nonRetryableGuard.maxRunsExceeded && safeOutputsPath && hasExpectedSafeOutputs(safeOutputsPath, { logger: log })) {
+          log(`attempt ${attempt + 1}: invocation cap saturated but safe-outputs already contain expected output — suppressing terminal verdict (false-red: core work succeeded)`);
+          return { action: "stop", exitCode: 0 };
+        }
         return { action: "stop" };
       }
 

actions/setup/js/codex_harness.test.cjs

@@ -675,6 +675,40 @@ process.exit(1);`,
       expect(result.status).toBe(0);
       expect(result.stderr).toContain("noop message found in safe-outputs — not retrying");
     });
+
+    it("exits 0 without retrying when the LLM invocation cap is saturated but the expected safe-output was already produced", () => {
+      const tempDir = makeHarnessTempDir("codex-invocation-cap-suppression-");
+      const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl");
+      const stubPath = path.join(tempDir, "stub.cjs");
+      const promptPath = path.join(tempDir, "prompt.txt");
+      const callsPath = path.join(tempDir, "calls.jsonl");
+      // Stub writes an expected safe-output then fails with the pooled invocation-cap error.
+      fs.writeFileSync(
+        stubPath,
+        `const fs = require("fs");
+const callsPath = process.env.CODEX_HARNESS_STUB_CALLS;
+const safeOutputsPath = process.env.GH_AW_SAFE_OUTPUTS;
+fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n");
+fs.appendFileSync(safeOutputsPath, JSON.stringify({type:"add_comment",body:"ADR reviewed"}) + "\\n");
+process.stderr.write('{"error":{"type":"max_runs_exceeded","message":"Maximum LLM invocations exceeded (20 / 20)."}}\\n');
+process.exit(1);`,
+        "utf8"
+      );
+      fs.writeFileSync(promptPath, "fix the bug", "utf8");
+
+      const result = spawnSync(process.execPath, ["codex_harness.cjs", process.execPath, stubPath, "exec", "--prompt-file", promptPath], {
+        cwd: path.dirname(require.resolve("./codex_harness.cjs")),
+        env: { ...process.env, CODEX_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, CODEX_API_KEY: "fake-key-for-test" },
+        encoding: "utf8",
+        timeout: 10000,
+      });
+      const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length;
+      // Only one attempt — invocation cap exhaustion is never retried
+      expect(callCount).toBe(1);
+      // Harness exits 0 because the core work (add_comment) already succeeded
+      expect(result.status).toBe(0);
+      expect(result.stderr).toContain("invocation cap saturated but safe-outputs already contain expected output");
+    });
   });
 
   describe("post-result watchdog suppression when terminal safe-output already produced", () => {

actions/setup/js/copilot_harness.cjs

@@ -1277,6 +1277,10 @@ async function main() {
               log(`attempt ${attempt + 1}: AI credits budget enforced — exiting 0 (budget control, not an error)`);
               return { action: "stop", exitCode: 0 };
             }
+            if (isInvocationCapExceeded && safeOutputsPath && hasTerminalSafeOutput(safeOutputsPath)) {
+              log(`attempt ${attempt + 1}: invocation cap saturated but safe-outputs already contain expected output — suppressing terminal verdict (false-red: core work succeeded)`);
+              return { action: "stop", exitCode: 0 };
+            }
             return { action: "stop" };
           }
 

actions/setup/js/copilot_harness.test.cjs

@@ -2498,6 +2498,40 @@ process.exit(1);`,
       expect(result.status).toBe(1);
       expect(result.stderr).toContain("detected numerous permission-denied issues — not retrying");
     });
+
+    it("exits 0 without retrying when the LLM invocation cap is saturated but a terminal safe-output was already produced", () => {
+      const tempDir = makeHarnessTempDir("copilot-invocation-cap-suppression-");
+      const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl");
+      const stubPath = path.join(tempDir, "stub.cjs");
+      const promptPath = path.join(tempDir, "prompt.txt");
+      const callsPath = path.join(tempDir, "calls.jsonl");
+      // Stub writes an expected safe-output then fails with the pooled invocation-cap error.
+      fs.writeFileSync(
+        stubPath,
+        `const fs = require("fs");
+const callsPath = process.env.COPILOT_HARNESS_STUB_CALLS;
+const safeOutputsPath = process.env.GH_AW_SAFE_OUTPUTS;
+fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n");
+fs.appendFileSync(safeOutputsPath, JSON.stringify({type:"add_comment",body:"ADR reviewed"}) + "\\n");
+process.stdout.write("Execution failed: CAPIError: 429 Maximum LLM invocations exceeded (20/20)\\n");
+process.exit(1);`,
+        "utf8"
+      );
+      fs.writeFileSync(promptPath, "fix the bug", "utf8");
+
+      const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, stubPath, "--prompt-file", promptPath], {
+        cwd: path.dirname(require.resolve("./copilot_harness.cjs")),
+        env: { ...process.env, COPILOT_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath },
+        encoding: "utf8",
+        timeout: 15000,
+      });
+      const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length;
+      // Only one attempt — invocation cap exhaustion is never retried
+      expect(callCount).toBe(1);
+      // Harness exits 0 because the core work (add_comment) already succeeded
+      expect(result.status).toBe(0);
+      expect(result.stderr).toContain("invocation cap saturated but safe-outputs already contain expected output");
+    });
   });
 
   describe("post-result watchdog suppression when terminal safe-output already produced", () => {