Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions actions/setup/js/claude_harness.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ const {
const { emitMissingToolPermissionIssue, hasExpectedSafeOutputs, hasNoopInSafeOutputs } = require("./safeoutputs_cli.cjs");
const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, buildMissingToolPermissionIssuePayload } = require("./permission_denied_helpers.cjs");
const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError } = require("./harness_retry_guard.cjs");
const { isCrashSignalExitCode, crashSignalNameForExitCode } = require("./harness_crash_signals.cjs");
const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs");
const { applyModelFallback } = require("./model_fallback.cjs");
const { parseMaxAICreditsExceededFromAuditLog } = require("./ai_credits_context.cjs");
Expand Down Expand Up @@ -260,7 +261,7 @@ function shouldRetryWithContinue({ attempt, maxRetries, exitCode, hasOutput, isN
if (attempt >= maxRetries || !hasOutput || continueDisabledPermanently) {
return false;
}
if (isSignalTerminationExitCode(exitCode)) {
if (isSignalTerminationExitCode(exitCode) || isCrashSignalExitCode(exitCode)) {
return false;
}
if (isNoDeferredMarker) {
Expand Down Expand Up @@ -502,9 +503,11 @@ async function main() {
sessionHasProgress = sessionHasProgress || hasClaudeSessionProgress(result.output);
const permissionDeniedCount = countPermissionDeniedIssues(result.output);
const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output);
const crashSignalName = crashSignalNameForExitCode(result.exitCode);
log(
`attempt ${attempt + 1} failed:` +
` exitCode=${result.exitCode}` +
(crashSignalName ? ` crashSignal=${crashSignalName}` : "") +
` isOverloadedError=${isOverloaded}` +
` isRateLimitError=${isRateLimit}` +
` isAuthenticationFailedError=${isAuthenticationFailed}` +
Expand Down Expand Up @@ -605,6 +608,8 @@ async function main() {

if (attempt < maxRetries && result.hasOutput) {
const isSignalTermination = isSignalTerminationExitCode(result.exitCode);
const isCrashSignal = isCrashSignalExitCode(result.exitCode);
const crashSignalName = crashSignalNameForExitCode(result.exitCode);
const retryWithContinue = shouldRetryWithContinue({
attempt,
maxRetries,
Expand All @@ -613,16 +618,18 @@ async function main() {
isNoDeferredMarker,
continueDisabledPermanently,
});
if (isSignalTermination) {
if (isSignalTermination || isCrashSignal) {
continueDisabledPermanently = true;
}
const reason = isSignalTermination
? `signal-style termination exitCode=${result.exitCode} (failure_reason=cancelled_or_timed_out)`
: isOverloaded
? "overloaded_error (transient)"
: isRateLimit
? "rate_limit_error (transient)"
: "partial execution";
const reason = isCrashSignal
? `fatal-signal crash exitCode=${result.exitCode} (signal=${crashSignalName}, failure_reason=sandbox_runtime_crash)`
: isSignalTermination
? `signal-style termination exitCode=${result.exitCode} (failure_reason=cancelled_or_timed_out)`
: isOverloaded
? "overloaded_error (transient)"
: isRateLimit
? "rate_limit_error (transient)"
: "partial execution";
useContinueOnRetry = retryWithContinue;
const retryMode = retryWithContinue ? "--continue" : "fresh run (--continue disabled permanently)";
log(`attempt ${attempt + 1}: ${reason} — will retry with ${retryMode} (attempt ${attempt + 2}/${maxRetries + 1})`);
Expand Down Expand Up @@ -670,6 +677,8 @@ if (typeof module !== "undefined" && module.exports) {
isConnectionRefusedError,
hasClaudeSessionProgress,
isSignalTerminationExitCode,
isCrashSignalExitCode,
crashSignalNameForExitCode,
shouldRetryWithContinue,
countPermissionDeniedIssues,
hasNumerousPermissionDeniedIssues,
Expand Down
41 changes: 41 additions & 0 deletions actions/setup/js/claude_harness.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ const {
isConnectionRefusedError,
hasClaudeSessionProgress,
isSignalTerminationExitCode,
isCrashSignalExitCode,
crashSignalNameForExitCode,
shouldRetryWithContinue,
countPermissionDeniedIssues,
hasNumerousPermissionDeniedIssues,
Expand Down Expand Up @@ -336,6 +338,31 @@ describe("claude_harness.cjs", () => {
});
});

describe("isCrashSignalExitCode / crashSignalNameForExitCode", () => {
it("identifies known fatal-signal crash exit codes", () => {
expect(isCrashSignalExitCode(139)).toBe(true); // SIGSEGV
expect(isCrashSignalExitCode(159)).toBe(true); // SIGSYS
expect(isCrashSignalExitCode(134)).toBe(true); // SIGABRT
});

it("returns false for non-crash exit codes, including timeout/cancellation signals", () => {
expect(isCrashSignalExitCode(0)).toBe(false);
expect(isCrashSignalExitCode(1)).toBe(false);
expect(isCrashSignalExitCode(137)).toBe(false);
expect(isCrashSignalExitCode(143)).toBe(false);
});

it("maps known crash exit codes to their signal name", () => {
expect(crashSignalNameForExitCode(139)).toBe("SIGSEGV");
expect(crashSignalNameForExitCode(159)).toBe("SIGSYS");
});

it("returns null for exit codes that are not recognized crash signals", () => {
expect(crashSignalNameForExitCode(1)).toBeNull();
expect(crashSignalNameForExitCode(137)).toBeNull();
});
});

describe("permission-denied classification helpers", () => {
it("counts repeated permission-denied signals", () => {
const output = "permission denied\nEACCES: permission denied\npermissions denied";
Expand Down Expand Up @@ -421,6 +448,20 @@ describe("claude_harness.cjs", () => {
}
});

it("does not use --continue for fatal-signal crash exit codes", () => {
for (const exitCode of [134, 139, 159]) {
const result = shouldRetryWithContinue({
attempt: 0,
maxRetries: 3,
exitCode,
hasOutput: true,
isNoDeferredMarker: false,
continueDisabledPermanently: false,
});
expect(result).toBe(false);
}
});

it("uses a fresh retry after a --continue attempt hits no-deferred-marker", () => {
const stubScript = `
const fs = require("fs");
Expand Down
16 changes: 15 additions & 1 deletion actions/setup/js/copilot_harness.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
* history and permanently disables `--continue` for the remainder of the run so the corrupt
* state can never be reloaded. Once `--continue` is disabled this way it is not re-enabled
* even if later retries produce output.
* - Exit codes that indicate the CLI subprocess was killed by a fatal OS-level signal
* (SIGILL/SIGABRT/SIGBUS/SIGFPE/SIGSEGV/SIGSYS — see harness_crash_signals.cjs) are treated
* like the null-type tool_call case: `--continue` is permanently disabled and the next retry
* starts a fresh session, since resuming the exact on-disk session risks immediately
* reproducing the same crash.
* - Retries use exponential backoff: 5s → 10s → 20s (capped at 60s) by default.
* - Maximum 3 retry attempts after the initial run by default.
*
Expand Down Expand Up @@ -75,6 +80,7 @@ const {
const { runSafeOutputsCLI, buildMissingToolAlternatives, emitMissingToolPermissionIssue, emitInfrastructureIncomplete, hasExpectedSafeOutputs, hasNoopInSafeOutputs } = require("./safeoutputs_cli.cjs");
const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, buildMissingToolPermissionIssuePayload } = require("./permission_denied_helpers.cjs");
const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError: isCommonAuthenticationFailedError } = require("./harness_retry_guard.cjs");
const { isCrashSignalExitCode, crashSignalNameForExitCode } = require("./harness_crash_signals.cjs");
const { isCAPIQuotaExceededError } = require("./detect_agent_errors.cjs");
const { applyModelFallback } = require("./model_fallback.cjs");
const { loadModelsJson } = require("./model_costs.cjs");
Expand Down Expand Up @@ -1386,10 +1392,16 @@ async function main() {

if (shouldRetryFailedExecution({ ...result, attempt, maxRetries })) {
const reason = isCAPIError ? "CAPIError 400 (transient)" : "partial execution";
const isCrashSignal = isCrashSignalExitCode(result.exitCode);
const crashSignalName = crashSignalNameForExitCode(result.exitCode);
if (isCrashSignal) {
continueDisabledPermanently = true;
}
// --continue is only meaningful in CLI mode; SDK mode always restarts fresh.
useContinueOnRetry = !copilotSDKMode && !continueDisabledPermanently;
const retryMode = useContinueOnRetry ? "--continue" : copilotSDKMode ? "fresh run" : "fresh run (--continue permanently disabled)";
log(`attempt ${attempt + 1}: ${reason} — will retry with ${retryMode} (attempt ${attempt + 2}/${maxRetries + 1})`);
const crashSuffix = isCrashSignal ? ` crashSignal=${crashSignalName}` : "";
log(`attempt ${attempt + 1}: ${reason} — will retry with ${retryMode} (attempt ${attempt + 2}/${maxRetries + 1})${crashSuffix}`);
return { action: "retry" };
}

Expand Down Expand Up @@ -1465,6 +1477,8 @@ if (typeof module !== "undefined" && module.exports) {
classifyCopilotFailure,
extractTokenCountFromOutput,
shouldRetryFailedExecution,
isCrashSignalExitCode,
crashSignalNameForExitCode,
extractOutputTail,
isRetryableProxyAuthenticationFailure,
hasNumerousPermissionDeniedIssues,
Expand Down
72 changes: 72 additions & 0 deletions actions/setup/js/copilot_harness.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ const {
resolvePromptFileArgs,
resolveRetryConfig,
shouldRetryFailedExecution,
isCrashSignalExitCode,
crashSignalNameForExitCode,
writeCopilotOutputs,
parseCopilotSDKServerArgsFromEnv,
applyCopilotWireAPI,
Expand Down Expand Up @@ -1776,6 +1778,76 @@ describe("copilot_harness.cjs", () => {
});
});

describe("fatal-signal crash exit codes disable --continue", () => {
it("recognizes known fatal-signal exit codes", () => {
expect(isCrashSignalExitCode(134)).toBe(true); // SIGABRT
expect(isCrashSignalExitCode(139)).toBe(true); // SIGSEGV
expect(isCrashSignalExitCode(159)).toBe(true); // SIGSYS
expect(crashSignalNameForExitCode(134)).toBe("SIGABRT");
expect(crashSignalNameForExitCode(139)).toBe("SIGSEGV");
expect(crashSignalNameForExitCode(159)).toBe("SIGSYS");
});

it("does not classify normal exit codes or timeout/cancellation signals as crashes", () => {
expect(isCrashSignalExitCode(1)).toBe(false);
expect(isCrashSignalExitCode(137)).toBe(false); // SIGKILL — timeout/cancellation, not a crash
expect(isCrashSignalExitCode(143)).toBe(false); // SIGTERM — timeout/cancellation, not a crash
expect(crashSignalNameForExitCode(1)).toBeNull();
expect(crashSignalNameForExitCode(137)).toBeNull();
});

// Inline the same retry logic as the driver's shouldRetryFailedExecution handler,
// including the crash-signal guard: a fatal-signal crash (SIGSEGV, SIGSYS, ...)
// must never be retried with --continue since resuming the on-disk session risks
// immediately reproducing the crash.
const MAX_RETRIES = 3;

/**
* @param {{hasOutput: boolean, exitCode: number}} result
* @param {number} attempt
* @param {boolean} continueDisabledPermanently
* @returns {{ shouldRetry: boolean, useContinueOnRetry: boolean, continueDisabledPermanently: boolean }}
*/
function applyRetryPolicy(result, attempt, continueDisabledPermanently = false) {
if (result.exitCode === 0) return { shouldRetry: false, useContinueOnRetry: false, continueDisabledPermanently };
if (!(attempt < MAX_RETRIES && result.hasOutput)) {
return { shouldRetry: false, useContinueOnRetry: false, continueDisabledPermanently };
}
const isCrashSignal = isCrashSignalExitCode(result.exitCode);
const nextContinueDisabledPermanently = continueDisabledPermanently || isCrashSignal;
return { shouldRetry: true, useContinueOnRetry: !nextContinueDisabledPermanently, continueDisabledPermanently: nextContinueDisabledPermanently };
}

it("disables --continue and restarts fresh after a SIGSYS (159) crash", () => {
const result = { exitCode: 159, hasOutput: true };
const { shouldRetry, useContinueOnRetry, continueDisabledPermanently } = applyRetryPolicy(result, 0, false);
expect(shouldRetry).toBe(true);
expect(useContinueOnRetry).toBe(false);
expect(continueDisabledPermanently).toBe(true);
});

it("disables --continue and restarts fresh after a SIGSEGV (139) crash", () => {
const result = { exitCode: 139, hasOutput: true };
const { shouldRetry, useContinueOnRetry, continueDisabledPermanently } = applyRetryPolicy(result, 0, false);
expect(shouldRetry).toBe(true);
expect(useContinueOnRetry).toBe(false);
expect(continueDisabledPermanently).toBe(true);
});

it("keeps --continue disabled on subsequent retries after a crash-signal restart", () => {
const crashResult = { exitCode: 134, hasOutput: true };
const after0 = applyRetryPolicy(crashResult, 0, false);
expect(after0.useContinueOnRetry).toBe(false);
expect(after0.continueDisabledPermanently).toBe(true);

const nextResult = { exitCode: 1, hasOutput: true };
const after1 = applyRetryPolicy(nextResult, 1, after0.continueDisabledPermanently);
expect(after1.shouldRetry).toBe(true);
expect(after1.useContinueOnRetry).toBe(false); // must not re-enable --continue
expect(after1.continueDisabledPermanently).toBe(true);
});
});

describe("permanent --continue disable guard", () => {
// Inline retry logic to verify that once continueDisabledPermanently is set,
// subsequent partial-execution retries never re-enable --continue.
Expand Down
47 changes: 47 additions & 0 deletions actions/setup/js/harness_crash_signals.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// @ts-check

"use strict";

// Exit codes (128 + signal number) that indicate an agentic CLI subprocess was
// killed by a fatal OS-level signal rather than exiting normally or being
// cancelled. These signify a sandbox/runtime-level crash (e.g. a bad syscall
// trapped by seccomp, a segfault, or an illegal instruction) rather than an
// application-level error, so resuming the same on-disk session with
// --continue risks immediately reproducing the same crash.
// Map: SIGILL=4, SIGABRT=6, SIGBUS=7, SIGFPE=8, SIGSEGV=11, SIGSYS=31.
const CRASH_SIGNAL_EXIT_CODES = new Map([
[132, "SIGILL"],
[134, "SIGABRT"],
[135, "SIGBUS"],
[136, "SIGFPE"],
[139, "SIGSEGV"],
[159, "SIGSYS"],
]);

/**
* Determines whether the exit code corresponds to a fatal-signal crash of a CLI
* subprocess (e.g. SIGSEGV=139, SIGSYS=159) as opposed to a normal application
* error or an expected timeout/cancellation signal (e.g. SIGKILL=137/SIGTERM=143).
* @param {number} exitCode
* @returns {boolean}
*/
function isCrashSignalExitCode(exitCode) {
return CRASH_SIGNAL_EXIT_CODES.has(exitCode);
}

/**
* Best-effort mapping of a fatal-signal exit code (128 + signal number) to its
* signal name, for diagnostic logging. Returns null when the exit code is not a
* recognized crash signal.
* @param {number} exitCode
* @returns {string | null}
*/
function crashSignalNameForExitCode(exitCode) {
return CRASH_SIGNAL_EXIT_CODES.get(exitCode) ?? null;
}

module.exports = {
CRASH_SIGNAL_EXIT_CODES,
isCrashSignalExitCode,
crashSignalNameForExitCode,
};
40 changes: 40 additions & 0 deletions actions/setup/js/harness_crash_signals.test.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// @ts-check

import { describe, expect, it } from "vitest";
import { createRequire } from "node:module";

const require = createRequire(import.meta.url);
const { CRASH_SIGNAL_EXIT_CODES, isCrashSignalExitCode, crashSignalNameForExitCode } = require("./harness_crash_signals.cjs");

describe("harness_crash_signals.cjs", () => {
it("classifies known fatal-signal exit codes as crashes", () => {
for (const [exitCode, signalName] of CRASH_SIGNAL_EXIT_CODES) {
expect(isCrashSignalExitCode(exitCode)).toBe(true);
expect(crashSignalNameForExitCode(exitCode)).toBe(signalName);
}
});

it("does not classify normal exit codes as crashes", () => {
expect(isCrashSignalExitCode(0)).toBe(false);
expect(isCrashSignalExitCode(1)).toBe(false);
expect(isCrashSignalExitCode(2)).toBe(false);
expect(crashSignalNameForExitCode(0)).toBeNull();
expect(crashSignalNameForExitCode(1)).toBeNull();
});

it("does not classify expected timeout/cancellation signals (SIGKILL/SIGTERM) as crashes", () => {
expect(isCrashSignalExitCode(137)).toBe(false); // SIGKILL
expect(isCrashSignalExitCode(143)).toBe(false); // SIGTERM
expect(crashSignalNameForExitCode(137)).toBeNull();
expect(crashSignalNameForExitCode(143)).toBeNull();
});

it("maps exit codes to the expected signal names", () => {
expect(crashSignalNameForExitCode(132)).toBe("SIGILL");
expect(crashSignalNameForExitCode(134)).toBe("SIGABRT");
expect(crashSignalNameForExitCode(135)).toBe("SIGBUS");
expect(crashSignalNameForExitCode(136)).toBe("SIGFPE");
expect(crashSignalNameForExitCode(139)).toBe("SIGSEGV");
expect(crashSignalNameForExitCode(159)).toBe("SIGSYS");
});
});
Loading
Loading