From 09e5f6902dcd155c1f2e7188ea08344aa17ff52c Mon Sep 17 00:00:00 2001 From: Kieran Brown Date: Sat, 25 Apr 2026 21:27:11 +0100 Subject: [PATCH 1/2] Retry EcsCredentialProvider on HTTP 429 responses The EKS Pod Identity Agent rate-limits the credentials endpoint with a token-bucket limiter and returns 429 when the bucket is empty. The previous retry check only matched ConnectException, so HTTP-level throttling responses bubbled up immediately and crashed the client. Treat a Guzzle BadResponseException whose status is 429 as retryable in addition to ConnectException, mirroring the explicit allowlist used by aws-sdk-go-v2's endpointcreds client. Other 4xx responses (e.g. 401) remain non-retryable. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Credentials/EcsCredentialProvider.php | 7 +++- .../Credentials/EcsCredentialProviderTest.php | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/Credentials/EcsCredentialProvider.php b/src/Credentials/EcsCredentialProvider.php index e95b2b005b..ed44f0e176 100644 --- a/src/Credentials/EcsCredentialProvider.php +++ b/src/Credentials/EcsCredentialProvider.php @@ -3,6 +3,7 @@ use Aws\Arn\Arn; use Aws\Exception\CredentialsException; +use GuzzleHttp\Exception\BadResponseException; use GuzzleHttp\Exception\ConnectException; use GuzzleHttp\Exception\GuzzleException; use GuzzleHttp\Psr7\Request; @@ -107,7 +108,11 @@ public function __invoke() })->otherwise(function ($reason) { $reason = is_array($reason) ? $reason['exception'] : $reason; - $isRetryable = $reason instanceof ConnectException; + $isRetryable = $reason instanceof ConnectException + || ( + $reason instanceof BadResponseException + && $reason->getResponse()->getStatusCode() === 429 + ); if ($isRetryable && ($this->attempts < $this->retries)) { sleep((int)pow(1.2, $this->attempts)); } else { diff --git a/tests/Credentials/EcsCredentialProviderTest.php b/tests/Credentials/EcsCredentialProviderTest.php index 6dd99916d6..57847bbe77 100644 --- a/tests/Credentials/EcsCredentialProviderTest.php +++ b/tests/Credentials/EcsCredentialProviderTest.php @@ -7,6 +7,7 @@ use Aws\Exception\CredentialsException; use Aws\Handler\Guzzle\GuzzleHandler; use GuzzleHttp\Client; +use GuzzleHttp\Exception\BadResponseException; use GuzzleHttp\Exception\ConnectException; use GuzzleHttp\Exception\GuzzleException; use GuzzleHttp\Exception\RequestException; @@ -414,6 +415,15 @@ public static function successDataProvider(): array 'exception' => $connectException, ]); + $tooManyRequestsException = new BadResponseException( + '429 Too Many Requests', + new Psr7\Request('GET', '/latest'), + new Psr7\Response(429) + ); + $rejectionTooManyRequests = Promise\Create::rejectionFor([ + 'exception' => $tooManyRequestsException, + ]); + $promiseCreds = Promise\Create::promiseFor( new Response(200, [], Psr7\Utils::streamFor( json_encode(call_user_func_array( @@ -453,6 +463,16 @@ public static function successDataProvider(): array ], $credsObject ], + 'With retries for HTTP 429 (Pod Identity Agent rate limit)' => [ + [ + 'responses' => [ + $rejectionTooManyRequests, + $promiseCreds + ], + 'credentials' => $creds + ], + $credsObject + ], ]; } @@ -501,6 +521,13 @@ public static function failureDataProvider(): array $rejectionConnection = Promise\Create::rejectionFor([ 'exception' => $connectException, ]); + $rejectionTooManyRequests = Promise\Create::rejectionFor([ + 'exception' => new BadResponseException( + '429 Too Many Requests', + $getRequest, + new Psr7\Response(429) + ) + ]); return [ 'Non-retryable error' => [ @@ -520,6 +547,15 @@ public static function failureDataProvider(): array 'Error retrieving credentials from container metadata after attempt 1/1 (cURL error 28: Connection timed out after 1000 milliseconds)' ) ], + 'Retryable HTTP 429 error exhausts retries' => [ + [ + $rejectionTooManyRequests, + $rejectionTooManyRequests, + ], + new CredentialsException( + 'Error retrieving credentials from container metadata after attempt 1/1 (429 Too Many Requests)' + ) + ], ]; } From 9be2775da17402f3c8440ec81bd3a22eb8a9c790 Mon Sep 17 00:00:00 2001 From: Kieran Brown Date: Sun, 3 May 2026 21:10:36 +0100 Subject: [PATCH 2/2] Make EcsCredentialProvider retry behavior configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add retryable_exceptions and retryable_error_codes options to the constructor config, replacing the hardcoded ConnectException + 429 check with a private isRetryable() helper that consults both lists. retryable_exceptions defaults to [ConnectException::class], preserving existing behavior. retryable_error_codes defaults to [], so HTTP 429 responses are no longer retried by default — callers running against the EKS Pod Identity Agent opt in via ['retryable_error_codes' => [429]]. Addresses review feedback on aws/aws-sdk-php#3279 — the SDK has not previously specified retry behavior for HTTP/container credential providers, so making 429 retry default would set a precedent the maintainers want to avoid. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Credentials/EcsCredentialProvider.php | 46 ++++- .../Credentials/EcsCredentialProviderTest.php | 173 ++++++++++++++++-- 2 files changed, 200 insertions(+), 19 deletions(-) diff --git a/src/Credentials/EcsCredentialProvider.php b/src/Credentials/EcsCredentialProvider.php index ed44f0e176..4d0f9e415f 100644 --- a/src/Credentials/EcsCredentialProvider.php +++ b/src/Credentials/EcsCredentialProvider.php @@ -42,11 +42,22 @@ class EcsCredentialProvider /** @var int */ private $attempts; + /** @var string[] */ + private $retryableExceptions; + + /** @var int[] */ + private $retryableErrorCodes; + /** * The constructor accepts following options: * - timeout: (optional) Connection timeout, in seconds, default 1.0 * - retries: Optional number of retries to be attempted, default 3. * - client: An EcsClient to make request from + * - retryable_exceptions: Optional array of exception class names that + * should be retried. Defaults to [ConnectException::class]. + * - retryable_error_codes: Optional array of HTTP status codes that + * should be retried when returned in a BadResponseException. Defaults + * to an empty array. * * @param array $config Configuration options */ @@ -60,6 +71,9 @@ public function __construct(array $config = []) : ((int) getenv(self::ENV_RETRIES) ?: self::DEFAULT_ENV_RETRIES); $this->client = $config['client'] ?? \Aws\default_http_handler(); + $this->retryableExceptions = $config['retryable_exceptions'] + ?? [ConnectException::class]; + $this->retryableErrorCodes = $config['retryable_error_codes'] ?? []; } /** @@ -108,12 +122,7 @@ public function __invoke() })->otherwise(function ($reason) { $reason = is_array($reason) ? $reason['exception'] : $reason; - $isRetryable = $reason instanceof ConnectException - || ( - $reason instanceof BadResponseException - && $reason->getResponse()->getStatusCode() === 429 - ); - if ($isRetryable && ($this->attempts < $this->retries)) { + if ($this->isRetryable($reason) && ($this->attempts < $this->retries)) { sleep((int)pow(1.2, $this->attempts)); } else { $msg = $reason->getMessage(); @@ -226,6 +235,31 @@ private function getEcsUri() return self::SERVER_URI . $credsUri; } + /** + * Determines whether a failed request should be retried, based on the + * configured retryable_exceptions and retryable_error_codes. + */ + private function isRetryable($reason): bool + { + foreach ($this->retryableExceptions as $exceptionClass) { + if ($reason instanceof $exceptionClass) { + return true; + } + } + + if ($reason instanceof BadResponseException + && in_array( + $reason->getResponse()->getStatusCode(), + $this->retryableErrorCodes, + true + ) + ) { + return true; + } + + return false; + } + private function decodeResult($response) { $result = json_decode($response, true); diff --git a/tests/Credentials/EcsCredentialProviderTest.php b/tests/Credentials/EcsCredentialProviderTest.php index 57847bbe77..11cd2612a5 100644 --- a/tests/Credentials/EcsCredentialProviderTest.php +++ b/tests/Credentials/EcsCredentialProviderTest.php @@ -463,19 +463,137 @@ public static function successDataProvider(): array ], $credsObject ], - 'With retries for HTTP 429 (Pod Identity Agent rate limit)' => [ - [ - 'responses' => [ - $rejectionTooManyRequests, - $promiseCreds - ], - 'credentials' => $creds - ], - $credsObject - ], ]; } + public function testRetriesOptedInErrorCode() + { + $expiry = time() + 1000; + $creds = ['foo_key', 'baz_secret', 'qux_token', "@{$expiry}"]; + + $rejectionTooManyRequests = Promise\Create::rejectionFor([ + 'exception' => new BadResponseException( + '429 Too Many Requests', + new Psr7\Request('GET', '/latest'), + new Psr7\Response(429) + ), + ]); + $promiseCreds = Promise\Create::promiseFor( + new Response(200, [], Psr7\Utils::streamFor( + json_encode(call_user_func_array( + [self::class, 'getCredentialArray'], + $creds + ))) + ) + ); + + $provider = new EcsCredentialProvider([ + 'client' => $this->getTestClient([ + $rejectionTooManyRequests, + $promiseCreds, + ], $creds), + 'retries' => 2, + 'retryable_error_codes' => [429], + ]); + + $credentials = $provider()->wait(); + $this->assertSame('foo_key', $credentials->getAccessKeyId()); + $this->assertSame('baz_secret', $credentials->getSecretKey()); + } + + public function testDoesNotRetry429ByDefault() + { + $rejectionTooManyRequests = Promise\Create::rejectionFor([ + 'exception' => new BadResponseException( + '429 Too Many Requests', + new Psr7\Request('GET', '/latest'), + new Psr7\Response(429) + ), + ]); + + $provider = new EcsCredentialProvider([ + 'client' => $this->getTestClient([ + $rejectionTooManyRequests, + ]), + 'retries' => 3, + ]); + + try { + $provider()->wait(); + $this->fail('Provider should have thrown an exception.'); + } catch (CredentialsException $e) { + $this->assertStringContainsString( + 'attempt 0/3', + $e->getMessage() + ); + $this->assertStringContainsString('429 Too Many Requests', $e->getMessage()); + } + + $this->assertSame(0, $provider->getAttempts()); + } + + public function testRetriesOptedInExceptionClass() + { + $expiry = time() + 1000; + $creds = ['foo_key', 'baz_secret', 'qux_token', "@{$expiry}"]; + + $rejectionRequest = Promise\Create::rejectionFor([ + 'exception' => new RequestException( + 'Boom', + new Psr7\Request('GET', '/latest'), + new Psr7\Response(500) + ), + ]); + $promiseCreds = Promise\Create::promiseFor( + new Response(200, [], Psr7\Utils::streamFor( + json_encode(call_user_func_array( + [self::class, 'getCredentialArray'], + $creds + ))) + ) + ); + + $provider = new EcsCredentialProvider([ + 'client' => $this->getTestClient([ + $rejectionRequest, + $promiseCreds, + ], $creds), + 'retries' => 2, + 'retryable_exceptions' => [ + ConnectException::class, + RequestException::class, + ], + ]); + + $credentials = $provider()->wait(); + $this->assertSame('foo_key', $credentials->getAccessKeyId()); + } + + public function testCustomRetryableExceptionsReplaceDefaults() + { + $rejectionConnection = Promise\Create::rejectionFor([ + 'exception' => new ConnectException( + 'cURL error 28: Connection timed out after 1000 milliseconds', + new Psr7\Request('GET', '/latest') + ), + ]); + + $provider = new EcsCredentialProvider([ + 'client' => $this->getTestClient([ + $rejectionConnection, + ]), + 'retries' => 3, + 'retryable_exceptions' => [], + ]); + + try { + $provider()->wait(); + $this->fail('Provider should have thrown an exception.'); + } catch (CredentialsException $e) { + $this->assertSame(0, $provider->getAttempts()); + } + } + /** * @param $client * @param \Exception $expected @@ -547,18 +665,47 @@ public static function failureDataProvider(): array 'Error retrieving credentials from container metadata after attempt 1/1 (cURL error 28: Connection timed out after 1000 milliseconds)' ) ], - 'Retryable HTTP 429 error exhausts retries' => [ + 'Non-retryable HTTP 429 by default' => [ [ $rejectionTooManyRequests, - $rejectionTooManyRequests, ], new CredentialsException( - 'Error retrieving credentials from container metadata after attempt 1/1 (429 Too Many Requests)' + 'Error retrieving credentials from container metadata after attempt 0/1 (429 Too Many Requests)' ) ], ]; } + public function testOptedInHTTP429RetryExhaustsAttempts() + { + $rejectionTooManyRequests = Promise\Create::rejectionFor([ + 'exception' => new BadResponseException( + '429 Too Many Requests', + new Psr7\Request('GET', '/latest'), + new Psr7\Response(429) + ), + ]); + + $provider = new EcsCredentialProvider([ + 'client' => $this->getTestClient([ + $rejectionTooManyRequests, + $rejectionTooManyRequests, + ]), + 'retries' => 1, + 'retryable_error_codes' => [429], + ]); + + try { + $provider()->wait(); + $this->fail('Provider should have thrown an exception.'); + } catch (CredentialsException $e) { + $this->assertSame( + 'Error retrieving credentials from container metadata after attempt 1/1 (429 Too Many Requests)', + $e->getMessage() + ); + } + } + public function testReadsRetriesFromEnvironment() { putenv('AWS_METADATA_SERVICE_NUM_ATTEMPTS=1');