diff --git a/apps/sim/lib/embeddings/client.test.ts b/apps/sim/lib/embeddings/client.test.ts index 0f2d4e66eaa..78ce526bce9 100644 --- a/apps/sim/lib/embeddings/client.test.ts +++ b/apps/sim/lib/embeddings/client.test.ts @@ -822,6 +822,87 @@ describe('knowledge embedding transport fallback', () => { expect(result.embeddings).toEqual([[4, 4]]) }) + /** + * OpenAI returns 429 for an exhausted balance as well as for a rate limit, but + * only one of them reopens. Retrying a spent account cannot succeed, and since + * the sweep re-queues failed documents every sync it turns into permanent load + * — this was observed burning every attempt on thousands of documents for + * weeks against an account with no credit. + */ + it('does not retry a 429 that reports an exhausted balance', async () => { + vi.useFakeTimers() + setEnv({ OPENAI_API_KEY: 'openai-test' }) + const fetchMock = vi.fn().mockImplementation( + async () => + ({ + ok: false, + status: 429, + statusText: 'Too Many Requests', + headers: new Headers(), + json: async () => ({}), + text: async () => + JSON.stringify({ + error: { + message: 'You have no credits remaining.', + type: 'insufficient_quota', + code: 'credit_balance_exhausted', + }, + }), + }) as Response + ) + vi.stubGlobal('fetch', fetchMock) + + const pending = embed(['hello'], { ...options, apiKey: 'openai-test' }).catch((e) => e) + await vi.runAllTimersAsync() + const error = await pending + + expect(error).toBeInstanceOf(EmbeddingAPIError) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + /** + * A rate limit with the same status must keep its retries — the two are only + * distinguishable by the body. + */ + it('still retries a 429 that reports a rate limit', async () => { + vi.useFakeTimers() + setEnv({ OPENAI_API_KEY: 'openai-test' }) + let call = 0 + const fetchMock = vi.fn().mockImplementation(async () => { + call++ + if (call === 1) { + return { + ok: false, + status: 429, + statusText: 'Too Many Requests', + headers: new Headers(), + json: async () => ({}), + text: async () => + JSON.stringify({ error: { message: 'slow down', type: 'rate_limit_exceeded' } }), + } as Response + } + return jsonResponse(openAIBody([[5, 5]], 2)) + }) + vi.stubGlobal('fetch', fetchMock) + + const pending = embed(['hello'], { ...options, apiKey: 'openai-test' }) + await vi.runAllTimersAsync() + const result = await pending + + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(result.embeddings).toEqual([[5, 5]]) + }) + + /** + * An exhausted key rules out the key just used, not the next one in the chain, + * so failover must still consider it. + */ + it('keeps an exhausted-balance error eligible for failover', () => { + const error = new EmbeddingAPIError('Embedding API failed: 429', 429) + error.quotaExhausted = true + expect(isTransientEmbeddingError(error)).toBe(true) + }) + it('classifies only transient embedding failures for failover', () => { expect(isTransientEmbeddingError(new EmbeddingAPIError('unavailable', 503))).toBe(true) expect(isTransientEmbeddingError(new EmbeddingAPIError('rate limited', 429))).toBe(true) diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index 3383de1eaf4..9e39a691225 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -79,6 +79,12 @@ const EMBEDDING_RETRY_BUDGET_MS = EMBEDDING_MAX_RETRIES * EMBEDDING_MAX_RETRY_DE export class EmbeddingAPIError extends Error { public status: number + /** + * The provider rejected this for an exhausted balance rather than a rate that + * will recover. Both arrive as 429. + */ + public quotaExhausted?: boolean + /** * Wait the provider asked for, read from the rejected response. Consumed by * {@link retryWithExponentialBackoff}, which prefers it over its own backoff. @@ -92,6 +98,31 @@ export class EmbeddingAPIError extends Error { } } +/** + * True when a rejection body reports an exhausted balance rather than a rate + * limit. + * + * OpenAI returns 429 for both, but only one of them reopens. `insufficient_quota` + * stands until somebody adds credit, so retrying it cannot succeed no matter how + * long the loop waits — and because a failed document is re-queued by the sweep + * on every sync, an account that has run out turns into a permanent load: the + * budget is spent per document, per attempt, forever. + */ +function isQuotaExhaustionBody(errorText: string): boolean { + try { + const body = JSON.parse(errorText) as { error?: { type?: string; code?: string } } + const type = body.error?.type + const code = body.error?.code + return ( + type === 'insufficient_quota' || + code === 'insufficient_quota' || + code === 'credit_balance_exhausted' + ) + } catch { + return false + } +} + /** * True when the provider's stated wait outlasts the entire retry budget. * @@ -116,6 +147,21 @@ function statedWaitOutlastsBudget(error: unknown): boolean { ) } +/** + * Whether another attempt against the same provider could plausibly succeed. + * + * Deliberately narrower than {@link isTransientEmbeddingError}, which also + * decides whether the fallback chain should try a *different* provider. Those + * two questions differ: an exhausted balance rules out the key we just used, but + * says nothing about the next one in the chain, so a quota rejection stops the + * retries here while remaining eligible for failover. + */ +function isWorthRetrying(error: unknown): boolean { + if (!isTransientEmbeddingError(error)) return false + if (error instanceof EmbeddingAPIError && error.quotaExhausted) return false + return !statedWaitOutlastsBudget(error) +} + export function isTransientEmbeddingError(error: unknown): boolean { if (error instanceof EmbeddingAPIError) { return error.status === 429 || error.status >= 500 @@ -251,6 +297,7 @@ async function callEmbeddingAPI( `Embedding API failed: ${response.status} ${response.statusText} - ${errorText}`, response.status ) + error.quotaExhausted = isQuotaExhaustionBody(errorText) /** * Carry the provider's own answer to "when may I retry" onto the error, @@ -298,8 +345,7 @@ async function callEmbeddingAPI( maxRetries: EMBEDDING_MAX_RETRIES, initialDelayMs: 1000, maxDelayMs: EMBEDDING_MAX_RETRY_DELAY_MS, - retryCondition: (error) => - isTransientEmbeddingError(error) && !statedWaitOutlastsBudget(error), + retryCondition: isWorthRetrying, } ) }