fix(knowledge): stop retrying an embedding key with no credit left (#6868)

OpenAI answers an exhausted balance with 429, the same status as a rate limit,
but the two are not alike: a rate limit reopens and a spent account does not.
Both were classified transient, so every document burned its full retry budget
against a key that could never accept it, and because the sweep re-queues failed
documents on every sync the account turned into permanent load rather than a
one-off failure. A connector sync ran the full hour and timed out doing this.

The rejection body is what separates them — insufficient_quota, or a
credit_balance_exhausted code — so it is read when the error is built and the
retries stop immediately.

Retrying and failing over are decided separately here. An exhausted balance
rules out the key just used but says nothing about the next provider in the
chain, so the error stays eligible for failover and only the retries against the
spent key are dropped.

Author: waleedlatif1

apps/sim/lib/embeddings/client.test.ts

@@ -822,6 +822,87 @@ describe('knowledge embedding transport fallback', () => {
     expect(result.embeddings).toEqual([[4, 4]])
   })
 
+  /**
+   * OpenAI returns 429 for an exhausted balance as well as for a rate limit, but
+   * only one of them reopens. Retrying a spent account cannot succeed, and since
+   * the sweep re-queues failed documents every sync it turns into permanent load
+   * — this was observed burning every attempt on thousands of documents for
+   * weeks against an account with no credit.
+   */
+  it('does not retry a 429 that reports an exhausted balance', async () => {
+    vi.useFakeTimers()
+    setEnv({ OPENAI_API_KEY: 'openai-test' })
+    const fetchMock = vi.fn().mockImplementation(
+      async () =>
+        ({
+          ok: false,
+          status: 429,
+          statusText: 'Too Many Requests',
+          headers: new Headers(),
+          json: async () => ({}),
+          text: async () =>
+            JSON.stringify({
+              error: {
+                message: 'You have no credits remaining.',
+                type: 'insufficient_quota',
+                code: 'credit_balance_exhausted',
+              },
+            }),
+        }) as Response
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const pending = embed(['hello'], { ...options, apiKey: 'openai-test' }).catch((e) => e)
+    await vi.runAllTimersAsync()
+    const error = await pending
+
+    expect(error).toBeInstanceOf(EmbeddingAPIError)
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  /**
+   * A rate limit with the same status must keep its retries — the two are only
+   * distinguishable by the body.
+   */
+  it('still retries a 429 that reports a rate limit', async () => {
+    vi.useFakeTimers()
+    setEnv({ OPENAI_API_KEY: 'openai-test' })
+    let call = 0
+    const fetchMock = vi.fn().mockImplementation(async () => {
+      call++
+      if (call === 1) {
+        return {
+          ok: false,
+          status: 429,
+          statusText: 'Too Many Requests',
+          headers: new Headers(),
+          json: async () => ({}),
+          text: async () =>
+            JSON.stringify({ error: { message: 'slow down', type: 'rate_limit_exceeded' } }),
+        } as Response
+      }
+      return jsonResponse(openAIBody([[5, 5]], 2))
+    })
+    vi.stubGlobal('fetch', fetchMock)
+
+    const pending = embed(['hello'], { ...options, apiKey: 'openai-test' })
+    await vi.runAllTimersAsync()
+    const result = await pending
+
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(result.embeddings).toEqual([[5, 5]])
+  })
+
+  /**
+   * An exhausted key rules out the key just used, not the next one in the chain,
+   * so failover must still consider it.
+   */
+  it('keeps an exhausted-balance error eligible for failover', () => {
+    const error = new EmbeddingAPIError('Embedding API failed: 429', 429)
+    error.quotaExhausted = true
+    expect(isTransientEmbeddingError(error)).toBe(true)
+  })
+
   it('classifies only transient embedding failures for failover', () => {
     expect(isTransientEmbeddingError(new EmbeddingAPIError('unavailable', 503))).toBe(true)
     expect(isTransientEmbeddingError(new EmbeddingAPIError('rate limited', 429))).toBe(true)

apps/sim/lib/embeddings/client.ts

@@ -79,6 +79,12 @@ const EMBEDDING_RETRY_BUDGET_MS = EMBEDDING_MAX_RETRIES * EMBEDDING_MAX_RETRY_DE
 export class EmbeddingAPIError extends Error {
   public status: number
 
+  /**
+   * The provider rejected this for an exhausted balance rather than a rate that
+   * will recover. Both arrive as 429.
+   */
+  public quotaExhausted?: boolean
+
   /**
    * Wait the provider asked for, read from the rejected response. Consumed by
    * {@link retryWithExponentialBackoff}, which prefers it over its own backoff.
@@ -92,6 +98,31 @@ export class EmbeddingAPIError extends Error {
   }
 }
 
+/**
+ * True when a rejection body reports an exhausted balance rather than a rate
+ * limit.
+ *
+ * OpenAI returns 429 for both, but only one of them reopens. `insufficient_quota`
+ * stands until somebody adds credit, so retrying it cannot succeed no matter how
+ * long the loop waits — and because a failed document is re-queued by the sweep
+ * on every sync, an account that has run out turns into a permanent load: the
+ * budget is spent per document, per attempt, forever.
+ */
+function isQuotaExhaustionBody(errorText: string): boolean {
+  try {
+    const body = JSON.parse(errorText) as { error?: { type?: string; code?: string } }
+    const type = body.error?.type
+    const code = body.error?.code
+    return (
+      type === 'insufficient_quota' ||
+      code === 'insufficient_quota' ||
+      code === 'credit_balance_exhausted'
+    )
+  } catch {
+    return false
+  }
+}
+
 /**
  * True when the provider's stated wait outlasts the entire retry budget.
  *
@@ -116,6 +147,21 @@ function statedWaitOutlastsBudget(error: unknown): boolean {
   )
 }
 
+/**
+ * Whether another attempt against the same provider could plausibly succeed.
+ *
+ * Deliberately narrower than {@link isTransientEmbeddingError}, which also
+ * decides whether the fallback chain should try a *different* provider. Those
+ * two questions differ: an exhausted balance rules out the key we just used, but
+ * says nothing about the next one in the chain, so a quota rejection stops the
+ * retries here while remaining eligible for failover.
+ */
+function isWorthRetrying(error: unknown): boolean {
+  if (!isTransientEmbeddingError(error)) return false
+  if (error instanceof EmbeddingAPIError && error.quotaExhausted) return false
+  return !statedWaitOutlastsBudget(error)
+}
+
 export function isTransientEmbeddingError(error: unknown): boolean {
   if (error instanceof EmbeddingAPIError) {
     return error.status === 429 || error.status >= 500
@@ -251,6 +297,7 @@ async function callEmbeddingAPI(
           `Embedding API failed: ${response.status} ${response.statusText} - ${errorText}`,
           response.status
         )
+        error.quotaExhausted = isQuotaExhaustionBody(errorText)
 
         /**
          * Carry the provider's own answer to "when may I retry" onto the error,
@@ -298,8 +345,7 @@ async function callEmbeddingAPI(
       maxRetries: EMBEDDING_MAX_RETRIES,
       initialDelayMs: 1000,
       maxDelayMs: EMBEDDING_MAX_RETRY_DELAY_MS,
-      retryCondition: (error) =>
-        isTransientEmbeddingError(error) && !statedWaitOutlastsBudget(error),
+      retryCondition: isWorthRetrying,
     }
   )
 }