fix(connectors): treat a zero-byte source file as nothing to index (#6848)

Observed in production after connectors began delivering source files: a
zero-byte PDF was stored and shipped to OCR, which answered `400 Bad Request`.
That bills an external call to discover the file was empty and reports it as an
API fault rather than as what it is. Before source files existed, an empty file
produced empty extracted text and was dropped at the empty-content check, so
this was a regression.

The emptiness rule now lives in one place, `hasIndexablePayload`, used by the
sync engine's classify and hydrate gates and by both connectors' `getDocument`.
It previously existed twice — the connectors asked whether a source file was
present while the sync engine asked the same question a second way — and a
source file with no bytes satisfied both.

Author: waleedlatif1

apps/sim/connectors/onedrive/onedrive.ts

@@ -8,6 +8,7 @@ import {
   ConnectorFileTooLargeError,
   connectorFileExtension,
   extractConnectorText,
+  hasIndexablePayload,
   isIndexableConnectorFile,
   isSkippedDocument,
   markSkipped,
@@ -383,7 +384,7 @@ export const onedriveConnector: ConnectorConfig = {
 
     try {
       const payload = await fetchFilePayload(accessToken, item.id, item.name)
-      if (!payload.sourceFile && !payload.content.trim()) return null
+      if (!hasIndexablePayload(payload)) return null
 
       const stub = fileToStub(item)
       return { ...stub, ...payload, contentDeferred: false }

apps/sim/connectors/sharepoint/sharepoint.ts

@@ -8,6 +8,7 @@ import {
   ConnectorFileTooLargeError,
   connectorFileExtension,
   extractConnectorText,
+  hasIndexablePayload,
   isIndexableConnectorFile,
   isSkippedDocument,
   markSkipped,
@@ -931,7 +932,7 @@ export const sharepointConnector: ConnectorConfig = {
 
     try {
       const payload = await fetchFilePayload(accessToken, driveId, item.id, item.name)
-      if (!payload.sourceFile && !payload.content.trim()) return null
+      if (!hasIndexablePayload(payload)) return null
 
       const stub = itemToStub(item, siteName ?? siteUrl)
       return { ...stub, ...payload, contentDeferred: false }

apps/sim/connectors/utils.test.ts

@@ -63,6 +63,7 @@ import { typeformConnector } from '@/connectors/typeform/typeform'
 import {
   ConnectorFileTooLargeError,
   extractConnectorText,
+  hasIndexablePayload,
   htmlToPlainText,
   isIndexableConnectorFile,
   isSkippedDocument,
@@ -1482,3 +1483,33 @@ describe('pipelineParsedMimeType', () => {
     expect(pipelineParsedMimeType('README')).toBeUndefined()
   })
 })
+
+describe('hasIndexablePayload', () => {
+  const bytes = (value: string) => ({
+    bytes: Buffer.from(value),
+    fileName: 'Report.pdf',
+    mimeType: 'application/pdf',
+  })
+
+  it('accepts a source file with bytes', () => {
+    expect(hasIndexablePayload({ content: '', sourceFile: bytes('%PDF') })).toBe(true)
+  })
+
+  it('accepts extracted text', () => {
+    expect(hasIndexablePayload({ content: 'notes' })).toBe(true)
+  })
+
+  /**
+   * Observed in production: a zero-byte PDF was stored and shipped to OCR, which
+   * answered `400 Bad Request` — an external call billed to discover the file was
+   * empty, reported as an API fault rather than as an empty file. Before source
+   * files existed this was dropped at the empty-content check.
+   */
+  it('rejects a zero-byte source file rather than sending it to OCR', () => {
+    expect(hasIndexablePayload({ content: '', sourceFile: bytes('') })).toBe(false)
+  })
+
+  it('rejects blank text', () => {
+    expect(hasIndexablePayload({ content: '   ' })).toBe(false)
+  })
+})

apps/sim/connectors/utils.ts

@@ -226,6 +226,21 @@ export function isIndexableConnectorFile(fileName: string): boolean {
   return extension !== undefined && CONNECTOR_INDEXABLE_EXTENSIONS.has(extension)
 }
 
+/**
+ * Whether a document carries anything worth indexing.
+ *
+ * A source file has to have bytes. A zero-byte file is not payload: it produces an
+ * empty stored object, and for a PDF that reaches OCR as an empty request and comes
+ * back as an opaque `400 Bad Request` — billing an external call to learn the file
+ * was empty, and reporting it as an API fault rather than as what it is.
+ */
+export function hasIndexablePayload(
+  doc: Pick<ExternalDocument, 'content' | 'sourceFile'>
+): boolean {
+  if (doc.sourceFile) return doc.sourceFile.bytes.length > 0
+  return doc.content.trim().length > 0
+}
+
 /**
  * MIME type to store a file under when the shared pipeline should parse it, or
  * `undefined` when the connector should decode it as text itself.

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -33,6 +33,7 @@ import type {
   ExternalDocument,
   SyncResult,
 } from '@/connectors/types'
+import { hasIndexablePayload } from '@/connectors/utils'
 
 const logger = createLogger('ConnectorSyncEngine')
 
@@ -157,7 +158,7 @@ export function classifyExternalDoc(
   if (extDoc.skippedReason) {
     return existing ? { type: 'unchanged' } : { type: 'skip' }
   }
-  if (!hasPayload(extDoc) && !extDoc.contentDeferred) {
+  if (!hasIndexablePayload(extDoc) && !extDoc.contentDeferred) {
     return { type: 'drop' }
   }
   if (!existing) {
@@ -203,11 +204,6 @@ export function mergeHydratedDocument(
   }
 }
 
-/** Whether a document carries anything to index — extracted text or the source file. */
-function hasPayload(extDoc: Pick<ExternalDocument, 'content' | 'sourceFile'>): boolean {
-  return extDoc.sourceFile !== undefined || extDoc.content.trim().length > 0
-}
-
 /** Estimated source bytes for a pending op, taken from its listing metadata. */
 function estimateOpSizeBytes(op: DocOp): number {
   // Skip ops load no content (just a row insert), so they do not count against the
@@ -1093,7 +1089,7 @@ export async function executeSync(
               }
               return null
             }
-            if (!fullDoc || !hasPayload(fullDoc)) {
+            if (!fullDoc || !hasIndexablePayload(fullDoc)) {
               // An empty re-fetch leaves an already-indexed update as last-known-good; count
               // it as unchanged so the totals still reconcile with documents seen. Not a
               // verified refresh, though — see failedExternalIds below.