fix(connectors): treat a zero-byte source file as nothing to index

apps/sim/connectors/onedrive/onedrive.ts

@@ -8,6 +8,7 @@ import {
   ConnectorFileTooLargeError,
   connectorFileExtension,
   extractConnectorText,
+  hasIndexablePayload,
   isIndexableConnectorFile,
   isSkippedDocument,
   markSkipped,
@@ -383,7 +384,7 @@ export const onedriveConnector: ConnectorConfig = {
 
     try {
       const payload = await fetchFilePayload(accessToken, item.id, item.name)
-      if (!payload.sourceFile && !payload.content.trim()) return null
+      if (!hasIndexablePayload(payload)) return null
 
       const stub = fileToStub(item)
       return { ...stub, ...payload, contentDeferred: false }

apps/sim/connectors/sharepoint/sharepoint.ts

@@ -8,6 +8,7 @@ import {
   ConnectorFileTooLargeError,
   connectorFileExtension,
   extractConnectorText,
+  hasIndexablePayload,
   isIndexableConnectorFile,
   isSkippedDocument,
   markSkipped,
@@ -931,7 +932,7 @@ export const sharepointConnector: ConnectorConfig = {
 
     try {
       const payload = await fetchFilePayload(accessToken, driveId, item.id, item.name)
-      if (!payload.sourceFile && !payload.content.trim()) return null
+      if (!hasIndexablePayload(payload)) return null
 
       const stub = itemToStub(item, siteName ?? siteUrl)
       return { ...stub, ...payload, contentDeferred: false }

apps/sim/connectors/utils.test.ts

@@ -63,6 +63,7 @@ import { typeformConnector } from '@/connectors/typeform/typeform'
 import {
   ConnectorFileTooLargeError,
   extractConnectorText,
+  hasIndexablePayload,
   htmlToPlainText,
   isIndexableConnectorFile,
   isSkippedDocument,
@@ -1482,3 +1483,33 @@ describe('pipelineParsedMimeType', () => {
     expect(pipelineParsedMimeType('README')).toBeUndefined()
   })
 })
+
+describe('hasIndexablePayload', () => {
+  const bytes = (value: string) => ({
+    bytes: Buffer.from(value),
+    fileName: 'Report.pdf',
+    mimeType: 'application/pdf',
+  })
+
+  it('accepts a source file with bytes', () => {
+    expect(hasIndexablePayload({ content: '', sourceFile: bytes('%PDF') })).toBe(true)
+  })
+
+  it('accepts extracted text', () => {
+    expect(hasIndexablePayload({ content: 'notes' })).toBe(true)
+  })
+
+  /**
+   * Observed in production: a zero-byte PDF was stored and shipped to OCR, which
+   * answered `400 Bad Request` — an external call billed to discover the file was
+   * empty, reported as an API fault rather than as an empty file. Before source
+   * files existed this was dropped at the empty-content check.
+   */
+  it('rejects a zero-byte source file rather than sending it to OCR', () => {
+    expect(hasIndexablePayload({ content: '', sourceFile: bytes('') })).toBe(false)
+  })
+
+  it('rejects blank text', () => {
+    expect(hasIndexablePayload({ content: '   ' })).toBe(false)
+  })
+})

apps/sim/connectors/utils.ts

@@ -226,6 +226,21 @@ export function isIndexableConnectorFile(fileName: string): boolean {
   return extension !== undefined && CONNECTOR_INDEXABLE_EXTENSIONS.has(extension)
 }
 
+/**
+ * Whether a document carries anything worth indexing.
+ *
+ * A source file has to have bytes. A zero-byte file is not payload: it produces an
+ * empty stored object, and for a PDF that reaches OCR as an empty request and comes
+ * back as an opaque `400 Bad Request` — billing an external call to learn the file
+ * was empty, and reporting it as an API fault rather than as what it is.
+ */
+export function hasIndexablePayload(
+  doc: Pick<ExternalDocument, 'content' | 'sourceFile'>
+): boolean {
+  if (doc.sourceFile) return doc.sourceFile.bytes.length > 0
+  return doc.content.trim().length > 0
+}
+
 /**
  * MIME type to store a file under when the shared pipeline should parse it, or
  * `undefined` when the connector should decode it as text itself.

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -33,6 +33,7 @@ import type {
   ExternalDocument,
   SyncResult,
 } from '@/connectors/types'
+import { hasIndexablePayload } from '@/connectors/utils'
 
 const logger = createLogger('ConnectorSyncEngine')
 
@@ -157,7 +158,7 @@ export function classifyExternalDoc(
   if (extDoc.skippedReason) {
     return existing ? { type: 'unchanged' } : { type: 'skip' }
   }
-  if (!hasPayload(extDoc) && !extDoc.contentDeferred) {
+  if (!hasIndexablePayload(extDoc) && !extDoc.contentDeferred) {
     return { type: 'drop' }
   }
   if (!existing) {
@@ -203,11 +204,6 @@ export function mergeHydratedDocument(
   }
 }
 
-/** Whether a document carries anything to index — extracted text or the source file. */
-function hasPayload(extDoc: Pick<ExternalDocument, 'content' | 'sourceFile'>): boolean {
-  return extDoc.sourceFile !== undefined || extDoc.content.trim().length > 0
-}
-
 /** Estimated source bytes for a pending op, taken from its listing metadata. */
 function estimateOpSizeBytes(op: DocOp): number {
   // Skip ops load no content (just a row insert), so they do not count against the
@@ -1093,7 +1089,7 @@ export async function executeSync(
               }
               return null
             }
-            if (!fullDoc || !hasPayload(fullDoc)) {
+            if (!fullDoc || !hasIndexablePayload(fullDoc)) {
               // An empty re-fetch leaves an already-indexed update as last-known-good; count
               // it as unchanged so the totals still reconcile with documents seen. Not a
               // verified refresh, though — see failedExternalIds below.