From eabfba72c1222ba54849fcccd18d7b7191f0f1d5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 19 Aug 2026 10:18:07 -0700 Subject: [PATCH] fix(connectors): treat a zero-byte source file as nothing to index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observed in production after connectors began delivering source files: a zero-byte PDF was stored and shipped to OCR, which answered `400 Bad Request`. That bills an external call to discover the file was empty and reports it as an API fault rather than as what it is. Before source files existed, an empty file produced empty extracted text and was dropped at the empty-content check, so this was a regression. The emptiness rule now lives in one place, `hasIndexablePayload`, used by the sync engine's classify and hydrate gates and by both connectors' `getDocument`. It previously existed twice — the connectors asked whether a source file was present while the sync engine asked the same question a second way — and a source file with no bytes satisfied both. --- apps/sim/connectors/onedrive/onedrive.ts | 3 +- apps/sim/connectors/sharepoint/sharepoint.ts | 3 +- apps/sim/connectors/utils.test.ts | 31 +++++++++++++++++++ apps/sim/connectors/utils.ts | 15 +++++++++ .../lib/knowledge/connectors/sync-engine.ts | 10 ++---- 5 files changed, 53 insertions(+), 9 deletions(-) diff --git a/apps/sim/connectors/onedrive/onedrive.ts b/apps/sim/connectors/onedrive/onedrive.ts index 63a427d59d5..6e25767f5ef 100644 --- a/apps/sim/connectors/onedrive/onedrive.ts +++ b/apps/sim/connectors/onedrive/onedrive.ts @@ -8,6 +8,7 @@ import { ConnectorFileTooLargeError, connectorFileExtension, extractConnectorText, + hasIndexablePayload, isIndexableConnectorFile, isSkippedDocument, markSkipped, @@ -383,7 +384,7 @@ export const onedriveConnector: ConnectorConfig = { try { const payload = await fetchFilePayload(accessToken, item.id, item.name) - if (!payload.sourceFile && !payload.content.trim()) return null + if (!hasIndexablePayload(payload)) return null const stub = fileToStub(item) return { ...stub, ...payload, contentDeferred: false } diff --git a/apps/sim/connectors/sharepoint/sharepoint.ts b/apps/sim/connectors/sharepoint/sharepoint.ts index 3ebd325555b..8a7fb352d29 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.ts @@ -8,6 +8,7 @@ import { ConnectorFileTooLargeError, connectorFileExtension, extractConnectorText, + hasIndexablePayload, isIndexableConnectorFile, isSkippedDocument, markSkipped, @@ -931,7 +932,7 @@ export const sharepointConnector: ConnectorConfig = { try { const payload = await fetchFilePayload(accessToken, driveId, item.id, item.name) - if (!payload.sourceFile && !payload.content.trim()) return null + if (!hasIndexablePayload(payload)) return null const stub = itemToStub(item, siteName ?? siteUrl) return { ...stub, ...payload, contentDeferred: false } diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index 6ba40e5200a..ca786fadc1d 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -63,6 +63,7 @@ import { typeformConnector } from '@/connectors/typeform/typeform' import { ConnectorFileTooLargeError, extractConnectorText, + hasIndexablePayload, htmlToPlainText, isIndexableConnectorFile, isSkippedDocument, @@ -1482,3 +1483,33 @@ describe('pipelineParsedMimeType', () => { expect(pipelineParsedMimeType('README')).toBeUndefined() }) }) + +describe('hasIndexablePayload', () => { + const bytes = (value: string) => ({ + bytes: Buffer.from(value), + fileName: 'Report.pdf', + mimeType: 'application/pdf', + }) + + it('accepts a source file with bytes', () => { + expect(hasIndexablePayload({ content: '', sourceFile: bytes('%PDF') })).toBe(true) + }) + + it('accepts extracted text', () => { + expect(hasIndexablePayload({ content: 'notes' })).toBe(true) + }) + + /** + * Observed in production: a zero-byte PDF was stored and shipped to OCR, which + * answered `400 Bad Request` — an external call billed to discover the file was + * empty, reported as an API fault rather than as an empty file. Before source + * files existed this was dropped at the empty-content check. + */ + it('rejects a zero-byte source file rather than sending it to OCR', () => { + expect(hasIndexablePayload({ content: '', sourceFile: bytes('') })).toBe(false) + }) + + it('rejects blank text', () => { + expect(hasIndexablePayload({ content: ' ' })).toBe(false) + }) +}) diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 608de615bd1..63674ae350c 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -226,6 +226,21 @@ export function isIndexableConnectorFile(fileName: string): boolean { return extension !== undefined && CONNECTOR_INDEXABLE_EXTENSIONS.has(extension) } +/** + * Whether a document carries anything worth indexing. + * + * A source file has to have bytes. A zero-byte file is not payload: it produces an + * empty stored object, and for a PDF that reaches OCR as an empty request and comes + * back as an opaque `400 Bad Request` — billing an external call to learn the file + * was empty, and reporting it as an API fault rather than as what it is. + */ +export function hasIndexablePayload( + doc: Pick +): boolean { + if (doc.sourceFile) return doc.sourceFile.bytes.length > 0 + return doc.content.trim().length > 0 +} + /** * MIME type to store a file under when the shared pipeline should parse it, or * `undefined` when the connector should decode it as text itself. diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 14e3bb7f13a..c35f3b3d303 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -33,6 +33,7 @@ import type { ExternalDocument, SyncResult, } from '@/connectors/types' +import { hasIndexablePayload } from '@/connectors/utils' const logger = createLogger('ConnectorSyncEngine') @@ -157,7 +158,7 @@ export function classifyExternalDoc( if (extDoc.skippedReason) { return existing ? { type: 'unchanged' } : { type: 'skip' } } - if (!hasPayload(extDoc) && !extDoc.contentDeferred) { + if (!hasIndexablePayload(extDoc) && !extDoc.contentDeferred) { return { type: 'drop' } } if (!existing) { @@ -203,11 +204,6 @@ export function mergeHydratedDocument( } } -/** Whether a document carries anything to index — extracted text or the source file. */ -function hasPayload(extDoc: Pick): boolean { - return extDoc.sourceFile !== undefined || extDoc.content.trim().length > 0 -} - /** Estimated source bytes for a pending op, taken from its listing metadata. */ function estimateOpSizeBytes(op: DocOp): number { // Skip ops load no content (just a row insert), so they do not count against the @@ -1093,7 +1089,7 @@ export async function executeSync( } return null } - if (!fullDoc || !hasPayload(fullDoc)) { + if (!fullDoc || !hasIndexablePayload(fullDoc)) { // An empty re-fetch leaves an already-indexed update as last-known-good; count // it as unchanged so the totals still reconcile with documents seen. Not a // verified refresh, though — see failedExternalIds below.