diff --git a/apps/sim/connectors/onedrive/onedrive.ts b/apps/sim/connectors/onedrive/onedrive.ts index 63a427d59d5..6e25767f5ef 100644 --- a/apps/sim/connectors/onedrive/onedrive.ts +++ b/apps/sim/connectors/onedrive/onedrive.ts @@ -8,6 +8,7 @@ import { ConnectorFileTooLargeError, connectorFileExtension, extractConnectorText, + hasIndexablePayload, isIndexableConnectorFile, isSkippedDocument, markSkipped, @@ -383,7 +384,7 @@ export const onedriveConnector: ConnectorConfig = { try { const payload = await fetchFilePayload(accessToken, item.id, item.name) - if (!payload.sourceFile && !payload.content.trim()) return null + if (!hasIndexablePayload(payload)) return null const stub = fileToStub(item) return { ...stub, ...payload, contentDeferred: false } diff --git a/apps/sim/connectors/sharepoint/sharepoint.ts b/apps/sim/connectors/sharepoint/sharepoint.ts index 3ebd325555b..8a7fb352d29 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.ts @@ -8,6 +8,7 @@ import { ConnectorFileTooLargeError, connectorFileExtension, extractConnectorText, + hasIndexablePayload, isIndexableConnectorFile, isSkippedDocument, markSkipped, @@ -931,7 +932,7 @@ export const sharepointConnector: ConnectorConfig = { try { const payload = await fetchFilePayload(accessToken, driveId, item.id, item.name) - if (!payload.sourceFile && !payload.content.trim()) return null + if (!hasIndexablePayload(payload)) return null const stub = itemToStub(item, siteName ?? siteUrl) return { ...stub, ...payload, contentDeferred: false } diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index 6ba40e5200a..ca786fadc1d 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -63,6 +63,7 @@ import { typeformConnector } from '@/connectors/typeform/typeform' import { ConnectorFileTooLargeError, extractConnectorText, + hasIndexablePayload, htmlToPlainText, isIndexableConnectorFile, isSkippedDocument, @@ -1482,3 +1483,33 @@ describe('pipelineParsedMimeType', () => { expect(pipelineParsedMimeType('README')).toBeUndefined() }) }) + +describe('hasIndexablePayload', () => { + const bytes = (value: string) => ({ + bytes: Buffer.from(value), + fileName: 'Report.pdf', + mimeType: 'application/pdf', + }) + + it('accepts a source file with bytes', () => { + expect(hasIndexablePayload({ content: '', sourceFile: bytes('%PDF') })).toBe(true) + }) + + it('accepts extracted text', () => { + expect(hasIndexablePayload({ content: 'notes' })).toBe(true) + }) + + /** + * Observed in production: a zero-byte PDF was stored and shipped to OCR, which + * answered `400 Bad Request` — an external call billed to discover the file was + * empty, reported as an API fault rather than as an empty file. Before source + * files existed this was dropped at the empty-content check. + */ + it('rejects a zero-byte source file rather than sending it to OCR', () => { + expect(hasIndexablePayload({ content: '', sourceFile: bytes('') })).toBe(false) + }) + + it('rejects blank text', () => { + expect(hasIndexablePayload({ content: ' ' })).toBe(false) + }) +}) diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 608de615bd1..63674ae350c 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -226,6 +226,21 @@ export function isIndexableConnectorFile(fileName: string): boolean { return extension !== undefined && CONNECTOR_INDEXABLE_EXTENSIONS.has(extension) } +/** + * Whether a document carries anything worth indexing. + * + * A source file has to have bytes. A zero-byte file is not payload: it produces an + * empty stored object, and for a PDF that reaches OCR as an empty request and comes + * back as an opaque `400 Bad Request` — billing an external call to learn the file + * was empty, and reporting it as an API fault rather than as what it is. + */ +export function hasIndexablePayload( + doc: Pick +): boolean { + if (doc.sourceFile) return doc.sourceFile.bytes.length > 0 + return doc.content.trim().length > 0 +} + /** * MIME type to store a file under when the shared pipeline should parse it, or * `undefined` when the connector should decode it as text itself. diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 14e3bb7f13a..c35f3b3d303 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -33,6 +33,7 @@ import type { ExternalDocument, SyncResult, } from '@/connectors/types' +import { hasIndexablePayload } from '@/connectors/utils' const logger = createLogger('ConnectorSyncEngine') @@ -157,7 +158,7 @@ export function classifyExternalDoc( if (extDoc.skippedReason) { return existing ? { type: 'unchanged' } : { type: 'skip' } } - if (!hasPayload(extDoc) && !extDoc.contentDeferred) { + if (!hasIndexablePayload(extDoc) && !extDoc.contentDeferred) { return { type: 'drop' } } if (!existing) { @@ -203,11 +204,6 @@ export function mergeHydratedDocument( } } -/** Whether a document carries anything to index — extracted text or the source file. */ -function hasPayload(extDoc: Pick): boolean { - return extDoc.sourceFile !== undefined || extDoc.content.trim().length > 0 -} - /** Estimated source bytes for a pending op, taken from its listing metadata. */ function estimateOpSizeBytes(op: DocOp): number { // Skip ops load no content (just a row insert), so they do not count against the @@ -1093,7 +1089,7 @@ export async function executeSync( } return null } - if (!fullDoc || !hasPayload(fullDoc)) { + if (!fullDoc || !hasIndexablePayload(fullDoc)) { // An empty re-fetch leaves an already-indexed update as last-known-good; count // it as unchanged so the totals still reconcile with documents seen. Not a // verified refresh, though — see failedExternalIds below.