Skip to content

Commit 6f37771

Browse files
committed
fix(connectors): close swallow-into-empty and cursor-taper regressions
Ship-gate pass over the connector audit. Every finding was re-verified against the provider's live documentation or machine-readable spec before being acted on; several pass-3 edits were reverted rather than extended. Correctness fixes: - fireflies: a 2xx with an unparseable body returned an empty listing instead of throwing. fireflies runs a full sync every time, so a fault persisting across two syncs would have tombstoned all indexed docs. - linear: same shape via `data.issues || {}` on a non-nullable connection. - greenhouse: a 403 from a key without scorecard permission was treated as transient, appending `:partial` to the hash. That never matches the list stub, forcing full re-hydration of every candidate on every sync forever. - google-meet: `fetchParticipants` carried a 404 swallow copied from its transcript siblings, freezing every speaker as "Unknown". - airtable, asana, ashby: reverted page-size tapers applied over opaque cursor tokens. The cap was already enforced server-side. - google-docs: response byte cap resolved to 800MB and could never fire. - google-forms, google-vault, notion, sharepoint, dropbox: `getDocument` now throws on transient failure instead of returning null, which the engine reads as absence. Security: - Retry headers are attached non-enumerably. TypeScript `private` is compile-time only, so `SecureFetchHeaders.setCookies` was an own enumerable property that the logger serialized into sync logs. Docs and dead code: - Corrected six fabricated doc citations (github, jira, jsm, linear, google-meet, dropbox) and removed the Evernote integration entirely.
1 parent ac4519f commit 6f37771

47 files changed

Lines changed: 694 additions & 228 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

apps/sim/connectors/airtable/airtable.ts

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -156,14 +156,18 @@ export const airtableConnector: ConnectorConfig = {
156156
const maxRecords = readMaxRecords(sourceConfig)
157157

158158
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
159-
/** Shrink the last page so a `maxRecords` cap never over-fetches past it. */
160-
const pageSize =
161-
maxRecords > 0 ? Math.max(1, Math.min(PAGE_SIZE, maxRecords - prevFetched)) : PAGE_SIZE
162159

163160
const tableId = await resolveTableId(accessToken, baseId, tableIdOrName, syncContext)
164161

162+
/**
163+
* `pageSize` is held at the documented maximum for every request of a sync.
164+
* Airtable already stops pagination itself once `maxRecords` is reached, and
165+
* its `offset` is an opaque iterator token whose validity across a changed
166+
* `pageSize` is undocumented — so shrinking the last page would buy nothing
167+
* and risk breaking iteration mid-sync.
168+
*/
165169
const params = new URLSearchParams()
166-
params.append('pageSize', String(pageSize))
170+
params.append('pageSize', String(PAGE_SIZE))
167171
if (viewId) params.append('view', viewId)
168172
if (maxRecords > 0) params.append('maxRecords', String(maxRecords))
169173

@@ -217,10 +221,12 @@ export const airtableConnector: ConnectorConfig = {
217221
const nextOffset = data.offset
218222
const hitLimit = maxRecords > 0 && totalFetched >= maxRecords
219223
/**
220-
* Airtable omits `offset` once `maxRecords` is reached, so an exhausted
221-
* source and a capped one are indistinguishable. Flag conservatively: a
222-
* capped listing must never let the engine hard-delete the records the cap
223-
* hid.
224+
* Airtable enforces `maxRecords` itself — "pagination will stop once you've
225+
* reached this maximum" — but does not document whether it still returns an
226+
* `offset` at that point, so an exhausted source and a capped one cannot be
227+
* told apart here. Flagged conservatively: a capped listing must never let
228+
* the engine hard-delete the records the cap hid. The cost is that deletion
229+
* reconciliation only runs for a capped source on an explicit full resync.
224230
*/
225231
if (hitLimit && syncContext) syncContext.listingCapped = true
226232

@@ -239,7 +245,10 @@ export const airtableConnector: ConnectorConfig = {
239245
): Promise<ExternalDocument | null> => {
240246
const baseId = readConfigString(sourceConfig, 'baseId')
241247
const tableIdOrName = readConfigString(sourceConfig, 'tableIdOrName')
242-
if (!baseId || !tableIdOrName) return null
248+
/** A broken config is not evidence the record is gone, so it must not read as absence. */
249+
if (!baseId || !tableIdOrName) {
250+
throw new Error('Airtable connector is missing baseId or tableIdOrName')
251+
}
243252
const titleField = readConfigString(sourceConfig, 'titleField')
244253

245254
const tableId = await resolveTableId(accessToken, baseId, tableIdOrName, syncContext)

apps/sim/connectors/asana/asana.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,7 @@ describe('asanaConnector.listDocuments', () => {
445445
expect(syncContext.listingCapped).toBeUndefined()
446446
})
447447

448-
it('shrinks the requested page size to the remaining cap on the last page', async () => {
448+
it('holds the requested page size constant regardless of how much cap is left', async () => {
449449
mockFetch.mockImplementation(async (url) => {
450450
if (url.includes('/projects')) {
451451
return jsonResponse({ data: [{ gid: 'p1', name: 'Live' }], next_page: null })
@@ -460,7 +460,7 @@ describe('asanaConnector.listDocuments', () => {
460460
totalDocsFetched: 497,
461461
})
462462

463-
expect(requestedUrls().find((url) => url.includes('/tasks?'))).toContain('limit=3')
463+
expect(requestedUrls().find((url) => url.includes('/tasks?'))).toContain('limit=100')
464464
})
465465

466466
it('keeps syncing an explicitly pinned project without listing workspace projects', async () => {

apps/sim/connectors/asana/asana.ts

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -326,16 +326,12 @@ export const asanaConnector: ConnectorConfig = {
326326
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
327327

328328
/**
329-
* Last-page precision: once a `maxTasks` cap exists, never ask Asana for
330-
* more rows than the cap still has room for.
329+
* Held constant across every request of a sync. Asana's `offset` is an
330+
* opaque token and the docs do not say it survives a changed `limit`, while
331+
* `decideTaskCap` already trims the cap exactly — so varying the page size
332+
* per call would buy nothing and risk invalidating a mid-project offset.
331333
*/
332-
const remaining = maxTasks > 0 ? Math.max(maxTasks - previouslyFetched, 0) : 0
333-
if (maxTasks > 0 && remaining === 0) {
334-
if (syncContext) syncContext.listingCapped = true
335-
return { documents: [], nextCursor: undefined, hasMore: false }
336-
}
337-
const pageSize =
338-
maxTasks > 0 ? Math.max(Math.min(remaining, ASANA_MAX_PAGE_SIZE), 1) : ASANA_MAX_PAGE_SIZE
334+
const pageSize = maxTasks > 0 ? Math.min(maxTasks, ASANA_MAX_PAGE_SIZE) : ASANA_MAX_PAGE_SIZE
339335

340336
/**
341337
* Cursor format:

apps/sim/connectors/ashby/ashby.ts

Lines changed: 34 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,11 @@ interface AshbyEnvelope {
5050
}
5151

5252
/**
53-
* Extracts a human-readable error message from an Ashby error envelope. Ashby documents
54-
* two shapes and uses both: `errorInfo.message`, and an `errors` array whose entries are
55-
* either plain strings or `{ message, parameter }` objects. An object entry stringifies
56-
* to `[object Object]` unless its message is read explicitly, which is the form a 403
57-
* for a missing module permission arrives in.
53+
* Extracts a human-readable error message from an Ashby error envelope. The documented
54+
* failure body is `{ success: false, errors: [{ message }] }`, but `errorInfo.message`
55+
* and plain-string `errors` entries also occur, so all three are handled. Reading the
56+
* object entry's `message` explicitly is what keeps it from stringifying to
57+
* `[object Object]`.
5858
*/
5959
function ashbyErrorMessage(data: AshbyEnvelope, fallback: string): string {
6060
if (data.errorInfo?.message) return data.errorInfo.message
@@ -550,9 +550,13 @@ export const ashbyConnector: ConnectorConfig = {
550550
return { documents: [], hasMore: false }
551551
}
552552

553-
/** Remaining budget under `maxCandidates`, so the last page requests only what is needed. */
553+
/**
554+
* `limit` is held constant for every request of a sync: Ashby's `cursor` is
555+
* opaque and the docs do not say it survives a changed `limit`, and the cap
556+
* is enforced below by trimming the page instead.
557+
*/
554558
const remaining = maxCandidates > 0 ? maxCandidates - prevFetched : Number.POSITIVE_INFINITY
555-
const body: UnknownRecord = { limit: Math.min(CANDIDATES_PER_PAGE, remaining) }
559+
const body: UnknownRecord = { limit: CANDIDATES_PER_PAGE }
556560
if (cursor) body.cursor = cursor
557561
if (createdAfterMs !== undefined) body.createdAfter = createdAfterMs
558562

@@ -565,8 +569,10 @@ export const ashbyConnector: ConnectorConfig = {
565569
const results = Array.isArray(data.results) ? data.results : []
566570
const candidates = results.map(mapCandidate).filter((c) => c.id)
567571

568-
let documents = candidates.map(candidateToStub)
569-
if (documents.length > remaining) documents = documents.slice(0, remaining)
572+
const stubs = candidates.map(candidateToStub)
573+
const documents = stubs.length > remaining ? stubs.slice(0, remaining) : stubs
574+
/** True when the cap hid candidates Ashby already returned on this very page. */
575+
const droppedInPage = documents.length < stubs.length
570576

571577
const totalFetched = prevFetched + documents.length
572578
if (syncContext) syncContext.totalCandidatesFetched = totalFetched
@@ -576,11 +582,13 @@ export const ashbyConnector: ConnectorConfig = {
576582
const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates
577583
/**
578584
* `listingCapped` blocks the sync engine's deletion reconciliation, so it is set only
579-
* when `maxCandidates` cut the listing short while Ashby still had more candidates
580-
* never when the cap coincides with genuine exhaustion, and never for the intentional
581-
* `createdAfter` scope filter.
585+
* when `maxCandidates` made the listing knowingly incomplete — candidates dropped from
586+
* this page, or pages left unread behind the cap. Never when the cap coincides with
587+
* genuine exhaustion, and never for the intentional `createdAfter` scope filter.
582588
*/
583-
if (syncContext && hitLimit && sourceHasMore) syncContext.listingCapped = true
589+
if (syncContext && (droppedInPage || (hitLimit && sourceHasMore))) {
590+
syncContext.listingCapped = true
591+
}
584592

585593
const hasMore = !hitLimit && sourceHasMore
586594

@@ -616,19 +624,20 @@ export const ashbyConnector: ConnectorConfig = {
616624
})
617625
}
618626

619-
const settled = await Promise.allSettled(
620-
applicationIds.map((applicationId) =>
621-
fetchFeedbackForApplication(accessToken, applicationId)
622-
)
623-
)
624-
for (let i = 0; i < settled.length; i++) {
625-
const outcome = settled[i]
626-
if (outcome.status === 'fulfilled') {
627-
feedback.push(...outcome.value)
628-
} else {
627+
/**
628+
* Sequential on purpose. The sync engine already hydrates SYNC_BATCH_SIZE
629+
* candidates concurrently, so fanning these out would multiply that into
630+
* a burst of up to `MAX_APPLICATIONS_FOR_FEEDBACK` × the batch size
631+
* simultaneous Ashby requests. A per-application catch keeps one failing
632+
* application from losing the rest of the candidate's feedback.
633+
*/
634+
for (const applicationId of applicationIds) {
635+
try {
636+
feedback.push(...(await fetchFeedbackForApplication(accessToken, applicationId)))
637+
} catch (error) {
629638
logger.warn('Failed to fetch Ashby feedback for application', {
630-
applicationId: applicationIds[i],
631-
error: toError(outcome.reason).message,
639+
applicationId,
640+
error: toError(error).message,
632641
})
633642
}
634643
}

apps/sim/connectors/azure-devops/azure-devops.ts

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -246,10 +246,11 @@ interface GitRepository {
246246
}
247247

248248
/**
249-
* Resolves the browsable base URL for a repository. `webUrl` is only present
250-
* when the repositories listing is requested with `includeAllUrls=true`;
251-
* `remoteUrl` is always returned and points at the same
252-
* `.../{project}/_git/{repo}` web route, so it is the fallback.
249+
* Resolves the browsable base URL for a repository. `webUrl` is declared on
250+
* GitRepository but absent from the documented sample responses, so it cannot be
251+
* relied on; `remoteUrl` appears in every sample as
252+
* `https://dev.azure.com/{org}/{project}/_git/{repo}`, which is the same web
253+
* route, and serves as the fallback.
253254
*/
254255
function repoBaseUrl(repo: GitRepository | undefined): string | undefined {
255256
return repo?.webUrl || repo?.remoteUrl
@@ -699,9 +700,10 @@ async function listRepositories(
699700
syncContext?: Record<string, unknown>
700701
): Promise<GitRepository[]> {
701702
/**
702-
* `includeAllUrls=true` is required for the response to carry `webUrl`; without
703-
* it the listing returns only `url`/`remoteUrl` and every repository-file
704-
* document would be indexed with no `sourceUrl`.
703+
* `includeAllUrls=true` — "True to include all remote URLs. The default value
704+
* is false." The docs do not say which of GitRepository's URL fields it gates,
705+
* and the sample listing omits `webUrl`, so it is requested to maximise the
706+
* chance of getting one; `repoBaseUrl` falls back to `remoteUrl` regardless.
705707
*/
706708
const url = `${ADO_BASE_URL}/${encodeURIComponent(organization)}/${encodeURIComponent(project)}/_apis/git/repositories?includeAllUrls=true&api-version=${GIT_API_VERSION}`
707709
const response = await fetchWithRetry(

apps/sim/connectors/docusign/docusign.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -435,10 +435,14 @@ export const docusignConnector: ConnectorConfig = {
435435

436436
/**
437437
* Remaining budget under `maxEnvelopes`. The last page requests only what is still
438-
* needed instead of a full {@link MAX_PAGE_SIZE} page.
438+
* needed instead of a full {@link MAX_PAGE_SIZE} page. Safe to shrink because
439+
* `start_position` is an offset the next page resumes from, not a page number.
440+
*
441+
* Floored to a positive integer: `maxEnvelopes` is free-form user input that
442+
* `validateConfig` only checks for sign, and DocuSign rejects a fractional `count`.
439443
*/
440444
const remaining = maxEnvelopes > 0 ? maxEnvelopes - prevFetched : Number.POSITIVE_INFINITY
441-
const pageSize = Math.min(MAX_PAGE_SIZE, remaining)
445+
const pageSize = Math.max(1, Math.min(MAX_PAGE_SIZE, Math.floor(remaining)))
442446

443447
const queryParams = new URLSearchParams({
444448
from_date: formatFromDate(fromDate),

apps/sim/connectors/dropbox/dropbox.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,12 @@ function isDownloadableFile(entry: DropboxEntry): entry is DropboxFileMetadata {
9090
}
9191

9292
/**
93-
* Normalizes a user-supplied folder path to what `/2/files/list_folder` expects:
94-
* the empty string for the Dropbox root, otherwise a leading slash and no trailing
95-
* slash (a trailing slash is rejected as `malformed_path`).
93+
* Normalizes a user-supplied folder path to the `PathROrId` format
94+
* `/2/files/list_folder` declares: the empty string for the Dropbox root (the
95+
* leading-slash branch of the pattern is optional precisely so `""` matches),
96+
* otherwise a leading slash. The trailing slash is stripped as defensive
97+
* tidying of free-form input, not because Dropbox documents rejecting it.
98+
* A path outside the format fails with `path/malformed_path`.
9699
*/
97100
function normalizeFolderPath(raw: unknown): string {
98101
const trimmed = typeof raw === 'string' ? raw.trim() : ''

apps/sim/connectors/fireflies/fireflies.test.ts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,30 @@ describe('fireflies listDocuments', () => {
134134
)
135135
})
136136

137+
it('throws rather than reporting an empty listing when a 200 body is unreadable', async () => {
138+
mockFetchWithRetry.mockResolvedValue({
139+
ok: true,
140+
status: 200,
141+
json: async () => {
142+
throw new SyntaxError('Unexpected token < in JSON at position 0')
143+
},
144+
text: async () => '<html>gateway</html>',
145+
} as unknown as Response)
146+
const syncContext: Record<string, unknown> = {}
147+
148+
await expect(
149+
firefliesConnector.listDocuments('key', {}, undefined, syncContext)
150+
).rejects.toThrow(/malformed/i)
151+
})
152+
153+
it('throws rather than reporting an empty listing when a 200 carries no data', async () => {
154+
mockGraphQL([{ body: {} }])
155+
156+
await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow(
157+
/malformed/i
158+
)
159+
})
160+
137161
it('surfaces the errors[] message on a non-2xx response', async () => {
138162
mockGraphQL([
139163
{ status: 403, body: { errors: [{ message: 'Upgrade required', code: 'paid_required' }] } },

apps/sim/connectors/fireflies/fireflies.ts

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,18 @@ async function firefliesGraphQL(
9191
throw new Error(`Fireflies API HTTP error: ${response.status}`)
9292
}
9393

94-
return data?.data ?? {}
94+
/**
95+
* A 2xx carrying neither `errors` nor a `data` object is unreadable — an
96+
* unparseable body, a truncated response, a proxy interstitial. It must raise
97+
* rather than degrade to an empty result: `listDocuments` would otherwise
98+
* report a confident empty listing and the sync engine would reconcile every
99+
* stored document as deleted.
100+
*/
101+
if (!data || typeof data.data !== 'object' || data.data === null) {
102+
throw new Error('Fireflies API returned a malformed response with no data')
103+
}
104+
105+
return data.data
95106
}
96107

97108
/**
@@ -287,7 +298,13 @@ export const firefliesConnector: ConnectorConfig = {
287298

288299
return {
289300
documents,
290-
nextCursor: hasMore ? String(skip + documents.length) : undefined,
301+
/**
302+
* `skip` is an offset over the raw API result set, so it must advance by the
303+
* rows Fireflies returned — not by the stubs kept. Advancing by the kept
304+
* count would re-request any row dropped for a missing `id`. `hasMore` is
305+
* only ever true on the uncapped path, where nothing is sliced off.
306+
*/
307+
nextCursor: hasMore ? String(skip + transcripts.length) : undefined,
291308
hasMore,
292309
}
293310
},

apps/sim/connectors/github/github.ts

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -176,10 +176,12 @@ async function fetchBlobContent(
176176
return buf.toString('utf8')
177177
}
178178
/**
179-
* Per https://docs.github.com/en/rest/git/blobs the Blobs API only ever
180-
* returns base64. Refuse to silently persist empty content for an
181-
* unexpected encoding so a sync surfaces the error instead.
179+
* https://docs.github.com/en/rest/git/blobs documents two encodings —
180+
* "Currently, 'utf-8' and 'base64' are supported" — so a utf-8 body is already
181+
* plain text and is used as written. Any other encoding would silently persist
182+
* empty content, so it throws and surfaces as a failed document instead.
182183
*/
184+
if (encoding === 'utf-8') return content
183185
throw new Error(`Unexpected git blob encoding for ${sha}: ${encoding ?? 'undefined'}`)
184186
}
185187

@@ -378,9 +380,11 @@ export const githubConnector: ConnectorConfig = {
378380
content = buf.toString('utf8')
379381
} else if (encoding === 'none' && data.sha && size > 0) {
380382
/**
381-
* The Contents API returns `content: ""` with `encoding: "none"` for files
382-
* over 1 MB (verified against a 2.3 MB blob). The Git Blobs API serves the
383-
* same blob base64-encoded up to 100 MB.
383+
* Per https://docs.github.com/en/rest/repos/contents, for files of 1-100 MB
384+
* "the content field will be an empty string and the encoding field will be
385+
* `none`". The Contents page points at the raw media type; the Git Blobs API
386+
* is used instead because it returns the same blob as JSON and is documented
387+
* to support blobs up to 100 MB.
384388
*/
385389
const blobContent = await fetchBlobContent(accessToken, owner, repo, data.sha as string)
386390
if (blobContent === null) {

0 commit comments

Comments
 (0)