diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index ea42ae779d5..5a40a535e09 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -7269,6 +7269,44 @@ export function GoogleGroupsIcon(props: SVGProps) { ) } +/** + * Google Chat product mark: a hollow four-colour speech bubble with a chamfered + * upper-right corner, traced from the official 2023 icon + * (https://commons.wikimedia.org/wiki/File:Google_Chat_icon_(2023).svg) and + * using the shared Google Workspace two-tone palette the other Google marks in + * this file already carry. + */ +export function GoogleChatIcon(props: SVGProps) { + return ( + + + + + + + + + ) +} + export function GoogleMeetIcon(props: SVGProps) { return ( diff --git a/apps/docs/content/docs/en/knowledgebase/connectors.mdx b/apps/docs/content/docs/en/knowledgebase/connectors.mdx index 659fd5dae8d..d95b9c2b88a 100644 --- a/apps/docs/content/docs/en/knowledgebase/connectors.mdx +++ b/apps/docs/content/docs/en/knowledgebase/connectors.mdx @@ -14,23 +14,24 @@ Connectors continuously sync documents from external services into your knowledg -Sim ships with 60 built-in connectors: +Sim ships with 64 built-in connectors: | Category | Connectors | |----------|-----------| | **Productivity** | Notion, Confluence, Asana, Linear, Jira, Jira Service Management, Monday, Trello, ClickUp, Google Calendar, Google Sheets, Google Forms, Microsoft Excel, Typeform | | **Cloud Storage** | Google Drive, Dropbox, OneDrive, SharePoint, Box, Amazon S3, SFTP | | **Documents** | Google Docs, Google Slides, Mintlify, WordPress, Webflow, DocuSign | -| **Development** | GitHub, GitLab, Azure DevOps, Sentry | -| **Communication** | Slack, Discord, Microsoft Teams, Reddit, X, YouTube | +| **Development** | GitHub, GitLab, Bitbucket, Azure DevOps, Sentry | +| **Communication** | Slack, Discord, Microsoft Teams, Google Chat, Reddit, X, YouTube | | **Email** | Gmail, Outlook | | **CRM** | HubSpot, Salesforce | | **Support** | Intercom, ServiceNow, Zendesk, Zoho Desk | | **Incident Management** | incident.io, Rootly, PagerDuty | -| **Data** | Airtable | +| **Data** | Airtable, Databricks | | **Note-taking** | Obsidian | | **Meetings** | Zoom, Google Meet, Gong, Grain, Granola, Fathom, Fireflies | | **Recruiting** | Greenhouse, Ashby | +| **HR** | Workday Help | | **Compliance** | Google Vault | ## Adding a Connector @@ -58,6 +59,8 @@ Other connectors use **API keys** or **personal access tokens** instead. The set | **PagerDuty** | REST API key from Integrations → API Access Keys | | **SFTP** | Password or unencrypted private key, plus a required SHA-256 host key fingerprint (host, port, username, and root path are entered as config fields) | | **Mintlify** | API key — optional for public documentation sites, which sync from `llms.txt` | +| **Databricks** | Personal access token from your workspace's user settings (the workspace host is entered as a config field) | +| **Workday Help** | Register an API client for integrations in your tenant, then enter the client secret and refresh token together as `clientSecret:refreshToken` (the client ID, tenant host, and tenant name are entered as config fields) | If you rotate an API key in the external service, update it in Sim as well — OAuth tokens refresh automatically, but API keys do not. diff --git a/apps/docs/content/docs/en/platform/self-hosting/integrations-oauth.mdx b/apps/docs/content/docs/en/platform/self-hosting/integrations-oauth.mdx index 03f4f1c5def..3bf2b8a769b 100644 --- a/apps/docs/content/docs/en/platform/self-hosting/integrations-oauth.mdx +++ b/apps/docs/content/docs/en/platform/self-hosting/integrations-oauth.mdx @@ -100,7 +100,7 @@ One OAuth client in [Google Cloud Console](https://console.cloud.google.com/apis | Environment variables | Provider IDs | |---|---| -| `GOOGLE_CLIENT_ID`
`GOOGLE_CLIENT_SECRET` | `google-email`, `google-drive`, `google-sheets`, `google-docs`, `google-calendar`, `google-contacts`, `google-forms`, `google-tasks`, `google-meet`, `google-groups`, `google-ads`, `google-bigquery`, `google-vault`, `vertex-ai` | +| `GOOGLE_CLIENT_ID`
`GOOGLE_CLIENT_SECRET` | `google-email`, `google-drive`, `google-sheets`, `google-docs`, `google-calendar`, `google-contacts`, `google-forms`, `google-tasks`, `google-meet`, `google-groups`, `google-chat`, `google-ads`, `google-bigquery`, `google-vault`, `vertex-ai` | The same variables also power "Sign in with Google". See [Authentication](/platform/self-hosting/authentication). diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index ea42ae779d5..5a40a535e09 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -7269,6 +7269,44 @@ export function GoogleGroupsIcon(props: SVGProps) { ) } +/** + * Google Chat product mark: a hollow four-colour speech bubble with a chamfered + * upper-right corner, traced from the official 2023 icon + * (https://commons.wikimedia.org/wiki/File:Google_Chat_icon_(2023).svg) and + * using the shared Google Workspace two-tone palette the other Google marks in + * this file already carry. + */ +export function GoogleChatIcon(props: SVGProps) { + return ( + + + + + + + + + ) +} + export function GoogleMeetIcon(props: SVGProps) { return ( diff --git a/apps/sim/connectors/bitbucket/bitbucket.test.ts b/apps/sim/connectors/bitbucket/bitbucket.test.ts new file mode 100644 index 00000000000..a40289405ad --- /dev/null +++ b/apps/sim/connectors/bitbucket/bitbucket.test.ts @@ -0,0 +1,718 @@ +/** + * @vitest-environment node + * + * The load-bearing logic here is the repository tree walk: a `max_depth` listing + * that Bitbucket may answer with a `555` timeout, an opaque `next` cursor that + * cannot be re-cut, and a per-run frontier of directories the walk has not reached. + * Every path that shortens the listing has to leave `syncContext.listingCapped` set, + * because the sync engine hard-deletes whatever a full listing omits. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { bitbucketConnector } from '@/connectors/bitbucket/bitbucket' + +const ACCESS_TOKEN = 'bitbucket-token' +const COMMIT = 'e7d158ff7ed5538c28f94cd97a9ad569680fc94e' +const CONFIG = { workspaceSlug: 'acme', repoSlug: 'widgets' } +const PR_CONFIG = { ...CONFIG, contentTypes: 'pullrequests' } + +const mockFetch = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +const REPOSITORY = { + full_name: 'acme/widgets', + mainbranch: { name: 'main', target: { hash: COMMIT } }, + links: { html: { href: 'https://bitbucket.org/acme/widgets' } }, +} + +function fileEntry(path: string, size = 10, attributes: string[] = []) { + return { type: 'commit_file', path, size, attributes, commit: { hash: COMMIT } } +} + +function dirEntry(path: string) { + return { type: 'commit_directory', path, commit: { hash: COMMIT } } +} + +function pullRequestFixture(id: number, overrides: Record = {}) { + return { + id, + title: `PR ${id}`, + state: 'OPEN', + summary: { raw: `Body of ${id}`, markup: 'markdown', html: `

Body of ${id}

` }, + author: { display_name: 'Ada Lovelace', nickname: 'ada' }, + created_on: '2026-01-01T00:00:00.000000+00:00', + updated_on: '2026-02-01T00:00:00.000000+00:00', + links: { html: { href: `https://bitbucket.org/acme/widgets/pull-requests/${id}` } }, + ...overrides, + } +} + +/** Routes by URL, because the number of lookups before a listing varies by phase. */ +function mockApi(routes: Array<[RegExp, () => Response]>) { + mockFetch.mockImplementation(async (url: string) => { + for (const [pattern, respond] of routes) { + if (pattern.test(url)) return respond() + } + if (/\/repositories\/acme\/widgets$/.test(url)) return jsonResponse(REPOSITORY) + throw new Error(`unrouted request: ${url}`) + }) +} + +function requestedUrls(pattern: RegExp): string[] { + return mockFetch.mock.calls.map(([url]) => url as string).filter((url) => pattern.test(url)) +} + +beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('bitbucket repository file listing', () => { + it('lists text files as deferred stubs and drops binary, symlink, and submodule entries', async () => { + mockApi([ + [ + /\/src\//, + () => + jsonResponse({ + values: [ + fileEntry('README.md'), + fileEntry('logo.png', 2048, ['binary']), + fileEntry('link.txt', 5, ['link']), + fileEntry('vendor', 40, ['subrepository']), + fileEntry('run.sh', 12, ['executable']), + ], + }), + ], + ]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + + expect(result.documents.map((d) => d.externalId)).toEqual(['file:README.md', 'file:run.sh']) + expect(result.documents[0].contentDeferred).toBe(true) + expect(result.documents[0].content).toBe('') + expect(result.documents[0].contentHash).toBe(`bitbucket:file:acme/widgets:${COMMIT}:README.md`) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('pins the listing to the resolved tip commit and asks for the documented depth', async () => { + mockApi([[/\/src\//, () => jsonResponse({ values: [] })]]) + + await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + + const url = requestedUrls(/\/src\//)[0] + expect(url).toContain(`/src/${COMMIT}/`) + expect(url).toContain('max_depth=5') + expect(url).toContain('pagelen=100') + }) + + it('applies the path prefix and extension filters to files and prunes unrelated directories', async () => { + mockApi([ + [ + /\/src\//, + () => + jsonResponse({ + values: [ + fileEntry('docs/guide.md'), + fileEntry('docs/guide.txt'), + fileEntry('src/index.md'), + dirEntry('docs/a/b/c/d'), + dirEntry('src/a/b/c/d'), + ], + }), + ], + ]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, pathPrefix: '/docs', fileExtensions: 'md' }, + undefined, + syncContext + ) + + expect(result.documents.map((d) => d.externalId)).toEqual(['file:docs/guide.md']) + expect(syncContext.pendingDirs).toEqual(['docs/a/b/c/d']) + }) + + it('queues only the frontier directories the walk actually stopped at', async () => { + mockApi([ + [ + /\/src\//, + () => + jsonResponse({ + values: [dirEntry('a'), dirEntry('a/b'), dirEntry('a/b/c/d/e'), fileEntry('a/b/x.md')], + }), + ], + ]) + + const syncContext: Record = {} + await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, syncContext) + + expect(syncContext.pendingDirs).toEqual(['a/b/c/d/e']) + }) + + it('walks the frontier on a following page and finishes without flagging the listing', async () => { + mockApi([ + [ + /\/src\/[a-f0-9]+\/\?/, + () => jsonResponse({ values: [dirEntry('a/b/c/d/e'), fileEntry('root.md')] }), + ], + [ + /\/src\/[a-f0-9]+\/a\/b\/c\/d\/e\/\?/, + () => jsonResponse({ values: [fileEntry('a/b/c/d/e/deep.md')] }), + ], + ]) + + const syncContext: Record = {} + const first = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + expect(first.hasMore).toBe(true) + + const second = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + first.nextCursor, + syncContext + ) + + expect(second.documents.map((d) => d.externalId)).toEqual(['file:a/b/c/d/e/deep.md']) + expect(second.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('replays the opaque next link verbatim instead of rebuilding it', async () => { + const nextUrl = + 'https://api.bitbucket.org/2.0/repositories/acme/widgets/src/abc/?page=2&opaque=xyz' + mockApi([ + [/page=2/, () => jsonResponse({ values: [fileEntry('second.md')] })], + [/\/src\//, () => jsonResponse({ values: [fileEntry('first.md')], next: nextUrl })], + ]) + + const syncContext: Record = {} + const first = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + expect(first.hasMore).toBe(true) + + await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, first.nextCursor, syncContext) + + expect(requestedUrls(/page=2/)).toEqual([nextUrl]) + }) + + it('rejects a next cursor that does not point at the Bitbucket Cloud API', async () => { + mockApi([ + [ + /\/src\//, + () => jsonResponse({ values: [], next: 'https://evil.example.com/2.0/repositories/x' }), + ], + ]) + + const syncContext: Record = {} + const first = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + + await expect( + bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, first.nextCursor, syncContext) + ).rejects.toThrow(/Bitbucket Cloud API/) + }) +}) + +describe('bitbucket source listing timeouts', () => { + it('retries a first-page 555 at depth 1 and keeps the listing reconcilable', async () => { + mockApi([ + [/max_depth=1/, () => jsonResponse({ values: [dirEntry('a'), fileEntry('root.md')] })], + [/max_depth=5/, () => new Response('timeout', { status: 555 })], + ]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + + expect(result.documents.map((d) => d.externalId)).toEqual(['file:root.md']) + expect(syncContext.listingCapped).toBeUndefined() + /** The depth-1 response only reached one level, so `a` is now unexplored. */ + expect(syncContext.pendingDirs).toEqual(['a']) + expect(result.hasMore).toBe(true) + }) + + it('flags the listing capped when even the depth-1 retry times out', async () => { + mockApi([[/\/src\//, () => new Response('timeout', { status: 555 })]]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + expect(result.documents).toEqual([]) + expect(requestedUrls(/\/src\//)).toHaveLength(2) + }) + + it('flags the listing capped when a replayed next cursor times out, without re-cutting it', async () => { + const nextUrl = + 'https://api.bitbucket.org/2.0/repositories/acme/widgets/src/abc/?page=2&max_depth=5' + mockApi([ + [/page=2/, () => new Response('timeout', { status: 555 })], + [/\/src\//, () => jsonResponse({ values: [fileEntry('first.md')], next: nextUrl })], + ]) + + const syncContext: Record = {} + const first = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + const second = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + first.nextCursor, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + expect(second.documents).toEqual([]) + /** Exactly one attempt: an opaque cursor is never rebuilt at another depth. */ + expect(requestedUrls(/page=2/)).toHaveLength(1) + }) + + it('leaves the listing reconcilable when a directory is simply absent', async () => { + mockApi([[/\/src\//, () => jsonResponse({ type: 'error' }, 404)]]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + expect(result.hasMore).toBe(false) + }) + + it('flags the listing capped when the token stops working mid-walk', async () => { + mockApi([[/\/src\//, () => jsonResponse({ type: 'error' }, 403)]]) + + const syncContext: Record = {} + await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, syncContext) + + expect(syncContext.listingCapped).toBe(true) + }) +}) + +describe('bitbucket maxItems cap', () => { + it('flags the listing capped when the cap truncates a page', async () => { + mockApi([ + [ + /\/src\//, + () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md'), fileEntry('c.md')] }), + ], + ]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxItems: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it('leaves the listing reconcilable when a complete listing ends exactly on the cap', async () => { + mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxItems: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags the listing capped when the code walk ends on the cap and the pull request phase never runs', async () => { + mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, contentTypes: 'all', maxItems: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it('flags the listing capped when the cap lands on a page boundary with more to come', async () => { + mockApi([ + [ + /\/src\//, + () => + jsonResponse({ + values: [fileEntry('a.md'), fileEntry('b.md')], + next: 'https://api.bitbucket.org/2.0/repositories/acme/widgets/src/abc/?page=2', + }), + ], + ]) + + const syncContext: Record = {} + await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxItems: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('leaves the listing reconcilable when no cap is configured', async () => { + mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('lets an oversized file ride along without consuming the cap', async () => { + mockApi([ + [ + /\/src\//, + () => + jsonResponse({ + values: [fileEntry('huge.md', 500 * 1024 * 1024), fileEntry('a.md'), fileEntry('b.md')], + }), + ], + ]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxItems: '2' }, + undefined, + syncContext + ) + + expect(result.documents.map((d) => d.externalId)).toEqual([ + 'file:huge.md', + 'file:a.md', + 'file:b.md', + ]) + expect(result.documents[0].skippedReason).toMatch(/size limit/) + }) +}) + +describe('bitbucket pull request listing', () => { + it('builds the documented collection query', async () => { + mockApi([[/\/pullrequests/, () => jsonResponse({ values: [pullRequestFixture(7)] })]]) + + await bitbucketConnector.listDocuments(ACCESS_TOKEN, PR_CONFIG, undefined, {}) + + const url = requestedUrls(/\/pullrequests/)[0] + expect(url).toContain('pagelen=50') + /** `+` must reach Bitbucket percent-encoded or it is read as a space. */ + expect(url).toContain('fields=%2Bvalues.summary') + expect(url).toContain('sort=-id') + expect(url).toContain('state=OPEN') + }) + + it('repeats the state parameter for each requested state', async () => { + mockApi([[/\/pullrequests/, () => jsonResponse({ values: [] })]]) + + await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...PR_CONFIG, pullRequestState: 'all' }, + undefined, + {} + ) + + const url = requestedUrls(/\/pullrequests/)[0] + expect(url).toContain('state=OPEN') + expect(url).toContain('state=MERGED') + expect(url).toContain('state=DECLINED') + expect(url).toContain('state=SUPERSEDED') + }) + + it('renders the incremental BBQL filter with an explicit numeric UTC offset', async () => { + mockApi([[/\/pullrequests/, () => jsonResponse({ values: [] })]]) + + await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + PR_CONFIG, + undefined, + {}, + new Date('2026-02-03T04:05:06.007Z') + ) + + const url = new URL(requestedUrls(/\/pullrequests/)[0]) + expect(url.searchParams.get('q')).toBe('updated_on > 2026-02-03T04:05:06.007+00:00') + }) + + it('maps the author-typed body and falls back to the rendered description', async () => { + mockApi([ + [ + /\/pullrequests/, + () => + jsonResponse({ + values: [ + pullRequestFixture(7), + pullRequestFixture(8, { + summary: undefined, + rendered: { description: { raw: 'Rendered body' } }, + }), + ], + }), + ], + ]) + + const result = await bitbucketConnector.listDocuments(ACCESS_TOKEN, PR_CONFIG, undefined, {}) + + expect(result.documents[0].content).toBe('PR 7\n\nBody of 7') + expect(result.documents[0].contentDeferred).toBe(false) + expect(result.documents[0].contentHash).toBe( + 'bitbucket:pr:acme/widgets:7:2026-02-01T00:00:00.000000+00:00' + ) + expect(result.documents[1].content).toBe('PR 8\n\nRendered body') + }) + + it('walks from the code phase into the pull request phase for combined content', async () => { + mockApi([ + [/\/src\//, () => jsonResponse({ values: [fileEntry('a.md')] })], + [/\/pullrequests/, () => jsonResponse({ values: [pullRequestFixture(7)] })], + ]) + + const syncContext: Record = {} + const first = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, contentTypes: 'all' }, + undefined, + syncContext + ) + expect(first.hasMore).toBe(true) + + const second = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, contentTypes: 'all' }, + first.nextCursor, + syncContext + ) + + expect(second.documents.map((d) => d.externalId)).toEqual(['pr:7']) + expect(second.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) +}) + +describe('bitbucket getDocument', () => { + it('hydrates a file with the same contentHash the listing produced', async () => { + mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('README.md')] })]]) + + const syncContext: Record = {} + const listed = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + + mockApi([[/\/src\/[a-f0-9]+\/README\.md$/, () => new Response('# Widgets', { status: 200 })]]) + const fetched = await bitbucketConnector.getDocument( + ACCESS_TOKEN, + CONFIG, + 'file:README.md', + syncContext + ) + + expect(fetched?.contentHash).toBe(listed.documents[0].contentHash) + expect(fetched?.contentDeferred).toBe(false) + expect(fetched?.content).toBe('README.md\n\n# Widgets') + expect(fetched?.metadata?.size).toBe(9) + }) + + it('surfaces an LFS-managed file as skipped rather than following the media redirect', async () => { + mockApi([ + [ + /\/src\/[a-f0-9]+\/big\.bin$/, + () => + new Response(null, { + status: 301, + headers: { Location: 'https://media.atlassian.com/file/abc' }, + }), + ], + ]) + + const doc = await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:big.bin', {}) + + expect(doc?.skippedReason).toMatch(/LFS/) + expect(doc?.content).toBe('') + }) + + it('surfaces a file with a NUL byte as a skipped binary', async () => { + mockApi([ + [ + /\/src\/[a-f0-9]+\/data\.md$/, + () => new Response(Buffer.from([0x61, 0x00, 0x62]), { status: 200 }), + ], + ]) + + const doc = await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:data.md', {}) + + expect(doc?.skippedReason).toMatch(/Binary/) + }) + + it('returns null for a file the ref no longer carries', async () => { + mockApi([[/\/src\/[a-f0-9]+\/gone\.md$/, () => jsonResponse({ type: 'error' }, 404)]]) + + expect( + await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:gone.md', {}) + ).toBeNull() + }) + + it('rethrows a transient failure so the sync records it instead of reading it as a deletion', async () => { + mockApi([[/\/src\/[a-f0-9]+\/flaky\.md$/, () => jsonResponse({ type: 'error' }, 500)]]) + + await expect( + bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:flaky.md', {}) + ).rejects.toThrow(/500/) + }) + + it('returns null for a pull request the repository no longer has', async () => { + mockApi([[/\/pullrequests\/9$/, () => jsonResponse({ type: 'error' }, 404)]]) + + expect(await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'pr:9', {})).toBeNull() + }) + + it('rejects an externalId with no recognized resource prefix', async () => { + mockApi([]) + + expect(await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'wiki:Home', {})).toBeNull() + }) +}) + +describe('bitbucket tag mapping', () => { + it('maps every declared file tag from the listing metadata', async () => { + mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('docs/guide.md', 42)] })]]) + + const result = await bitbucketConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + const tags = bitbucketConnector.mapTags?.(result.documents[0].metadata ?? {}) + + expect(tags).toEqual({ + contentType: 'file', + title: 'guide.md', + repository: 'acme/widgets', + path: 'docs/guide.md', + size: 42, + }) + }) + + it('maps every declared pull request tag from the listing metadata', async () => { + mockApi([[/\/pullrequests/, () => jsonResponse({ values: [pullRequestFixture(7)] })]]) + + const result = await bitbucketConnector.listDocuments(ACCESS_TOKEN, PR_CONFIG, undefined, {}) + const tags = bitbucketConnector.mapTags?.(result.documents[0].metadata ?? {}) + + expect(tags).toEqual({ + contentType: 'pull_request', + title: 'PR 7', + repository: 'acme/widgets', + state: 'OPEN', + author: 'Ada Lovelace', + createdAt: new Date('2026-01-01T00:00:00.000000+00:00'), + updatedAt: new Date('2026-02-01T00:00:00.000000+00:00'), + }) + }) +}) + +describe('bitbucket validateConfig', () => { + it('rejects a repository the credential cannot read', async () => { + mockApi([[/\/repositories\/acme\/widgets$/, () => jsonResponse({ type: 'error' }, 404)]]) + + expect(await bitbucketConnector.validateConfig(ACCESS_TOKEN, CONFIG)).toEqual({ + valid: false, + error: 'Repository "acme/widgets" not found', + }) + }) + + it('rejects a branch or tag that resolves to neither', async () => { + mockApi([[/\/refs\//, () => jsonResponse({ type: 'error' }, 404)]]) + + const result = await bitbucketConnector.validateConfig(ACCESS_TOKEN, { + ...CONFIG, + ref: 'nope', + }) + + expect(result.valid).toBe(false) + expect(result.error).toMatch(/Branch or tag "nope" not found/) + }) + + it('accepts a tag when no branch by that name exists', async () => { + mockApi([ + [/\/refs\/branches\//, () => jsonResponse({ type: 'error' }, 404)], + [/\/refs\/tags\//, () => jsonResponse({ name: 'v1.0', target: { hash: COMMIT } })], + ]) + + expect( + await bitbucketConnector.validateConfig(ACCESS_TOKEN, { ...CONFIG, ref: 'v1.0' }) + ).toEqual({ valid: true }) + }) + + it('rejects a non-positive maxItems before spending a request', async () => { + mockApi([]) + + expect( + await bitbucketConnector.validateConfig(ACCESS_TOKEN, { ...CONFIG, maxItems: '0' }) + ).toEqual({ valid: false, error: 'Max items must be a positive number' }) + }) +}) diff --git a/apps/sim/connectors/bitbucket/bitbucket.ts b/apps/sim/connectors/bitbucket/bitbucket.ts new file mode 100644 index 00000000000..686392662ca --- /dev/null +++ b/apps/sim/connectors/bitbucket/bitbucket.ts @@ -0,0 +1,1221 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { + CONNECTOR_MAX_FILE_BYTES, + markSkipped, + parseTagDate, + readBodyWithLimit, + sizeLimitSkipReason, + stubOrSkipBySize, + takeIndexableWithinCap, +} from '@/connectors/utils' +import { + BITBUCKET_API_BASE, + encodeBitbucketRepositoryPath, + encodeBitbucketSegment, + validateBitbucketOpaqueUrl, +} from '@/tools/bitbucket/utils' + +const logger = createLogger('BitbucketConnector') + +/** + * Scope of this connector, and the two Bitbucket surfaces it deliberately leaves out. + * + * Wikis: Bitbucket Cloud REST 2.0 exposes no wiki resource at all. The published + * OpenAPI 3 description declares a `wiki` OAuth scope but not a single path under + * it, so wiki pages are unreachable over the API rather than merely unimplemented + * here — `has_wiki` on the repository record is the only wiki-related field there is. + * + * Issues: the issue tracker does have endpoints, and they require the `issue` scope. + * Sim's Bitbucket OAuth service does not request it, and adding it would invalidate + * every already-issued Bitbucket credential until its owner re-consented. Nothing in + * this connector touches an issue endpoint, so no configuration of it can 403 for a + * missing scope: every request it makes needs only `repository` or `pullrequest`. + */ + +/** Bitbucket caps `pagelen` at 100 globally (REST 2.0 pagination reference). */ +const PAGE_SIZE = 100 +/** Pull request payloads are far heavier than tree entries, so they page smaller. */ +const PULL_REQUEST_PAGE_SIZE = 50 +/** + * Partial-response selector for the pull request collection. + * + * Bitbucket documents that "a resource's `self` URL, as well its 'collection' URL + * typically return the full object with all its fields, [but] there are some + * exceptions for fields that are overly verbose or costly to generate", and only + * enumerates `reviewers`/`participants` as such. The pull request body is exactly + * the kind of field that carve-out covers and the reference never promises it on + * the collection, so rather than assume, the connector asks for it explicitly: + * `+` additive syntax keeps every default field and adds `values.summary` on top. + * Without it a listing that omits the body would index title-only documents. + */ +const PULL_REQUEST_LIST_FIELDS = '+values.summary' +/** + * Sort key for the pull request collection. + * + * The listing feeds deletion reconciliation, so it has to page over a stable + * order: with `-updated_on` any pull request touched between two page requests + * moves to the front of the collection and shifts a later one past the page + * boundary, dropping it from the listing entirely. `id` never changes once + * assigned, so descending id is both immutable and newest-first — which is also + * the useful half to keep when a `maxItems` cap truncates the listing. + */ +const PULL_REQUEST_SORT = '-id' +const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES +/** Bytes sniffed for NUL when detecting binary files (matches git's heuristic). */ +const BINARY_SNIFF_BYTES = 8000 +/** + * Recursion depth requested from the source browsing endpoint. Bitbucket documents + * `max_depth` as a breadth-first walk that "will time out and return a 555" when the + * value is too large, so the connector asks for a modest depth and queues the + * directories at the frontier of each response for their own listing. A 555 on the + * first page of a directory falls back to a single-level listing, which is lossless + * because the frontier walk covers whatever the shallower listing did not reach; a + * 555 that survives the fallback skips that directory and flags the listing capped. + */ +const MAX_TREE_DEPTH = 5 +const BINARY_SKIP_REASON = 'Binary file was not indexed' +/** + * Bitbucket answers a raw read of an LFS-managed file with a 301 to Atlassian's + * media services platform, which the connector does not follow — the redirect is + * cross-origin, so the bearer token would be stripped and the request would fail + * anyway. The file surfaces once as a skipped row instead of being dropped. + */ +const LFS_SKIP_REASON = 'Git LFS file was not indexed' + +/** + * Prefix encoded into each document's externalId so getDocument can route to the + * correct Bitbucket resource. Repository files are addressed by their + * repo-relative path, pull requests by their repository-scoped numeric ID. + */ +const FILE_PREFIX = 'file:' +const PULL_REQUEST_PREFIX = 'pr:' + +/** Selects which Bitbucket resources to sync. */ +type ContentTypeChoice = 'code' | 'pullrequests' | 'all' + +/** Listing phases, walked in order: repository files ➜ pull requests. */ +type SyncPhase = 'code' | 'pullrequests' + +interface BitbucketCommitRef { + hash?: string +} + +interface BitbucketTreeEntry { + type?: string + path?: string + size?: number + /** `link`, `executable`, `subrepository`, or `binary` per the source API. */ + attributes?: string[] | string + commit?: BitbucketCommitRef +} + +interface BitbucketPagedResponse { + values?: T[] + /** Opaque absolute URL for the next page; absent on the last page. */ + next?: string +} + +interface BitbucketRenderedText { + raw?: string + markup?: string + html?: string +} + +interface BitbucketAccount { + display_name?: string + nickname?: string + uuid?: string +} + +interface BitbucketLink { + href?: string +} + +interface BitbucketPullRequest { + id?: number + title?: string + state?: string + /** The pull request body as typed by the author. */ + summary?: BitbucketRenderedText + rendered?: { description?: BitbucketRenderedText } + author?: BitbucketAccount + created_on?: string + updated_on?: string + links?: { html?: BitbucketLink } +} + +interface BitbucketRef { + name?: string + target?: BitbucketCommitRef +} + +interface BitbucketRepositoryRecord { + full_name?: string + mainbranch?: BitbucketRef + links?: { html?: BitbucketLink } +} + +/** Repository facts resolved once per sync and cached on syncContext. */ +interface ResolvedRepository { + /** Canonical `workspace/repo`, always lowercase in Bitbucket responses. */ + fullName: string + htmlUrl: string + mainBranch: string + mainBranchHash: string +} + +/** + * Heuristic binary detection: a NUL byte in the first 8 KB marks the file as + * binary, matching `git diff` / `git grep` semantics. Bitbucket already flags most + * binaries with the `binary` source attribute; this catches the rest. + */ +function isBinaryBuffer(buf: Buffer): boolean { + const len = Math.min(buf.length, BINARY_SNIFF_BYTES) + for (let i = 0; i < len; i++) { + if (buf[i] === 0) return true + } + return false +} + +/** + * Standard request headers carrying the OAuth bearer token. + */ +function authHeaders(accessToken: string, json = true): Record { + return { + Accept: json ? 'application/json' : '*/*', + Authorization: `Bearer ${accessToken}`, + } +} + +/** + * Reads a required slug (or curly-brace UUID) from sourceConfig. + */ +function readSlug(value: unknown): string { + return typeof value === 'string' ? value.trim() : '' +} + +/** + * Reads the parsed content-type choice from sourceConfig (defaults to 'code'). + */ +function getContentTypeChoice(sourceConfig: Record): ContentTypeChoice { + const value = typeof sourceConfig.contentTypes === 'string' ? sourceConfig.contentTypes : 'code' + if (value === 'code' || value === 'pullrequests' || value === 'all') return value + return 'code' +} + +/** + * Returns the ordered list of active sync phases for a content-type choice. + */ +function activePhases(choice: ContentTypeChoice): SyncPhase[] { + const phases: SyncPhase[] = [] + if (choice === 'code' || choice === 'all') phases.push('code') + if (choice === 'pullrequests' || choice === 'all') phases.push('pullrequests') + return phases +} + +/** + * Returns the phase following `current`, or undefined when `current` is last. + */ +function nextPhase(current: SyncPhase, choice: ContentTypeChoice): SyncPhase | undefined { + const phases = activePhases(choice) + const idx = phases.indexOf(current) + return idx >= 0 && idx + 1 < phases.length ? phases[idx + 1] : undefined +} + +/** + * Maps the pull request state config value to the documented `state` values. + * An empty result means "do not filter", which Bitbucket answers with OPEN only. + */ +function pullRequestStates(sourceConfig: Record): string[] { + const value = + typeof sourceConfig.pullRequestState === 'string' ? sourceConfig.pullRequestState.trim() : '' + switch (value) { + case 'merged': + return ['MERGED'] + case 'openMerged': + return ['OPEN', 'MERGED'] + case 'all': + return ['OPEN', 'MERGED', 'DECLINED', 'SUPERSEDED'] + default: + return ['OPEN'] + } +} + +/** + * Renders a timestamp as a BBQL datetime literal. + * + * BBQL datetimes are unquoted ISO-8601 strings whose grammar the filtering + * reference illustrates with an explicit numeric offset (`2015-03-04T14:08:59.123+02:00`); + * the bare `Z` designator appears nowhere in it. The offset is spelled out + * numerically so the literal matches the documented form exactly. + */ +function bbqlDateTime(value: Date): string { + return value.toISOString().replace(/Z$/, '+00:00') +} + +/** + * Parses a comma-separated extension filter into a normalized set (leading dot, + * lowercased). Returns null when no filter is configured (accept all files). + */ +function parseExtensions(raw: unknown): Set | null { + const trimmed = typeof raw === 'string' ? raw.trim() : '' + if (!trimmed) return null + const exts = trimmed + .split(',') + .map((e) => e.trim().toLowerCase()) + .filter(Boolean) + .map((e) => (e.startsWith('.') ? e : `.${e}`)) + return exts.length > 0 ? new Set(exts) : null +} + +/** + * Returns true when the file path matches the extension filter (or no filter set). + * The extension is read from the basename so a dot in a directory segment + * (`docs/v1.2/CHANGELOG`) is not mistaken for the file's extension. + */ +function matchesExtension(filePath: string, extSet: Set | null): boolean { + if (!extSet) return true + const fileName = filePath.slice(filePath.lastIndexOf('/') + 1) + const lastDot = fileName.lastIndexOf('.') + if (lastDot === -1) return false + return extSet.has(fileName.slice(lastDot).toLowerCase()) +} + +/** + * Normalizes the configured path filter: no leading slash, one trailing slash. + * Returns '' when unset. + */ +function normalizePathPrefix(raw: unknown): string { + const trimmed = typeof raw === 'string' ? raw.trim().replace(/^\/+/, '') : '' + if (!trimmed) return '' + return trimmed.endsWith('/') ? trimmed : `${trimmed}/` +} + +/** + * Whether a directory is worth descending into given the configured path filter. + * A directory qualifies when it is an ancestor of the prefix or lives inside it; + * everything else can be pruned before a request is spent on it. + */ +function directoryMayContainPrefix(dirPath: string, pathPrefix: string): boolean { + if (!pathPrefix) return true + const dir = dirPath ? `${dirPath}/` : '' + return dir.startsWith(pathPrefix) || pathPrefix.startsWith(dir) +} + +/** Number of path segments in a repository-relative path ('' is depth 0). */ +function pathDepth(path: string): number { + return path ? path.split('/').length : 0 +} + +/** Normalizes the `attributes` element, which the API returns as an array. */ +function entryAttributes(entry: BitbucketTreeEntry): Set { + const raw = entry.attributes + if (Array.isArray(raw)) return new Set(raw) + if (typeof raw === 'string') return new Set([raw]) + return new Set() +} + +/** + * Composes the document body as "Title\n\n". + */ +function composeBody(title: string, content: string): string { + const trimmedTitle = title.trim() + const trimmedContent = content.trim() + if (!trimmedTitle) return trimmedContent + if (!trimmedContent) return trimmedTitle + return `${trimmedTitle}\n\n${trimmedContent}` +} + +/** + * Builds the API URL for a repository sub-resource. + */ +function repositoryUrl(workspaceSlug: string, repoSlug: string, suffix = ''): string { + const workspace = encodeBitbucketSegment(workspaceSlug, 'workspaceSlug') + const repo = encodeBitbucketSegment(repoSlug, 'repoSlug') + return `${BITBUCKET_API_BASE}/repositories/${workspace}/${repo}${suffix}` +} + +/** + * Builds the source-browsing URL for a directory at a pinned commit. + * + * Bitbucket requires a trailing slash when listing the repository root, so the + * root case is built explicitly rather than by joining an empty path. + */ +function sourceListingUrl( + workspaceSlug: string, + repoSlug: string, + commit: string, + dirPath: string, + maxDepth: number +): string { + const encodedCommit = encodeBitbucketSegment(commit, 'commit') + const encodedPath = dirPath ? `${encodeBitbucketRepositoryPath(dirPath)}/` : '' + const url = new URL( + repositoryUrl(workspaceSlug, repoSlug, `/src/${encodedCommit}/${encodedPath}`) + ) + url.searchParams.set('pagelen', String(PAGE_SIZE)) + url.searchParams.set('max_depth', String(maxDepth)) + return url.toString() +} + +/** + * Builds the raw-content URL for a single file at a pinned commit. + */ +function sourceFileUrl( + workspaceSlug: string, + repoSlug: string, + commit: string, + path: string +): string { + const encodedCommit = encodeBitbucketSegment(commit, 'commit') + return repositoryUrl( + workspaceSlug, + repoSlug, + `/src/${encodedCommit}/${encodeBitbucketRepositoryPath(path)}` + ) +} + +/** + * Builds the change-detection hash for a repository file. + * + * Bitbucket's source listing carries no per-file content identifier: the `commit` + * embedded in each tree entry is documented as "merely the commit that was used in + * the URL... *not* the commit that last modified the file", and the only + * content-addressed value Bitbucket exposes is the ETag on a raw file read, which + * costs one request per file and so cannot be used while listing. + * + * The hash is therefore pinned to the resolved tip commit, which means every + * in-scope file re-hydrates when the branch advances. Path and extension filters + * are the intended way to keep that cost bounded. + */ +function buildFileContentHash(fullName: string, commit: string, path: string): string { + return `bitbucket:file:${fullName}:${commit}:${path}` +} + +/** + * Builds the change-detection hash for a pull request. `updated_on` advances on + * every edit, comment, approval, and state change, so it needs no content fetch. + */ +function buildPullRequestContentHash(fullName: string, id: number, updatedOn: string): string { + return `bitbucket:pr:${fullName}:${id}:${updatedOn}` +} + +/** + * Builds a deferred stub for a repository file. Content is empty and fetched + * lazily via getDocument for new/changed files only. + */ +function fileToStub( + repository: ResolvedRepository, + ref: string, + commit: string, + path: string, + size: number | undefined +): ExternalDocument { + const title = path.slice(path.lastIndexOf('/') + 1) || path + const encodedRef = ref.split('/').map(encodeURIComponent).join('/') + const encodedPath = path.split('/').map(encodeURIComponent).join('/') + return { + externalId: `${FILE_PREFIX}${path}`, + title, + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: `${repository.htmlUrl}/src/${encodedRef}/${encodedPath}`, + contentHash: buildFileContentHash(repository.fullName, commit, path), + metadata: { + contentType: 'file', + title, + repository: repository.fullName, + path, + size, + }, + } +} + +/** + * Builds a pull request document from either the collection or the `self` record. + * + * The body is read from `summary.raw` — the text as the author typed it — which the + * listing requests explicitly via {@link PULL_REQUEST_LIST_FIELDS}, so no per-document + * fetch is needed. `rendered.description.raw` is the same text under the endpoint's + * rendered-markup element and stands in on the rare record that carries only that. + */ +function pullRequestToDocument( + repository: ResolvedRepository, + pullRequest: BitbucketPullRequest +): ExternalDocument | null { + const id = pullRequest.id + if (typeof id !== 'number') return null + + const title = pullRequest.title?.trim() || `Pull request #${id}` + const description = + pullRequest.summary?.raw?.trim() || pullRequest.rendered?.description?.raw?.trim() || '' + const body = composeBody(title, description) + if (!body.trim()) return null + + const updatedOn = pullRequest.updated_on ?? pullRequest.created_on ?? '' + const author = + pullRequest.author?.display_name?.trim() || pullRequest.author?.nickname?.trim() || '' + + return { + externalId: `${PULL_REQUEST_PREFIX}${id}`, + title, + content: body, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: pullRequest.links?.html?.href || `${repository.htmlUrl}/pull-requests/${id}`, + contentHash: buildPullRequestContentHash(repository.fullName, id, updatedOn), + metadata: { + contentType: 'pull_request', + title, + repository: repository.fullName, + state: pullRequest.state, + author, + createdAt: pullRequest.created_on ?? '', + updatedAt: updatedOn, + }, + } +} + +/** + * Fetches the repository record, used to resolve the canonical full name, the web + * UI base URL, and the default branch — and to confirm access during validation. + */ +async function fetchRepository( + workspaceSlug: string, + repoSlug: string, + accessToken: string, + retryOptions?: typeof VALIDATE_RETRY_OPTIONS +): Promise { + return fetchWithRetry( + repositoryUrl(workspaceSlug, repoSlug), + { method: 'GET', headers: authHeaders(accessToken) }, + retryOptions + ) +} + +/** + * Resolves the repository facts once per sync run and caches them on syncContext. + * + * Throws when the repository cannot be read. That is not cosmetic: a token that + * lost access would otherwise produce an empty but apparently successful listing + * and let deletion reconciliation hard-delete every previously synced document. + * Proving the repository is readable here is also what lets the per-phase 404 + * handling below be read as "this ref or resource is genuinely absent". + */ +async function resolveRepository( + syncContext: Record | undefined, + workspaceSlug: string, + repoSlug: string, + accessToken: string +): Promise { + const cached = syncContext?.repository as ResolvedRepository | undefined + if (cached?.fullName) return cached + + const response = await fetchRepository(workspaceSlug, repoSlug, accessToken) + if (!response.ok) { + throw new Error( + `Cannot access Bitbucket repository ${workspaceSlug}/${repoSlug}: ${response.status}` + ) + } + + const record = (await response.json()) as BitbucketRepositoryRecord + /** + * Bitbucket accepts workspace and repository slugs case-insensitively but echoes + * the canonical lowercase form, so the resolved `full_name` — not the user's + * input — is what identifies the repository in hashes and URLs. + */ + const fullName = record.full_name?.trim() || `${workspaceSlug}/${repoSlug}`.toLowerCase() + const repository: ResolvedRepository = { + fullName, + htmlUrl: record.links?.html?.href?.replace(/\/+$/, '') || `https://bitbucket.org/${fullName}`, + mainBranch: record.mainbranch?.name?.trim() ?? '', + mainBranchHash: record.mainbranch?.target?.hash?.trim() ?? '', + } + if (syncContext) syncContext.repository = repository + return repository +} + +/** + * Resolves a branch or tag name to its tip commit SHA. Returns null when neither a + * branch nor a tag by that name exists. + */ +async function resolveRefHash( + workspaceSlug: string, + repoSlug: string, + ref: string, + accessToken: string, + retryOptions?: typeof VALIDATE_RETRY_OPTIONS +): Promise { + const encodedRef = encodeBitbucketRepositoryPath(ref) + for (const kind of ['branches', 'tags'] as const) { + const response = await fetchWithRetry( + repositoryUrl(workspaceSlug, repoSlug, `/refs/${kind}/${encodedRef}`), + { method: 'GET', headers: authHeaders(accessToken) }, + retryOptions + ) + if (response.status === 404) continue + if (!response.ok) { + throw new Error(`Failed to resolve Bitbucket ref "${ref}": ${response.status}`) + } + const record = (await response.json()) as BitbucketRef + const hash = record.target?.hash?.trim() + if (hash) return hash + } + return null +} + +/** + * Resolves the commit the code phase is pinned to, caching it on syncContext so + * every listing page and every deferred hydration in one run reads the same tree. + * + * Pinning matters twice over: the source endpoint's `next` links embed the commit, + * and the file content hash is derived from it, so a branch that advances mid-sync + * must not split the run across two trees. + */ +async function resolveCommit( + sourceConfig: Record, + syncContext: Record | undefined, + repository: ResolvedRepository, + workspaceSlug: string, + repoSlug: string, + accessToken: string +): Promise<{ commit: string; ref: string }> { + const cachedCommit = syncContext?.commit + const cachedRef = syncContext?.ref + if (typeof cachedCommit === 'string' && cachedCommit && typeof cachedRef === 'string') { + return { commit: cachedCommit, ref: cachedRef } + } + + const configuredRef = typeof sourceConfig.ref === 'string' ? sourceConfig.ref.trim() : '' + const ref = configuredRef || repository.mainBranch + if (!ref) { + throw new Error('Repository has no default branch; set a branch or tag explicitly') + } + + let commit = !configuredRef && repository.mainBranchHash ? repository.mainBranchHash : '' + if (!commit) { + const resolved = await resolveRefHash(workspaceSlug, repoSlug, ref, accessToken) + if (!resolved) { + throw new Error(`Branch or tag "${ref}" not found in ${repository.fullName}`) + } + commit = resolved + } + + if (syncContext) { + syncContext.commit = commit + syncContext.ref = ref + } + return { commit, ref } +} + +/** + * Issues a source listing GET, retrying at depth 1 when Bitbucket answers the + * documented `555` timeout for a `max_depth` it considers too large. Falling back + * is lossless because unexplored directories are queued and walked separately. + * + * The fallback is only offered for the first page of a directory, where `shallowUrl` + * is a URL this connector built itself. A `555` on a replayed `next` link cannot be + * retried the same way: `next` is documented as an opaque location "not to be + * constructed by clients", and rewriting its `max_depth` would pair a cursor cut + * from one tree walk with the results of a different one. The caller treats that + * case as an unlistable directory instead. + * + * The effective depth travels back with the response because the caller derives the + * frontier from it: after a fallback the listing only reached one level, so + * directories one level down are unexplored even though the requested depth was + * deeper. Reusing the requested depth there would silently drop every file below + * them — and, worse, let deletion reconciliation remove the ones already indexed. + */ +async function fetchSourceListing( + url: string, + requestedDepth: number, + accessToken: string, + shallowUrl?: string +): Promise<{ response: Response; depth: number }> { + const response = await fetchWithRetry(url, { method: 'GET', headers: authHeaders(accessToken) }) + if (response.status !== 555 || !shallowUrl) { + return { response, depth: requestedDepth } + } + + logger.warn('Bitbucket source listing timed out; retrying at depth 1', { url }) + const shallow = await fetchWithRetry(shallowUrl, { + method: 'GET', + headers: authHeaders(accessToken), + }) + return { response: shallow, depth: 1 } +} + +/** + * Cursor state. The cursor packs the resource phase and, within a phase, the + * opaque Bitbucket `next` URL to fetch. Bitbucket documents `next` as a location + * "not to be constructed by clients", so it is stored and replayed verbatim after + * being re-validated as an api.bitbucket.org 2.0 URL. + * + * The code phase's frontier of not-yet-walked directories lives on syncContext + * rather than in the cursor: it is per-run state that can grow unboundedly, and + * every page of a run shares one syncContext. + */ +interface CursorState { + phase: SyncPhase + nextUrl?: string + /** Directory the in-flight code listing is rooted at, for frontier depth math. */ + dir?: string + /** Effective `max_depth` the in-flight code listing was served at. */ + depth?: number +} + +function encodeCursor(state: CursorState): string { + return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url') +} + +function decodeCursor(cursor: string | undefined, initialPhase: SyncPhase): CursorState { + if (!cursor) return { phase: initialPhase } + try { + const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as Partial<{ + phase: SyncPhase + nextUrl: string + dir: string + depth: number + }> + return { + phase: + parsed.phase === 'code' || parsed.phase === 'pullrequests' ? parsed.phase : initialPhase, + nextUrl: typeof parsed.nextUrl === 'string' ? parsed.nextUrl : undefined, + dir: typeof parsed.dir === 'string' ? parsed.dir : undefined, + depth: + Number.isSafeInteger(parsed.depth) && Number(parsed.depth) > 0 ? parsed.depth : undefined, + } + } catch { + return { phase: initialPhase } + } +} + +/** + * Reads the frontier of directories the code phase has not walked yet, seeding it + * with the repository root on the first page of a run. + * + * The frontier is per-run state that only a syncContext can carry between pages. + * Without one the caller must not hand back a "there is more of the tree" cursor: + * the array it just filled is discarded, the next call re-seeds the root, and the + * walk would loop over the root listing until the engine's page ceiling stops it. + */ +function pendingDirectories(syncContext: Record | undefined): string[] { + if (!syncContext) return [''] + if (!Array.isArray(syncContext.pendingDirs)) syncContext.pendingDirs = [''] + return syncContext.pendingDirs as string[] +} + +/** + * Applies the optional maxItems cap to a page, tracking the running total in + * syncContext and flagging `listingCapped` when the cap truncates the listing. + * Skipped (oversized) documents ride along without consuming the cap. + * + * `moreToEnumerate` is whether anything the connector was configured to list + * remains unlisted beyond this page: a `next` link, directories still queued on + * the frontier, or a later phase that the cap is about to stop us reaching. + * + * It is required because `takeIndexableWithinCap` reports `capReached` as soon as + * the running total *equals* maxItems, which is also true of a listing that ended + * at exactly that count. Setting `listingCapped` unconditionally there suppresses + * deletion reconciliation for a complete listing; not setting it when a later + * phase is skipped is worse still, because the engine would treat the run as a + * complete enumeration and hard-delete that phase's previously indexed documents. + * `maxItems` is shared across phases, so the code walk ending exactly on the cap + * is precisely when the pull-request phase never runs. + */ +function applyMaxItemsCap( + documents: ExternalDocument[], + maxItems: number, + syncContext: Record | undefined, + moreToEnumerate: boolean +): { documents: ExternalDocument[]; capped: boolean } { + if (maxItems <= 0) return { documents, capped: false } + const alreadyIndexed = (syncContext?.totalDocsFetched as number) ?? 0 + const { + documents: taken, + indexableCount, + capReached, + } = takeIndexableWithinCap( + documents, + (doc) => doc.skippedReason !== undefined, + maxItems, + alreadyIndexed + ) + if (syncContext) { + syncContext.totalDocsFetched = alreadyIndexed + indexableCount + const withheld = taken.length < documents.length || moreToEnumerate + if (capReached && withheld) syncContext.listingCapped = true + } + return { documents: taken, capped: capReached } +} + +export const bitbucketConnector: ConnectorConfig = { + ...bitbucketConnectorMeta, + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date + ): Promise => { + const workspaceSlug = readSlug(sourceConfig.workspaceSlug) + const repoSlug = readSlug(sourceConfig.repoSlug) + if (!workspaceSlug || !repoSlug) { + throw new Error('Workspace and repository are required') + } + + const choice = getContentTypeChoice(sourceConfig) + const phases = activePhases(choice) + if (phases.length === 0) return { documents: [], hasMore: false } + + const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : 0 + const repository = await resolveRepository(syncContext, workspaceSlug, repoSlug, accessToken) + + let state = decodeCursor(cursor, phases[0]) + if (!phases.includes(state.phase)) state = { phase: phases[0] } + + /** Cursor that advances to the first page of the phase after `current`, if any. */ + const advance = (current: SyncPhase): { nextCursor?: string; hasMore: boolean } => { + const next = nextPhase(current, choice) + if (!next) return { hasMore: false } + return { nextCursor: encodeCursor({ phase: next }), hasMore: true } + } + + /** + * Cursor that stays in the code phase to walk the next frontier directory. + * Falls through to the following phase when the frontier is empty, or when + * there is no syncContext to carry it — see {@link pendingDirectories}. + * Checking the frontier here matters on the skip paths (a timed-out or absent + * directory): claiming `hasMore` with nothing left to walk costs the sync + * engine a whole extra listing round-trip that can only return zero documents. + */ + const continueCode = (frontier: string[]): { nextCursor?: string; hasMore: boolean } => + syncContext && frontier.length > 0 + ? { nextCursor: encodeCursor({ phase: 'code' }), hasMore: true } + : advance('code') + + if (state.phase === 'code') { + const { commit, ref } = await resolveCommit( + sourceConfig, + syncContext, + repository, + workspaceSlug, + repoSlug, + accessToken + ) + const pathPrefix = normalizePathPrefix(sourceConfig.pathPrefix) + const extSet = parseExtensions(sourceConfig.fileExtensions) + const frontier = pendingDirectories(syncContext) + + let url: string + let dir: string + let requestedDepth: number + /** Depth-1 retry target, offered only for URLs this connector built itself. */ + let shallowUrl: string | undefined + if (state.nextUrl) { + url = validateBitbucketOpaqueUrl(state.nextUrl) + dir = state.dir ?? '' + requestedDepth = state.depth ?? MAX_TREE_DEPTH + } else { + const nextDir = frontier.shift() + if (nextDir === undefined) { + const adv = advance('code') + return { documents: [], nextCursor: adv.nextCursor, hasMore: adv.hasMore } + } + dir = nextDir + requestedDepth = MAX_TREE_DEPTH + url = sourceListingUrl(workspaceSlug, repoSlug, commit, dir, requestedDepth) + shallowUrl = sourceListingUrl(workspaceSlug, repoSlug, commit, dir, 1) + } + + logger.info('Listing Bitbucket repository files', { + repository: repository.fullName, + ref, + dir, + continued: Boolean(state.nextUrl), + }) + + const { response, depth } = await fetchSourceListing( + url, + requestedDepth, + accessToken, + shallowUrl + ) + if (!response.ok) { + if (response.status === 555) { + /** + * The walk still timed out after the depth-1 fallback, or timed out on a + * `next` link that cannot be re-cut at a shallower depth. Either way this + * directory is unlistable on this run: its subtree is skipped and the + * listing is flagged incomplete so reconciliation cannot read the missing + * files as deletions. The rest of the frontier is still walked. + */ + logger.warn('Bitbucket source listing timed out; skipping directory', { + repository: repository.fullName, + dir, + }) + if (syncContext) syncContext.listingCapped = true + const skipped = continueCode(frontier) + return { documents: [], nextCursor: skipped.nextCursor, hasMore: skipped.hasMore } + } + if (response.status === 404) { + /** + * `resolveRepository` already proved the repository is readable, so a 404 + * here can only mean the directory (or an empty repository's root) is + * absent. Reconciliation is left enabled so its files delete normally. + */ + logger.warn('Bitbucket source path not found; skipping', { + repository: repository.fullName, + dir, + }) + const skipped = continueCode(frontier) + return { documents: [], nextCursor: skipped.nextCursor, hasMore: skipped.hasMore } + } + if (response.status === 401 || response.status === 403) { + /** + * The token stopped working mid-sync. Flag the listing as incomplete so + * deletion reconciliation does not hard-delete previously synced files. + */ + if (syncContext) syncContext.listingCapped = true + const adv = advance('code') + return { documents: [], nextCursor: adv.nextCursor, hasMore: adv.hasMore } + } + const errorText = await response.text().catch(() => '') + logger.error('Failed to list Bitbucket repository files', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list Bitbucket repository files: ${response.status}`) + } + + const page = (await response.json()) as BitbucketPagedResponse + const entries = Array.isArray(page.values) ? page.values : [] + const dirDepth = pathDepth(dir) + const documents: ExternalDocument[] = [] + + for (const entry of entries) { + const path = typeof entry.path === 'string' ? entry.path : '' + if (!path) continue + + if (entry.type === 'commit_directory') { + /** + * A `max_depth` walk returns intermediate directories as well as the ones + * it stopped at, so only the frontier — directories at the depth this + * response actually reached — is queued. Anything shallower was already + * expanded in this same response. + */ + if (pathDepth(path) - dirDepth < depth) continue + if (!directoryMayContainPrefix(path, pathPrefix)) continue + frontier.push(path) + continue + } + + if (entry.type !== 'commit_file') continue + + const attributes = entryAttributes(entry) + if (attributes.has('binary') || attributes.has('link') || attributes.has('subrepository')) { + continue + } + if (pathPrefix && !path.startsWith(pathPrefix)) continue + if (!matchesExtension(path, extSet)) continue + + documents.push( + stubOrSkipBySize( + fileToStub(repository, ref, commit, path, entry.size), + entry.size, + MAX_FILE_SIZE + ) + ) + } + + const { documents: capped, capped: hitLimit } = applyMaxItemsCap( + documents, + maxItems, + syncContext, + Boolean(page.next) || frontier.length > 0 || Boolean(nextPhase('code', choice)) + ) + if (hitLimit) return { documents: capped, hasMore: false } + + if (page.next) { + return { + documents: capped, + nextCursor: encodeCursor({ phase: 'code', nextUrl: page.next, dir, depth }), + hasMore: true, + } + } + const cont = continueCode(frontier) + return { documents: capped, nextCursor: cont.nextCursor, hasMore: cont.hasMore } + } + + let url: string + if (state.nextUrl) { + url = validateBitbucketOpaqueUrl(state.nextUrl) + } else { + const prUrl = new URL(repositoryUrl(workspaceSlug, repoSlug, '/pullrequests')) + prUrl.searchParams.set('pagelen', String(PULL_REQUEST_PAGE_SIZE)) + prUrl.searchParams.set('fields', PULL_REQUEST_LIST_FIELDS) + prUrl.searchParams.set('sort', PULL_REQUEST_SORT) + for (const prState of pullRequestStates(sourceConfig)) { + prUrl.searchParams.append('state', prState) + } + if (lastSyncAt) { + /** + * The filtering reference is explicit that the paginated envelope's + * `values.` prefix must not appear in a query field, so the field is + * named bare even though the partial response above addresses it as + * `values.summary`. + */ + prUrl.searchParams.set('q', `updated_on > ${bbqlDateTime(lastSyncAt)}`) + } + url = prUrl.toString() + } + + logger.info('Listing Bitbucket pull requests', { + repository: repository.fullName, + continued: Boolean(state.nextUrl), + incremental: Boolean(lastSyncAt), + }) + + const response = await fetchWithRetry(url, { + method: 'GET', + headers: authHeaders(accessToken), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list Bitbucket pull requests', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list Bitbucket pull requests: ${response.status}`) + } + + const page = (await response.json()) as BitbucketPagedResponse + const documents: ExternalDocument[] = [] + for (const pullRequest of Array.isArray(page.values) ? page.values : []) { + const doc = pullRequestToDocument(repository, pullRequest) + if (doc) documents.push(doc) + } + + const { documents: capped, capped: hitLimit } = applyMaxItemsCap( + documents, + maxItems, + syncContext, + Boolean(page.next) + ) + if (hitLimit) return { documents: capped, hasMore: false } + + if (page.next) { + return { + documents: capped, + nextCursor: encodeCursor({ phase: 'pullrequests', nextUrl: page.next }), + hasMore: true, + } + } + return { documents: capped, hasMore: false } + }, + + getDocument: async ( + accessToken: string, + sourceConfig: Record, + externalId: string, + syncContext?: Record + ): Promise => { + const workspaceSlug = readSlug(sourceConfig.workspaceSlug) + const repoSlug = readSlug(sourceConfig.repoSlug) + if (!workspaceSlug || !repoSlug || !externalId) return null + + try { + const repository = await resolveRepository(syncContext, workspaceSlug, repoSlug, accessToken) + + if (externalId.startsWith(PULL_REQUEST_PREFIX)) { + const rawId = externalId.slice(PULL_REQUEST_PREFIX.length) + const id = Number(rawId) + if (!rawId || !Number.isSafeInteger(id) || id < 1) return null + + const response = await fetchWithRetry( + repositoryUrl(workspaceSlug, repoSlug, `/pullrequests/${id}`), + { method: 'GET', headers: authHeaders(accessToken) } + ) + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to fetch Bitbucket pull request: ${response.status}`) + } + return pullRequestToDocument(repository, (await response.json()) as BitbucketPullRequest) + } + + if (!externalId.startsWith(FILE_PREFIX)) return null + + const path = externalId.slice(FILE_PREFIX.length) + if (!path) return null + + const { commit, ref } = await resolveCommit( + sourceConfig, + syncContext, + repository, + workspaceSlug, + repoSlug, + accessToken + ) + const response = await fetchWithRetry(sourceFileUrl(workspaceSlug, repoSlug, commit, path), { + method: 'GET', + headers: authHeaders(accessToken, false), + /** + * LFS files answer with a 301 to Atlassian media services. Following it + * would drop the bearer token (cross-origin) and, on success, would store + * an opaquely redirected body, so the redirect is surfaced instead. + */ + redirect: 'manual', + }) + + if (response.status === 404) return null + + const stub = fileToStub(repository, ref, commit, path, undefined) + + if (response.status >= 300 && response.status < 400) { + logger.info('Skipping Bitbucket LFS-managed file', { path }) + return markSkipped(stub, LFS_SKIP_REASON) + } + if (!response.ok) { + throw new Error(`Failed to fetch Bitbucket file ${path}: ${response.status}`) + } + + const buffer = await readBodyWithLimit(response, MAX_FILE_SIZE) + if (buffer === null) { + logger.info('Skipping oversized Bitbucket file', { path, limit: MAX_FILE_SIZE }) + return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) + } + if (isBinaryBuffer(buffer)) { + logger.info('Skipping binary Bitbucket file', { path }) + return markSkipped(stub, BINARY_SKIP_REASON) + } + + const body = composeBody(stub.title, buffer.toString('utf8')) + if (!body.trim()) return null + + return { + ...stub, + content: body, + contentDeferred: false, + metadata: { ...stub.metadata, size: buffer.byteLength }, + } + } catch (error) { + /** + * Only the 404 checks above (and an unrecognized externalId prefix) mean the + * object is genuinely gone. Every other failure is rethrown so the sync engine + * records a visible `docsFailed` row — returning `null` would report a + * transient Bitbucket fault as success, silently counting an already-indexed + * document as unchanged and dropping a new one from the run entirely. + */ + logger.warn(`Failed to fetch Bitbucket document ${externalId}`, { + error: toError(error).message, + }) + throw toError(error) + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const workspaceSlug = readSlug(sourceConfig.workspaceSlug) + if (!workspaceSlug) return { valid: false, error: 'Workspace is required' } + + const repoSlug = readSlug(sourceConfig.repoSlug) + if (!repoSlug) return { valid: false, error: 'Repository is required' } + + const maxItems = sourceConfig.maxItems as string | undefined + if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) <= 0)) { + return { valid: false, error: 'Max items must be a positive number' } + } + + const choice = getContentTypeChoice(sourceConfig) + + try { + const response = await fetchRepository( + workspaceSlug, + repoSlug, + accessToken, + VALIDATE_RETRY_OPTIONS + ) + + if (response.status === 404) { + return { + valid: false, + error: `Repository "${workspaceSlug}/${repoSlug}" not found`, + } + } + if (response.status === 401 || response.status === 403) { + return { valid: false, error: 'Invalid credential or insufficient permissions' } + } + if (!response.ok) { + return { valid: false, error: `Cannot access repository: ${response.status}` } + } + + const record = (await response.json()) as BitbucketRepositoryRecord + const configuredRef = typeof sourceConfig.ref === 'string' ? sourceConfig.ref.trim() : '' + + if (activePhases(choice).includes('code')) { + if (!configuredRef && !record.mainbranch?.name?.trim()) { + return { + valid: false, + error: 'Repository has no default branch; set a branch or tag explicitly', + } + } + if (configuredRef) { + const hash = await resolveRefHash( + workspaceSlug, + repoSlug, + configuredRef, + accessToken, + VALIDATE_RETRY_OPTIONS + ) + if (!hash) { + return { + valid: false, + error: `Branch or tag "${configuredRef}" not found in "${workspaceSlug}/${repoSlug}"`, + } + } + } + } + + return { valid: true } + } catch (error) { + return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') } + } + }, + + /** + * Maps document metadata to tag slots. `contentType`, `title`, and `repository` + * apply to every document type. `path`/`size` are repository-file-only and + * `state`/`author`/`createdAt`/`updatedAt` are pull-request-only; each document + * type leaves the others' fields empty and the guards below skip them. + */ + mapTags: (metadata: Record): Record => { + const result: Record = {} + + for (const key of ['contentType', 'title', 'repository', 'path', 'state', 'author'] as const) { + const value = metadata[key] + if (typeof value === 'string' && value.trim()) result[key] = value + } + + if (metadata.size != null) { + const size = Number(metadata.size) + if (!Number.isNaN(size)) result.size = size + } + + const createdAt = parseTagDate(metadata.createdAt) + if (createdAt) result.createdAt = createdAt + + const updatedAt = parseTagDate(metadata.updatedAt) + if (updatedAt) result.updatedAt = updatedAt + + return result + }, +} diff --git a/apps/sim/connectors/bitbucket/index.ts b/apps/sim/connectors/bitbucket/index.ts new file mode 100644 index 00000000000..ffb6f03e592 --- /dev/null +++ b/apps/sim/connectors/bitbucket/index.ts @@ -0,0 +1 @@ +export { bitbucketConnector } from '@/connectors/bitbucket/bitbucket' diff --git a/apps/sim/connectors/bitbucket/meta.ts b/apps/sim/connectors/bitbucket/meta.ts new file mode 100644 index 00000000000..c250d59743a --- /dev/null +++ b/apps/sim/connectors/bitbucket/meta.ts @@ -0,0 +1,156 @@ +import { BitbucketIcon } from '@/components/icons' +import type { ConnectorMeta } from '@/connectors/types' + +export const bitbucketConnectorMeta: ConnectorMeta = { + id: 'bitbucket', + name: 'Bitbucket', + description: + 'Sync repository files and pull request descriptions from a Bitbucket Cloud repository into your knowledge base', + version: '1.0.0', + icon: BitbucketIcon, + + /** + * Deliberately disabled. Incremental sync would only ever apply to pull requests, + * via a BBQL `q=updated_on > ` filter derived from lastSyncAt; repository + * files carry no per-file change timestamp in the source listing and are always + * re-listed in full regardless. The sync engine disables deletion reconciliation for + * every incremental run, so declaring support would leave files deleted upstream + * indexed indefinitely on the default `code` configuration -- a complete listing whose + * deletions are never applied. Re-listing pull requests in full each run is the + * cheaper tradeoff. + */ + supportsIncrementalSync: false, + + /** + * Bitbucket Cloud REST API 2.0 authenticates with an OAuth 2.0 bearer token, the + * same credential the Bitbucket tools and selectors already use. The scopes named + * here are the ones Sim's Bitbucket OAuth service already requests + * (`lib/oauth/oauth.ts`); the connector deliberately indexes nothing that would + * need a scope outside that set. + */ + auth: { + mode: 'oauth', + provider: 'bitbucket', + requiredScopes: ['repository', 'pullrequest'], + }, + + configFields: [ + { + id: 'workspaceSelector', + title: 'Workspace', + type: 'selector', + selectorKey: 'bitbucket.workspaces', + canonicalParamId: 'workspaceSlug', + mode: 'basic', + placeholder: 'Select a workspace', + required: true, + }, + { + id: 'workspaceSlug', + title: 'Workspace Slug', + type: 'short-input', + canonicalParamId: 'workspaceSlug', + mode: 'advanced', + placeholder: 'e.g. my-team', + required: true, + description: 'Workspace ID (slug) or workspace UUID in curly braces.', + }, + { + id: 'repoSelector', + title: 'Repository', + type: 'selector', + selectorKey: 'bitbucket.repositories', + canonicalParamId: 'repoSlug', + mode: 'basic', + dependsOn: ['workspaceSelector'], + placeholder: 'Select a repository', + required: true, + }, + { + id: 'repoSlug', + title: 'Repository Slug', + type: 'short-input', + canonicalParamId: 'repoSlug', + mode: 'advanced', + placeholder: 'e.g. my-repo', + required: true, + description: 'Repository slug or repository UUID in curly braces.', + }, + { + id: 'contentTypes', + title: 'Content', + type: 'dropdown', + required: false, + options: [ + { label: 'Code (repository files) only', id: 'code' }, + { label: 'Pull requests only', id: 'pullrequests' }, + { label: 'Code & Pull Requests', id: 'all' }, + ], + placeholder: 'Code (repository files) only', + description: + 'Which content to index. Defaults to repository files when left unset. Bitbucket Cloud wikis and the issue tracker are not indexed — see the connector notes.', + }, + { + id: 'ref', + title: 'Branch or Tag', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'Default branch', + description: 'Branch or tag to sync repository files from. Applies only when syncing Code.', + }, + { + id: 'pathPrefix', + title: 'Path Filter', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. docs/', + description: + 'Only sync repository files under this path prefix. Applies only when syncing Code.', + }, + { + id: 'fileExtensions', + title: 'File Extensions', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. .md, .txt, .mdx', + description: + 'Only sync repository files with these extensions (comma-separated). Leave blank for all text files. Applies only when syncing Code.', + }, + { + id: 'pullRequestState', + title: 'Pull Request State', + type: 'dropdown', + required: false, + mode: 'advanced', + options: [ + { label: 'Open only', id: 'open' }, + { label: 'Merged only', id: 'merged' }, + { label: 'Open & Merged', id: 'openMerged' }, + { label: 'All', id: 'all' }, + ], + description: 'Which pull requests to sync by state. Applies only when syncing pull requests.', + }, + { + id: 'maxItems', + title: 'Max Items', + type: 'short-input', + required: false, + placeholder: 'e.g. 500 (default: unlimited)', + }, + ], + + tagDefinitions: [ + { id: 'contentType', displayName: 'Content Type', fieldType: 'text' }, + { id: 'title', displayName: 'Title', fieldType: 'text' }, + { id: 'repository', displayName: 'Repository', fieldType: 'text' }, + { id: 'path', displayName: 'File Path', fieldType: 'text' }, + { id: 'state', displayName: 'State', fieldType: 'text' }, + { id: 'author', displayName: 'Author', fieldType: 'text' }, + { id: 'size', displayName: 'File Size (bytes)', fieldType: 'number' }, + { id: 'createdAt', displayName: 'Created At', fieldType: 'date' }, + { id: 'updatedAt', displayName: 'Updated At', fieldType: 'date' }, + ], +} diff --git a/apps/sim/connectors/databricks/databricks.test.ts b/apps/sim/connectors/databricks/databricks.test.ts new file mode 100644 index 00000000000..d062264a537 --- /dev/null +++ b/apps/sim/connectors/databricks/databricks.test.ts @@ -0,0 +1,650 @@ +/** + * @vitest-environment node + * + * `GET /api/2.0/workspace/list` is unpaginated and returns one directory at a + * time, so the notebook side of this connector is a hand-rolled breadth-first + * walk whose pending-directory queue rides in the cursor. Everything that can + * silently shrink a listing lives in that walk — a container type the walk does + * not recognise, a directory the token cannot read, the document cap — and the + * sync engine hard-deletes whatever a listing leaves out unless + * `syncContext.listingCapped` says the listing was partial. Those paths are the + * bulk of what is asserted here, alongside the stub/`getDocument` hash equality + * that keeps every sync from re-indexing every notebook. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { databricksConnector } from '@/connectors/databricks/databricks' +import { databricksConnectorMeta } from '@/connectors/databricks/meta' + +const TOKEN = 'dapi-personal-access-token' +const HOST = 'dbc-1234abcd-5678.cloud.databricks.com' +const ORIGIN = `https://${HOST}` + +const NOTEBOOK_CONFIG = { workspaceHost: HOST, contentType: 'notebooks', rootPath: '/Shared' } +const QUERY_CONFIG = { workspaceHost: HOST, contentType: 'queries' } + +const fetchMock = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +function dir(path: string) { + return { object_type: 'DIRECTORY', path, object_id: path.length } +} + +function repo(path: string) { + return { object_type: 'REPO', path, object_id: path.length } +} + +function notebook(path: string, modifiedAt?: number) { + return { + object_type: 'NOTEBOOK', + path, + object_id: path.length * 1000, + language: 'PYTHON', + ...(modifiedAt === undefined ? {} : { modified_at: modifiedAt }), + } +} + +function query(id: string, overrides: Record = {}) { + return { + id, + display_name: `Query ${id}`, + query_text: `SELECT ${id}`, + catalog: 'main', + schema: 'default', + owner_user_name: 'ada@example.com', + lifecycle_state: 'ACTIVE', + update_time: '2026-01-01T00:00:00Z', + ...overrides, + } +} + +/** Reads the `path` query parameter off a recorded `workspace/list` call. */ +function listedPaths(): string[] { + return fetchMock.mock.calls + .map(([url]) => new URL(url as string)) + .filter((url) => url.pathname === '/api/2.0/workspace/list') + .map((url) => url.searchParams.get('path') ?? '') +} + +function queryListingUrls(): URL[] { + return fetchMock.mock.calls + .map(([url]) => new URL(url as string)) + .filter((url) => url.pathname === '/api/2.0/sql/queries') +} + +/** + * Serves `workspace/list` from a path→objects map. An unmapped path is a 404, + * which is the Databricks answer for a path that no longer exists. + */ +function mockWorkspace(tree: Record Response)>) { + fetchMock.mockImplementation(async (url: string) => { + const parsed = new URL(url) + if (parsed.pathname === '/api/2.0/workspace/list') { + const path = parsed.searchParams.get('path') ?? '' + const entry = tree[path] + if (entry === undefined) { + return jsonResponse({ error_code: 'RESOURCE_DOES_NOT_EXIST', message: 'not found' }, 404) + } + /** A factory, not a value: a retried request must not re-read a used body. */ + if (typeof entry === 'function') return entry() + return jsonResponse({ objects: entry }) + } + throw new Error(`Unexpected request: ${url}`) + }) +} + +/** Wraps an error body so each (possibly retried) request gets a fresh Response. */ +function errorResponse(body: Record, status: number): () => Response { + return () => jsonResponse(body, status) +} + +beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +describe('databricks notebook traversal', () => { + it('walks DIRECTORY and REPO containers and indexes only notebooks', async () => { + mockWorkspace({ + '/Shared': [dir('/Shared/team'), repo('/Shared/git-folder')], + '/Shared/team': [ + notebook('/Shared/team/etl', 1), + { object_type: 'FILE', path: '/Shared/team/readme.md', object_id: 5 }, + { object_type: 'DASHBOARD', path: '/Shared/team/board.lvdash.json', object_id: 6 }, + { object_type: 'LIBRARY', path: '/Shared/team/lib.jar', object_id: 7 }, + ], + '/Shared/git-folder': [notebook('/Shared/git-folder/model', 2)], + }) + + const syncContext: Record = {} + const result = await databricksConnector.listDocuments( + TOKEN, + NOTEBOOK_CONFIG, + undefined, + syncContext + ) + + expect(listedPaths().sort()).toEqual(['/Shared', '/Shared/git-folder', '/Shared/team']) + expect(result.documents.map((doc) => doc.externalId).sort()).toEqual([ + 'notebook:/Shared/git-folder/model', + 'notebook:/Shared/team/etl', + ]) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + /** + * `workspace/list` answers with the object itself when the path is not a + * directory. Re-queueing that self-reference would list the same path forever. + */ + it('does not re-queue a container that lists itself', async () => { + mockWorkspace({ + '/Shared': [dir('/Shared'), notebook('/Shared/etl', 1)], + }) + + const result = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + + expect(listedPaths()).toEqual(['/Shared']) + expect(result.documents).toHaveLength(1) + expect(result.hasMore).toBe(false) + }) + + it('carries the pending queue through the cursor without re-listing or duplicating', async () => { + /** 60 sibling directories: more than the 50 a single call will list. */ + const children = Array.from({ length: 60 }, (_, i) => dir(`/Shared/d${i}`)) + const tree: Record = { '/Shared': children } + for (let i = 0; i < 60; i++) { + tree[`/Shared/d${i}`] = [notebook(`/Shared/d${i}/nb`, i + 1)] + } + mockWorkspace(tree) + + const first = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + expect(first.hasMore).toBe(true) + expect(first.nextCursor).toBeDefined() + expect(first.documents).toHaveLength(49) + + const firstPaths = listedPaths() + fetchMock.mockClear() + + const second = await databricksConnector.listDocuments( + TOKEN, + NOTEBOOK_CONFIG, + first.nextCursor, + {} + ) + expect(second.hasMore).toBe(false) + expect(second.documents).toHaveLength(11) + + const secondPaths = listedPaths() + expect(secondPaths.some((path) => firstPaths.includes(path))).toBe(false) + + const ids = [...first.documents, ...second.documents].map((doc) => doc.externalId) + expect(new Set(ids).size).toBe(ids.length) + expect(ids).toHaveLength(60) + }) +}) + +describe('databricks notebook listingCapped', () => { + it('flags listingCapped when a subdirectory is unreadable', async () => { + mockWorkspace({ + '/Shared': [dir('/Shared/open'), dir('/Shared/locked')], + '/Shared/open': [notebook('/Shared/open/nb', 1)], + '/Shared/locked': errorResponse({ error_code: 'PERMISSION_DENIED' }, 403), + }) + + const syncContext: Record = {} + const result = await databricksConnector.listDocuments( + TOKEN, + NOTEBOOK_CONFIG, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(1) + expect(syncContext.listingCapped).toBe(true) + }) + + it('flags listingCapped when a subdirectory has been deleted mid-walk', async () => { + mockWorkspace({ + '/Shared': [dir('/Shared/open'), dir('/Shared/gone')], + '/Shared/open': [notebook('/Shared/open/nb', 1)], + }) + + const syncContext: Record = {} + await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, syncContext) + + expect(syncContext.listingCapped).toBe(true) + }) + + /** + * A denied root yields an empty listing, not a partial one. Reporting that as + * a successful sync of zero documents is worse than failing. + */ + it('throws rather than reporting an empty sync when the root itself is denied', async () => { + mockWorkspace({ '/Shared': errorResponse({ error_code: 'PERMISSION_DENIED' }, 403) }) + + await expect( + databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + ).rejects.toThrow(/denied access to \/Shared/) + }) + + it('flags listingCapped when maxDocuments hides notebooks from the same page', async () => { + mockWorkspace({ + '/Shared': [notebook('/Shared/a', 1), notebook('/Shared/b', 2), notebook('/Shared/c', 3)], + }) + + const syncContext: Record = {} + const result = await databricksConnector.listDocuments( + TOKEN, + { ...NOTEBOOK_CONFIG, maxDocuments: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it('leaves listingCapped unset when maxDocuments lands exactly on an exhausted walk', async () => { + mockWorkspace({ + '/Shared': [notebook('/Shared/a', 1), notebook('/Shared/b', 2)], + }) + + const syncContext: Record = {} + const result = await databricksConnector.listDocuments( + TOKEN, + { ...NOTEBOOK_CONFIG, maxDocuments: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when maxDocuments stops the walk with directories pending', async () => { + mockWorkspace({ + '/Shared': [dir('/Shared/a'), dir('/Shared/b')], + '/Shared/a': [notebook('/Shared/a/nb', 1)], + '/Shared/b': [notebook('/Shared/b/nb', 2)], + }) + + const syncContext: Record = { totalDocsFetched: 0 } + const result = await databricksConnector.listDocuments( + TOKEN, + { ...NOTEBOOK_CONFIG, maxDocuments: '1' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(1) + expect(syncContext.listingCapped).toBe(true) + }) +}) + +describe('databricks notebook error semantics under concurrency', () => { + /** + * The walk lists siblings in parallel. A permission failure on one sibling has + * to stay a skip while a server failure on another still aborts the whole + * listing — a `Promise.all` that swallowed either would be invisible in the + * happy path. + */ + it('propagates a 500 on one sibling even when another sibling is a skippable 403', async () => { + mockWorkspace({ + '/Shared': [dir('/Shared/locked'), dir('/Shared/broken')], + '/Shared/locked': errorResponse({ error_code: 'PERMISSION_DENIED' }, 403), + '/Shared/broken': errorResponse({ error_code: 'INTERNAL_ERROR', message: 'boom' }, 500), + }) + + await expect( + databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + ).rejects.toThrow(/Failed to list Databricks workspace path \/Shared\/broken: 500/) + }) + + it('propagates a 401 rather than treating a bad token as an empty directory', async () => { + mockWorkspace({ + '/Shared': [dir('/Shared/a')], + '/Shared/a': errorResponse({ error_code: 'PERMISSION_DENIED' }, 401), + }) + + await expect( + databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + ).rejects.toThrow(/401/) + }) + + it('propagates a 429 after the shared retry budget is spent', async () => { + vi.useFakeTimers() + mockWorkspace({ + '/Shared': [dir('/Shared/a')], + '/Shared/a': errorResponse({ error_code: 'REQUEST_LIMIT_EXCEEDED' }, 429), + }) + + const pending = databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + const assertion = expect(pending).rejects.toThrow(/429/) + await vi.runAllTimersAsync() + await assertion + }) +}) + +describe('databricks saved SQL queries', () => { + it('drops trashed queries without suppressing deletion reconciliation', async () => { + fetchMock.mockResolvedValue( + jsonResponse({ + results: [query('a'), query('b', { lifecycle_state: 'TRASHED' })], + }) + ) + + const syncContext: Record = {} + const result = await databricksConnector.listDocuments( + TOKEN, + QUERY_CONFIG, + undefined, + syncContext + ) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['query:a']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when the cap stops paging while a page token remains', async () => { + fetchMock.mockResolvedValue( + jsonResponse({ results: [query('a'), query('b')], next_page_token: 'tok' }) + ) + + const syncContext: Record = {} + const result = await databricksConnector.listDocuments( + TOKEN, + { ...QUERY_CONFIG, maxDocuments: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it('leaves listingCapped unset when the cap lands on the last page', async () => { + fetchMock.mockResolvedValue(jsonResponse({ results: [query('a'), query('b')] })) + + const syncContext: Record = {} + await databricksConnector.listDocuments( + TOKEN, + { ...QUERY_CONFIG, maxDocuments: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + /** + * An echoed token on an empty tail would re-request the same page until the + * engine truncates pagination, which permanently disables deletion + * reconciliation. + */ + it('stops paging on an empty page even when a token is echoed back', async () => { + fetchMock.mockResolvedValue(jsonResponse({ results: [], next_page_token: 'tok' })) + + const result = await databricksConnector.listDocuments(TOKEN, QUERY_CONFIG, undefined, {}) + + expect(result.hasMore).toBe(false) + expect(result.nextCursor).toBeUndefined() + }) + + it('asks for the full page size when no cap is configured', async () => { + fetchMock.mockResolvedValue(jsonResponse({ results: [query('a')] })) + + await databricksConnector.listDocuments(TOKEN, QUERY_CONFIG, undefined, {}) + + expect(queryListingUrls()[0].searchParams.get('page_size')).toBe('100') + }) + + it('shrinks the last page to what the cap still allows', async () => { + fetchMock.mockResolvedValue(jsonResponse({ results: [query('a')] })) + + await databricksConnector.listDocuments( + TOKEN, + { ...QUERY_CONFIG, maxDocuments: '10' }, + undefined, + { totalDocsFetched: 7 } + ) + + expect(queryListingUrls()[0].searchParams.get('page_size')).toBe('3') + }) + + /** + * `page_size=0` is not a valid request. The cap arithmetic must floor at one + * even when the running total has already reached (or overshot) the cap. + */ + it.each([ + ['exactly at the cap', 10], + ['past the cap', 12], + ])('never requests a non-positive page size when already %s', async (_label, fetched) => { + fetchMock.mockResolvedValue(jsonResponse({ results: [] })) + + await databricksConnector.listDocuments( + TOKEN, + { ...QUERY_CONFIG, maxDocuments: '10' }, + undefined, + { totalDocsFetched: fetched } + ) + + const pageSize = Number(queryListingUrls()[0].searchParams.get('page_size')) + expect(pageSize).toBeGreaterThan(0) + }) +}) + +describe('databricks contentHash consistency', () => { + it('produces the same notebook hash from the listing stub and from getDocument', async () => { + mockWorkspace({ '/Shared': [notebook('/Shared/etl', 1700000000000)] }) + + const listed = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + const stub = listed.documents[0] + expect(stub.contentDeferred).toBe(true) + expect(stub.content).toBe('') + + fetchMock.mockImplementation(async (url: string) => { + const parsed = new URL(url) + if (parsed.pathname === '/api/2.0/workspace/get-status') { + return jsonResponse(notebook('/Shared/etl', 1700000000000)) + } + if (parsed.pathname === '/api/2.0/workspace/export') { + expect(parsed.searchParams.get('format')).toBe('SOURCE') + return jsonResponse({ + content: Buffer.from('print("hi")', 'utf8').toString('base64'), + file_type: 'py', + }) + } + throw new Error(`Unexpected request: ${url}`) + }) + + const hydrated = await databricksConnector.getDocument(TOKEN, NOTEBOOK_CONFIG, stub.externalId) + + expect(hydrated?.contentHash).toBe(stub.contentHash) + expect(hydrated?.externalId).toBe(stub.externalId) + expect(hydrated?.contentDeferred).toBe(false) + expect(hydrated?.content).toBe('print("hi")') + expect(hydrated?.sourceUrl).toBe(`${ORIGIN}/#notebook/${'/Shared/etl'.length * 1000}`) + }) + + /** + * `modified_at` is documented as file-only, so a notebook routinely carries no + * timestamp at all. Its hash must still be stable across syncs — a clock- or + * fetch-derived component would make every sync re-index every notebook. The + * cost of that stability is that edits go unseen until a full resync, which is + * exactly what `rehydrateOnFullSync` covers. + */ + it('keeps an undated notebook hash stable across repeated listings', async () => { + mockWorkspace({ '/Shared': [notebook('/Shared/etl')] }) + + const first = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + const second = await databricksConnector.listDocuments(TOKEN, NOTEBOOK_CONFIG, undefined, {}) + + expect(first.documents[0].contentHash).toBe('databricks:notebook:/Shared/etl:') + expect(second.documents[0].contentHash).toBe(first.documents[0].contentHash) + }) + + it('produces the same query hash from the listing and from getDocument', async () => { + fetchMock.mockResolvedValue(jsonResponse({ results: [query('q1')] })) + const listed = await databricksConnector.listDocuments(TOKEN, QUERY_CONFIG, undefined, {}) + const stub = listed.documents[0] + + fetchMock.mockResolvedValue(jsonResponse(query('q1'))) + const refetched = await databricksConnector.getDocument(TOKEN, QUERY_CONFIG, stub.externalId) + + expect(refetched?.contentHash).toBe(stub.contentHash) + expect(refetched?.content).toBe(stub.content) + }) + + it('returns null for a notebook that has been deleted since the listing', async () => { + fetchMock.mockResolvedValue( + jsonResponse({ error_code: 'RESOURCE_DOES_NOT_EXIST', message: 'gone' }, 404) + ) + + await expect( + databricksConnector.getDocument(TOKEN, NOTEBOOK_CONFIG, 'notebook:/Shared/gone') + ).resolves.toBeNull() + }) + + it('reports an oversize export as a skip rather than losing the notebook', async () => { + fetchMock.mockImplementation(async (url: string) => { + const parsed = new URL(url) + if (parsed.pathname === '/api/2.0/workspace/get-status') { + return jsonResponse(notebook('/Shared/big', 5)) + } + return jsonResponse({ error_code: 'MAX_NOTEBOOK_SIZE_EXCEEDED', message: 'too big' }, 400) + }) + + const doc = await databricksConnector.getDocument( + TOKEN, + NOTEBOOK_CONFIG, + 'notebook:/Shared/big' + ) + + expect(doc?.skippedReason).toMatch(/10MB size limit/) + expect(doc?.contentHash).toBe(`databricks:notebook:/Shared/big:5`) + }) +}) + +describe('databricks configuration', () => { + it('declares rehydrateOnFullSync so undated notebook edits stay recoverable', () => { + expect(databricksConnectorMeta.rehydrateOnFullSync).toBe(true) + }) + + it('refuses a workspace host outside the Databricks-owned domains', async () => { + const result = await databricksConnector.validateConfig(TOKEN, { + workspaceHost: 'evil.example.com', + contentType: 'notebooks', + }) + + expect(result.valid).toBe(false) + expect(result.error).toMatch(/Databricks-hosted domain/) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('refuses to list against a host outside the allowlist', async () => { + await expect( + databricksConnector.listDocuments( + TOKEN, + { ...NOTEBOOK_CONFIG, workspaceHost: 'evil.example.com' }, + undefined, + {} + ) + ).rejects.toThrow(/Databricks-hosted domain/) + }) + + it('rejects a root path that names a notebook instead of a folder', async () => { + fetchMock.mockResolvedValue(jsonResponse(notebook('/Shared/etl', 1))) + + const result = await databricksConnector.validateConfig(TOKEN, { + ...NOTEBOOK_CONFIG, + rootPath: '/Shared/etl', + }) + + expect(result.valid).toBe(false) + expect(result.error).toMatch(/is a notebook, not a folder/) + }) + + it('accepts a Git folder as the notebook root', async () => { + fetchMock.mockResolvedValue(jsonResponse(repo('/Repos/ada/project'))) + + await expect( + databricksConnector.validateConfig(TOKEN, { + ...NOTEBOOK_CONFIG, + rootPath: '/Repos/ada/project', + }) + ).resolves.toEqual({ valid: true }) + }) + + it('normalizes a relative, trailing-slashed root path to an absolute one', async () => { + fetchMock.mockResolvedValue(jsonResponse(dir('/Shared/team'))) + + await databricksConnector.validateConfig(TOKEN, { + ...NOTEBOOK_CONFIG, + rootPath: ' Shared/team/ ', + }) + + const url = new URL(fetchMock.mock.calls[0][0] as string) + expect(url.searchParams.get('path')).toBe('/Shared/team') + }) +}) + +describe('databricks mapTags', () => { + it('maps notebook metadata to the declared tag slots', () => { + const tags = databricksConnector.mapTags?.({ + language: 'PYTHON', + lastModified: '2026-01-01T00:00:00.000Z', + path: '/Shared/etl', + }) + + expect(tags?.language).toBe('PYTHON') + expect(tags?.lastModified).toBeInstanceOf(Date) + }) + + it('maps query metadata, including the joined tag array', () => { + const tags = databricksConnector.mapTags?.({ + owner: 'ada@example.com', + catalog: 'main', + schema: 'default', + labels: ['finance', 'daily'], + lastModified: '2026-01-01T00:00:00Z', + }) + + expect(tags).toMatchObject({ + owner: 'ada@example.com', + catalog: 'main', + schema: 'default', + labels: 'finance, daily', + }) + }) + + it('emits only tag ids the connector declares', () => { + const declared = new Set(databricksConnectorMeta.tagDefinitions?.map((tag) => tag.id)) + const produced = databricksConnector.mapTags?.({ + language: 'SQL', + owner: 'ada@example.com', + catalog: 'main', + schema: 'default', + labels: ['a'], + lastModified: '2026-01-01T00:00:00Z', + }) + + for (const key of Object.keys(produced ?? {})) { + expect(declared.has(key)).toBe(true) + } + }) +}) diff --git a/apps/sim/connectors/databricks/databricks.ts b/apps/sim/connectors/databricks/databricks.ts new file mode 100644 index 00000000000..eb127d8dac8 --- /dev/null +++ b/apps/sim/connectors/databricks/databricks.ts @@ -0,0 +1,814 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { truncate } from '@sim/utils/string' +import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { + DATABRICKS_CONTENT_TYPES, + type DatabricksContentType, + DEFAULT_NOTEBOOK_ROOT_PATH, + databricksConnectorMeta, +} from '@/connectors/databricks/meta' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { + CONNECTOR_MAX_FILE_BYTES, + isSkippedDocument, + joinTagArray, + markSkipped, + parseTagDate, + sizeLimitSkipReason, + takeIndexableWithinCap, +} from '@/connectors/utils' + +const logger = createLogger('DatabricksConnector') + +/** Page size for `GET /api/2.0/sql/queries`. */ +const QUERY_PAGE_SIZE = 100 + +/** + * Directories listed per `listDocuments` call. + * + * `GET /api/2.0/workspace/list` is not paginated — it returns a whole directory + * in one response and offers no page token — so the connector walks the tree + * breadth-first and carries the pending-directory queue in the cursor. This + * bounds how much of the walk happens before the engine gets a batch to persist. + */ +const DIRECTORIES_PER_CALL = 50 + +/** Sibling directories listed in parallel while walking the tree. */ +const DIRECTORY_CONCURRENCY = 5 + +/** Notebooks collected before a `listDocuments` call yields, even mid-walk. */ +const MAX_NOTEBOOKS_PER_CALL = 500 + +/** + * Workspace object types that contain other objects and are therefore walked. + * + * `REPO` is a Git folder. It is a container just like `DIRECTORY` — every + * notebook inside a Git folder is only reachable by listing the repo path — so + * omitting it would silently skip every notebook under `/Repos` and every Git + * folder in a user's home. + */ +const TRAVERSABLE_OBJECT_TYPES = new Set(['DIRECTORY', 'REPO']) + +/** + * Maximum size of a single `workspace/export`, documented alongside the + * `MAX_NOTEBOOK_SIZE_EXCEEDED` error. Independent of (and far below) the + * knowledge base's own document cap, so the skip reason has to quote it rather + * than `CONNECTOR_MAX_FILE_BYTES`. + */ +const DATABRICKS_MAX_EXPORT_BYTES = 10 * 1024 * 1024 + +/** `externalId` prefixes, so a document always names the API it came from. */ +const NOTEBOOK_ID_PREFIX = 'notebook:' +const QUERY_ID_PREFIX = 'query:' + +/** + * `ObjectInfo` as returned by `GET /api/2.0/workspace/list` and + * `GET /api/2.0/workspace/get-status`. + * + * `created_at`, `modified_at` and `size` are documented as "only applicable to + * files", so they are all optional here — a `NOTEBOOK` entry frequently carries + * none of them. + */ +interface DatabricksObjectInfo { + object_type?: string + path?: string + object_id?: number + language?: string + created_at?: number + modified_at?: number + resource_id?: string + size?: number +} + +interface DatabricksListResponse { + objects?: DatabricksObjectInfo[] +} + +/** `GET /api/2.0/workspace/export` response (no `direct_download`, so JSON). */ +interface DatabricksExportResponse { + content?: string + file_type?: string +} + +/** A `results` entry from `GET /api/2.0/sql/queries`. */ +interface DatabricksQuery { + id?: string + display_name?: string + description?: string + query_text?: string + catalog?: string + schema?: string + warehouse_id?: string + owner_user_name?: string + last_modifier_user_name?: string + lifecycle_state?: string + tags?: string[] + create_time?: string + update_time?: string +} + +interface DatabricksQueryListResponse { + results?: DatabricksQuery[] + next_page_token?: string +} + +/** Databricks REST error envelope: `{ error_code, message }`. */ +interface DatabricksErrorBody { + error_code?: string + message?: string +} + +/** + * Traversal position across pages of a single notebook sync run. + * + * Only the pending-directory queue needs carrying: each entry is listed in full + * by a single unpaginated `workspace/list` call, so there is no intra-directory + * marker to resume from. + */ +interface DatabricksTraversalState { + queue: string[] +} + +function encodeCursor(state: DatabricksTraversalState): string { + return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url') +} + +function decodeCursor(cursor: string): DatabricksTraversalState | null { + try { + const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as unknown + if (typeof parsed !== 'object' || parsed === null) return null + const candidate = parsed as Partial + if (!Array.isArray(candidate.queue)) return null + return { queue: candidate.queue.filter((path): path is string => typeof path === 'string') } + } catch { + return null + } +} + +/** + * Normalizes the configured workspace host to an https origin, rejecting hosts + * outside Databricks-owned domains. + * + * Validated here rather than only in `validateConfig` because `listDocuments` + * and `getDocument` run against a stored `sourceConfig` that may have been + * written after validation, and the host is interpolated into every request URL. + */ +function resolveWorkspaceOrigin(sourceConfig: Record): string { + const result = validateDatabricksWorkspaceHost(sourceConfig.workspaceHost as string | undefined) + if (!result.isValid || !result.sanitized) { + throw new Error(result.error || 'Invalid Databricks workspace host') + } + return result.sanitized +} + +function resolveContentType(sourceConfig: Record): DatabricksContentType { + return sourceConfig.contentType === DATABRICKS_CONTENT_TYPES.queries + ? DATABRICKS_CONTENT_TYPES.queries + : DATABRICKS_CONTENT_TYPES.notebooks +} + +/** + * Normalizes the notebook root to an absolute workspace path. + * + * Every Workspace API path parameter is documented as absolute, so a relative + * value is corrected rather than sent through as a guaranteed 404. Trailing + * slashes are dropped (except on the root itself) so the path matches what + * `workspace/list` echoes back in `ObjectInfo.path`. + */ +function normalizeRootPath(value: unknown): string { + const raw = typeof value === 'string' ? value.trim() : '' + if (!raw) return DEFAULT_NOTEBOOK_ROOT_PATH + const absolute = raw.startsWith('/') ? raw : `/${raw}` + const trimmed = absolute.replace(/\/+$/, '') + return trimmed || DEFAULT_NOTEBOOK_ROOT_PATH +} + +function resolveMaxDocuments(value: unknown): number { + const parsed = Number(value) + return Number.isFinite(parsed) && parsed > 0 ? parsed : 0 +} + +async function readErrorBody(response: Response): Promise { + try { + return (await response.json()) as DatabricksErrorBody + } catch { + return {} + } +} + +function describeError(status: number, body: DatabricksErrorBody): string { + const detail = body.message?.trim() || body.error_code?.trim() + return detail ? `${status} — ${truncate(detail, 200)}` : String(status) +} + +/** + * Issues an authenticated GET against a workspace REST endpoint. + * + * Databricks authenticates every workspace API with a bearer personal access + * token, which is exactly what the sync engine hands connectors as + * `accessToken`. + */ +async function databricksGet( + origin: string, + path: string, + accessToken: string, + params: Record, + retryOptions?: Parameters[2] +): Promise { + const url = new URL(`${origin}${path}`) + for (const [key, value] of Object.entries(params)) { + url.searchParams.set(key, value) + } + return fetchWithRetry( + url.toString(), + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }, + retryOptions + ) +} + +/** + * Builds the listing stub for a workspace notebook. + * + * `externalId` carries the notebook's absolute path rather than its `object_id` + * because `workspace/export` is addressed by path only — an id-keyed document + * could never be re-hydrated. A moved notebook therefore reads as a delete plus + * an add, which is the same behaviour every path-addressed connector has. + * + * The hash is metadata-only so change detection never needs an export. When + * `modified_at` is absent (it is documented as file-only) the hash degenerates + * to a constant and only an explicit full resync — see `rehydrateOnFullSync` on + * the meta — will pick up later edits. + */ +function notebookToStub(origin: string, object: DatabricksObjectInfo): ExternalDocument { + const path = object.path ?? '' + const title = path.slice(path.lastIndexOf('/') + 1) || path || 'Untitled notebook' + + return { + externalId: `${NOTEBOOK_ID_PREFIX}${path}`, + title, + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: object.object_id ? `${origin}/#notebook/${object.object_id}` : undefined, + contentHash: `databricks:notebook:${path}:${object.modified_at ?? ''}`, + metadata: { + path, + objectId: object.object_id, + language: object.language, + lastModified: object.modified_at ? new Date(object.modified_at).toISOString() : undefined, + }, + } +} + +/** + * Renders a saved SQL query as indexable text. + * + * The description is the human-authored half (Databricks surfaces it as "usage + * notes") and the query text is the machine half; both are indexed so a search + * matches either. + */ +function formatQueryContent(query: DatabricksQuery): string { + const sections: string[] = [] + const description = query.description?.trim() + if (description) sections.push(description) + const text = query.query_text?.trim() + if (text) sections.push(text) + return sections.join('\n\n') +} + +/** + * Builds a document for a saved SQL query. + * + * `GET /api/2.0/sql/queries` returns `query_text` inline, so there is nothing + * left to defer — the listing already carries the full content. + * + * No `sourceUrl` is set: Databricks documents no stable URL for opening a saved + * query by id, and inventing one would produce dead links on every document. + */ +function queryToDocument(query: DatabricksQuery): ExternalDocument | null { + const id = query.id?.trim() + if (!id) return null + + const content = formatQueryContent(query) + if (!content) return null + + return { + externalId: `${QUERY_ID_PREFIX}${id}`, + title: query.display_name?.trim() || `Query ${id}`, + content, + contentDeferred: false, + mimeType: 'text/plain', + contentHash: `databricks:query:${id}:${query.update_time ?? ''}`, + metadata: { + queryId: id, + catalog: query.catalog, + schema: query.schema, + warehouseId: query.warehouse_id, + owner: query.owner_user_name ?? query.last_modifier_user_name, + labels: query.tags, + lastModified: query.update_time, + }, + } +} + +/** + * Lists one workspace directory. + * + * Returns `null` when the directory is unreadable (deleted, or the token lacks + * permission on that subtree) so the caller can decide whether that is fatal. + */ +async function listDirectory( + origin: string, + accessToken: string, + accessPath: string +): Promise { + const response = await databricksGet(origin, '/api/2.0/workspace/list', accessToken, { + path: accessPath, + }) + + if (!response.ok) { + if (response.status === 403 || response.status === 404) { + const body = await readErrorBody(response) + logger.warn('Skipping unreadable Databricks workspace directory', { + path: accessPath, + error: describeError(response.status, body), + }) + return null + } + const body = await readErrorBody(response) + throw new Error( + `Failed to list Databricks workspace path ${accessPath}: ${describeError(response.status, body)}` + ) + } + + const data = (await response.json()) as DatabricksListResponse + return data.objects ?? [] +} + +/** Walks the workspace tree breadth-first, one bounded slice per call. */ +async function listNotebooks( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record +): Promise { + const origin = resolveWorkspaceOrigin(sourceConfig) + const rootPath = normalizeRootPath(sourceConfig.rootPath) + const maxDocuments = resolveMaxDocuments(sourceConfig.maxDocuments) + + const state = cursor ? (decodeCursor(cursor) ?? { queue: [rootPath] }) : { queue: [rootPath] } + const queue = [...state.queue] + const notebooks: DatabricksObjectInfo[] = [] + + logger.info('Listing Databricks notebooks', { + rootPath, + pending: queue.length, + resumed: Boolean(cursor), + }) + + for (let listed = 0; listed < DIRECTORIES_PER_CALL && queue.length > 0; ) { + const batch = queue.splice(0, Math.min(DIRECTORY_CONCURRENCY, DIRECTORIES_PER_CALL - listed)) + listed += batch.length + + const listings = await Promise.all( + batch.map(async (path) => ({ + path, + objects: await listDirectory(origin, accessToken, path), + })) + ) + + for (const { path: currentPath, objects } of listings) { + /** + * Losing access to a *sub*directory is survivable, but losing the configured + * root means the listing is empty. Failing loudly beats reporting a + * successful sync that indexed nothing. + */ + if (!objects) { + if (currentPath === rootPath) { + throw new Error( + `Databricks denied access to ${rootPath}. Check the token's workspace permissions or choose another root path.` + ) + } + /** + * A directory was skipped, so notebooks that still exist in the workspace + * are absent from this listing. Without this flag the engine would + * reconcile them as deleted. + */ + if (syncContext) syncContext.listingCapped = true + continue + } + + for (const object of objects) { + const path = object.path + if (!path) continue + if (object.object_type && TRAVERSABLE_OBJECT_TYPES.has(object.object_type)) { + /** + * `workspace/list` returns the object itself when the path is not a + * directory; re-queueing a self-reference would loop forever. + */ + if (path !== currentPath) queue.push(path) + } else if (object.object_type === 'NOTEBOOK') { + notebooks.push(object) + } + } + } + + if (notebooks.length >= MAX_NOTEBOOKS_PER_CALL) break + } + + /** + * A notebook with no `modified_at` gets a constant `contentHash`, so an edit to + * it will never be detected by an ordinary sync — only the "Full resync" action + * re-exports it (see `rehydrateOnFullSync`). Surface that here so the condition + * is diagnosable from the sync logs instead of presenting as silently stale + * content. + */ + const undatedCount = notebooks.reduce((count, object) => count + (object.modified_at ? 0 : 1), 0) + if (undatedCount > 0) { + logger.warn( + 'Databricks listed notebooks without a modification timestamp; their edits are only picked up by a full resync', + { rootPath, undatedCount, listedNotebooks: notebooks.length } + ) + } + + const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const stubs = notebooks.map((object) => notebookToStub(origin, object)) + const { documents, indexableCount, capReached } = takeIndexableWithinCap( + stubs, + isSkippedDocument, + maxDocuments, + previouslyFetched + ) + + if (syncContext) syncContext.totalDocsFetched = previouslyFetched + indexableCount + + /** + * The cap truncates the listing only when it stopped the walk with directories + * still pending, or dropped notebooks from this very page. Reaching the cap on + * the last notebook of an exhausted walk hides nothing, so deletion + * reconciliation stays enabled in that case. + */ + const hitLimit = capReached && (queue.length > 0 || documents.length < stubs.length) + if (hitLimit && syncContext) syncContext.listingCapped = true + + const hasMore = !hitLimit && queue.length > 0 + + return { + documents, + nextCursor: hasMore ? encodeCursor({ queue }) : undefined, + hasMore, + } +} + +/** Pages through the workspace's saved SQL queries. */ +async function listQueries( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record +): Promise { + const origin = resolveWorkspaceOrigin(sourceConfig) + const maxDocuments = resolveMaxDocuments(sourceConfig.maxDocuments) + + const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + + /** + * Ask for only what the cap still allows, so the last page of a capped sync + * does not pull a full 100 rows to discard most of them. Trashed and empty + * rows are dropped after the fetch, so a short page can still leave the cap + * unfilled — `next_page_token` then simply carries the sync into another page. + */ + const remaining = + maxDocuments > 0 ? Math.max(1, maxDocuments - previouslyFetched) : QUERY_PAGE_SIZE + const params: Record = { + page_size: String(Math.min(QUERY_PAGE_SIZE, remaining)), + } + if (cursor) params.page_token = cursor + + logger.info('Listing Databricks SQL queries', { cursor: cursor ?? 'initial', maxDocuments }) + + const response = await databricksGet(origin, '/api/2.0/sql/queries', accessToken, params) + + if (!response.ok) { + const body = await readErrorBody(response) + throw new Error( + `Failed to list Databricks SQL queries: ${describeError(response.status, body)}` + ) + } + + const data = (await response.json()) as DatabricksQueryListResponse + const results = data.results ?? [] + + /** + * A trashed query is Databricks' delete: the row stays listable but is no + * longer part of the workspace. Dropping it here lets deletion reconciliation + * remove any copy already in the knowledge base, so `listingCapped` must NOT + * be set for it. + */ + const stubs = results + .filter((query) => query.lifecycle_state !== 'TRASHED') + .map(queryToDocument) + .filter((document): document is ExternalDocument => document !== null) + + const { documents, indexableCount, capReached } = takeIndexableWithinCap( + stubs, + isSkippedDocument, + maxDocuments, + previouslyFetched + ) + + if (syncContext) syncContext.totalDocsFetched = previouslyFetched + indexableCount + + /** + * `next_page_token` is only a usable cursor when this page actually returned + * results — an echoed token on an empty tail would re-request the same page + * until the engine truncates pagination, permanently disabling deletion + * reconciliation. + */ + const nextPageToken = data.next_page_token?.trim() || undefined + const sourceHasMore = Boolean(nextPageToken) && results.length > 0 + + const hitLimit = capReached && (sourceHasMore || documents.length < stubs.length) + if (hitLimit && syncContext) syncContext.listingCapped = true + + const hasMore = !hitLimit && sourceHasMore + + return { + documents, + nextCursor: hasMore ? nextPageToken : undefined, + hasMore, + } +} + +/** + * Exports a notebook and returns its decoded source. + * + * `workspace/export` answers with base64 in `content` (no `direct_download`, so + * the JSON form) and caps a single export at `DATABRICKS_MAX_EXPORT_BYTES`, past + * which it fails with `MAX_NOTEBOOK_SIZE_EXCEEDED`. That case is reported as an + * oversize skip rather than an error so the notebook stays visible in the + * knowledge base. + */ +async function exportNotebook( + origin: string, + accessToken: string, + path: string +): Promise<{ content: string } | { skippedReason: string } | null> { + const response = await databricksGet(origin, '/api/2.0/workspace/export', accessToken, { + path, + format: 'SOURCE', + }) + + if (!response.ok) { + const body = await readErrorBody(response) + if (response.status === 404 || body.error_code === 'RESOURCE_DOES_NOT_EXIST') return null + if (body.error_code === 'MAX_NOTEBOOK_SIZE_EXCEEDED') { + return { skippedReason: sizeLimitSkipReason(DATABRICKS_MAX_EXPORT_BYTES) } + } + throw new Error( + `Failed to export Databricks notebook ${path}: ${describeError(response.status, body)}` + ) + } + + const data = (await response.json()) as DatabricksExportResponse + if (!data.content) return null + + const decoded = Buffer.from(data.content, 'base64') + if (decoded.byteLength > CONNECTOR_MAX_FILE_BYTES) { + return { skippedReason: sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES) } + } + + return { content: decoded.toString('utf8') } +} + +/** + * Re-hydrates one notebook. + * + * The stub is rebuilt from `workspace/get-status` — the same `ObjectInfo` shape + * `workspace/list` returns — rather than assembled by hand, so the hash produced + * here is identical to the listing's by construction. + */ +async function getNotebook( + accessToken: string, + sourceConfig: Record, + path: string +): Promise { + const origin = resolveWorkspaceOrigin(sourceConfig) + + const statusResponse = await databricksGet(origin, '/api/2.0/workspace/get-status', accessToken, { + path, + }) + + if (!statusResponse.ok) { + const body = await readErrorBody(statusResponse) + if (statusResponse.status === 404 || body.error_code === 'RESOURCE_DOES_NOT_EXIST') return null + throw new Error( + `Failed to read Databricks notebook status for ${path}: ${describeError(statusResponse.status, body)}` + ) + } + + const object = (await statusResponse.json()) as DatabricksObjectInfo + if (object.object_type !== 'NOTEBOOK') return null + + const stub = notebookToStub(origin, { ...object, path: object.path ?? path }) + + const exported = await exportNotebook(origin, accessToken, path) + if (exported === null) return null + if ('skippedReason' in exported) return markSkipped(stub, exported.skippedReason) + if (!exported.content.trim()) return null + + return { ...stub, content: exported.content, contentDeferred: false } +} + +/** Re-fetches one saved SQL query by id. */ +async function getQuery( + accessToken: string, + sourceConfig: Record, + queryId: string +): Promise { + const origin = resolveWorkspaceOrigin(sourceConfig) + + const response = await databricksGet( + origin, + `/api/2.0/sql/queries/${encodeURIComponent(queryId)}`, + accessToken, + {} + ) + + if (!response.ok) { + const body = await readErrorBody(response) + if (response.status === 404 || body.error_code === 'RESOURCE_DOES_NOT_EXIST') return null + throw new Error( + `Failed to fetch Databricks SQL query ${queryId}: ${describeError(response.status, body)}` + ) + } + + const query = (await response.json()) as DatabricksQuery + if (query.lifecycle_state === 'TRASHED') return null + + return queryToDocument({ ...query, id: query.id ?? queryId }) +} + +export const databricksConnector: ConnectorConfig = { + ...databricksConnectorMeta, + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record + ): Promise => { + return resolveContentType(sourceConfig) === DATABRICKS_CONTENT_TYPES.queries + ? listQueries(accessToken, sourceConfig, cursor, syncContext) + : listNotebooks(accessToken, sourceConfig, cursor, syncContext) + }, + + getDocument: async ( + accessToken: string, + sourceConfig: Record, + externalId: string + ): Promise => { + try { + if (externalId.startsWith(QUERY_ID_PREFIX)) { + return await getQuery(accessToken, sourceConfig, externalId.slice(QUERY_ID_PREFIX.length)) + } + if (externalId.startsWith(NOTEBOOK_ID_PREFIX)) { + return await getNotebook( + accessToken, + sourceConfig, + externalId.slice(NOTEBOOK_ID_PREFIX.length) + ) + } + return null + } catch (error) { + /** + * Only an explicit "gone" above returns null. Everything else — 429, 5xx, + * network faults — is rethrown so the sync engine records a failed row and + * keeps the already-indexed document out of deletion reconciliation. + */ + logger.warn('Failed to get Databricks document', { + externalId, + error: toError(error).message, + }) + throw toError(error) + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const hostResult = validateDatabricksWorkspaceHost( + sourceConfig.workspaceHost as string | undefined + ) + if (!hostResult.isValid || !hostResult.sanitized) { + return { valid: false, error: hostResult.error || 'Invalid Databricks workspace host' } + } + const origin = hostResult.sanitized + + const maxDocuments = sourceConfig.maxDocuments + if (maxDocuments !== undefined && maxDocuments !== null && String(maxDocuments).trim()) { + const parsed = Number(maxDocuments) + if (!Number.isFinite(parsed) || parsed <= 0) { + return { valid: false, error: 'Max documents must be a positive number' } + } + } + + const contentType = resolveContentType(sourceConfig) + + try { + if (contentType === DATABRICKS_CONTENT_TYPES.queries) { + const response = await databricksGet( + origin, + '/api/2.0/sql/queries', + accessToken, + { page_size: '1' }, + VALIDATE_RETRY_OPTIONS + ) + if (!response.ok) { + const body = await readErrorBody(response) + return { + valid: false, + error: `Databricks SQL queries access failed: ${describeError(response.status, body)}`, + } + } + return { valid: true } + } + + const rootPath = normalizeRootPath(sourceConfig.rootPath) + const response = await databricksGet( + origin, + '/api/2.0/workspace/get-status', + accessToken, + { path: rootPath }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const body = await readErrorBody(response) + return { + valid: false, + error: `Databricks workspace access failed for ${rootPath}: ${describeError(response.status, body)}`, + } + } + + const object = (await response.json()) as DatabricksObjectInfo + if ( + object.object_type && + object.object_type !== 'DIRECTORY' && + object.object_type !== 'REPO' + ) { + return { + valid: false, + error: `${rootPath} is a ${object.object_type.toLowerCase()}, not a folder. Enter the folder that contains the notebooks.`, + } + } + + return { valid: true } + } catch (error) { + return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') } + } + }, + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.language === 'string' && metadata.language.trim()) { + result.language = metadata.language + } + + if (typeof metadata.owner === 'string' && metadata.owner.trim()) { + result.owner = metadata.owner + } + + if (typeof metadata.catalog === 'string' && metadata.catalog.trim()) { + result.catalog = metadata.catalog + } + + if (typeof metadata.schema === 'string' && metadata.schema.trim()) { + result.schema = metadata.schema + } + + const labels = joinTagArray(metadata.labels) + if (labels) result.labels = labels + + /** + * Both stubs normalize their source timestamp to an ISO string — notebooks + * from `modified_at` (epoch milliseconds), queries from `update_time` — so + * `parseTagDate`, which only accepts strings, resolves both. Notebooks that + * carry no timestamp simply produce no tag. + */ + const lastModified = parseTagDate(metadata.lastModified) + if (lastModified) result.lastModified = lastModified + + return result + }, +} diff --git a/apps/sim/connectors/databricks/index.ts b/apps/sim/connectors/databricks/index.ts new file mode 100644 index 00000000000..c51f2c1d9fd --- /dev/null +++ b/apps/sim/connectors/databricks/index.ts @@ -0,0 +1 @@ +export { databricksConnector } from '@/connectors/databricks/databricks' diff --git a/apps/sim/connectors/databricks/meta.ts b/apps/sim/connectors/databricks/meta.ts new file mode 100644 index 00000000000..969b28b7479 --- /dev/null +++ b/apps/sim/connectors/databricks/meta.ts @@ -0,0 +1,86 @@ +import { DatabricksIcon } from '@/components/icons' +import type { ConnectorMeta } from '@/connectors/types' + +/** Default root of the notebook walk when the user leaves `rootPath` empty. */ +export const DEFAULT_NOTEBOOK_ROOT_PATH = '/' + +/** Content kinds this connector can index, keyed by the `contentType` config field. */ +export const DATABRICKS_CONTENT_TYPES = { + notebooks: 'notebooks', + queries: 'queries', +} as const + +export type DatabricksContentType = + (typeof DATABRICKS_CONTENT_TYPES)[keyof typeof DATABRICKS_CONTENT_TYPES] + +export const databricksConnectorMeta: ConnectorMeta = { + id: 'databricks', + name: 'Databricks', + description: 'Sync notebooks and saved SQL queries from Databricks into your knowledge base', + version: '1.0.0', + icon: DatabricksIcon, + + auth: { + mode: 'apiKey', + label: 'Personal Access Token', + placeholder: 'Enter your Databricks personal access token', + }, + + /** + * `GET /api/2.0/workspace/list` documents `created_at`, `modified_at` and `size` + * as "only applicable to files", so a `NOTEBOOK` entry can come back with no + * change indicator at all. Its metadata-derived `contentHash` then stays + * identical across edits and an incremental sync would never re-export it. + * Opting into full-resync rehydration gives users a way to pick those edits up. + */ + rehydrateOnFullSync: true, + + configFields: [ + { + id: 'workspaceHost', + title: 'Workspace Host', + type: 'short-input', + placeholder: 'dbc-1234abcd-5678.cloud.databricks.com', + required: true, + description: + 'Your per-workspace Databricks URL. Azure and GCP hosts (adb-*.azuredatabricks.net, *.gcp.databricks.com) are also accepted.', + }, + { + id: 'contentType', + title: 'Content Type', + type: 'dropdown', + required: true, + description: 'What to sync from the workspace. Add a second connector to sync both.', + options: [ + { label: 'Notebooks', id: DATABRICKS_CONTENT_TYPES.notebooks }, + { label: 'Saved SQL Queries', id: DATABRICKS_CONTENT_TYPES.queries }, + ], + }, + { + id: 'rootPath', + title: 'Notebook Root Path', + type: 'short-input', + required: false, + placeholder: '/Users/you@example.com (default: /)', + description: + 'Absolute workspace path to walk for notebooks. Ignored when syncing saved SQL queries.', + }, + { + id: 'maxDocuments', + title: 'Max Documents', + type: 'short-input', + required: false, + placeholder: 'e.g. 500 (default: unlimited)', + description: 'Cap the number of documents synced. Leave empty to sync everything.', + }, + ], + + tagDefinitions: [ + { id: 'language', displayName: 'Language', fieldType: 'text' }, + { id: 'owner', displayName: 'Owner', fieldType: 'text' }, + { id: 'catalog', displayName: 'Catalog', fieldType: 'text' }, + { id: 'schema', displayName: 'Schema', fieldType: 'text' }, + { id: 'labels', displayName: 'Tags', fieldType: 'text' }, + { id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' }, + ], +} diff --git a/apps/sim/connectors/google-chat/google-chat.test.ts b/apps/sim/connectors/google-chat/google-chat.test.ts new file mode 100644 index 00000000000..2b68fb36b39 --- /dev/null +++ b/apps/sim/connectors/google-chat/google-chat.test.ts @@ -0,0 +1,359 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { googleChatConnector } from '@/connectors/google-chat/google-chat' +import { googleChatConnectorMeta } from '@/connectors/google-chat/meta' + +const SPACE_NAME = 'spaces/AAAA1111' + +const SPACE = { + name: SPACE_NAME, + displayName: 'Platform Team', + spaceType: 'SPACE' as const, + spaceUri: 'https://mail.google.com/chat/u/0/#chat/space/AAAA1111', + lastActiveTime: '2026-02-01T10:30:00Z', +} + +const MESSAGES = [ + { + name: `${SPACE_NAME}/messages/m2`, + sender: { name: 'users/2', displayName: 'Grace Hopper' }, + createTime: '2026-02-01T10:30:00Z', + text: 'Shipping today', + }, + { + name: `${SPACE_NAME}/messages/m1`, + sender: { name: 'users/1', displayName: 'Ada Lovelace' }, + createTime: '2026-02-01T10:00:00Z', + text: 'Morning', + }, +] + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +const requestedUrls: string[] = [] +const fetchMock = vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise>() + +/** Spaces returned by `spaces.list`; per-test overridable. */ +let listedSpaces: Record[] = [SPACE] +/** `nextPageToken` returned by `spaces.list`; per-test overridable. */ +let listNextPageToken: string | undefined +/** Messages returned by `spaces.messages.list`; per-test overridable. */ +let listedMessages: Record[] = MESSAGES +/** Space returned by `spaces.list` / `spaces.get`; per-test overridable. */ +let fetchedSpace: Record = SPACE + +beforeEach(() => { + requestedUrls.length = 0 + listedSpaces = [SPACE] + listNextPageToken = undefined + listedMessages = MESSAGES + fetchedSpace = SPACE + fetchMock.mockReset() + fetchMock.mockImplementation(async (input) => { + const url = String(input) + requestedUrls.push(url) + if (url.includes('/messages?')) return jsonResponse({ messages: listedMessages }) + if (url.includes('/spaces?')) { + return jsonResponse({ spaces: listedSpaces, nextPageToken: listNextPageToken }) + } + if (url.endsWith(`/${SPACE_NAME}`)) return jsonResponse(fetchedSpace) + return jsonResponse({ error: { message: 'not found' } }, 404) + }) + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +/** The decoded `filter` the last `spaces.list` request carried, if any. */ +function listFilter(): string | null { + const listUrl = requestedUrls.find((url) => url.includes('/spaces?')) + return listUrl ? new URL(listUrl).searchParams.get('filter') : null +} + +/** Search params of the first `spaces.messages.list` request. */ +function messagesParams(): URLSearchParams { + const url = requestedUrls.find((requested) => requested.includes('/messages?')) + if (!url) throw new Error('no messages request was made') + return new URL(url).searchParams +} + +describe('google-chat space scope', () => { + it('filters to named spaces only when spaceTypes is unset', async () => { + await googleChatConnector.listDocuments('token', {}) + expect(listFilter()).toBe('spaceType = "SPACE"') + }) + + it('filters to named spaces only when spaceTypes is an empty string', async () => { + await googleChatConnector.listDocuments('token', { spaceTypes: '' }) + expect(listFilter()).toBe('spaceType = "SPACE"') + }) + + it('widens the filter to group chats without reaching direct messages', async () => { + await googleChatConnector.listDocuments('token', { spaceTypes: 'SPACE_AND_GROUP_CHAT' }) + expect(listFilter()).toBe('spaceType = "SPACE" OR spaceType = "GROUP_CHAT"') + }) + + it('omits the filter only when every conversation type is requested', async () => { + await googleChatConnector.listDocuments('token', { spaceTypes: 'ALL' }) + expect(listFilter()).toBeNull() + }) + + it('rejects an unsupported space type selection', async () => { + await expect( + googleChatConnector.validateConfig('token', { spaceTypes: 'DIRECT_MESSAGE' }) + ).resolves.toEqual({ valid: false, error: 'Unsupported space type selection' }) + }) +}) + +describe('google-chat change detection', () => { + it('hashes on lastActiveTime and keeps the stub and hydrated hash identical', async () => { + const syncContext: Record = {} + const listed = await googleChatConnector.listDocuments('token', {}, undefined, syncContext) + const stub = listed.documents[0] + expect(stub.contentDeferred).toBe(true) + expect(stub.contentHash).toContain(SPACE.lastActiveTime) + + const hydrated = await googleChatConnector.getDocument('token', {}, SPACE_NAME, syncContext) + expect(hydrated?.contentHash).toBe(stub.contentHash) + expect(hydrated?.contentDeferred).toBe(false) + expect(hydrated?.content).toContain('Ada Lovelace: Morning') + expect(hydrated?.content).toContain('Grace Hopper: Shipping today') + }) + + it('reuses the listed space rather than re-reading it during hydration', async () => { + const syncContext: Record = {} + await googleChatConnector.listDocuments('token', {}, undefined, syncContext) + requestedUrls.length = 0 + + await googleChatConnector.getDocument('token', {}, SPACE_NAME, syncContext) + expect(requestedUrls.some((url) => url.endsWith(`/${SPACE_NAME}`))).toBe(false) + }) + + it('produces a stable hash across syncs while lastActiveTime is unchanged', async () => { + const first = await googleChatConnector.listDocuments('token', {}, undefined, {}) + const second = await googleChatConnector.listDocuments('token', {}, undefined, {}) + expect(second.documents[0].contentHash).toBe(first.documents[0].contentHash) + }) + + it('re-hydrates every sync when a space has no lastActiveTime', async () => { + listedSpaces = [{ ...SPACE, lastActiveTime: undefined }] + + const runOne: Record = {} + const firstPage = await googleChatConnector.listDocuments('token', {}, undefined, runOne) + const runTwo: Record = {} + const secondPage = await googleChatConnector.listDocuments('token', {}, undefined, runTwo) + + // Stable within a run, so the hydrated document keeps the stub's hash … + const hydrated = await googleChatConnector.getDocument('token', {}, SPACE_NAME, runOne) + expect(hydrated?.contentHash).toBe(firstPage.documents[0].contentHash) + + // … and different on the next run, so content never silently goes stale. + expect(secondPage.documents[0].contentHash).not.toBe(firstPage.documents[0].contentHash) + }) + + it('declares rehydrateOnFullSync so edits and deletions are recoverable', () => { + expect(googleChatConnectorMeta.rehydrateOnFullSync).toBe(true) + }) +}) + +describe('google-chat listing caps', () => { + it('flags the listing as capped only when the cap truncated a larger source', async () => { + listedSpaces = [SPACE, { ...SPACE, name: 'spaces/BBBB2222' }] + const syncContext: Record = {} + + const result = await googleChatConnector.listDocuments( + 'token', + { maxSpaces: '1' }, + undefined, + syncContext + ) + expect(result.documents).toHaveLength(1) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it('stops paginating at the cap even while the source offers another page', async () => { + listNextPageToken = 'page-2' + const syncContext: Record = {} + + const result = await googleChatConnector.listDocuments( + 'token', + { maxSpaces: '1' }, + undefined, + syncContext + ) + expect(result.hasMore).toBe(false) + expect(result.nextCursor).toBeUndefined() + expect(syncContext.listingCapped).toBe(true) + }) + + it('leaves deletion reconciliation enabled when the source is exhausted at the cap', async () => { + const syncContext: Record = {} + await googleChatConnector.listDocuments('token', { maxSpaces: '1' }, undefined, syncContext) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('paginates with the returned nextPageToken', async () => { + listNextPageToken = 'page-2' + const result = await googleChatConnector.listDocuments('token', {}) + expect(result.hasMore).toBe(true) + expect(result.nextCursor).toBe('page-2') + }) +}) + +describe('google-chat empty windows', () => { + it('indexes a space whose only prose is its description when no message has text', async () => { + fetchedSpace = { ...SPACE, spaceDetails: { description: 'Release coordination' } } + listedSpaces = [fetchedSpace] + listedMessages = [] + + const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME) + expect(doc).not.toBeNull() + expect(doc?.content).toContain('Release coordination') + }) + + it('indexes a space whose only prose is its guidelines when no message has text', async () => { + fetchedSpace = { ...SPACE, spaceDetails: { guidelines: 'Be excellent to each other' } } + listedSpaces = [fetchedSpace] + listedMessages = [] + + const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME) + expect(doc?.content).toContain('Be excellent to each other') + }) + + it('returns a document rather than null when the window is empty, so a cleared space does not keep a stale transcript', async () => { + listedMessages = [] + const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME) + expect(doc).not.toBeNull() + expect(doc?.content).not.toContain('Shipping today') + expect(doc?.metadata?.messageCount).toBe(0) + }) + + it('omits the transcript header entirely when no message contributed text', async () => { + listedMessages = [] + const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME) + expect(doc?.content).not.toContain('--- Messages ---') + }) + + it('still returns null when the space itself is gone', async () => { + const doc = await googleChatConnector.getDocument('token', {}, 'spaces/MISSING') + expect(doc).toBeNull() + }) +}) + +describe('google-chat message window', () => { + it('requests messages newest-first so the cap keeps the most recent conversation', async () => { + await googleChatConnector.getDocument('token', {}, SPACE_NAME) + expect(messagesParams().get('orderBy')).toBe('createTime DESC') + }) + + it('renders the newest-first page back into chronological order', async () => { + const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME) + const content = doc?.content ?? '' + expect(content.indexOf('Ada Lovelace')).toBeLessThan(content.indexOf('Grace Hopper')) + }) + + it('bounds the message page to the remaining window rather than the API maximum', async () => { + await googleChatConnector.getDocument('token', { maxMessages: '5' }, SPACE_NAME) + expect(messagesParams().get('pageSize')).toBe('5') + }) + + it('sends the lookback cutoff as an RFC-3339 timestamp without fractional seconds', async () => { + await googleChatConnector.getDocument('token', { lookbackDays: '30' }, SPACE_NAME) + const filter = messagesParams().get('filter') + expect(filter).toMatch(/^createTime > "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z"$/) + }) + + it('omits the filter entirely when no lookback window is configured', async () => { + await googleChatConnector.getDocument('token', {}, SPACE_NAME) + expect(messagesParams().get('filter')).toBeNull() + }) + + it('indexes a card-only message through its fallbackText', async () => { + fetchMock.mockImplementation(async (input) => { + const url = String(input) + requestedUrls.push(url) + if (url.includes('/messages?')) { + return jsonResponse({ + messages: [ + { + name: `${SPACE_NAME}/messages/card`, + sender: { name: 'users/9', type: 'BOT' }, + createTime: '2026-02-01T09:00:00Z', + fallbackText: 'Deploy finished', + }, + ], + }) + } + if (url.endsWith(`/${SPACE_NAME}`)) return jsonResponse(SPACE) + return jsonResponse({}, 404) + }) + + const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME) + expect(doc?.content).toContain('Deploy finished') + expect(doc?.metadata?.messageCount).toBe(1) + }) + + it('labels a sender by resource name when user auth omits displayName', async () => { + fetchMock.mockImplementation(async (input) => { + const url = String(input) + requestedUrls.push(url) + if (url.includes('/messages?')) { + return jsonResponse({ + messages: [ + { + name: `${SPACE_NAME}/messages/m1`, + sender: { name: 'users/104512345678', type: 'HUMAN' }, + createTime: '2026-02-01T10:00:00Z', + text: 'Morning', + }, + ], + }) + } + if (url.endsWith(`/${SPACE_NAME}`)) return jsonResponse(SPACE) + return jsonResponse({}, 404) + }) + + const doc = await googleChatConnector.getDocument('token', {}, SPACE_NAME) + expect(doc?.content).toContain('users/104512345678: Morning') + }) + + it('rehashes when the configured window changes so the stored transcript is refetched', async () => { + const base = await googleChatConnector.listDocuments('token', {}, undefined, {}) + const narrower = await googleChatConnector.listDocuments( + 'token', + { maxMessages: '50' }, + undefined, + {} + ) + const windowed = await googleChatConnector.listDocuments( + 'token', + { lookbackDays: '30' }, + undefined, + {} + ) + + expect(narrower.documents[0].contentHash).not.toBe(base.documents[0].contentHash) + expect(windowed.documents[0].contentHash).not.toBe(base.documents[0].contentHash) + }) +}) + +describe('google-chat getDocument', () => { + it('returns null for a space that no longer exists', async () => { + await expect(googleChatConnector.getDocument('token', {}, 'spaces/MISSING')).resolves.toBeNull() + }) + + it('accepts a bare space id and normalizes it to a resource name', async () => { + const doc = await googleChatConnector.getDocument('token', {}, 'AAAA1111') + expect(doc?.externalId).toBe(SPACE_NAME) + }) +}) diff --git a/apps/sim/connectors/google-chat/google-chat.ts b/apps/sim/connectors/google-chat/google-chat.ts new file mode 100644 index 00000000000..1dcca49d7b3 --- /dev/null +++ b/apps/sim/connectors/google-chat/google-chat.ts @@ -0,0 +1,559 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { + DEFAULT_MAX_MESSAGES, + googleChatConnectorMeta, + MESSAGES_PAGE_SIZE, + SPACES_PAGE_SIZE, +} from '@/connectors/google-chat/meta' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { parseTagDate } from '@/connectors/utils' + +const logger = createLogger('GoogleChatConnector') + +const CHAT_API_BASE = 'https://chat.googleapis.com/v1' +const MS_PER_DAY = 24 * 60 * 60 * 1000 + +/** `syncContext` key holding the spaces this run listed, keyed by resource name. */ +const SPACE_CACHE_KEY = 'googleChatSpaces' + +/** `syncContext` key holding this run's fallback activity token. */ +const ACTIVITY_TOKEN_KEY = 'googleChatActivityToken' + +/** + * A Google Chat space as returned by `spaces.list` / `spaces.get`. Only the + * fields this connector reads are modeled. + * + * https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces + */ +interface Space { + name: string + displayName?: string + spaceType?: 'SPACE_TYPE_UNSPECIFIED' | 'SPACE' | 'GROUP_CHAT' | 'DIRECT_MESSAGE' + spaceUri?: string + createTime?: string + /** Output only. Timestamp of the last message in the space. */ + lastActiveTime?: string + spaceDetails?: { description?: string; guidelines?: string } +} + +interface SpacesListResponse { + spaces?: Space[] + nextPageToken?: string +} + +/** + * The author of a message. + * + * This connector authenticates as a user, and the User reference states that + * "if your Chat app authenticates as a user, the output for a User resource + * only populates the user's `name` and `type`" — so `displayName` is expected + * to be absent here and `name` (a `users/{id}` resource name) is the label that + * actually reaches the transcript. Resolving human names would need + * `spaces.members.list` and the `chat.memberships.readonly` scope, which this + * connector does not request. `displayName` is still read first so the + * transcript improves for free if a call ever does carry it. + * + * https://developers.google.com/workspace/chat/api/reference/rest/v1/User + */ +interface ChatUser { + name?: string + displayName?: string + type?: 'TYPE_UNSPECIFIED' | 'HUMAN' | 'BOT' +} + +/** + * A message in a space. `text` is the plain-text body; `fallbackText` is the + * plain-text description of a card-only message. + * + * https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces.messages + */ +interface ChatMessage { + name: string + sender?: ChatUser + createTime?: string + lastUpdateTime?: string + text?: string + fallbackText?: string +} + +interface MessagesListResponse { + messages?: ChatMessage[] + nextPageToken?: string +} + +function chatHeaders(accessToken: string): Record { + return { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' } +} + +/** Trailing id segment of a `spaces/{space}` resource name. */ +function resourceIdSegment(resourceName: string): string { + return resourceName.slice(resourceName.lastIndexOf('/') + 1) +} + +/** Normalizes a space identifier to its full resource name, tolerating a bare id. */ +function spaceResourceName(externalId: string): string { + const trimmed = externalId.trim() + return trimmed.startsWith('spaces/') ? trimmed : `spaces/${trimmed}` +} + +/** + * Builds the `spaces.list` filter from the configured scope. The API only + * supports filtering on `spaceType`, with `OR` between values; `ALL` omits the + * filter entirely so every conversation type is listed. + */ +function buildSpacesFilter(sourceConfig: Record): string | undefined { + const scope = typeof sourceConfig.spaceTypes === 'string' ? sourceConfig.spaceTypes : 'SPACE' + if (scope === 'ALL') return undefined + if (scope === 'SPACE_AND_GROUP_CHAT') return 'spaceType = "SPACE" OR spaceType = "GROUP_CHAT"' + return 'spaceType = "SPACE"' +} + +/** + * Resolves the per-space message window, falling back to the default for + * missing, non-numeric, or non-positive values. `validateConfig` rejects those + * inputs, but a config saved before validation existed would otherwise yield + * `NaN` and index nothing. + */ +function resolveMaxMessages(value: unknown): number { + const parsed = Number(value) + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : DEFAULT_MAX_MESSAGES +} + +/** Resolves the lookback window in days, or 0 when unset/invalid (no cutoff). */ +function resolveLookbackDays(value: unknown): number { + const parsed = Number(value) + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : 0 +} + +/** + * Human-readable title for a space. `displayName` is optional and documented as + * possibly empty for direct messages, so the conversation type plus the space id + * is used as a stable fallback rather than an anonymous "Untitled". + */ +function spaceTitle(space: Space): string { + const displayName = space.displayName?.trim() + if (displayName) return displayName + const id = resourceIdSegment(space.name) + if (space.spaceType === 'DIRECT_MESSAGE') return `Direct message ${id}` + if (space.spaceType === 'GROUP_CHAT') return `Group chat ${id}` + return `Google Chat space ${id}` +} + +/** + * Change-detection token for a space. + * + * `lastActiveTime` is documented as the timestamp of the last message in the + * space, so it moves whenever new content arrives without any message having to + * be fetched. The `spaces.list` reference documents `permissionSettings` as the + * only field omitted from list responses, but it does not positively guarantee + * `lastActiveTime` is populated for every space — and a space missing it would + * hash to a constant, so its content would never be refreshed again. + * + * A space without the field therefore falls back to a token that is stable for + * one sync run and different on the next, so the space re-hydrates every sync + * rather than going silently stale. `listDocuments` warns when that branch is + * taken, since it re-indexes the space's content on every sync. + */ +function activityToken(space: Space, syncContext?: Record): string { + const lastActiveTime = space.lastActiveTime?.trim() + if (lastActiveTime) return lastActiveTime + + const cached = syncContext?.[ACTIVITY_TOKEN_KEY] + if (typeof cached === 'string') return cached + + const token = `unknown-${generateId()}` + if (syncContext) syncContext[ACTIVITY_TOKEN_KEY] = token + return token +} + +/** + * Metadata-based change-detection hash. The window settings are folded in + * because changing them changes which messages the document contains. + * + * Known limitation: editing or deleting an existing message does not advance + * `lastActiveTime`, so such a change is only picked up once a newer message + * lands or an explicit full resync runs — see `rehydrateOnFullSync` in `meta.ts`. + */ +function buildContentHash( + space: Space, + maxMessages: number, + lookbackDays: number, + syncContext?: Record +): string { + const token = activityToken(space, syncContext) + return `gchat:v1:${space.name}:${token}:${maxMessages}:${lookbackDays}` +} + +/** Records the spaces a listing page returned so `getDocument` can reuse them. */ +function cacheSpaces(spaces: Space[], syncContext?: Record): void { + if (!syncContext) return + const existing = syncContext[SPACE_CACHE_KEY] + const cache = (existing && typeof existing === 'object' ? existing : {}) as Record + for (const space of spaces) cache[space.name] = space + syncContext[SPACE_CACHE_KEY] = cache +} + +/** The listed space for a resource name, when this sync run listed it. */ +function cachedSpace(name: string, syncContext?: Record): Space | undefined { + const cache = syncContext?.[SPACE_CACHE_KEY] + if (!cache || typeof cache !== 'object') return undefined + return (cache as Record)[name] +} + +/** + * Builds the deferred listing stub for a space. Messages are fetched lazily in + * `getDocument`; only metadata and the change hash are computed here. + */ +function spaceToStub( + space: Space, + maxMessages: number, + lookbackDays: number, + syncContext?: Record +): ExternalDocument { + return { + externalId: space.name, + title: spaceTitle(space), + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: space.spaceUri, + contentHash: buildContentHash(space, maxMessages, lookbackDays, syncContext), + metadata: { + spaceName: spaceTitle(space), + spaceType: space.spaceType, + lastActivity: space.lastActiveTime, + }, + } +} + +/** Fetches a single space. Returns null on 404 (space deleted or left). */ +async function fetchSpace(accessToken: string, name: string): Promise { + const response = await fetchWithRetry(`${CHAT_API_BASE}/${name}`, { + method: 'GET', + headers: chatHeaders(accessToken), + }) + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to fetch Google Chat space: ${response.status}`) + } + return (await response.json()) as Space +} + +/** + * An RFC-3339 timestamp without fractional seconds, matching the form every + * example in the `spaces.messages.list` filter reference uses. `toISOString()` + * alone emits milliseconds, and the API rejects a filter it cannot parse with + * `INVALID_ARGUMENT` rather than ignoring it. + * + * https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces.messages/list + */ +function rfc3339(date: Date): string { + return `${date.toISOString().slice(0, 19)}Z` +} + +/** + * Fetches the newest `maxMessages` messages of a space, optionally bounded by a + * lookback window. Messages are requested newest-first so the cap keeps the most + * recent conversation, then returned in chronological order. + * + * `orderBy` takes a full ordering expression, not a bare direction: the reference + * documents the default as `createTime ASC` and lists ASC/DESC as the ordering + * *operations* usable within one. `createTime` is the only orderable field here. + */ +async function fetchSpaceMessages( + accessToken: string, + spaceName: string, + maxMessages: number, + lookbackDays: number +): Promise { + const filter = + lookbackDays > 0 + ? `createTime > "${rfc3339(new Date(Date.now() - lookbackDays * MS_PER_DAY))}"` + : undefined + + const collected: ChatMessage[] = [] + let pageToken: string | undefined + + while (collected.length < maxMessages) { + const params = new URLSearchParams({ + pageSize: String(Math.min(MESSAGES_PAGE_SIZE, maxMessages - collected.length)), + orderBy: 'createTime DESC', + }) + if (filter) params.set('filter', filter) + if (pageToken) params.set('pageToken', pageToken) + + const response = await fetchWithRetry( + `${CHAT_API_BASE}/${spaceName}/messages?${params.toString()}`, + { method: 'GET', headers: chatHeaders(accessToken) } + ) + if (!response.ok) { + if (response.status === 404) break + throw new Error(`Failed to list Google Chat messages: ${response.status}`) + } + + const data = (await response.json()) as MessagesListResponse + const messages = data.messages ?? [] + if (messages.length === 0) break + collected.push(...messages) + + pageToken = data.nextPageToken?.trim() || undefined + if (!pageToken) break + } + + return collected.slice(0, maxMessages).reverse() +} + +/** Resolves a message author's label, preferring the display name. */ +function senderLabel(sender: ChatUser | undefined): string { + return sender?.displayName?.trim() || sender?.name?.trim() || 'Unknown' +} + +/** + * Renders a space's messages as one plain-text transcript. Card-only messages + * fall back to `fallbackText`, the documented plain-text description of a + * message's cards; messages with neither are skipped as they carry no text. + * + * Message attachments are deliberately not indexed. Downloading them needs the + * separate `media.download` endpoint (and, for Drive-hosted attachments, Drive + * scopes this connector does not request), which would turn one document per + * space into an unbounded per-message download fan-out. A file shared in Chat + * that belongs in a knowledge base is synced through the Google Drive connector + * instead, which already handles size caps, OCR, and format parsing. + */ +function formatSpaceContent(space: Space, messages: ChatMessage[]): string { + const parts: string[] = [`Space: ${spaceTitle(space)}`] + const description = space.spaceDetails?.description?.trim() + if (description) parts.push(`Description: ${description}`) + const guidelines = space.spaceDetails?.guidelines?.trim() + if (guidelines) parts.push(`Guidelines: ${guidelines}`) + + const lines: string[] = [] + for (const message of messages) { + const text = message.text?.trim() || message.fallbackText?.trim() + if (!text) continue + const timestamp = message.createTime ?? '' + lines.push(`[${timestamp}] ${senderLabel(message.sender)}: ${text}`) + } + + if (lines.length > 0) { + parts.push('') + parts.push('--- Messages ---') + parts.push(...lines) + } + + return parts.join('\n') +} + +/** Number of messages that actually contributed text to the transcript. */ +function countIndexedMessages(messages: ChatMessage[]): number { + let count = 0 + for (const message of messages) { + if (message.text?.trim() || message.fallbackText?.trim()) count++ + } + return count +} + +export const googleChatConnector: ConnectorConfig = { + ...googleChatConnectorMeta, + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record + ): Promise => { + const maxSpaces = sourceConfig.maxSpaces ? Number(sourceConfig.maxSpaces) : 0 + const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) + const lookbackDays = resolveLookbackDays(sourceConfig.lookbackDays) + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + + const pageSize = + maxSpaces > 0 + ? Math.min(SPACES_PAGE_SIZE, Math.max(1, maxSpaces - prevFetched)) + : SPACES_PAGE_SIZE + const params = new URLSearchParams({ pageSize: String(pageSize) }) + if (cursor) params.set('pageToken', cursor) + const filter = buildSpacesFilter(sourceConfig) + if (filter) params.set('filter', filter) + + logger.info('Listing Google Chat spaces', { + hasCursor: Boolean(cursor), + hasFilter: Boolean(filter), + }) + + const response = await fetchWithRetry(`${CHAT_API_BASE}/spaces?${params.toString()}`, { + method: 'GET', + headers: chatHeaders(accessToken), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + logger.error('Failed to list Google Chat spaces', { + status: response.status, + error: errorText.slice(0, 500), + }) + throw new Error(`Failed to list Google Chat spaces: ${response.status}`) + } + + const data = (await response.json()) as SpacesListResponse + const spaces = (data.spaces ?? []).filter((space) => Boolean(space.name)) + const nextPageToken = data.nextPageToken?.trim() || undefined + + const missingActivity = spaces.filter((space) => !space.lastActiveTime?.trim()).length + if (missingActivity > 0) { + logger.warn( + 'Google Chat spaces listed without lastActiveTime; their content re-indexes every sync', + { spacesMissingLastActiveTime: missingActivity, spacesInPage: spaces.length } + ) + } + + cacheSpaces(spaces, syncContext) + + const allDocuments = spaces.map((space) => + spaceToStub(space, maxMessages, lookbackDays, syncContext) + ) + + let documents = allDocuments + if (maxSpaces > 0) { + const remaining = Math.max(0, maxSpaces - prevFetched) + if (allDocuments.length > remaining) documents = allDocuments.slice(0, remaining) + } + + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const reachedCap = maxSpaces > 0 && totalFetched >= maxSpaces + + /** + * Only flag the listing as capped when the cap actually truncated a larger + * source — either more pages remain, or spaces were dropped from this page. + * A source that was fully listed and merely happens to equal the cap stays + * unflagged so the sync engine still reconciles deletions. + */ + const truncated = + reachedCap && (Boolean(nextPageToken) || allDocuments.length > documents.length) + if (truncated && syncContext) syncContext.listingCapped = true + + const hasMore = !reachedCap && Boolean(nextPageToken) + + return { + documents, + nextCursor: hasMore ? nextPageToken : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + sourceConfig: Record, + externalId: string, + syncContext?: Record + ): Promise => { + if (!externalId) return null + const spaceName = spaceResourceName(externalId) + + /** + * Reuses the space the listing already fetched. Beyond saving a `spaces.get` + * per document, it is what keeps the hydrated `contentHash` identical to the + * stub's: re-reading the space here could observe a `lastActiveTime` that + * advanced after the listing, storing a hash for messages this call never + * fetched and hiding them until the *next* message arrives. + */ + const space = cachedSpace(spaceName, syncContext) ?? (await fetchSpace(accessToken, spaceName)) + if (!space) return null + + const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) + const lookbackDays = resolveLookbackDays(sourceConfig.lookbackDays) + const messages = await fetchSpaceMessages(accessToken, spaceName, maxMessages, lookbackDays) + + /** + * A space with no messages in the window is still a live space, so it is + * indexed rather than skipped. `null` is this connector's "document is gone" + * signal, and the engine treats it as last-known-good: returning it here would + * both drop spaces whose only prose is their description or guidelines, and + * leave a previously indexed transcript in place after the space was cleared + * or `lookbackDays` was tightened past every message. + */ + const messageCount = countIndexedMessages(messages) + const stub = spaceToStub(space, maxMessages, lookbackDays, syncContext) + + return { + ...stub, + content: formatSpaceContent(space, messages), + contentDeferred: false, + metadata: { ...stub.metadata, messageCount }, + } + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + const maxMessages = sourceConfig.maxMessages as string | undefined + if (maxMessages && (Number.isNaN(Number(maxMessages)) || Number(maxMessages) <= 0)) { + return { valid: false, error: 'Max messages per space must be a positive number' } + } + + const maxSpaces = sourceConfig.maxSpaces as string | undefined + if (maxSpaces && (Number.isNaN(Number(maxSpaces)) || Number(maxSpaces) <= 0)) { + return { valid: false, error: 'Max spaces must be a positive number' } + } + + const lookbackDays = sourceConfig.lookbackDays as string | undefined + if (lookbackDays && (Number.isNaN(Number(lookbackDays)) || Number(lookbackDays) <= 0)) { + return { valid: false, error: 'Lookback window must be a positive number of days' } + } + + const spaceTypes = sourceConfig.spaceTypes + if ( + spaceTypes != null && + spaceTypes !== '' && + !['SPACE', 'SPACE_AND_GROUP_CHAT', 'ALL'].includes(String(spaceTypes)) + ) { + return { valid: false, error: 'Unsupported space type selection' } + } + + try { + const params = new URLSearchParams({ pageSize: '1' }) + const filter = buildSpacesFilter(sourceConfig) + if (filter) params.set('filter', filter) + + const response = await fetchWithRetry( + `${CHAT_API_BASE}/spaces?${params.toString()}`, + { method: 'GET', headers: chatHeaders(accessToken) }, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + return { + valid: false, + error: `Google Chat access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`, + } + } + + return { valid: true } + } catch (error) { + return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') } + } + }, + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + if (typeof metadata.spaceName === 'string') result.spaceName = metadata.spaceName + if (typeof metadata.spaceType === 'string') result.spaceType = metadata.spaceType + + if (metadata.messageCount != null) { + const count = Number(metadata.messageCount) + if (!Number.isNaN(count)) result.messageCount = count + } + + const lastActivity = parseTagDate(metadata.lastActivity) + if (lastActivity) result.lastActivity = lastActivity + + return result + }, +} diff --git a/apps/sim/connectors/google-chat/index.ts b/apps/sim/connectors/google-chat/index.ts new file mode 100644 index 00000000000..a7838770654 --- /dev/null +++ b/apps/sim/connectors/google-chat/index.ts @@ -0,0 +1 @@ +export { googleChatConnector } from '@/connectors/google-chat/google-chat' diff --git a/apps/sim/connectors/google-chat/meta.ts b/apps/sim/connectors/google-chat/meta.ts new file mode 100644 index 00000000000..f20ad56cce0 --- /dev/null +++ b/apps/sim/connectors/google-chat/meta.ts @@ -0,0 +1,106 @@ +import { GoogleChatIcon } from '@/components/icons' +import type { ConnectorMeta } from '@/connectors/types' + +/** + * Default per-space message window. Google Chat spaces are long-lived and a + * single document holds one space's history, so the newest N messages are + * indexed rather than the entire backlog. + */ +export const DEFAULT_MAX_MESSAGES = 1000 + +/** + * Page size for `spaces.list`. The API defaults to 100 and accepts up to 1000; + * 100 keeps each listing round-trip small, since a sync walks every page anyway. + * + * https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces/list + */ +export const SPACES_PAGE_SIZE = 100 + +/** + * Page size for `spaces.messages.list`. The API defaults to 25 and accepts up to + * 1000; the documented maximum is used so a space's message window is fetched in + * as few round-trips as possible. + * + * https://developers.google.com/workspace/chat/api/reference/rest/v1/spaces.messages/list + */ +export const MESSAGES_PAGE_SIZE = 1000 + +export const googleChatConnectorMeta: ConnectorMeta = { + id: 'google_chat', + name: 'Google Chat', + description: 'Sync space conversations from Google Chat into your knowledge base', + version: '1.0.0', + icon: GoogleChatIcon, + + auth: { + mode: 'oauth', + provider: 'google-chat', + requiredScopes: [ + 'https://www.googleapis.com/auth/chat.spaces.readonly', + 'https://www.googleapis.com/auth/chat.messages.readonly', + ], + }, + + /** + * A space document's `contentHash` is keyed on `lastActiveTime`, documented as + * the timestamp of the last message in the space. Editing or deleting an + * existing message does not move it, and the Chat API exposes no space-level + * revision counter to key on instead, so a routine hash-gated sync cannot see + * an edit-only or delete-only change until a newer message lands. + * + * Routine syncs therefore stay cheap, and an explicit full resync re-hydrates + * every space (one messages listing per space) so edits and deletions are + * picked up on demand. That cost is paid on every full resync, not once. + */ + rehydrateOnFullSync: true, + + configFields: [ + { + id: 'spaceTypes', + title: 'Space Types', + type: 'dropdown', + required: false, + options: [ + { label: 'Named spaces only (default)', id: 'SPACE' }, + { label: 'Named spaces and group chats', id: 'SPACE_AND_GROUP_CHAT' }, + { label: 'All conversations, including direct messages', id: 'ALL' }, + ], + description: + 'Which conversations to sync. Direct messages are one-to-one conversations of the connected account — indexing them makes their contents searchable by everyone with access to this knowledge base.', + }, + { + id: 'maxMessages', + title: 'Max Messages Per Space', + type: 'short-input', + required: false, + placeholder: `e.g. 500 (default: ${DEFAULT_MAX_MESSAGES})`, + description: + 'Number of most recent messages indexed per space. Older messages are not included.', + }, + { + id: 'maxSpaces', + title: 'Max Spaces', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 100 (default: unlimited)', + description: 'Cap the total number of spaces synced. Leave blank to sync all of them.', + }, + { + id: 'lookbackDays', + title: 'Lookback Window (days)', + type: 'short-input', + required: false, + mode: 'advanced', + placeholder: 'e.g. 90 (default: all available)', + description: 'Only index messages created in the last N days.', + }, + ], + + tagDefinitions: [ + { id: 'spaceName', displayName: 'Space Name', fieldType: 'text' }, + { id: 'spaceType', displayName: 'Space Type', fieldType: 'text' }, + { id: 'messageCount', displayName: 'Message Count', fieldType: 'number' }, + { id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' }, + ], +} diff --git a/apps/sim/connectors/registry.server.ts b/apps/sim/connectors/registry.server.ts index f184fd6e77f..5493db98118 100644 --- a/apps/sim/connectors/registry.server.ts +++ b/apps/sim/connectors/registry.server.ts @@ -2,9 +2,11 @@ import { airtableConnector } from '@/connectors/airtable' import { asanaConnector } from '@/connectors/asana' import { ashbyConnector } from '@/connectors/ashby' import { azureDevopsConnector } from '@/connectors/azure-devops' +import { bitbucketConnector } from '@/connectors/bitbucket' import { boxConnector } from '@/connectors/box' import { clickupConnector } from '@/connectors/clickup' import { confluenceConnector } from '@/connectors/confluence' +import { databricksConnector } from '@/connectors/databricks' import { discordConnector } from '@/connectors/discord' import { docusignConnector } from '@/connectors/docusign' import { dropboxConnector } from '@/connectors/dropbox' @@ -15,6 +17,7 @@ import { gitlabConnector } from '@/connectors/gitlab' import { gmailConnector } from '@/connectors/gmail' import { gongConnector } from '@/connectors/gong' import { googleCalendarConnector } from '@/connectors/google-calendar' +import { googleChatConnector } from '@/connectors/google-chat' import { googleDocsConnector } from '@/connectors/google-docs' import { googleDriveConnector } from '@/connectors/google-drive' import { googleFormsConnector } from '@/connectors/google-forms' @@ -54,6 +57,7 @@ import { typeformConnector } from '@/connectors/typeform' import type { ConnectorRegistry } from '@/connectors/types' import { webflowConnector } from '@/connectors/webflow' import { wordpressConnector } from '@/connectors/wordpress' +import { workdayConnector } from '@/connectors/workday' import { xConnector } from '@/connectors/x' import { youtubeConnector } from '@/connectors/youtube' import { zendeskConnector } from '@/connectors/zendesk' @@ -72,9 +76,11 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { asana: asanaConnector, ashby: ashbyConnector, azure_devops: azureDevopsConnector, + bitbucket: bitbucketConnector, box: boxConnector, clickup: clickupConnector, confluence: confluenceConnector, + databricks: databricksConnector, discord: discordConnector, docusign: docusignConnector, dropbox: dropboxConnector, @@ -85,6 +91,7 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { gmail: gmailConnector, gong: gongConnector, google_calendar: googleCalendarConnector, + google_chat: googleChatConnector, google_docs: googleDocsConnector, google_drive: googleDriveConnector, google_forms: googleFormsConnector, @@ -123,6 +130,7 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { typeform: typeformConnector, webflow: webflowConnector, wordpress: wordpressConnector, + workday: workdayConnector, x: xConnector, youtube: youtubeConnector, zendesk: zendeskConnector, diff --git a/apps/sim/connectors/registry.ts b/apps/sim/connectors/registry.ts index de7f123cc42..d903635048a 100644 --- a/apps/sim/connectors/registry.ts +++ b/apps/sim/connectors/registry.ts @@ -2,9 +2,11 @@ import { airtableConnectorMeta } from '@/connectors/airtable/meta' import { asanaConnectorMeta } from '@/connectors/asana/meta' import { ashbyConnectorMeta } from '@/connectors/ashby/meta' import { azureDevopsConnectorMeta } from '@/connectors/azure-devops/meta' +import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta' import { boxConnectorMeta } from '@/connectors/box/meta' import { clickupConnectorMeta } from '@/connectors/clickup/meta' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' +import { databricksConnectorMeta } from '@/connectors/databricks/meta' import { discordConnectorMeta } from '@/connectors/discord/meta' import { docusignConnectorMeta } from '@/connectors/docusign/meta' import { dropboxConnectorMeta } from '@/connectors/dropbox/meta' @@ -15,6 +17,7 @@ import { gitlabConnectorMeta } from '@/connectors/gitlab/meta' import { gmailConnectorMeta } from '@/connectors/gmail/meta' import { gongConnectorMeta } from '@/connectors/gong/meta' import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' +import { googleChatConnectorMeta } from '@/connectors/google-chat/meta' import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' import { googleFormsConnectorMeta } from '@/connectors/google-forms/meta' @@ -54,6 +57,7 @@ import { typeformConnectorMeta } from '@/connectors/typeform/meta' import type { ConnectorMeta, ConnectorMetaRegistry } from '@/connectors/types' import { webflowConnectorMeta } from '@/connectors/webflow/meta' import { wordpressConnectorMeta } from '@/connectors/wordpress/meta' +import { workdayConnectorMeta } from '@/connectors/workday/meta' import { xConnectorMeta } from '@/connectors/x/meta' import { youtubeConnectorMeta } from '@/connectors/youtube/meta' import { zendeskConnectorMeta } from '@/connectors/zendesk/meta' @@ -72,9 +76,11 @@ export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = { asana: asanaConnectorMeta, ashby: ashbyConnectorMeta, azure_devops: azureDevopsConnectorMeta, + bitbucket: bitbucketConnectorMeta, box: boxConnectorMeta, clickup: clickupConnectorMeta, confluence: confluenceConnectorMeta, + databricks: databricksConnectorMeta, discord: discordConnectorMeta, docusign: docusignConnectorMeta, dropbox: dropboxConnectorMeta, @@ -85,6 +91,7 @@ export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = { gmail: gmailConnectorMeta, gong: gongConnectorMeta, google_calendar: googleCalendarConnectorMeta, + google_chat: googleChatConnectorMeta, google_docs: googleDocsConnectorMeta, google_drive: googleDriveConnectorMeta, google_forms: googleFormsConnectorMeta, @@ -123,6 +130,7 @@ export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = { typeform: typeformConnectorMeta, webflow: webflowConnectorMeta, wordpress: wordpressConnectorMeta, + workday: workdayConnectorMeta, x: xConnectorMeta, youtube: youtubeConnectorMeta, zendesk: zendeskConnectorMeta, diff --git a/apps/sim/connectors/workday/index.ts b/apps/sim/connectors/workday/index.ts new file mode 100644 index 00000000000..478a20425e9 --- /dev/null +++ b/apps/sim/connectors/workday/index.ts @@ -0,0 +1 @@ +export { workdayConnector } from '@/connectors/workday/workday' diff --git a/apps/sim/connectors/workday/meta.ts b/apps/sim/connectors/workday/meta.ts new file mode 100644 index 00000000000..ac33364cdeb --- /dev/null +++ b/apps/sim/connectors/workday/meta.ts @@ -0,0 +1,144 @@ +import { WorkdayIcon } from '@/components/icons' +import type { ConnectorMeta } from '@/connectors/types' + +/** + * Workday Help knowledge articles, read through the public `helpArticle` v1 REST + * service (`GET /ccx/api/helpArticle/v1/{tenant}/articleVersions`). + * + * Scope is deliberately narrow, and the connector is named for it. `helpArticle` + * is the only Workday REST service whose documented response carries a title + * alongside the article body as plain text, which is what a knowledge base + * indexes. Worker, organization, and compensation data — what Sim's Workday + * *tools* read over SOAP — are records rather than documents and are out of scope. + * + * ## Why the credential is a customer API client rather than `mode: 'oauth'` + * + * Workday REST is OAuth-2.0-only; it does not accept the WS-Security username + * and password the SOAP tools use. It still cannot become a Sim OAuth provider: + * + * - Better Auth's `genericOAuth` takes **static** `authorizationUrl`/`tokenUrl` + * strings registered once at module load — see the comment on + * `salesforceConnector` in `lib/auth/connectors/providers.ts`, which enumerates + * Salesforce's two fixed login hosts for exactly this reason. Workday's + * endpoints are `https://{tenantHost}/ccx/oauth2/{tenant}/token`, where both + * the host and the tenant segment are per-customer and unbounded, so there is + * no finite set to enumerate. + * - Every provider in that file supplies Sim's own `env.*_CLIENT_ID/SECRET`. + * Workday API clients are registered by the customer inside their own tenant + * (the "Register API Client for Integrations" task), so no Sim-owned Workday + * application exists whose credentials could be configured there. + * + * The connector therefore carries the customer's API client itself. The + * non-secret client ID is a config field; the client secret and the refresh + * token are both secrets and so share the connector's single encrypted key as + * `clientSecret:refreshToken` — the packing the Gong connector already uses for + * its `accessKey:accessKeySecret` pair. They buy a short-lived bearer token once + * per sync run. + * + * ## Why the article status filter is required + * + * `/articleVersions` lists article *versions*, not articles: every row carries a + * `version` number and a `parentArticle` reference to the article it belongs to, + * and the service publishes no "latest version only" filter or sort order — the + * whole document has eleven paths and none of them addresses an article. An + * unfiltered sync therefore indexes each historical revision as its own + * document, so the status filter is required and has no default: the operator + * picks the scope explicitly, and the option that indexes every revision says so + * on its label. + * + * The three options are the ones the service's own prose enumerates — "an + * article version can have a status of Published, Draft, or Archived". What the + * `status` *query parameter* accepts for them is the one thing the published + * spec does not settle: it is an untyped `array` of `string` with no enum and no + * `x-workday-populated-by`, unlike the sibling `audience` parameter, whose model + * names `/values/common/audiences` as its value source. The connector resolves + * the chosen name against `/articleStatuses` and sends the Workday ID; + * `validateConfig` reads the filtered response back and refuses the + * configuration if the tenant answered with a version in another status. + */ +export const workdayConnectorMeta: ConnectorMeta = { + id: 'workday', + name: 'Workday Help', + description: 'Sync Workday Help knowledge articles into your knowledge base', + version: '1.0.0', + icon: WorkdayIcon, + + auth: { + mode: 'apiKey', + label: 'Client Secret & Refresh Token', + placeholder: 'clientSecret:refreshToken', + }, + + configFields: [ + { + id: 'tenantUrl', + title: 'Tenant Host', + type: 'short-input', + placeholder: 'e.g. https://wd5-impl-services1.workday.com', + required: true, + description: + 'Host of your Workday instance, without the tenant or any path. Must be a Workday-hosted domain.', + }, + { + id: 'tenant', + title: 'Tenant', + type: 'short-input', + placeholder: 'e.g. acme_pt1', + required: true, + description: 'Workday tenant name, as it appears in your Workday URLs.', + }, + { + id: 'clientId', + title: 'Client ID', + type: 'short-input', + placeholder: 'e.g. NDdiMGE0ZmQtZjk1YS00...', + required: true, + description: + 'Client ID of the Workday API Client for Integrations whose client secret and refresh token you entered above. Register it with a non-expiring refresh token: Sim stores the token you enter and cannot replace one the tenant rotates.', + }, + { + id: 'status', + title: 'Article Status', + type: 'dropdown', + required: true, + options: [ + { label: 'Published', id: 'Published' }, + { label: 'Draft', id: 'Draft' }, + { label: 'Archived', id: 'Archived' }, + { label: 'Every status — indexes every historical revision', id: 'all' }, + ], + description: + 'Which article versions to sync, resolved against the statuses your tenant returns. Workday exposes no "latest version only" filter, so an article that has held the chosen status more than once still contributes one document per revision.', + }, + { + id: 'audience', + title: 'Audience', + type: 'short-input', + required: false, + multi: true, + placeholder: 'e.g. All Employees (optional)', + description: + 'Comma-separated audience names to sync, resolved against the audiences your tenant returns. A Workday ID is accepted in place of a name. Leave blank to sync every audience.', + }, + { + id: 'maxVersions', + title: 'Max Article Versions', + type: 'short-input', + required: false, + placeholder: 'e.g. 500 (default: unlimited)', + description: 'Stop syncing after this many article versions.', + }, + ], + + tagDefinitions: [ + { id: 'article', displayName: 'Article', fieldType: 'text' }, + { id: 'category', displayName: 'Category', fieldType: 'text' }, + { id: 'status', displayName: 'Status', fieldType: 'text' }, + { id: 'audience', displayName: 'Audience', fieldType: 'text' }, + { id: 'articleTags', displayName: 'Tags', fieldType: 'text' }, + { id: 'language', displayName: 'Language', fieldType: 'text' }, + { id: 'version', displayName: 'Version', fieldType: 'number' }, + { id: 'created', displayName: 'Created', fieldType: 'date' }, + { id: 'lastUpdated', displayName: 'Last Updated', fieldType: 'date' }, + ], +} diff --git a/apps/sim/connectors/workday/workday.test.ts b/apps/sim/connectors/workday/workday.test.ts new file mode 100644 index 00000000000..3bd51ea630e --- /dev/null +++ b/apps/sim/connectors/workday/workday.test.ts @@ -0,0 +1,481 @@ +/** + * @vitest-environment node + * + * `GET /articleVersions` returns one row per article *revision* and offers no + * latest-version filter, so what this connector indexes is decided entirely by + * the required status choice and the version cap. Both are exercised here, along + * with the `listingCapped` flag the sync engine reads before hard-deleting the + * documents a partial listing left out. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { workdayConnector } from '@/connectors/workday/workday' + +const ACCESS_TOKEN = 'client-secret:refresh-token' +const CONFIG = { + tenantUrl: 'https://wd5-impl-services1.workday.com', + tenant: 'acme_pt1', + clientId: 'client-id', + status: 'Published', +} + +const PUBLISHED_ID = '0d75a5e37a411000167d21b9239f0001' +const AUDIENCE_ID = '8ac3f16c1fff10000c53e90920940001' + +const mockFetch = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +function versionFixture(id: string, version = 1) { + return { + id, + title: `Article ${id}`, + content: 'Body text', + version, + lastUpdatedDate: '2026-01-01T00:00:00.000Z', + parentArticle: { id: `parent-${id}`, descriptor: `Article ${id}` }, + status: { id: PUBLISHED_ID, descriptor: 'Published' }, + } +} + +/** + * Routes by URL rather than call order, because the number of lookups before the + * listing depends on which filters the config asks for. + */ +function mockApi(listing: unknown, options: { audiences?: unknown } = {}) { + mockFetch.mockImplementation(async (url: string) => { + if (url.includes('/ccx/oauth2/')) { + return jsonResponse({ access_token: 'bearer', expires_in: 3600 }) + } + if (url.includes('/articleStatuses')) { + return jsonResponse({ + total: 3, + data: [ + { id: PUBLISHED_ID, descriptor: 'Published' }, + { id: 'aaaa5a5e37a411000167d21b9239f002', descriptor: 'Draft' }, + { id: 'bbbb5a5e37a411000167d21b9239f003', descriptor: 'Archived' }, + ], + }) + } + if (url.includes('/values/common/audiences')) { + return jsonResponse( + options.audiences ?? { + total: 1, + data: [{ id: AUDIENCE_ID, descriptor: 'All Employees' }], + } + ) + } + return jsonResponse(listing) + }) +} + +function tokenExchanges(): string[] { + return mockFetch.mock.calls + .map(([url]) => url as string) + .filter((url) => url.includes('/ccx/oauth2/')) +} + +function listingUrls(): string[] { + return mockFetch.mock.calls + .map(([url]) => url as string) + .filter((url) => url.includes('/articleVersions')) +} + +describe('workday listDocuments', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('resolves the status name to the tenant Workday ID and filters the listing by it', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + + expect(listingUrls()[0]).toContain(`status=${PUBLISHED_ID}`) + expect(result.documents).toHaveLength(1) + expect(result.documents[0].externalId).toBe('a') + }) + + it('omits the status filter only when the operator explicitly chose every status', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + await workdayConnector.listDocuments(ACCESS_TOKEN, { ...CONFIG, status: 'all' }, undefined, {}) + + expect(listingUrls()[0]).not.toContain('status=') + expect(mockFetch.mock.calls.some(([url]) => (url as string).includes('/articleStatuses'))).toBe( + false + ) + }) + + it('refuses to list when no status was chosen, rather than indexing every revision', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + await expect( + workdayConnector.listDocuments(ACCESS_TOKEN, { ...CONFIG, status: '' }, undefined, {}) + ).rejects.toThrow(/Article Status is required/) + }) + + it('resolves audience names to Workday IDs', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + await workdayConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, audience: 'All Employees' }, + undefined, + {} + ) + + expect(listingUrls()[0]).toContain(`audience=${AUDIENCE_ID}`) + }) + + it('names the available audiences when one cannot be resolved', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + await expect( + workdayConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, audience: 'Contractors' }, + undefined, + {} + ) + ).rejects.toThrow(/no audience named "Contractors". Available: All Employees/) + }) + + it('leaves listingCapped unset when the cap lands exactly on source exhaustion', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + const syncContext: Record = {} + await workdayConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxVersions: '1' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when the cap stops short of the tenant total', async () => { + mockApi({ total: 10, data: [versionFixture('a')] }) + + const syncContext: Record = {} + await workdayConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxVersions: '1' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('leaves listingCapped unset when an uncapped page exhausts the source', async () => { + mockApi({ total: 2, data: [versionFixture('a'), versionFixture('b')] }) + + const syncContext: Record = {} + const result = await workdayConnector.listDocuments( + ACCESS_TOKEN, + CONFIG, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + expect(result.hasMore).toBe(false) + }) + + it('requests only the versions the cap still allows on the final page', async () => { + mockApi({ total: 500, data: [versionFixture('a')] }) + + await workdayConnector.listDocuments(ACCESS_TOKEN, { ...CONFIG, maxVersions: '101' }, '100', {}) + + expect(listingUrls()[0]).toContain('limit=1') + expect(listingUrls()[0]).toContain('offset=100') + }) + + it('strips a path and trailing slash from the configured tenant host', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + await workdayConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, tenantUrl: 'https://wd5-impl-services1.workday.com/acme_pt1/d/home.htmld' }, + undefined, + {} + ) + + expect(listingUrls()[0]).toContain( + 'https://wd5-impl-services1.workday.com/ccx/api/helpArticle/v1/acme_pt1/articleVersions?' + ) + }) + + it('accepts a tenant host written without a scheme', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + await workdayConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, tenantUrl: 'wd5-impl-services1.workday.com' }, + undefined, + {} + ) + + expect(listingUrls()[0]).toContain( + 'https://wd5-impl-services1.workday.com/ccx/api/helpArticle/v1/acme_pt1/articleVersions?' + ) + }) + + it('passes a Workday ID through instead of looking it up as a display name', async () => { + mockApi({ total: 1, data: [versionFixture('a')] }) + + await workdayConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, audience: AUDIENCE_ID }, + undefined, + {} + ) + + expect(listingUrls()[0]).toContain(`audience=${AUDIENCE_ID}`) + expect( + mockFetch.mock.calls.some(([url]) => (url as string).includes('/values/common/audiences')) + ).toBe(false) + }) + + it('caps how many available values an unresolved-name error names', async () => { + const many = Array.from({ length: 30 }, (_, index) => ({ + id: `${index}`.padStart(32, '0'), + descriptor: `Audience ${index}`, + })) + mockApi({ total: 1, data: [versionFixture('a')] }, { audiences: { total: 30, data: many } }) + + const error = await workdayConnector + .listDocuments(ACCESS_TOKEN, { ...CONFIG, audience: 'Nope' }, undefined, {}) + .catch((thrown: Error) => thrown) + + expect((error as Error).message).toContain('and 10 more') + expect((error as Error).message).not.toContain('Audience 25') + }) + + it('reads a version cap persisted as a number rather than a string', async () => { + mockApi({ total: 500, data: [versionFixture('a')] }) + + await workdayConnector.listDocuments(ACCESS_TOKEN, { ...CONFIG, maxVersions: 1 }, undefined, {}) + + expect(listingUrls()[0]).toContain('limit=1') + }) + + it('rejects a tenant host that is not a Workday domain', async () => { + mockApi({ total: 0, data: [] }) + + await expect( + workdayConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, tenantUrl: 'https://evil.example.com' }, + undefined, + {} + ) + ).rejects.toThrow(/Workday-hosted domain/) + }) +}) + +describe('workday credential failures', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('explains that a rejected refresh token has to be re-entered', async () => { + mockFetch.mockResolvedValue(jsonResponse({ error: 'invalid_grant' }, 400)) + + const result = await workdayConnector.validateConfig(ACCESS_TOKEN, CONFIG) + + expect(result.valid).toBe(false) + expect(result.error).toMatch(/Manage Refresh Tokens for Integrations/) + }) + + it('buys one bearer token for a whole sync run and reuses it across pages', async () => { + mockApi({ total: 500, data: [versionFixture('a')] }) + const syncContext: Record = {} + + await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, syncContext) + await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, '100', syncContext) + + expect(tokenExchanges()).toHaveLength(1) + }) + + it('re-authenticates once and replays the request when a cached token has expired', async () => { + let listingCalls = 0 + mockFetch.mockImplementation(async (url: string) => { + if (url.includes('/ccx/oauth2/')) { + return jsonResponse({ access_token: 'bearer', expires_in: 3600 }) + } + if (url.includes('/articleStatuses')) { + return jsonResponse({ total: 1, data: [{ id: PUBLISHED_ID, descriptor: 'Published' }] }) + } + listingCalls += 1 + return listingCalls === 1 + ? jsonResponse({ error: 'expired token' }, 401) + : jsonResponse({ total: 1, data: [versionFixture('a')] }) + }) + + const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + + expect(result.documents).toHaveLength(1) + expect(listingCalls).toBe(2) + expect(tokenExchanges()).toHaveLength(2) + }) + + it('surfaces a persistent 401 instead of retrying the exchange forever', async () => { + mockFetch.mockImplementation(async (url: string) => { + if (url.includes('/ccx/oauth2/')) { + return jsonResponse({ access_token: 'bearer', expires_in: 3600 }) + } + if (url.includes('/articleStatuses')) { + return jsonResponse({ total: 1, data: [{ id: PUBLISHED_ID, descriptor: 'Published' }] }) + } + return jsonResponse({ error: 'still unauthorized' }, 401) + }) + + await expect( + workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + ).rejects.toThrow(/still unauthorized/) + expect(tokenExchanges()).toHaveLength(2) + }) + + it('rejects a version cap that is not a positive number', async () => { + mockApi({ total: 0, data: [] }) + + const result = await workdayConnector.validateConfig(ACCESS_TOKEN, { + ...CONFIG, + maxVersions: '0', + }) + + expect(result.valid).toBe(false) + expect(result.error).toMatch(/positive number/) + }) + + it('refuses a tenant that answered the status filter with another status', async () => { + mockApi({ + total: 1, + data: [{ ...versionFixture('a'), status: { id: 'other', descriptor: 'Draft' } }], + }) + + const result = await workdayConnector.validateConfig(ACCESS_TOKEN, CONFIG) + + expect(result.valid).toBe(false) + expect(result.error).toMatch(/ignored the article status filter/) + }) + + it('rejects a credential that is not a clientSecret:refreshToken pair', async () => { + mockApi({ total: 0, data: [] }) + + const result = await workdayConnector.validateConfig('only-one-secret', CONFIG) + + expect(result.valid).toBe(false) + expect(result.error).toMatch(/clientSecret:refreshToken/) + }) +}) + +describe('workday document mapping', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('produces the same contentHash from the listing and from getDocument', async () => { + mockApi({ total: 1, data: [versionFixture('a', 4)] }) + const listed = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + + vi.clearAllMocks() + mockFetch.mockImplementation(async (url: string) => + url.includes('/ccx/oauth2/') + ? jsonResponse({ access_token: 'bearer', expires_in: 3600 }) + : jsonResponse(versionFixture('a', 4)) + ) + const fetched = await workdayConnector.getDocument(ACCESS_TOKEN, CONFIG, 'a', {}) + + expect(fetched?.contentHash).toBe(listed.documents[0].contentHash) + expect(fetched?.contentHash).toBe('workday:a:4:2026-01-01T00:00:00.000Z') + }) + + it('returns null for an article version the tenant no longer has', async () => { + mockFetch.mockImplementation(async (url: string) => + url.includes('/ccx/oauth2/') + ? jsonResponse({ access_token: 'bearer', expires_in: 3600 }) + : jsonResponse({ error: 'not found' }, 404) + ) + + expect(await workdayConnector.getDocument(ACCESS_TOKEN, CONFIG, 'gone', {})).toBeNull() + }) + + it('indexes documented plain-text content verbatim', async () => { + mockApi({ + total: 1, + data: [{ ...versionFixture('a'), content: 'Email support for 5 < 10 cases' }], + }) + + const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + + expect(result.documents[0].content).toBe('Email support for 5 < 10 cases') + }) + + it('reduces a tenant that emits real markup to plain text', async () => { + mockApi({ + total: 1, + data: [{ ...versionFixture('a'), content: '

Reset your password.

' }], + }) + + const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + + expect(result.documents[0].content).toBe('Reset your password .') + }) + + it('maps every declared tag from the metadata the listing stores', async () => { + mockApi({ + total: 1, + data: [ + { + ...versionFixture('a', 2), + createdDate: '2025-06-01T00:00:00.000Z', + category: { id: 'c1', descriptor: 'Benefits' }, + language: { id: 'l1', descriptor: 'English' }, + audience: [{ id: AUDIENCE_ID, descriptor: 'All Employees' }], + tags: [ + { id: 't1', descriptor: 'payroll' }, + { id: 't2', descriptor: 'benefits' }, + ], + }, + ], + }) + + const result = await workdayConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + const tags = workdayConnector.mapTags?.(result.documents[0].metadata ?? {}) + + expect(tags).toEqual({ + article: 'Article a', + category: 'Benefits', + status: 'Published', + language: 'English', + audience: 'All Employees', + articleTags: 'payroll, benefits', + version: 2, + created: new Date('2025-06-01T00:00:00.000Z'), + lastUpdated: new Date('2026-01-01T00:00:00.000Z'), + }) + }) +}) diff --git a/apps/sim/connectors/workday/workday.ts b/apps/sim/connectors/workday/workday.ts new file mode 100644 index 00000000000..a19b9bf0ef6 --- /dev/null +++ b/apps/sim/connectors/workday/workday.ts @@ -0,0 +1,776 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { validatePathSegment, validateWorkdayTenantUrl } from '@/lib/core/security/input-validation' +import type { RetryOptions } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { + htmlToPlainText, + joinTagArray, + looksLikeHtml, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' +import { workdayConnectorMeta } from '@/connectors/workday/meta' + +const logger = createLogger('WorkdayConnector') + +/** + * Maximum `limit` the `helpArticle` v1 service accepts on `GET /articleVersions` + * and `GET /articleStatuses`; both document a default of 20 and a maximum of 100. + */ +const PAGE_SIZE = 100 + +/** `GET /values/common/audiences/` documents a default *and* maximum of 1000. */ +const VALUES_PAGE_SIZE = 1000 + +/** + * Hard stop on the value-lookup pagination that turns audience and status names + * into Workday IDs, so a tenant with an unexpectedly large prompt-value list + * cannot pull an unbounded number of rows into memory. + */ +const MAX_VALUE_LOOKUP_ROWS = 5000 + +/** How many available values an unresolved-name error is allowed to name. */ +const MAX_ERROR_DESCRIPTORS = 20 + +/** + * Workday resolves REST services as `/ccx/api/{service}/{version}/{tenant}/...`, + * where `{service}/{version}` is the `basePath` the service's OpenAPI document + * declares — `/helpArticle/v1` here. + */ +const API_BASE_PATH = '/ccx/api/helpArticle/v1' + +/** Sentinel status choice meaning "do not filter by status at all". */ +const ALL_STATUSES = 'all' + +/** + * A Workday instance identifier as `INSTANCE_MODEL_REFERENCE` declares it: a + * 32-character lowercase Workday ID, or a `Type=Value` reference ID. Config + * values matching it are passed to the API untouched instead of being looked up + * as display names. + */ +const WORKDAY_INSTANCE_ID = /^(?:[0-9a-f]{32}|\S+=\S+)$/ + +/** Refresh a bearer token this far before it actually expires. */ +const TOKEN_EXPIRY_SKEW_MS = 60_000 + +/** Workday documents its access tokens as valid for 3600 seconds. */ +const DEFAULT_TOKEN_LIFETIME_SECONDS = 3600 + +/** + * A Workday instance reference (`INSTANCE_MODEL_REFERENCE`): a Workday ID or + * reference ID, plus the display name Workday renders for it. + */ +interface WorkdayInstance { + id?: string + descriptor?: string + href?: string +} + +/** The `{ total, data }` envelope every collection in this service returns. */ +interface WorkdayCollection { + total?: number + data?: T[] +} + +/** + * One article version from `GET /articleVersions` and `GET /articleVersions/{ID}`, + * which share the `articleVersionDetails` schema. Every field is optional because + * the schema marks none of them required and Workday omits what the caller is not + * permitted to see. + */ +interface WorkdayArticleVersion { + id?: string + title?: string + content?: string + version?: number + createdDate?: string + lastUpdatedDate?: string + viewLink?: string + latestPublishedVersionViewURL?: string + status?: WorkdayInstance + category?: WorkdayInstance + language?: WorkdayInstance + location?: WorkdayInstance + parentArticle?: WorkdayInstance + tags?: WorkdayInstance[] + audience?: WorkdayInstance[] +} + +/** + * Workday's error body: a message in `error`, plus per-field detail in `errors` + * for validation failures. + */ +interface WorkdayErrorBody { + error?: string + errors?: { error?: string; field?: string }[] +} + +/** A bearer token cached for the remainder of a sync run, with its expiry. */ +interface WorkdayBearerToken { + value: string + expiresAt: number +} + +/** + * Tenant coordinates resolved from `sourceConfig`, with the host already checked + * against the Workday domain allowlist and reduced to a bare origin. + */ +interface WorkdayTenant { + origin: string + tenant: string + clientId: string +} + +/** The status and audience filters, resolved from display names to Workday IDs. */ +interface WorkdayFilters { + status: string[] + audience: string[] +} + +/** + * Reduces the configured tenant host to an origin the API paths can be appended + * to. `validateWorkdayTenantUrl` only reports whether a URL is well-formed and + * on a Workday domain — it returns the string as written — so a value carrying a + * path, a query, or a trailing slash would otherwise be spliced into every + * request URL. A scheme-less host is accepted and assumed HTTPS, which is the + * only scheme the service's OpenAPI document declares. + */ +function normalizeTenantOrigin(raw: unknown): string { + const trimmed = typeof raw === 'string' ? raw.trim() : '' + const candidate = trimmed && !/^https?:\/\//i.test(trimmed) ? `https://${trimmed}` : trimmed + const validation = validateWorkdayTenantUrl(candidate || undefined) + if (!validation.isValid) { + throw new Error(validation.error ?? 'Invalid Workday tenant host') + } + return new URL(validation.sanitized ?? candidate).origin +} + +function resolveTenant(sourceConfig: Record): WorkdayTenant { + const origin = normalizeTenantOrigin(sourceConfig.tenantUrl) + + const tenant = (sourceConfig.tenant as string | undefined)?.trim() + const tenantValidation = validatePathSegment(tenant, { paramName: 'tenant', maxLength: 128 }) + if (!tenantValidation.isValid) { + throw new Error(tenantValidation.error ?? 'Invalid Workday tenant') + } + + const clientId = (sourceConfig.clientId as string | undefined)?.trim() + if (!clientId) { + throw new Error('Client ID is required') + } + + return { origin, tenant: tenant as string, clientId } +} + +/** + * Splits the stored key into the client secret and refresh token. Both are + * secrets, so they share the connector's single encrypted field rather than one + * of them sitting in plaintext `sourceConfig`. Only the first colon separates + * them — a secret may legitimately contain further colons. + */ +function splitCredentials(accessToken: string): { clientSecret: string; refreshToken: string } { + const separator = accessToken.indexOf(':') + if (separator <= 0 || separator === accessToken.length - 1) { + throw new Error('Credential must be in the form clientSecret:refreshToken') + } + return { + clientSecret: accessToken.slice(0, separator), + refreshToken: accessToken.slice(separator + 1), + } +} + +/** + * Exchanges the refresh token for a bearer access token at the tenant's token + * endpoint (`POST /ccx/oauth2/{tenant}/token`), authenticating the API client + * with HTTP Basic. Workday access tokens live an hour, so the result is cached + * on `syncContext` with its expiry and reused across the pages of a sync run. + */ +async function getBearerToken( + accessToken: string, + wd: WorkdayTenant, + syncContext?: Record, + retryOptions?: RetryOptions, + forceRefresh = false +): Promise { + if (!forceRefresh) { + const cached = syncContext?.workdayBearerToken as WorkdayBearerToken | undefined + if (cached?.value && cached.expiresAt > Date.now()) return cached.value + } + + const { clientSecret, refreshToken } = splitCredentials(accessToken) + const basic = Buffer.from(`${wd.clientId}:${clientSecret}`, 'utf8').toString('base64') + + const response = await fetchWithRetry( + `${wd.origin}/ccx/oauth2/${encodeURIComponent(wd.tenant)}/token`, + { + method: 'POST', + headers: { + Authorization: `Basic ${basic}`, + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json', + }, + body: new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: refreshToken, + }).toString(), + }, + retryOptions + ) + + if (!response.ok) { + const detail = (await response.text().catch(() => '')).slice(0, 300) + /** + * A refresh token is the one credential this connector cannot repair on its + * own: it lives in the encrypted key the user typed, and a sync has nowhere + * to write a replacement. Say so, rather than reporting a bare 400 that + * looks like a transient outage. + */ + if (response.status === 400 || response.status === 401) { + throw new Error( + `Workday rejected the stored refresh token (HTTP ${response.status}). Sim cannot replace a refresh token that the tenant rotated, expired, or revoked — reissue one with the "Manage Refresh Tokens for Integrations" task and re-enter the credential as clientSecret:refreshToken.${detail ? ` Workday said: ${detail}` : ''}` + ) + } + throw new Error( + `Workday token exchange failed (HTTP ${response.status})${detail ? `: ${detail}` : ''}` + ) + } + + const body = (await response.json()) as { + access_token?: string + expires_in?: number + refresh_token?: string + } + if (!body.access_token) { + throw new Error('Workday token response did not include an access token') + } + + /** + * A tenant configured to rotate refresh tokens invalidates the stored one the + * first time it is redeemed, so the next sync fails no matter what this run + * does. Nothing here can persist the replacement, so record the cause up front + * — the failure it produces an hour later reads as an unexplained 400. + */ + if (body.refresh_token && body.refresh_token !== refreshToken) { + logger.warn( + 'Workday returned a rotated refresh token that Sim cannot persist; register the API client with a non-expiring refresh token or this connector will stop syncing', + { tenant: wd.tenant } + ) + } + + const lifetimeSeconds = + typeof body.expires_in === 'number' && body.expires_in > 0 + ? body.expires_in + : DEFAULT_TOKEN_LIFETIME_SECONDS + + if (syncContext) { + syncContext.workdayBearerToken = { + value: body.access_token, + expiresAt: Date.now() + lifetimeSeconds * 1000 - TOKEN_EXPIRY_SKEW_MS, + } satisfies WorkdayBearerToken + } + return body.access_token +} + +/** + * Issues an authenticated GET, retrying once with a fresh bearer token on a 401. + * A sync run can outlive the hour-long token, and a cached token that expires + * between pages would otherwise fail the run rather than the request. + */ +async function workdayGet( + url: string, + accessToken: string, + wd: WorkdayTenant, + syncContext?: Record, + retryOptions?: RetryOptions +): Promise { + const send = async (bearer: string) => + fetchWithRetry( + url, + { method: 'GET', headers: { Authorization: `Bearer ${bearer}`, Accept: 'application/json' } }, + retryOptions + ) + + const response = await send(await getBearerToken(accessToken, wd, syncContext, retryOptions)) + if (response.status !== 401) return response + + return send(await getBearerToken(accessToken, wd, syncContext, retryOptions, true)) +} + +async function readErrorMessage(response: Response): Promise { + const text = await response.text().catch(() => '') + if (!text) return `HTTP ${response.status}` + try { + const body = JSON.parse(text) as WorkdayErrorBody + if (body.error) { + const details = (body.errors ?? []) + .map((entry) => (entry.field ? `${entry.field}: ${entry.error}` : entry.error)) + .filter((detail): detail is string => Boolean(detail)) + return details.length > 0 ? `${body.error} (${details.join('; ')})` : body.error + } + } catch { + /* Workday returns HTML for some gateway failures; fall through to the raw body. */ + } + return text.slice(0, 300) +} + +function tenantResourceUrl(wd: WorkdayTenant, resource: string, query?: URLSearchParams): string { + const suffix = query ? `?${query.toString()}` : '' + return `${wd.origin}${API_BASE_PATH}/${encodeURIComponent(wd.tenant)}${resource}${suffix}` +} + +/** + * Reads a whole prompt-value collection so display names can be resolved to + * Workday IDs. Both `/articleStatuses` and `/values/common/audiences/` use the + * service's `limit`/`offset` paging and `{ total, data }` envelope. + */ +async function fetchInstances( + resource: string, + pageSize: number, + accessToken: string, + wd: WorkdayTenant, + syncContext?: Record, + retryOptions?: RetryOptions +): Promise { + const instances: WorkdayInstance[] = [] + + for (let offset = 0; offset < MAX_VALUE_LOOKUP_ROWS; offset += pageSize) { + const query = new URLSearchParams({ limit: String(pageSize), offset: String(offset) }) + const response = await workdayGet( + tenantResourceUrl(wd, resource, query), + accessToken, + wd, + syncContext, + retryOptions + ) + if (!response.ok) { + throw new Error(`Workday ${resource} lookup failed: ${await readErrorMessage(response)}`) + } + + const body = (await response.json()) as WorkdayCollection + const page = body.data ?? [] + instances.push(...page) + + if (page.length < pageSize) break + if (typeof body.total === 'number' && instances.length >= body.total) break + } + + if (instances.length >= MAX_VALUE_LOOKUP_ROWS) { + logger.warn('Workday value lookup hit its row cap; names beyond it cannot be resolved', { + resource, + rows: instances.length, + }) + } + + return instances +} + +/** + * Names the values the lookup did return, capped. An audience list runs to as + * many rows as {@link MAX_VALUE_LOOKUP_ROWS} allows, and the whole list would + * otherwise be interpolated into an error string the knowledge base UI renders. + */ +function availableSuffix(descriptors: string[]): string { + if (descriptors.length === 0) return '' + const shown = descriptors.slice(0, MAX_ERROR_DESCRIPTORS) + const omitted = descriptors.length - shown.length + return `. Available: ${shown.join(', ')}${omitted > 0 ? `, and ${omitted} more` : ''}` +} + +/** + * Turns configured display names into the Workday IDs the `status` and + * `audience` query parameters take. A value already shaped like a Workday ID or + * a `Type=Value` reference ID is passed through, so an operator who has the ID + * to hand never has to match a descriptor exactly. + */ +async function resolveInstanceIds( + values: string[], + loadInstances: () => Promise, + label: string +): Promise { + /** + * Every value already being an ID is the common case for an operator who + * pasted Workday IDs, and the lookup it would otherwise run reads the tenant's + * whole prompt-value collection. + */ + if (values.every((value) => WORKDAY_INSTANCE_ID.test(value))) return values + + const instances = await loadInstances() + const idByDescriptor = new Map() + const descriptors: string[] = [] + for (const instance of instances) { + const descriptor = instance.descriptor?.trim() + if (!descriptor || !instance.id) continue + const key = descriptor.toLowerCase() + if (idByDescriptor.has(key)) continue + idByDescriptor.set(key, instance.id) + descriptors.push(descriptor) + } + + return values.map((value) => { + if (WORKDAY_INSTANCE_ID.test(value)) return value + const resolved = idByDescriptor.get(value.toLowerCase()) + if (resolved) return resolved + throw new Error( + `Your Workday tenant has no ${label} named "${value}"${availableSuffix(descriptors)}` + ) + }) +} + +/** + * Resolves the configured status and audience filters once per sync run and + * caches them on `syncContext`, so later pages reuse the lookup. + * + * Status is required and has no default: `/articleVersions` returns one row per + * article *revision*, and the service offers no latest-version filter, so the + * scope of a sync has to be a decision the operator made rather than one this + * code made silently. + */ +async function resolveFilters( + accessToken: string, + wd: WorkdayTenant, + sourceConfig: Record, + syncContext?: Record, + retryOptions?: RetryOptions +): Promise { + const cached = syncContext?.workdayFilters as WorkdayFilters | undefined + if (cached) return cached + + const statusChoice = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : '' + if (!statusChoice) { + throw new Error( + 'Article Status is required: choose which article versions to sync, or "Every status" to index every historical revision.' + ) + } + + const status = + statusChoice === ALL_STATUSES + ? [] + : await resolveInstanceIds( + [statusChoice], + () => + fetchInstances( + '/articleStatuses', + PAGE_SIZE, + accessToken, + wd, + syncContext, + retryOptions + ), + 'article status' + ) + + const audienceNames = parseMultiValue(sourceConfig.audience) + const audience = + audienceNames.length === 0 + ? [] + : await resolveInstanceIds( + audienceNames, + () => + fetchInstances( + '/values/common/audiences/', + VALUES_PAGE_SIZE, + accessToken, + wd, + syncContext, + retryOptions + ), + 'audience' + ) + + const filters: WorkdayFilters = { status, audience } + if (syncContext) syncContext.workdayFilters = filters + return filters +} + +function descriptorsOf(instances: WorkdayInstance[] | undefined): string[] { + return (instances ?? []) + .map((instance) => instance.descriptor) + .filter((descriptor): descriptor is string => Boolean(descriptor)) +} + +/** + * Builds the indexable document for one article version. + * + * `contentHash` is derived only from fields the list response already carries — + * the version number Workday increments on edit, and `lastUpdatedDate` — and + * `GET /articleVersions/{ID}` returns the same `articleVersionDetails` schema, so + * the hash is identical by construction whichever call produced the row. + */ +function articleToDocument(article: WorkdayArticleVersion): ExternalDocument | null { + const externalId = article.id + if (!externalId) return null + + /** + * The service documents `content` as the article body "displayed in plain + * text", so it is indexed as written. `looksLikeHtml` is the shared markup + * test rather than an unconditional strip: a tenant that emits real markup has + * it reduced, while plain text that merely contains angle brackets — an email + * address, a bare autolink — survives untouched. + */ + const rawContent = article.content ?? '' + const content = looksLikeHtml(rawContent) ? htmlToPlainText(rawContent) : rawContent + + /** + * `viewLink` addresses this version and is populated only once the article is + * published; `latestPublishedVersionViewURL` points at the article's newest + * published version, which is the closest link a draft or archived revision has. + */ + const sourceUrl = article.viewLink || article.latestPublishedVersionViewURL || undefined + + return { + externalId, + title: article.title || 'Untitled article', + content, + mimeType: 'text/plain', + sourceUrl, + contentHash: `workday:${externalId}:${article.version ?? ''}:${article.lastUpdatedDate ?? ''}`, + metadata: { + article: article.parentArticle?.descriptor, + category: article.category?.descriptor, + status: article.status?.descriptor, + language: article.language?.descriptor, + audience: descriptorsOf(article.audience), + articleTags: descriptorsOf(article.tags), + version: article.version, + created: article.createdDate, + lastUpdated: article.lastUpdatedDate, + }, + } +} + +function buildListUrl( + wd: WorkdayTenant, + filters: WorkdayFilters, + offset: number, + limit: number +): string { + const query = new URLSearchParams({ limit: String(limit), offset: String(offset) }) + for (const status of filters.status) query.append('status', status) + for (const audience of filters.audience) query.append('audience', audience) + return tenantResourceUrl(wd, '/articleVersions', query) +} + +/** + * Reads the optional version cap: `0` when none was configured, the floored + * count when one was, and `-1` when a value was supplied that is not a positive + * number, which {@link workdayConnector.validateConfig} rejects. + * + * The field is a short-input so the value normally arrives as a string, but a + * `sourceConfig` persisted with a JSON number must not crash the sync — reading + * it as a string and calling `.trim()` on it would. + */ +function parseMaxVersions(value: unknown): number { + const raw = typeof value === 'string' ? value.trim() : value + if (raw === '' || raw === null || raw === undefined) return 0 + const parsed = Number(raw) + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : -1 +} + +export const workdayConnector: ConnectorConfig = { + ...workdayConnectorMeta, + + listDocuments: async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record + ): Promise => { + const wd = resolveTenant(sourceConfig) + const filters = await resolveFilters(accessToken, wd, sourceConfig, syncContext) + + const offset = cursor ? Number(cursor) : 0 + if (!Number.isFinite(offset) || offset < 0) { + throw new Error(`Invalid pagination cursor: ${cursor}`) + } + + const maxVersions = Math.max(parseMaxVersions(sourceConfig.maxVersions), 0) + const alreadyFetched = offset + const remaining = maxVersions > 0 ? maxVersions - alreadyFetched : PAGE_SIZE + const limit = Math.min(PAGE_SIZE, Math.max(remaining, 0)) + + if (limit === 0) { + if (syncContext) syncContext.listingCapped = true + return { documents: [], hasMore: false } + } + + const response = await workdayGet( + buildListUrl(wd, filters, offset, limit), + accessToken, + wd, + syncContext + ) + + if (!response.ok) { + throw new Error(`Workday article listing failed: ${await readErrorMessage(response)}`) + } + + const body = (await response.json()) as WorkdayCollection + const page = body.data ?? [] + + const documents: ExternalDocument[] = [] + for (const article of page) { + const document = articleToDocument(article) + if (document) documents.push(document) + } + + const fetched = alreadyFetched + page.length + const total = typeof body.total === 'number' ? body.total : undefined + const sourceExhausted = page.length < limit || (total !== undefined && fetched >= total) + const hitCap = maxVersions > 0 && fetched >= maxVersions + + /** + * The sync engine hard-deletes any stored document the listing did not return, + * so a listing the cap truncated while Workday still has more article versions + * must say so. Reaching the cap exactly at source exhaustion is a complete + * listing and must still reconcile deletions. + */ + if (syncContext && hitCap && !sourceExhausted) { + syncContext.listingCapped = true + } + + const hasMore = !sourceExhausted && !hitCap + + logger.info('Listed Workday article versions', { + tenant: wd.tenant, + offset, + returned: page.length, + /** Rows Workday returned without an `id`, which cannot be addressed or indexed. */ + unidentified: page.length - documents.length, + total, + hasMore, + }) + + return { + documents, + nextCursor: hasMore ? String(fetched) : undefined, + hasMore, + } + }, + + getDocument: async ( + accessToken: string, + sourceConfig: Record, + externalId: string, + syncContext?: Record + ): Promise => { + const wd = resolveTenant(sourceConfig) + + const response = await workdayGet( + tenantResourceUrl(wd, `/articleVersions/${encodeURIComponent(externalId)}`), + accessToken, + wd, + syncContext + ) + + if (response.status === 404) return null + if (!response.ok) { + throw new Error(`Workday article fetch failed: ${await readErrorMessage(response)}`) + } + + const article = (await response.json()) as WorkdayArticleVersion + return articleToDocument(article) + }, + + validateConfig: async ( + accessToken: string, + sourceConfig: Record + ): Promise<{ valid: boolean; error?: string }> => { + if (parseMaxVersions(sourceConfig.maxVersions) < 0) { + return { valid: false, error: 'Max article versions must be a positive number' } + } + + const statusChoice = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : '' + + try { + const wd = resolveTenant(sourceConfig) + + /** Scratch context so validation buys one bearer token, not one per call. */ + const validationContext: Record = {} + + /** + * Resolving the filters exercises the token exchange and the status and + * audience lookups, so a mistyped audience name or a credential without the + * Help Article REST API domain fails here rather than mid-sync. + */ + const filters = await resolveFilters( + accessToken, + wd, + sourceConfig, + validationContext, + VALIDATE_RETRY_OPTIONS + ) + + const response = await workdayGet( + buildListUrl(wd, filters, 0, 1), + accessToken, + wd, + validationContext, + VALIDATE_RETRY_OPTIONS + ) + + if (!response.ok) { + return { + valid: false, + error: `Workday returned ${response.status}: ${await readErrorMessage(response)}`, + } + } + + /** + * The `helpArticle` OpenAPI document declares `status` as an untyped + * `array` of `string` with no enum, no `$ref` and — unlike the sibling + * `audience` parameter, whose model carries + * `x-workday-populated-by: /values/common/audiences` — no declared value + * source. Whether it binds to an `/articleStatuses` Workday ID cannot be + * settled from the published spec, so the one request validation already + * makes is read back: a tenant that ignored the filter answers with a + * version in some other status, and the connector says so at configuration + * time instead of silently indexing the wrong scope on every sync. + */ + const sample = ((await response.json()) as WorkdayCollection).data?.[0] + const returnedStatus = sample?.status?.descriptor + if ( + statusChoice !== ALL_STATUSES && + returnedStatus && + returnedStatus.toLowerCase() !== statusChoice.toLowerCase() + ) { + return { + valid: false, + error: `Workday ignored the article status filter: the first article version it returned is "${returnedStatus}", not "${statusChoice}". Choose "Every status" to sync every revision, or ask your Workday administrator whether the Help Article REST API accepts a status filter in this tenant.`, + } + } + + return { valid: true } + } catch (error) { + logger.warn('Workday connector validation failed', { error: getErrorMessage(error) }) + return { valid: false, error: getErrorMessage(error, 'Failed to reach Workday') } + } + }, + + mapTags: (metadata: Record): Record => { + const result: Record = {} + + for (const key of ['article', 'category', 'status', 'language'] as const) { + const value = metadata[key] + if (typeof value === 'string' && value) result[key] = value + } + + for (const key of ['audience', 'articleTags'] as const) { + const value = joinTagArray(metadata[key]) + if (value) result[key] = value + } + + if (metadata.version != null) { + const version = Number(metadata.version) + if (!Number.isNaN(version)) result.version = version + } + + for (const key of ['created', 'lastUpdated'] as const) { + const value = parseTagDate(metadata[key]) + if (value) result[key] = value + } + + return result + }, +} diff --git a/apps/sim/lib/auth/connectors/providers.ts b/apps/sim/lib/auth/connectors/providers.ts index d561e5ac068..2020dc7c4d8 100644 --- a/apps/sim/lib/auth/connectors/providers.ts +++ b/apps/sim/lib/auth/connectors/providers.ts @@ -567,6 +567,43 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, }, + { + providerId: 'google-chat', + clientId: env.GOOGLE_CLIENT_ID as string, + clientSecret: env.GOOGLE_CLIENT_SECRET as string, + discoveryUrl: 'https://accounts.google.com/.well-known/openid-configuration', + accessType: 'offline', + scopes: getCanonicalScopesForProvider('google-chat'), + prompt: 'consent', + redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-chat`, + getUserInfo: async (tokens) => { + try { + const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + headers: { Authorization: `Bearer ${tokens.accessToken}` }, + }) + if (!response.ok) { + await response.text().catch(() => {}) + logger.error('Failed to fetch Google user info', { status: response.status }) + throw new Error(`Failed to fetch Google user info: ${response.statusText}`) + } + const profile = await response.json() + const now = new Date() + return { + id: `${profile.sub}-${generateId()}`, + name: profile.name || 'Google User', + email: profile.email, + image: profile.picture || undefined, + emailVerified: profile.email_verified || false, + createdAt: now, + updatedAt: now, + } + } catch (error) { + logger.error('Error in Google getUserInfo', { error }) + throw error + } + }, + }, + { providerId: 'google-meet', clientId: env.GOOGLE_CLIENT_ID as string, diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index f69cf7f735b..fb448e04a56 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1695,6 +1695,89 @@ export function validateWorkdayTenantUrl( return { isValid: true, sanitized: url as string } } +/** + * Every production Databricks control-plane DNS zone, mirroring `ALL_ENVS` in the + * Databricks SDK (`databricks/sdk/environments.py`). The SDK's `.dev.*`/`.staging.*` + * zones are internal and deliberately omitted; the ones that are subdomains of a + * zone listed here (e.g. `.staging.cloud.databricks.com`) match by suffix anyway. + */ +const DATABRICKS_ALLOWED_HOST_SUFFIXES = [ + '.cloud.databricks.com', + '.cloud.databricks.us', + '.gcp.databricks.com', + '.azuredatabricks.net', + '.databricks.azure.us', + '.databricks.azure.cn', +] as const + +/** + * Validates a Databricks workspace host to prevent SSRF attacks. + * + * Databricks is host-scoped: every workspace has its own per-workspace URL, and + * every REST call is made against it. Example valid hosts: + * - dbc-1234abcd-5678.cloud.databricks.com (AWS) + * - dbc-1234abcd-5678.cloud.databricks.us (AWS GovCloud) + * - adb-1234567890123456.7.azuredatabricks.net (Azure) + * - adb-1234567890123456.7.databricks.azure.us (Azure US Government) + * - adb-1234567890123456.7.databricks.azure.cn (Azure China) + * - 1234567890123456.7.gcp.databricks.com (GCP) + * + * The value is user-supplied and fetched server-side, so it is normalized to an + * https origin and then checked against a Databricks-owned domain allowlist. + * Users routinely paste a full console URL (a `#notebook/123` deep link, a + * trailing slash, a `?o=` query); every API path is built by + * appending `/api/2.0/...`, so any surviving path, query or fragment would + * produce a 404. `URL.origin` also lower-cases the host and drops the default + * port. + * + * Note: legacy regional URLs (`https://oregon.cloud.databricks.com`) match the + * allowlist but are not recommended by Databricks — point the connector at the + * per-workspace URL instead. + * + * @param host - The workspace host or URL to validate, with or without a scheme + * @param paramName - Name of the parameter for error messages + * @returns ValidationResult whose `sanitized` value is the https origin + * + * @example + * ```typescript + * const result = validateDatabricksWorkspaceHost(workspaceHost) + * if (!result.isValid) { + * throw new Error(result.error) + * } + * ``` + */ +export function validateDatabricksWorkspaceHost( + host: string | null | undefined, + paramName = 'workspaceHost' +): ValidationResult { + const raw = typeof host === 'string' ? host.trim() : '' + if (!raw) { + return { isValid: false, error: `${paramName} is required` } + } + + const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw}` + + const urlResult = validateExternalUrl(withScheme, paramName) + if (!urlResult.isValid) return urlResult + + const parsed = new URL(withScheme) + const hostname = parsed.hostname.toLowerCase() + const isAllowedHost = DATABRICKS_ALLOWED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix)) + + if (!isAllowedHost) { + logger.warn('Databricks workspace host not on allowlist', { + paramName, + hostname: hostname.substring(0, 100), + }) + return { + isValid: false, + error: `${paramName} must be a Databricks-hosted domain (e.g., *.cloud.databricks.com, *.azuredatabricks.net, or *.gcp.databricks.com)`, + } + } + + return { isValid: true, sanitized: parsed.origin } +} + /** * Validates a database identifier (table or column name) to prevent SQL injection. * diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index 4d927c1d0c2..d4e560f85f7 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -19,6 +19,7 @@ import { GoogleAdsIcon, GoogleBigQueryIcon, GoogleCalendarIcon, + GoogleChatIcon, GoogleContactsIcon, GoogleDocsIcon, GoogleDriveIcon, @@ -286,6 +287,27 @@ export const OAUTH_PROVIDERS: Record = { ], serviceAccountProviderId: 'google-service-account', }, + /** + * Deliberately declares no `serviceAccountProviderId`, unlike every sibling + * Google service. A Google service-account JWT cannot reach user-scoped Chat + * data without domain-wide delegation, so offering service-account auth here + * would surface a credential path that always fails. Enterprises that + * authenticate other Google connectors through a delegated service account must + * attach a per-user OAuth credential for Chat. + */ + 'google-chat': { + name: 'Google Chat', + description: 'Read Google Chat spaces and messages the signed-in user can access.', + providerId: 'google-chat', + icon: GoogleChatIcon, + baseProviderIcon: GoogleIcon, + scopes: [ + 'https://www.googleapis.com/auth/userinfo.email', + 'https://www.googleapis.com/auth/userinfo.profile', + 'https://www.googleapis.com/auth/chat.spaces.readonly', + 'https://www.googleapis.com/auth/chat.messages.readonly', + ], + }, 'google-meet': { name: 'Google Meet', description: 'Create and manage Google Meet meeting spaces and conferences.', diff --git a/apps/sim/lib/oauth/types.ts b/apps/sim/lib/oauth/types.ts index 129164cb10f..edfb2380042 100644 --- a/apps/sim/lib/oauth/types.ts +++ b/apps/sim/lib/oauth/types.ts @@ -46,6 +46,7 @@ export type OAuthProvider = | 'google-forms' | 'google-groups' | 'google-meet' + | 'google-chat' | 'vertex-ai' | 'x' | 'tiktok' @@ -103,6 +104,7 @@ export type OAuthService = | 'google-forms' | 'google-groups' | 'google-meet' + | 'google-chat' | 'vertex-ai' | 'x' | 'tiktok' diff --git a/apps/sim/lib/oauth/utils.ts b/apps/sim/lib/oauth/utils.ts index 83452b68560..367f6ae829a 100644 --- a/apps/sim/lib/oauth/utils.ts +++ b/apps/sim/lib/oauth/utils.ts @@ -47,6 +47,10 @@ export const SCOPE_DESCRIPTIONS: Record = { 'https://www.googleapis.com/auth/admin.directory.group.readonly': 'View Google Workspace groups', 'https://www.googleapis.com/auth/admin.directory.group.member.readonly': 'View Google Workspace group memberships', + 'https://www.googleapis.com/auth/chat.spaces.readonly': + 'View Google Chat spaces you are a member of', + 'https://www.googleapis.com/auth/chat.messages.readonly': + 'View messages in Google Chat spaces you are a member of', 'https://www.googleapis.com/auth/meetings.space.created': 'Create and manage Google Meet meeting spaces', 'https://www.googleapis.com/auth/meetings.space.readonly': diff --git a/packages/deployment-config/src/env-capabilities.ts b/packages/deployment-config/src/env-capabilities.ts index e0e7821ca68..9a4d473f8f9 100644 --- a/packages/deployment-config/src/env-capabilities.ts +++ b/packages/deployment-config/src/env-capabilities.ts @@ -1376,6 +1376,7 @@ const GOOGLE_OAUTH_SERVICES = new Set([ 'google-forms', 'google-groups', 'google-meet', + 'google-chat', 'vertex-ai', ])