From 50d9df67cddbe92216b1e5f747d1ff00d2cacf02 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 16 Sep 2026 15:28:48 -0700 Subject: [PATCH 1/2] fix(search): preserve Google sync diagnostics and retry transient failures --- apps/docs/content/docs/search/gmail.mdx | 2 + .../content/docs/search/google-calendar.mdx | 2 + .../docs/content/docs/search/google-drive.mdx | 2 + .../background/knowledge-processing.test.ts | 17 ++ apps/sim/background/knowledge-processing.ts | 8 +- apps/sim/connectors/gmail/gmail.test.ts | 18 ++- apps/sim/connectors/gmail/gmail.ts | 132 ++++++--------- .../google-calendar/google-calendar.ts | 98 ++++++------ .../connectors/google-drive/directory.test.ts | 19 +++ apps/sim/connectors/google-drive/directory.ts | 43 +++-- .../google-drive/google-drive-errors.ts | 88 +++------- .../google-drive/google-drive.test.ts | 6 +- .../connectors/google-drive/google-drive.ts | 90 +++++++---- .../google-drive/workspace-drives.ts | 4 +- .../google-workspace/api-errors.test.ts | 147 +++++++++++++++++ .../connectors/google-workspace/api-errors.ts | 150 ++++++++++++++++++ apps/sim/connectors/google-workspace/users.ts | 6 +- apps/sim/connectors/source-error.ts | 6 +- .../connectors/connector-error.test.ts | 29 ++++ .../knowledge/connectors/connector-error.ts | 40 ++++- .../connectors/external-group-sync.test.ts | 6 + .../connectors/external-group-sync.ts | 5 +- .../document-processing-source.test.ts | 46 ++++++ apps/sim/lib/knowledge/documents/service.ts | 7 +- 24 files changed, 710 insertions(+), 261 deletions(-) create mode 100644 apps/sim/connectors/google-workspace/api-errors.test.ts create mode 100644 apps/sim/connectors/google-workspace/api-errors.ts diff --git a/apps/docs/content/docs/search/gmail.mdx b/apps/docs/content/docs/search/gmail.mdx index e5c3f63adbe..0d86d7a59e9 100644 --- a/apps/docs/content/docs/search/gmail.mdx +++ b/apps/docs/content/docs/search/gmail.mdx @@ -139,6 +139,8 @@ Updates, removals, and access refresh in the background. Empty mailboxes and fil ## Troubleshooting +An individual thread failure does not mean the whole mailbox failed. Sim retries temporary server and rate-limit errors with bounded backoff. Error diagnostics record the Google API operation, HTTP status, and a recognized error reason when available. + | What you see | What to do | | --- | --- | | A different email is requested | Use the Google account matching your verified Sim email. A separate personal account or alias does not satisfy the match. | diff --git a/apps/docs/content/docs/search/google-calendar.mdx b/apps/docs/content/docs/search/google-calendar.mdx index 5ce4684134c..a2317d9d941 100644 --- a/apps/docs/content/docs/search/google-calendar.mdx +++ b/apps/docs/content/docs/search/google-calendar.mdx @@ -135,6 +135,8 @@ Search schedules syncs hourly. Event edits, cancellations, access changes, inact ## Troubleshooting +When a sync fails, **Sync history** includes the Google API operation, HTTP status, and a recognized reason when available. A `403` alone does not establish a missing scope: `rateLimitExceeded` and `userRateLimitExceeded` are retried with bounded backoff. A persistent access error requires checking the affected user’s Calendar access. + | What you see | What to do | | --- | --- | | No events | Check the date range, search query, and calendar IDs. Use an empty calendar selection or `primary` for each person's own calendar. | diff --git a/apps/docs/content/docs/search/google-drive.mdx b/apps/docs/content/docs/search/google-drive.mdx index e647b5544c8..3c70e3ccb62 100644 --- a/apps/docs/content/docs/search/google-drive.mdx +++ b/apps/docs/content/docs/search/google-drive.mdx @@ -155,6 +155,8 @@ Search schedules syncs hourly. Central crawls revisit the selected users' files ## Troubleshooting +**Directory permission sync failed** means Sim could not fully verify group membership. Check the Directory administrator’s access to the affected group and any nested groups; this is separate from file-download access. An incomplete membership read does not replace the last verified membership, which remains subject to freshness checks. + | Problem | Next step | | --- | --- | | Google rejects authorization (`unauthorized_client`) | In **Manage Domain Wide Delegation**, verify the numeric **Client ID** matches `client_id` in the JSON key uploaded to Sim and all required scopes appear under **View details**. Check pending approval and allow time for recent changes to propagate. Changing the OAuth consent screen alone does not authorize delegation. | diff --git a/apps/sim/background/knowledge-processing.test.ts b/apps/sim/background/knowledge-processing.test.ts index 6fda5c229bb..64f5f7c556f 100644 --- a/apps/sim/background/knowledge-processing.test.ts +++ b/apps/sim/background/knowledge-processing.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { DrizzleQueryError } from 'drizzle-orm/errors' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { @@ -566,6 +567,22 @@ describe('knowledge processing worker', () => { expect(mockTrigger).not.toHaveBeenCalled() }) + it('keeps database failures retryable without sending SQL or parameters to Trigger', async () => { + const error = new DrizzleQueryError( + 'insert private SQL', + ['private bound content'], + Object.assign(new Error('private database detail'), { code: '57014' }) + ) + mockProcessDocumentAsync.mockRejectedValueOnce(error) + const failure = await runDocumentProcessing(WORKSPACE_PAYLOAD).catch( + (caught: unknown) => caught + ) + expect(failure).toBeInstanceOf(Error) + expect(failure).toMatchObject({ message: 'Database request failed (SQLSTATE 57014).' }) + expect(failure).not.toHaveProperty('cause') + expect(JSON.stringify(failure)).not.toContain('private') + }) + it('retries failed provider continuation dispatch instead of reporting a successful deferral', async () => { const error = new Error('Trigger dispatch unavailable') mockTrigger.mockRejectedValue(error) diff --git a/apps/sim/background/knowledge-processing.ts b/apps/sim/background/knowledge-processing.ts index 4016e7defdd..7981c351208 100644 --- a/apps/sim/background/knowledge-processing.ts +++ b/apps/sim/background/knowledge-processing.ts @@ -7,6 +7,7 @@ import { isBYOKEmbeddingCredentialRejection, isEmbeddingQuotaExhaustion, } from '@/lib/embeddings' +import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' import { getOcrRequestRejection, isPermanentDocumentProcessingError, @@ -194,7 +195,12 @@ export async function runDocumentProcessing( processingTime: Date.now() - startedAt, } } - logger.error(`[${requestId}] Failed to process document: ${docData.filename}`, error) + const diagnostic = getConnectorFailureDiagnostic(error) + logger.error( + `[${requestId}] Failed to process document: ${docData.filename}`, + diagnostic ?? error + ) + if (diagnostic?.category === 'database') throw new Error(diagnostic.message) throw error } } diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts index f4d80fd1fd4..58595617115 100644 --- a/apps/sim/connectors/gmail/gmail.test.ts +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -5,9 +5,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ - fetchWithRetry: mockFetchWithRetry, + fetchWithRetry: ( + url: string, + init: RequestInit, + options: { + fetcher?: (url: string, init: RequestInit, transport: typeof fetch) => Promise + } + ) => + options.fetcher + ? options.fetcher(url, init, mockFetchWithRetry) + : mockFetchWithRetry(url, init), })) vi.mock('@/components/icons', () => ({ GmailIcon: () => null })) vi.mock('@/lib/knowledge/documents/service', () => ({ @@ -618,7 +626,9 @@ describe('Gmail separately stored message bodies', () => { .catch((caught: unknown) => caught) expect(error).toBeInstanceOf(Error) - expect(error).toMatchObject({ message: `Failed to fetch Gmail message body: ${status}` }) + expect(error).toMatchObject({ + message: `gmail.messages.attachments.get failed (HTTP ${status}).`, + }) expect(gmailConnector.isCredentialInvalidError?.(error)).toBe(status === 401) } ) @@ -899,7 +909,7 @@ describe('Gmail thread revisions and deferred content', () => { .mockResolvedValueOnce(Response.json({ threads: [{ id: 'thread-1' }] })) .mockResolvedValueOnce(new Response(null, { status })) await expect(gmailConnector.listDocuments('token', {}, undefined, {})).rejects.toThrow( - `Failed to fetch thread thread-1: ${status}` + `gmail.threads.get failed (HTTP ${status}).` ) } ) diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index f63ea781624..4f2be5cdde1 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -3,9 +3,9 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isPlainRecord } from '@sim/utils/object' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' -import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta' +import { fetchGoogleApiWithRetry, GoogleApiError } from '@/connectors/google-workspace/api-errors' import { getGoogleWorkspaceDocument, InvalidGoogleWorkspaceCursor, @@ -53,16 +53,6 @@ const CHANGED_THREAD_CONCURRENCY = 5 /** Gmail's thread listing omits these unless `includeSpamTrash` is set; the feed must agree. */ const HIDDEN_LABEL_IDS = new Set(['SPAM', 'TRASH']) -class GmailApiError extends Error { - constructor( - message: string, - readonly status: number - ) { - super(`${message}: ${status}`) - this.name = 'GmailApiError' - } -} - interface GmailHeader { name: string value: string @@ -229,26 +219,23 @@ async function getLabelIndex( let index: GmailLabelIndex | null = null try { - const response = await fetchWithRetry(`${GMAIL_API_BASE}/labels`, { - method: 'GET', - signal, - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) + const response = await fetchGoogleApiWithRetry( + 'gmail.labels.list', + `${GMAIL_API_BASE}/labels`, + { + method: 'GET', + signal, + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + } + ) - if (response.status === 401) { - throw new GmailApiError('Failed to fetch Gmail labels', response.status) - } - if (response.ok) { - index = buildLabelIndex(await readLabels(response)) - } else { - logger.warn('Failed to fetch Gmail labels', { status: response.status }) - } + index = buildLabelIndex(await readLabels(response)) } catch (error) { signal?.throwIfAborted() - if (error instanceof GmailApiError && error.status === 401) throw error + if (error instanceof GoogleApiError && error.status === 401) throw error logger.warn('Failed to fetch Gmail labels', { error: toError(error).message }) } @@ -387,7 +374,8 @@ async function readMessageBody( if (!body.attachmentId) return body.data ? decodeBase64Url(body.data, context) : '' const params = new URLSearchParams({ fields: 'data,size' }) - const response = await fetchWithRetry( + const response = await fetchGoogleApiWithRetry( + 'gmail.messages.attachments.get', `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}/attachments/${encodeURIComponent(body.attachmentId)}?${params}`, { method: 'GET', @@ -395,9 +383,6 @@ async function readMessageBody( headers: { Authorization: `Bearer ${context.accessToken}`, Accept: 'application/json' }, } ) - if (!response.ok) { - throw new GmailApiError('Failed to fetch Gmail message body', response.status) - } let fetchedBody: unknown try { @@ -591,18 +576,16 @@ async function fetchThread( params.set('fields', 'id,historyId,snippet,messages(id,labelIds,internalDate)') const url = `${GMAIL_API_BASE}/threads/${encodeURIComponent(threadId)}?${params}` - const response = await fetchWithRetry(url, { - method: 'GET', - signal, - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - if (response.status === 404) return null - throw new GmailApiError(`Failed to fetch thread ${threadId}`, response.status) + let response: Response + try { + response = await fetchGoogleApiWithRetry('gmail.threads.get', url, { + method: 'GET', + signal, + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }) + } catch (error) { + if (error instanceof GoogleApiError && error.status === 404) return null + throw error } const thread = await readResponseJsonWithLimit(response, { @@ -790,18 +773,21 @@ function threadInScope(thread: GmailThread, scope: GmailChangeScope): boolean { const gmailMailboxConnector: ConnectorConfig = { ...gmailConnectorMeta, - isCredentialInvalidError: (error) => error instanceof GmailApiError && error.status === 401, + isCredentialInvalidError: (error) => error instanceof GoogleApiError && error.status === 401, /** The mailbox's current history id; `users.history.list` replays everything after it. */ getChangeCursor: async (accessToken, _sourceConfig, syncContext): Promise => { if (syncContext?.mirrorsSourceAcls === true) { throw new Error('Company-wide Gmail indexing uses complete mailbox listings') } - const response = await fetchWithRetry(`${GMAIL_API_BASE}/profile?fields=historyId`, { - method: 'GET', - headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, - }) - if (!response.ok) throw new GmailApiError('Failed to read the Gmail profile', response.status) + const response = await fetchGoogleApiWithRetry( + 'gmail.users.getProfile', + `${GMAIL_API_BASE}/profile?fields=historyId`, + { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + } + ) const data: unknown = await response.json() if ( !isPlainRecord(data) || @@ -851,14 +837,14 @@ const gmailMailboxConnector: ConnectorConfig = { for (const type of HISTORY_TYPES) params.append('historyTypes', type) if (pageToken) params.set('pageToken', pageToken) - const response = await fetchWithRetry(`${GMAIL_API_BASE}/history?${params.toString()}`, { - method: 'GET', - headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, - }) - if (!response.ok) { - logger.warn('Failed to list Gmail history', { status: response.status }) - throw new GmailApiError('Failed to list Gmail history', response.status) - } + const response = await fetchGoogleApiWithRetry( + 'gmail.history.list', + `${GMAIL_API_BASE}/history?${params.toString()}`, + { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + } + ) const page = parseHistoryList(await response.json()) const changes = await mapWithConcurrency( @@ -881,7 +867,7 @@ const gmailMailboxConnector: ConnectorConfig = { /** Gmail answers 404 once `startHistoryId` falls outside the history it retains. */ isChangeCursorInvalidError: (error) => error instanceof InvalidGmailChangeCursorError || - (error instanceof GmailApiError && error.status === 404), + (error instanceof GoogleApiError && error.status === 404), listDocuments: async ( accessToken: string, @@ -953,7 +939,7 @@ const gmailMailboxConnector: ConnectorConfig = { maxThreads, }) - const response = await fetchWithRetry(url, { + const response = await fetchGoogleApiWithRetry('gmail.threads.list', url, { method: 'GET', signal, headers: { @@ -962,11 +948,6 @@ const gmailMailboxConnector: ConnectorConfig = { }, }) - if (!response.ok) { - logger.error('Failed to list Gmail threads', { status: response.status }) - throw new GmailApiError('Failed to list Gmail threads', response.status) - } - /** Gmail can return 204 when an empty listing has no requested metadata fields. */ const { threads, nextPageToken } = parseThreadList( response.status === 204 @@ -1102,7 +1083,8 @@ const gmailMailboxConnector: ConnectorConfig = { try { const profileUrl = `${GMAIL_API_BASE}/profile` - const profileResponse = await fetchWithRetry( + await fetchGoogleApiWithRetry( + 'gmail.users.getProfile', profileUrl, { method: 'GET', @@ -1115,10 +1097,6 @@ const gmailMailboxConnector: ConnectorConfig = { VALIDATE_RETRY_OPTIONS ) - if (!profileResponse.ok) { - return { valid: false, error: `Failed to access Gmail: ${profileResponse.status}` } - } - /** * Labels may arrive as ids (from the `gmail.labels` selector) or as names * (typed into the advanced input), so both forms are accepted here and the @@ -1128,7 +1106,8 @@ const gmailMailboxConnector: ConnectorConfig = { let labelIndex = EMPTY_LABEL_INDEX if (configuredLabels.length > 0) { const labelsUrl = `${GMAIL_API_BASE}/labels` - const labelsResponse = await fetchWithRetry( + const labelsResponse = await fetchGoogleApiWithRetry( + 'gmail.labels.list', labelsUrl, { method: 'GET', @@ -1141,10 +1120,6 @@ const gmailMailboxConnector: ConnectorConfig = { VALIDATE_RETRY_OPTIONS ) - if (!labelsResponse.ok) { - return { valid: false, error: 'Failed to fetch labels' } - } - const labels = await readLabels(labelsResponse) labelIndex = buildLabelIndex(labels) const missing = configuredLabels.filter( @@ -1171,7 +1146,8 @@ const gmailMailboxConnector: ConnectorConfig = { if (query?.trim()) { const searchQuery = buildSearchQuery(sourceConfig, labelIndex) const testUrl = `${GMAIL_API_BASE}/threads?q=${encodeURIComponent(searchQuery)}&maxResults=1` - const testResponse = await fetchWithRetry( + await fetchGoogleApiWithRetry( + 'gmail.threads.list', testUrl, { method: 'GET', @@ -1183,10 +1159,6 @@ const gmailMailboxConnector: ConnectorConfig = { }, VALIDATE_RETRY_OPTIONS ) - - if (!testResponse.ok) { - return { valid: false, error: 'Invalid search query. Check Gmail search syntax.' } - } } return { valid: true } diff --git a/apps/sim/connectors/google-calendar/google-calendar.ts b/apps/sim/connectors/google-calendar/google-calendar.ts index 52405ccb028..9847525d292 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.ts @@ -1,9 +1,9 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { z } from 'zod' -import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_EVENTS, googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' +import { fetchGoogleApiWithRetry, GoogleApiError } from '@/connectors/google-workspace/api-errors' import { getGoogleWorkspaceDocument, InvalidGoogleWorkspaceCursor, @@ -35,8 +35,6 @@ const CALENDAR_PAGE_MAX_BYTES = 16 * 1024 * 1024 const EVENT_FIELDS = 'id,status,htmlLink,created,updated,summary,description,location,creator(email,displayName),organizer(email,displayName,self),start(date,dateTime,timeZone),end(date,dateTime,timeZone),attendees(email,displayName,responseStatus,self,resource,optional),recurringEventId,eventType' -class GoogleCalendarCredentialInvalidError extends Error {} - const calendarEventTimeSchema = z.object({ date: z.string().optional(), dateTime: z.string().optional(), @@ -403,7 +401,7 @@ const userCalendarConnector: ConnectorConfig = { ...googleCalendarConnectorMeta, isListingScopeUnavailableError: isListingScopeUnavailableError, - isCredentialInvalidError: (error) => error instanceof GoogleCalendarCredentialInvalidError, + isCredentialInvalidError: (error) => error instanceof GoogleApiError && error.status === 401, listDocuments: async ( accessToken: string, @@ -504,24 +502,24 @@ const userCalendarConnector: ConnectorConfig = { hasPageToken: Boolean(pageToken), }) - const response = await fetchWithRetry(url, { - method: 'GET', - signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - if (response.status === 401) { - throw new GoogleCalendarCredentialInvalidError('Reconnect your Google Calendar account') - } + let response: Response + try { + response = await fetchGoogleApiWithRetry('calendar.events.list', url, { + method: 'GET', + signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }) + } catch (providerError) { + if (!(providerError instanceof GoogleApiError)) throw providerError logger.error('Failed to list Google Calendar events', { - status: response.status, + status: providerError.status, calendarId, + ...providerError.diagnostic, }) - const error = listingRequestError('Failed to list Google Calendar events', response.status) + const error = + providerError.status === 404 + ? listingRequestError('Failed to list Google Calendar events', providerError.status) + : providerError /** * One of several calendars a member cannot reach is absent from their * listing, not the end of it: move on to the next calendar so the rest of @@ -537,7 +535,7 @@ const userCalendarConnector: ConnectorConfig = { ) { logger.warn('Skipping a Google Calendar the member cannot reach', { calendarId, - status: response.status, + status: providerError.status, }) return calendarIndex + 1 < calendarIds.length ? { @@ -668,21 +666,17 @@ const userCalendarConnector: ConnectorConfig = { const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events/${encodeURIComponent(eventId)}?fields=${encodeURIComponent(EVENT_FIELDS)}` - const response = await fetchWithRetry(url, { - method: 'GET', - signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - if (response.status === 404 || response.status === 410) return null - if (response.status === 401) { - throw new GoogleCalendarCredentialInvalidError('Reconnect your Google Calendar account') - } - throw new Error(`Failed to get Google Calendar event: ${response.status}`) + let response: Response + try { + response = await fetchGoogleApiWithRetry('calendar.events.get', url, { + method: 'GET', + signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }) + } catch (error) { + if (error instanceof GoogleApiError && (error.status === 404 || error.status === 410)) + return null + throw error } const event = calendarEventSchema.parse(await readCalendarJson(response)) @@ -717,30 +711,28 @@ const userCalendarConnector: ConnectorConfig = { for (const calendarId of calendarIds) { const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events?maxResults=1&singleEvents=true&orderBy=startTime&timeMin=${encodeURIComponent(new Date().toISOString())}` - const response = await fetchWithRetry( - url, - { - method: 'GET', - signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', + try { + await fetchGoogleApiWithRetry( + 'calendar.events.list', + url, + { + method: 'GET', + signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, }, - }, - VALIDATE_RETRY_OPTIONS - ) - - if (!response.ok) { - if (response.status === 404) { + VALIDATE_RETRY_OPTIONS + ) + } catch (error) { + if (error instanceof GoogleApiError && error.status === 404) { return { valid: false, error: `Calendar not found: ${calendarId}. Check the calendar ID.`, } } - return { - valid: false, - error: `Failed to access Google Calendar "${calendarId}": ${response.status}`, - } + throw error } } diff --git a/apps/sim/connectors/google-drive/directory.test.ts b/apps/sim/connectors/google-drive/directory.test.ts index 58bcc4c55c3..660cd191bf3 100644 --- a/apps/sim/connectors/google-drive/directory.test.ts +++ b/apps/sim/connectors/google-drive/directory.test.ts @@ -215,6 +215,25 @@ describe('the membership a directory reports', () => { await expect(membersOf(GROUP)).rejects.toThrow() }) + it('preserves the denied nested-group operation instead of returning partial membership', async () => { + directory({ 'eng@corp.com': [USER('alice@corp.com'), NESTED('restricted@corp.com')] }) + const healthy = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (url: string) => { + if (decodeURIComponent(new URL(url).pathname).includes('/restricted@corp.com/members')) { + return jsonResponse( + { error: { errors: [{ reason: 'forbidden' }], message: 'private detail' } }, + 403 + ) + } + return healthy(url) + }) + + await expect(membersOf(GROUP)).rejects.toMatchObject({ + status: 403, + diagnostic: { operation: 'directory.members.list', reasons: ['forbidden'] }, + }) + }) + /** A directory that hiccups must not cost a group its membership; transient errors are retried. */ it('retries a transient directory error before giving up', async () => { directory({ 'eng@corp.com': [USER('alice@corp.com')] }) diff --git a/apps/sim/connectors/google-drive/directory.ts b/apps/sim/connectors/google-drive/directory.ts index 92b9ae54c54..dd2b9ec1eda 100644 --- a/apps/sim/connectors/google-drive/directory.ts +++ b/apps/sim/connectors/google-drive/directory.ts @@ -65,20 +65,27 @@ export async function validateGoogleDirectoryAccess( throw new Error('Enter a Directory administrator email to mirror Drive permissions.') } - const probe = async (path: string) => + const probe = async (path: string, operation: string) => fetchGoogleDriveWithRetry( `${DIRECTORY_BASE}/${path}`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' } }, - VALIDATE_RETRY_OPTIONS + VALIDATE_RETRY_OPTIONS, + operation ) try { - const groupsResponse = await probe('groups?customer=my_customer&maxResults=1&fields=groups(id)') + const groupsResponse = await probe( + 'groups?customer=my_customer&maxResults=1&fields=groups(id)', + 'directory.groups.list' + ) const groups = (await groupsResponse.json()) as { groups?: { id?: string }[] } - await probe('customer/my_customer/domains?fields=domains(domainName)') + await probe('customer/my_customer/domains?fields=domains(domainName)', 'directory.domains.list') const groupId = groups.groups?.[0]?.id if (groupId) { - await probe(`groups/${encodeURIComponent(groupId)}/members?maxResults=1&fields=members(id)`) + await probe( + `groups/${encodeURIComponent(groupId)}/members?maxResults=1&fields=members(id)`, + 'directory.members.list' + ) } } catch (error) { const guidance = @@ -96,15 +103,20 @@ export async function validateGoogleDirectoryAccess( } } -function directoryFetch(url: string, accessToken: string): Promise { - return fetchGoogleDriveWithRetry(url, { - method: 'GET', - headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, - }) +function directoryFetch(url: string, accessToken: string, operation: string): Promise { + return fetchGoogleDriveWithRetry( + url, + { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }, + {}, + operation + ) } async function getJson(url: string, accessToken: string): Promise { - const response = await directoryFetch(url, accessToken) + const response = await directoryFetch(url, accessToken, 'directory.domains.list') return (await response.json()) as T } @@ -128,7 +140,7 @@ async function listAll( if (pageToken) query.set('pageToken', pageToken) return `${url}?${query.toString()}` }, - fetch: (pageUrl) => directoryFetch(pageUrl, accessToken), + fetch: (pageUrl) => directoryFetch(pageUrl, accessToken, `directory.${itemsKey}.list`), parseError: (response) => response.json().catch(() => null), getItems: (body) => body[itemsKey] as T[] | undefined, getNextPageToken: (body) => body.nextPageToken as string | undefined, @@ -296,6 +308,13 @@ export function openGoogleDirectory( return members } catch (error) { const failure = toError(error) + if (failure instanceof GoogleDriveApiError) { + logger.warn('Failed to read Google group membership', { + groupId, + status: failure.status, + ...failure.diagnostic, + }) + } directMembers.set(groupId, failure) throw failure } diff --git a/apps/sim/connectors/google-drive/google-drive-errors.ts b/apps/sim/connectors/google-drive/google-drive-errors.ts index db9c993b5e5..86a582a68b2 100644 --- a/apps/sim/connectors/google-drive/google-drive-errors.ts +++ b/apps/sim/connectors/google-drive/google-drive-errors.ts @@ -5,13 +5,15 @@ import { resolveRetryDelayMs, retryWithExponentialBackoff, } from '@/lib/knowledge/documents/utils' +import { + readGoogleErrorReasons, + safeGoogleErrorReasons, +} from '@/connectors/google-workspace/api-errors' import { ConnectorSourceError, type ConnectorSourceFailureCategory, } from '@/connectors/source-error' -import { readBodyWithLimit } from '@/connectors/utils' -const GOOGLE_ERROR_BODY_MAX_BYTES = 64 * 1024 const GOOGLE_ERROR_REASON_MAX_COUNT = 16 const EXPORT_TOO_LARGE_REASONS = new Set(['exportSizeLimitExceeded']) @@ -52,50 +54,6 @@ export type GoogleDriveErrorKind = | 'unknown' | 'unsupported_export' -interface GoogleErrorEntry { - reason?: string -} - -interface ParsedGoogleErrorBody { - error?: { - errors?: GoogleErrorEntry[] - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null -} - -function optionalString(value: unknown): string | undefined { - return typeof value === 'string' && value.trim() ? value.trim() : undefined -} - -function parseErrorBody(value: unknown): ParsedGoogleErrorBody | undefined { - if (!isRecord(value) || !isRecord(value.error)) return undefined - - const entries = Array.isArray(value.error.errors) - ? value.error.errors.flatMap((entry): GoogleErrorEntry[] => { - if (!isRecord(entry)) return [] - return [ - { - reason: optionalString(entry.reason), - }, - ] - }) - : undefined - - return { - error: { - errors: entries, - }, - } -} - -function normalizeReason(reason: string): string | undefined { - const normalized = reason.trim() - return /^[A-Za-z][A-Za-z0-9_.-]{0,99}$/.test(normalized) ? normalized : undefined -} - function classifyGoogleDriveError( status: number, reasons: readonly string[] @@ -147,14 +105,18 @@ export class GoogleDriveApiError extends ConnectorSourceError { readonly kind: GoogleDriveErrorKind readonly rateLimited: boolean - constructor(status: number, normalizedReasons: readonly string[]) { - const diagnosticReasons = normalizedReasons.slice(0, GOOGLE_ERROR_REASON_MAX_COUNT) + constructor(status: number, normalizedReasons: readonly string[], operation = 'drive.request') { + const diagnosticReasons = safeGoogleErrorReasons(normalizedReasons).slice( + 0, + GOOGLE_ERROR_REASON_MAX_COUNT + ) const reasonSuffix = diagnosticReasons.length > 0 ? ` (${diagnosticReasons.join(', ')})` : '' const kind = classifyGoogleDriveError(status, normalizedReasons) super( `Google Drive API request failed with HTTP ${status}${reasonSuffix}`, status, - diagnosticCategory(kind, status) + diagnosticCategory(kind, status), + { operation, reasons: diagnosticReasons } ) this.name = 'GoogleDriveApiError' this.reasons = diagnosticReasons @@ -169,24 +131,11 @@ export class GoogleDriveApiError extends ConnectorSourceError { * response body. Error payloads are byte-bounded, free-form provider messages * are omitted, and only validated machine-readable reason tokens survive. */ -export async function readGoogleDriveApiError(response: Response): Promise { - const body = await readBodyWithLimit(response, GOOGLE_ERROR_BODY_MAX_BYTES).catch(() => null) - let parsedBody: ParsedGoogleErrorBody | undefined - - if (body) { - try { - parsedBody = parseErrorBody(JSON.parse(body.toString('utf8'))) - } catch { - parsedBody = undefined - } - } - - const entries = parsedBody?.error?.errors ?? [] - const rawReasons = [...new Set(entries.flatMap((entry) => (entry.reason ? [entry.reason] : [])))] - const normalizedReasons = [ - ...new Set(rawReasons.flatMap((reason) => normalizeReason(reason) ?? [])), - ] - return new GoogleDriveApiError(response.status, normalizedReasons) +export async function readGoogleDriveApiError( + response: Response, + operation = 'drive.request' +): Promise { + return new GoogleDriveApiError(response.status, await readGoogleErrorReasons(response), operation) } /** @@ -198,14 +147,15 @@ export async function readGoogleDriveApiError(response: Response): Promise { return retryWithExponentialBackoff( async () => { const response = await fetch(url, options) if (response.ok) return response - const error = await readGoogleDriveApiError(response) + const error = await readGoogleDriveApiError(response, operation) attachRetryHeaders(error, response.headers) const waitMs = resolveRetryDelayMs(response.headers) if (waitMs !== undefined) error.retryAfterMs = waitMs diff --git a/apps/sim/connectors/google-drive/google-drive.test.ts b/apps/sim/connectors/google-drive/google-drive.test.ts index 06444b02cf4..f86a0b210cd 100644 --- a/apps/sim/connectors/google-drive/google-drive.test.ts +++ b/apps/sim/connectors/google-drive/google-drive.test.ts @@ -436,6 +436,7 @@ describe('Google Drive recursive folders and raw files', () => { .mockResolvedValueOnce(driveErrorResponse('insufficientFilePermissions', 'No download')) await expect(googleDriveConnector.getDocument('token', {}, FILE_ID)).rejects.toMatchObject({ kind: 'permission', + diagnostic: { operation: 'drive.files.get', reasons: ['insufficientFilePermissions'] }, }) }) }) @@ -512,7 +513,7 @@ describe('Google Drive API error parsing', () => { ) ) - expect(error.reasons).toEqual(reasons.slice(0, 16)) + expect(error.reasons).toEqual(['userRateLimitExceeded']) expect(error.kind).toBe('transient') expect(error.rateLimited).toBe(true) }) @@ -754,7 +755,8 @@ describe('Google Drive export failures', () => { name: 'GoogleDriveApiError', status: 403, kind: 'unknown', - reasons: ['newGoogleReason'], + reasons: [], + diagnostic: { operation: 'drive.files.export', reasons: [] }, }) }) diff --git a/apps/sim/connectors/google-drive/google-drive.ts b/apps/sim/connectors/google-drive/google-drive.ts index 88d9413f5a4..2d1539b7171 100644 --- a/apps/sim/connectors/google-drive/google-drive.ts +++ b/apps/sim/connectors/google-drive/google-drive.ts @@ -115,6 +115,7 @@ function googleDriveErrorLogFields(error: unknown): Record { error: error.message, status: error.status, reasons: error.reasons, + operation: error.diagnostic?.operation, } } return { error: toError(error).message } @@ -166,10 +167,12 @@ async function exportGoogleWorkspaceFile( let response: Response try { - response = await fetchGoogleDriveWithRetry(url, { - method: 'GET', - headers: driveRequestHeaders(accessToken, fileId, resourceKey), - }) + response = await fetchGoogleDriveWithRetry( + url, + { method: 'GET', headers: driveRequestHeaders(accessToken, fileId, resourceKey) }, + {}, + 'drive.files.export' + ) } catch (error) { if (error instanceof GoogleDriveApiError && error.kind === 'export_too_large') { throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE) @@ -194,10 +197,12 @@ async function downloadFile( // metadata fetch in getDocument already does. (`files.export` takes no such param.) const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media&supportsAllDrives=true` - const response = await fetchGoogleDriveWithRetry(url, { - method: 'GET', - headers: driveRequestHeaders(accessToken, fileId, resourceKey), - }) + const response = await fetchGoogleDriveWithRetry( + url, + { method: 'GET', headers: driveRequestHeaders(accessToken, fileId, resourceKey) }, + {}, + 'drive.files.get' + ) // Stream with a hard byte cap so a file with missing/under-reported listing // size metadata is never fully buffered into memory. Oversized files raise @@ -610,10 +615,12 @@ async function listFilePermissions( return `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}/permissions?${query.toString()}` }, fetch: (url) => - fetchGoogleDriveWithRetry(url, { - method: 'GET', - headers: driveRequestHeaders(accessToken, fileId, resourceKey), - }), + fetchGoogleDriveWithRetry( + url, + { method: 'GET', headers: driveRequestHeaders(accessToken, fileId, resourceKey) }, + {}, + 'drive.permissions.list' + ), parseError: (response) => response.json().catch(() => null), getItems: (body) => body.permissions, getNextPageToken: (body) => body.nextPageToken, @@ -755,7 +762,9 @@ async function readDriveFile( const fields = `${DRIVE_FILE_FIELDS}${permissions ? `,permissions(${DRIVE_PERMISSION_FIELDS})` : ''}` const response = await fetchGoogleDriveWithRetry( `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`, - { method: 'GET', headers: driveRequestHeaders(accessToken, fileId, resourceKey) } + { method: 'GET', headers: driveRequestHeaders(accessToken, fileId, resourceKey) }, + {}, + 'drive.files.get' ) return parseDriveFileMetadata(await readDriveJson(response, DRIVE_METADATA_MAX_BYTES), fileId) } @@ -942,7 +951,9 @@ async function listShortcutChanges( { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, - } + }, + {}, + 'drive.files.list' ) const page = parseDriveFileListResponse(await readDriveJson(response, DRIVE_PAGE_MAX_BYTES)) if (page.incompleteSearch) throw new Error('Google Drive shortcut search was incomplete') @@ -1114,14 +1125,16 @@ const listGoogleDriveDocuments: ConnectorConfig['listDocuments'] = async ( let response: Response try { - response = await fetchGoogleDriveWithRetry(url, { - method: 'GET', - signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', + response = await fetchGoogleDriveWithRetry( + url, + { + method: 'GET', + signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, }, - }) + {}, + 'drive.files.list' + ) } catch (error) { if ( (traversal || sharedDriveId) && @@ -1327,7 +1340,8 @@ export const googleDriveConnector: ConnectorConfig = { await fetchGoogleDriveWithRetry( 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)&corpora=user&supportsAllDrives=true&includeItemsFromAllDrives=true', { headers: { Authorization: `Bearer ${sampleToken}`, Accept: 'application/json' } }, - VALIDATE_RETRY_OPTIONS + VALIDATE_RETRY_OPTIONS, + 'drive.files.list' ) } return { valid: true } @@ -1347,7 +1361,8 @@ export const googleDriveConnector: ConnectorConfig = { Accept: 'application/json', }, }, - VALIDATE_RETRY_OPTIONS + VALIDATE_RETRY_OPTIONS, + 'drive.files.get' ) } catch (error) { if (error instanceof GoogleDriveApiError) { @@ -1383,7 +1398,8 @@ export const googleDriveConnector: ConnectorConfig = { Accept: 'application/json', }, }, - VALIDATE_RETRY_OPTIONS + VALIDATE_RETRY_OPTIONS, + 'drive.files.list' ) } catch (error) { if (error instanceof GoogleDriveApiError) { @@ -1434,10 +1450,15 @@ export const googleDriveConnector: ConnectorConfig = { getChangeCursor: async (accessToken: string): Promise => { const url = 'https://www.googleapis.com/drive/v3/changes/startPageToken?supportsAllDrives=true' - const response = await fetchGoogleDriveWithRetry(url, { - method: 'GET', - headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, - }) + const response = await fetchGoogleDriveWithRetry( + url, + { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }, + {}, + 'drive.changes.getStartPageToken' + ) const data: unknown = await response.json() if ( !isPlainRecord(data) || @@ -1481,10 +1502,15 @@ export const googleDriveConnector: ConnectorConfig = { let response: Response try { - response = await fetchGoogleDriveWithRetry(url, { - method: 'GET', - headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, - }) + response = await fetchGoogleDriveWithRetry( + url, + { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }, + {}, + 'drive.changes.list' + ) } catch (error) { logger.error('Failed to list Google Drive changes', googleDriveErrorLogFields(error)) throw error diff --git a/apps/sim/connectors/google-drive/workspace-drives.ts b/apps/sim/connectors/google-drive/workspace-drives.ts index 6722a552edf..4a45d30b745 100644 --- a/apps/sim/connectors/google-drive/workspace-drives.ts +++ b/apps/sim/connectors/google-drive/workspace-drives.ts @@ -41,7 +41,9 @@ export async function listGoogleWorkspaceDrives( { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, signal, - } + }, + {}, + 'drive.drives.list' ) const body = await readBodyWithLimit(response, SHARED_DRIVES_PAGE_MAX_BYTES) if (!body) throw new Error('Google Drive shared-drive metadata exceeded its size limit') diff --git a/apps/sim/connectors/google-workspace/api-errors.test.ts b/apps/sim/connectors/google-workspace/api-errors.test.ts new file mode 100644 index 00000000000..f6e4ace1b8f --- /dev/null +++ b/apps/sim/connectors/google-workspace/api-errors.test.ts @@ -0,0 +1,147 @@ +/** @vitest-environment node */ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' +import { + fetchGoogleApiWithRetry, + readGoogleApiError, +} from '@/connectors/google-workspace/api-errors' + +const OPERATION = 'gmail.messages.attachments.get' +const RESPONSE_SECRET = 'private-customer-data' +function failure(status: number, reason: string, headers?: Record): Response { + return Response.json( + { error: { message: RESPONSE_SECRET, errors: [{ reason }] } }, + { status, headers } + ) +} + +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +describe('Google API diagnostics', () => { + it.each([ + [400, 'badRequest', 'request_rejected'], + [403, 'forbidden', 'authorization'], + [403, 'userRateLimitExceeded', 'rate_limit'], + [500, 'backendError', 'provider_unavailable'], + ] as const)('retains safe %s %s evidence through wrapping', async (status, reason, category) => { + const error = await readGoogleApiError(failure(status, reason), OPERATION) + const diagnostic = getConnectorFailureDiagnostic( + new Error('outer private detail', { cause: error }) + ) + expect(diagnostic).toMatchObject({ status, category, operation: OPERATION, reasons: [reason] }) + expect(JSON.stringify(diagnostic)).not.toContain(RESPONSE_SECRET) + expect(JSON.stringify(diagnostic)).not.toContain('outer private detail') + }) + + it('omits unknown reason tokens even when they look like machine codes', async () => { + const error = await readGoogleApiError(failure(403, RESPONSE_SECRET), OPERATION) + expect(error.diagnostic?.reasons).toEqual([]) + expect(JSON.stringify(error)).not.toContain(RESPONSE_SECRET) + }) + + it('reads structured ErrorInfo without its sensitive metadata', async () => { + const error = await readGoogleApiError( + Response.json( + { + error: { + details: [ + { + '@type': 'type.googleapis.com/google.rpc.ErrorInfo', + reason: 'SERVICE_DISABLED', + metadata: { credential: RESPONSE_SECRET }, + }, + ], + }, + }, + { status: 403 } + ), + 'calendar.events.list' + ) + expect(error.diagnostic?.reasons).toEqual(['SERVICE_DISABLED']) + expect(JSON.stringify(error)).not.toContain(RESPONSE_SECRET) + }) + + it.each(['not-json', 'x'.repeat(65 * 1024)])( + 'retains status for malformed or oversized bodies', + async (body) => { + const error = await readGoogleApiError(new Response(body, { status: 400 }), OPERATION) + expect(error.status).toBe(400) + expect(error.diagnostic?.reasons).toEqual([]) + } + ) +}) + +describe('Google API retries', () => { + it('preserves the caller admission hook and diagnoses its response', async () => { + const fetch = vi.fn().mockResolvedValueOnce(failure(403, 'forbidden')) + const fetcher = vi.fn( + (input: RequestInfo | URL, init: RequestInit, transport: typeof globalThis.fetch) => + transport(input, init) + ) + vi.stubGlobal('fetch', fetch) + await expect( + fetchGoogleApiWithRetry(OPERATION, 'https://gmail.googleapis.com/example', {}, { fetcher }) + ).rejects.toMatchObject({ + status: 403, + diagnostic: { operation: OPERATION, reasons: ['forbidden'] }, + }) + expect(fetcher).toHaveBeenCalledTimes(1) + expect(fetch).toHaveBeenCalledTimes(1) + expect(fetcher.mock.calls[0]?.[1].signal).toBeInstanceOf(AbortSignal) + }) + + it.each([ + [500, 'backendError'], + [403, 'rateLimitExceeded'], + [429, 'userRateLimitExceeded'], + ] as const)('retries %s %s through the bounded transport', async (status, reason) => { + vi.useFakeTimers() + const fetch = vi + .fn() + .mockResolvedValueOnce(failure(status, reason)) + .mockResolvedValueOnce(Response.json({ ok: true })) + vi.stubGlobal('fetch', fetch) + const request = fetchGoogleApiWithRetry( + OPERATION, + 'https://gmail.googleapis.com/example', + {}, + { maxRetries: 1, initialDelayMs: 1 } + ) + const checked = expect(request).resolves.toMatchObject({ status: 200 }) + await vi.runAllTimersAsync() + await checked + expect(fetch).toHaveBeenCalledTimes(2) + }) + + it.each([ + [400, 'failedPrecondition'], + [403, 'forbidden'], + [404, 'notFound'], + ] as const)('does not retry or suppress %s %s', async (status, reason) => { + const fetch = vi.fn().mockResolvedValueOnce(failure(status, reason)) + vi.stubGlobal('fetch', fetch) + await expect( + fetchGoogleApiWithRetry(OPERATION, 'https://gmail.googleapis.com/example', {}) + ).rejects.toMatchObject({ status, diagnostic: { reasons: [reason] } }) + expect(fetch).toHaveBeenCalledTimes(1) + }) + + it('retains Retry-After without exceeding the caller retry budget', async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(failure(429, 'rateLimitExceeded', { 'Retry-After': '300' })) + vi.stubGlobal('fetch', fetch) + await expect( + fetchGoogleApiWithRetry( + OPERATION, + 'https://gmail.googleapis.com/example', + {}, + { retryBudgetMs: 1000 } + ) + ).rejects.toMatchObject({ status: 429, retryAfterMs: 300000 }) + expect(fetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/connectors/google-workspace/api-errors.ts b/apps/sim/connectors/google-workspace/api-errors.ts new file mode 100644 index 00000000000..12653fd37c8 --- /dev/null +++ b/apps/sim/connectors/google-workspace/api-errors.ts @@ -0,0 +1,150 @@ +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { + attachRetryHeaders, + isRetryableError, + type RetryOptions, + resolveRetryDelayMs, +} from '@/lib/knowledge/documents/utils' +import { ConnectorSourceError } from '@/connectors/source-error' +import { readBodyWithLimit } from '@/connectors/utils' + +const ERROR_BODY_MAX_BYTES = 64 * 1024 +const MAX_REASONS = 16 +/** Only known provider codes may reach logs; messages and unknown strings may contain data. */ +const SAFE_REASONS = new Set([ + 'accessNotConfigured', + 'appNotAuthorizedToFile', + 'authError', + 'backendError', + 'badRequest', + 'cannotDownloadFile', + 'cannotExportFile', + 'dailyLimitExceeded', + 'domainPolicy', + 'download_restricted_for_revision', + 'exportSizeLimitExceeded', + 'failedPrecondition', + 'fileNotDownloadable', + 'fileNotExportable', + 'forbidden', + 'insufficientFilePermissions', + 'insufficientPermissions', + 'internalError', + 'invalid', + 'invalidArgument', + 'notFound', + 'quotaExceeded', + 'rateLimitExceeded', + 'required', + 'serviceDisabled', + 'sharingRateLimitExceeded', + 'teamDriveMembershipRequired', + 'userRateLimitExceeded', + 'ACCESS_TOKEN_SCOPE_INSUFFICIENT', + 'API_KEY_SERVICE_BLOCKED', + 'SERVICE_DISABLED', + 'RATE_LIMIT_EXCEEDED', + 'USER_PROJECT_DENIED', +]) +const RATE_LIMIT_REASONS = new Set([ + 'rateLimitExceeded', + 'userRateLimitExceeded', + 'sharingRateLimitExceeded', + 'RATE_LIMIT_EXCEEDED', +]) + +export function safeGoogleErrorReasons(reasons: readonly string[]): string[] { + return [...new Set(reasons.filter((reason) => SAFE_REASONS.has(reason)))] +} + +/** Reads the bounded Google envelope without retaining provider messages or request data. */ +export async function readGoogleErrorReasons(response: Response): Promise { + const body = await readBodyWithLimit(response, ERROR_BODY_MAX_BYTES).catch(() => null) + if (!body) return [] + try { + const payload: unknown = JSON.parse(body.toString('utf8')) + if (!payload || typeof payload !== 'object' || !('error' in payload)) return [] + const error = payload.error + if (!error || typeof error !== 'object') return [] + const entries = [ + ...('errors' in error && Array.isArray(error.errors) ? error.errors : []), + ...('details' in error && Array.isArray(error.details) ? error.details : []), + ] + return safeGoogleErrorReasons( + entries.flatMap((entry: unknown) => + entry && typeof entry === 'object' && 'reason' in entry && typeof entry.reason === 'string' + ? [entry.reason] + : [] + ) + ) + } catch { + return [] + } +} + +export class GoogleApiError extends ConnectorSourceError { + readonly rateLimited: boolean + retryAfterMs?: number + constructor(operation: string, status: number, reasons: readonly string[]) { + const safeReasons = safeGoogleErrorReasons(reasons) + const suffix = safeReasons.length ? ` (${safeReasons.join(', ')})` : '' + const category = + status === 429 || + safeReasons.some( + (reason) => + RATE_LIMIT_REASONS.has(reason) || + reason === 'dailyLimitExceeded' || + reason === 'quotaExceeded' + ) + ? 'rate_limit' + : status >= 500 + ? 'provider_unavailable' + : undefined + super(`${operation} failed (HTTP ${status})${suffix}.`, status, category, { + operation, + reasons: safeReasons.slice(0, MAX_REASONS), + }) + this.name = 'GoogleApiError' + this.rateLimited = + status === 429 || safeReasons.some((reason) => RATE_LIMIT_REASONS.has(reason)) + } +} + +export async function readGoogleApiError( + response: Response, + operation: string +): Promise { + return new GoogleApiError(operation, response.status, await readGoogleErrorReasons(response)) +} + +/** Preserves Google diagnostics when the shared transport retries a transient HTTP response. */ +function googleApiRetryOptions(operation: string, options: RetryOptions = {}): RetryOptions { + return { + ...options, + fetcher: async (input, init, transport) => { + const response = options.fetcher + ? await options.fetcher(input, init, transport) + : await transport(input, init) + if (!response.ok) { + const error = await readGoogleApiError(response, operation) + attachRetryHeaders(error, response.headers) + error.retryAfterMs = resolveRetryDelayMs(response.headers) + throw error + } + return response + }, + retryCondition: (error) => + error instanceof GoogleApiError && (error.status >= 500 || error.rateLimited) + ? true + : (options.retryCondition?.(error) ?? isRetryableError(error)), + } +} + +export function fetchGoogleApiWithRetry( + operation: string, + url: string, + options: RequestInit, + retryOptions: RetryOptions = {} +): Promise { + return fetchWithRetry(url, options, googleApiRetryOptions(operation, retryOptions)) +} diff --git a/apps/sim/connectors/google-workspace/users.ts b/apps/sim/connectors/google-workspace/users.ts index 762b43b1b2c..f5c2baf3204 100644 --- a/apps/sim/connectors/google-workspace/users.ts +++ b/apps/sim/connectors/google-workspace/users.ts @@ -110,7 +110,8 @@ export async function listGoogleWorkspaceUsers( headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, signal, }, - options.validate ? VALIDATE_RETRY_OPTIONS : undefined + options.validate ? VALIDATE_RETRY_OPTIONS : undefined, + 'directory.users.list' ) const data = await readDirectoryJson(response) if ( @@ -154,7 +155,8 @@ export async function getGoogleWorkspaceUser( headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, signal: options.signal, }, - options.validate ? VALIDATE_RETRY_OPTIONS : undefined + options.validate ? VALIDATE_RETRY_OPTIONS : undefined, + 'directory.users.get' ) return parseUser(await readDirectoryJson(response)) } catch (error) { diff --git a/apps/sim/connectors/source-error.ts b/apps/sim/connectors/source-error.ts index dd4745021d5..09e0c854e7f 100644 --- a/apps/sim/connectors/source-error.ts +++ b/apps/sim/connectors/source-error.ts @@ -11,9 +11,13 @@ export class ConnectorSourceError extends Error { constructor( message: string, readonly status: number, - readonly category?: ConnectorSourceFailureCategory + readonly category?: ConnectorSourceFailureCategory, + readonly diagnostic?: { operation: string; reasons: readonly string[] } ) { super(message) this.name = 'ConnectorSourceError' } } + +/** Keeps directory failures distinct from document-content failures through cause wrapping. */ +export class ConnectorDirectoryError extends Error {} diff --git a/apps/sim/lib/knowledge/connectors/connector-error.test.ts b/apps/sim/lib/knowledge/connectors/connector-error.test.ts index a1d52f97540..69a995c95e6 100644 --- a/apps/sim/lib/knowledge/connectors/connector-error.test.ts +++ b/apps/sim/lib/knowledge/connectors/connector-error.test.ts @@ -3,6 +3,7 @@ import { DrizzleQueryError } from 'drizzle-orm/errors' import { describe, expect, it } from 'vitest' import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' import { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' +import { ConnectorDirectoryError } from '@/connectors/source-error' describe('connector failure diagnostics', () => { it('retains the SQLSTATE while discarding SQL, bound values and driver detail', () => { @@ -85,6 +86,34 @@ describe('connector failure diagnostics', () => { expect(getConnectorFailureDiagnostic(error)?.message).not.toContain('access was denied') }) + it('reports a wrapped group-membership failure without suggesting file download permissions', () => { + const error = new Error('private outer message', { + cause: new ConnectorDirectoryError('private group detail', { + cause: new GoogleDriveApiError(403, ['forbidden'], 'directory.members.list'), + }), + }) + const diagnostic = getConnectorFailureDiagnostic(error) + expect(diagnostic).toMatchObject({ + phase: 'directory', + status: 403, + operation: 'directory.members.list', + reasons: ['forbidden'], + }) + expect(diagnostic?.message).toContain('Directory permission sync failed') + expect(diagnostic?.message).not.toContain('file access') + expect(JSON.stringify(diagnostic)).not.toContain('private') + }) + + it('keeps directory context when the failure has no HTTP status', () => { + expect( + getConnectorFailureDiagnostic(new ConnectorDirectoryError('private directory details')) + ).toMatchObject({ + category: 'directory', + phase: 'directory', + message: expect.stringContaining('Directory permission sync failed'), + }) + }) + it('does not infer status or permanence from a free-form message', () => { expect(getConnectorFailureDiagnostic(new Error('HTTP 403 permission denied'))).toBeNull() expect( diff --git a/apps/sim/lib/knowledge/connectors/connector-error.ts b/apps/sim/lib/knowledge/connectors/connector-error.ts index 7f8e4ac5179..6c7c09b3cc6 100644 --- a/apps/sim/lib/knowledge/connectors/connector-error.ts +++ b/apps/sim/lib/knowledge/connectors/connector-error.ts @@ -1,15 +1,19 @@ import { findCause, getPostgresErrorCode } from '@sim/utils/errors' import { DrizzleQueryError } from 'drizzle-orm/errors' import { + ConnectorDirectoryError, ConnectorSourceError, type ConnectorSourceFailureCategory, } from '@/connectors/source-error' export interface ConnectorFailureDiagnostic { - category: 'database' | ConnectorSourceFailureCategory | 'transport' + category: 'directory' | 'database' | ConnectorSourceFailureCategory | 'transport' message: string status?: number code?: string + operation?: string + reasons?: readonly string[] + phase?: 'directory' } const TRANSPORT_CODES = new Set([ @@ -30,7 +34,7 @@ const TRANSPORT_CODES = new Set([ * SQL, bound parameters, URLs and arbitrary exception messages never enter the * result. Unknown failures retain the caller's domain-specific fallback. */ -export function getConnectorFailureDiagnostic(error: unknown): ConnectorFailureDiagnostic | null { +function classifyFailure(error: unknown): ConnectorFailureDiagnostic | null { const code = getPostgresErrorCode(error) const databaseError = findCause( error, @@ -106,3 +110,35 @@ export function getConnectorFailureDiagnostic(error: unknown): ConnectorFailureD message: `Source content request was rejected (HTTP ${status}). Check the source's download restrictions and supported content.`, } } + +/** Preserves safe provider context and directory scope across wrapped failures. */ +export function getConnectorFailureDiagnostic(error: unknown): ConnectorFailureDiagnostic | null { + const diagnostic = classifyFailure(error) + const directoryError = findCause( + error, + (value): value is ConnectorDirectoryError => value instanceof ConnectorDirectoryError + ) + const sourceError = findCause( + error, + (value): value is ConnectorSourceError => value instanceof ConnectorSourceError + ) + const context = sourceError?.diagnostic + if (directoryError) { + const status = diagnostic?.status ? ` (HTTP ${diagnostic.status})` : '' + const reason = context?.reasons.length ? ` Google reason: ${context.reasons.join(', ')}.` : '' + return { + ...diagnostic, + ...context, + category: diagnostic?.category ?? 'directory', + phase: 'directory', + message: `Directory permission sync failed${status}.${context ? ` Operation: ${context.operation}.` : ''}${reason} Group membership could not be fully verified.`, + } + } + if (!diagnostic || !context) return diagnostic + const reason = context.reasons.length ? ` Google reason: ${context.reasons.join(', ')}.` : '' + return { + ...diagnostic, + ...context, + message: `Google request failed (HTTP ${diagnostic.status}). Operation: ${context.operation}.${reason}`, + } +} diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts index 8dd68d764c0..90ae45c85c4 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts @@ -3,6 +3,7 @@ */ import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' import type { ConnectorDirectory } from '@/connectors/types' const { mockResolveTokenUserId, mockResolveToken, mockOpenDirectory, mockAvailability } = @@ -295,6 +296,11 @@ describe('refreshConnectorDirectory', () => { syncContext: {}, accessToken: 'token', }).catch((error: unknown) => error) + expect(getConnectorFailureDiagnostic(failure)).toMatchObject({ + phase: 'directory', + status: 429, + category: 'rate_limit', + }) expect(getRetryAfterMs(failure)).toBe(60_000) expect(isRateLimitError(failure)).toBe(true) }) diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.ts index 221c6802483..5fbfc7c0bff 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.ts @@ -31,6 +31,7 @@ import { import { RUNNABLE_CONNECTOR_STATUSES } from '@/lib/knowledge/connectors/sync-lock' import { isRateLimitError } from '@/lib/knowledge/documents/utils' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' +import { ConnectorDirectoryError } from '@/connectors/source-error' import type { ConnectorConfig, ConnectorDirectory, @@ -382,7 +383,9 @@ export async function refreshMirroredDirectory(input: { connector: connectorConfig.id, error: getErrorMessage(error), }) - throw new Error(`${DIRECTORY_ERROR_PREFIX}${getErrorMessage(error)}`, { cause: error }) + throw new ConnectorDirectoryError(`${DIRECTORY_ERROR_PREFIX}${getErrorMessage(error)}`, { + cause: error, + }) } } diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 24b77c321fa..0b61147160b 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -11,6 +11,7 @@ import { schemaMock, setEnvFlags, } from '@sim/testing' +import { DrizzleQueryError } from 'drizzle-orm/errors' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { @@ -693,6 +694,51 @@ describe('processDocumentAsync write guards', () => { expect(guardForStatusWrite('failed')).toBeDefined() }) + it('stores bounded database diagnostics while retaining the original error for retry classification', async () => { + dbChainMockFns.limit + .mockResolvedValueOnce([PERSISTED_CONTEXT]) + .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW]) + .mockResolvedValueOnce([{ id: 'document-1' }]) + mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING]) + mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue( + new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]]) + ) + const databaseError = new DrizzleQueryError( + 'insert private SQL', + ['private bound content'], + Object.assign(new Error('private driver detail'), { code: '57014' }) + ) + mockProcessDocument.mockRejectedValueOnce(databaseError) + const onClaimed = vi.fn() + + await expect( + processDocumentAsync( + 'knowledge-base-1', + 'document-1', + { + filename: 'a.pdf', + fileUrl: 'https://example.com/a.pdf', + fileSize: 1, + mimeType: 'text/plain', + }, + {}, + BILLING_ATTRIBUTION, + 'request-1', + { chargedAtDispatch: true, onClaimed } + ) + ).rejects.toBe(databaseError) + + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ + processingStatus: 'failed', + processingError: 'Database request failed (SQLSTATE 57014).', + }) + ) + expect(onClaimed).toHaveBeenCalledTimes(1) + expect(guardForStatusWrite('processing')).toBeDefined() + expect(guardForStatusWrite('failed')).toBeDefined() + }) + it('accepts a legacy queuedAt-only payload only while the row has no token', async () => { dbChainMockFns.limit .mockResolvedValueOnce([PERSISTED_CONTEXT]) diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 4b5d9cf2761..c0248eb49f3 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -80,6 +80,7 @@ import { MAX_KNOWLEDGE_ACCESS_CANDIDATES, SYSTEM_ACCESS_SCOPE, } from '@/lib/knowledge/access/types' +import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' import { documentConnectorIsActive } from '@/lib/knowledge/documents/connector-lifecycle' import { @@ -2060,15 +2061,19 @@ export async function processDocumentAsync( const providerContinuationExhausted = recordedError instanceof ProviderCapacityContinuationExhaustedError const quotaContinuationFailed = quotaContinuationAttempted && !deferredUntil + const failureDiagnostic = getConnectorFailureDiagnostic(recordedError) const errorMessage = byokCredentialRejected ? BYOK_EMBEDDING_CREDENTIAL_REJECTION_MESSAGE : embeddingQuotaExhausted ? quotaContinuationFailed ? getErrorMessage(recordedError, 'Embedding quota continuation dispatch failed') : EMBEDDING_QUOTA_EXHAUSTED_MESSAGE - : getErrorMessage(recordedError, 'Unknown error') + : failureDiagnostic?.category === 'database' + ? failureDiagnostic.message + : getErrorMessage(recordedError, 'Unknown error') const logContext = { errorType: toError(recordedError).name, + ...(failureDiagnostic ? { diagnostic: failureDiagnostic } : {}), knowledgeBaseId, mimeType: docData.mimeType, fileSize: docData.fileSize, From 51c655e5e5c91161747b2e7703ae0b5982c24dca Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 16 Sep 2026 15:38:01 -0700 Subject: [PATCH 2/2] fix(search): sanitize in-process database failure logs --- .../connectors/gmail/company-crawl.test.ts | 12 ++++--- .../document-processing-source.test.ts | 35 +++++++++++++++++++ apps/sim/lib/knowledge/documents/service.ts | 4 ++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/apps/sim/connectors/gmail/company-crawl.test.ts b/apps/sim/connectors/gmail/company-crawl.test.ts index 5030a0f4948..2bf06db0956 100644 --- a/apps/sim/connectors/gmail/company-crawl.test.ts +++ b/apps/sim/connectors/gmail/company-crawl.test.ts @@ -9,8 +9,13 @@ const { fetchProvider, listUsers, getUser } = vi.hoisted(() => ({ getUser: vi.fn(), })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) -vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: fetchProvider })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ + fetchWithRetry: ( + url: string, + init: RequestInit, + options?: import('@/lib/knowledge/documents/utils').RetryOptions + ) => (options?.fetcher ? options.fetcher(url, init, fetchProvider) : fetchProvider(url, init)), +})) vi.mock('@/components/icons', () => ({ GmailIcon: () => null })) vi.mock('@/connectors/google-workspace/users', () => ({ GOOGLE_WORKSPACE_USERS_PAGE_SIZE: 100, @@ -180,8 +185,7 @@ describe('company-wide Gmail indexing', () => { ).resolves.toEqual({ valid: true }) expect(fetchProvider).toHaveBeenCalledWith( expect.stringContaining('/profile'), - expect.objectContaining({ signal: controller.signal }), - expect.any(Object) + expect.objectContaining({ signal: controller.signal }) ) }) diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 0b61147160b..a84e8fa96d5 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -21,6 +21,7 @@ const { mockGetBoundWorkspaceFileSecretProvenanceByMetadata, mockGetEmbeddingModelInfo, mockGetFileMetadataByKeys, + mockLogError, mockProcessDocument, mockTrigger, } = vi.hoisted(() => ({ @@ -30,10 +31,16 @@ const { mockGetBoundWorkspaceFileSecretProvenanceByMetadata: vi.fn(), mockGetEmbeddingModelInfo: vi.fn(), mockGetFileMetadataByKeys: vi.fn(), + mockLogError: vi.fn(), mockProcessDocument: vi.fn(), mockTrigger: vi.fn(), })) +vi.mock('@sim/logger', async () => { + const { createMockLogger, loggerMock } = await import('@sim/testing/mocks/logger.mock') + return { ...loggerMock, createLogger: () => ({ ...createMockLogger(), error: mockLogError }) } +}) + vi.mock('@trigger.dev/sdk', () => ({ tasks: { batchTrigger: mockBatchTrigger, trigger: mockTrigger }, })) @@ -1487,6 +1494,34 @@ describe('in-process quota continuation dispatch', () => { ) }) + it('redacts database query details in the in-process worker without changing acceptance', async () => { + const databaseError = new DrizzleQueryError( + 'insert private-query', + ['private-parameter'], + Object.assign(new Error('private-driver-message'), { code: '57014' }) + ) + mockGenerateEmbeddings.mockRejectedValue(databaseError) + + await expect( + processDocumentsWithQueue( + [queuedDocument], + 'knowledge-base-1', + {}, + 'request-1', + BILLING_ATTRIBUTION + ) + ).resolves.toEqual({ requested: 1, accepted: 1, failed: 0, failedDocumentIds: [] }) + + expect(mockLogError).toHaveBeenCalledWith( + '[request-1] In-process document processing failed', + expect.objectContaining({ + error: 'Database request failed (SQLSTATE 57014).', + diagnostic: expect.objectContaining({ category: 'database', code: '57014' }), + }) + ) + expect(JSON.stringify(mockLogError.mock.calls)).not.toContain('private-') + }) + it('resumes an OCR-throttled regular KB from the durable outbox to a completed index', async () => { mockProcessDocument.mockRejectedValueOnce( new ProviderCapacityDeferredError('rate_limit', { retryAfterMs: 600_000 }) diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index c0248eb49f3..efc0e9e9fda 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -1389,9 +1389,11 @@ async function dispatchInProcess( const message = processingClaimed ? 'In-process document processing failed' : 'In-process document dispatch failed before claiming the document' + const diagnostic = getConnectorFailureDiagnostic(error) logger.error(`[${requestId}] ${message}`, { documentId: p.documentId, - error: getErrorMessage(error), + error: diagnostic?.message ?? getErrorMessage(error), + ...(diagnostic ? { diagnostic } : {}), }) return processingClaimed }