fix(search): preserve Google sync diagnostics and retry transient failures

Author: waleedlatif1

apps/docs/content/docs/search/gmail.mdx

@@ -139,6 +139,8 @@ Updates, removals, and access refresh in the background. Empty mailboxes and fil
 
 ## Troubleshooting
 
+An individual thread failure does not mean the whole mailbox failed. Sim retries temporary server and rate-limit errors with bounded backoff. Error diagnostics record the Google API operation, HTTP status, and a recognized error reason when available.
+
 | What you see | What to do |
 | --- | --- |
 | A different email is requested | Use the Google account matching your verified Sim email. A separate personal account or alias does not satisfy the match. |

apps/docs/content/docs/search/google-calendar.mdx

@@ -135,6 +135,8 @@ Search schedules syncs hourly. Event edits, cancellations, access changes, inact
 
 ## Troubleshooting
 
+When a sync fails, **Sync history** includes the Google API operation, HTTP status, and a recognized reason when available. A `403` alone does not establish a missing scope: `rateLimitExceeded` and `userRateLimitExceeded` are retried with bounded backoff. A persistent access error requires checking the affected user’s Calendar access.
+
 | What you see | What to do |
 | --- | --- |
 | No events | Check the date range, search query, and calendar IDs. Use an empty calendar selection or `primary` for each person's own calendar. |

apps/docs/content/docs/search/google-drive.mdx

@@ -155,6 +155,8 @@ Search schedules syncs hourly. Central crawls revisit the selected users' files
 
 ## Troubleshooting
 
+**Directory permission sync failed** means Sim could not fully verify group membership. Check the Directory administrator’s access to the affected group and any nested groups; this is separate from file-download access. An incomplete membership read does not replace the last verified membership, which remains subject to freshness checks.
+
 | Problem | Next step |
 | --- | --- |
 | Google rejects authorization (`unauthorized_client`) | In **Manage Domain Wide Delegation**, verify the numeric **Client ID** matches `client_id` in the JSON key uploaded to Sim and all required scopes appear under **View details**. Check pending approval and allow time for recent changes to propagate. Changing the OAuth consent screen alone does not authorize delegation. |

apps/sim/background/knowledge-processing.test.ts

@@ -1,6 +1,7 @@
 /**
  * @vitest-environment node
  */
+import { DrizzleQueryError } from 'drizzle-orm/errors'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const {
@@ -566,6 +567,22 @@ describe('knowledge processing worker', () => {
     expect(mockTrigger).not.toHaveBeenCalled()
   })
 
+  it('keeps database failures retryable without sending SQL or parameters to Trigger', async () => {
+    const error = new DrizzleQueryError(
+      'insert private SQL',
+      ['private bound content'],
+      Object.assign(new Error('private database detail'), { code: '57014' })
+    )
+    mockProcessDocumentAsync.mockRejectedValueOnce(error)
+    const failure = await runDocumentProcessing(WORKSPACE_PAYLOAD).catch(
+      (caught: unknown) => caught
+    )
+    expect(failure).toBeInstanceOf(Error)
+    expect(failure).toMatchObject({ message: 'Database request failed (SQLSTATE 57014).' })
+    expect(failure).not.toHaveProperty('cause')
+    expect(JSON.stringify(failure)).not.toContain('private')
+  })
+
   it('retries failed provider continuation dispatch instead of reporting a successful deferral', async () => {
     const error = new Error('Trigger dispatch unavailable')
     mockTrigger.mockRejectedValue(error)

apps/sim/background/knowledge-processing.ts

@@ -7,6 +7,7 @@ import {
   isBYOKEmbeddingCredentialRejection,
   isEmbeddingQuotaExhaustion,
 } from '@/lib/embeddings'
+import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error'
 import {
   getOcrRequestRejection,
   isPermanentDocumentProcessingError,
@@ -194,7 +195,12 @@ export async function runDocumentProcessing(
         processingTime: Date.now() - startedAt,
       }
     }
-    logger.error(`[${requestId}] Failed to process document: ${docData.filename}`, error)
+    const diagnostic = getConnectorFailureDiagnostic(error)
+    logger.error(
+      `[${requestId}] Failed to process document: ${docData.filename}`,
+      diagnostic ?? error
+    )
+    if (diagnostic?.category === 'database') throw new Error(diagnostic.message)
     throw error
   }
 }

apps/sim/connectors/gmail/gmail.test.ts

@@ -5,9 +5,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() }))
 
-vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} }))
 vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({
-  fetchWithRetry: mockFetchWithRetry,
+  fetchWithRetry: (
+    url: string,
+    init: RequestInit,
+    options: {
+      fetcher?: (url: string, init: RequestInit, transport: typeof fetch) => Promise<Response>
+    }
+  ) =>
+    options.fetcher
+      ? options.fetcher(url, init, mockFetchWithRetry)
+      : mockFetchWithRetry(url, init),
 }))
 vi.mock('@/components/icons', () => ({ GmailIcon: () => null }))
 vi.mock('@/lib/knowledge/documents/service', () => ({
@@ -618,7 +626,9 @@ describe('Gmail separately stored message bodies', () => {
         .catch((caught: unknown) => caught)
 
       expect(error).toBeInstanceOf(Error)
-      expect(error).toMatchObject({ message: `Failed to fetch Gmail message body: ${status}` })
+      expect(error).toMatchObject({
+        message: `gmail.messages.attachments.get failed (HTTP ${status}).`,
+      })
       expect(gmailConnector.isCredentialInvalidError?.(error)).toBe(status === 401)
     }
   )
@@ -899,7 +909,7 @@ describe('Gmail thread revisions and deferred content', () => {
         .mockResolvedValueOnce(Response.json({ threads: [{ id: 'thread-1' }] }))
         .mockResolvedValueOnce(new Response(null, { status }))
       await expect(gmailConnector.listDocuments('token', {}, undefined, {})).rejects.toThrow(
-        `Failed to fetch thread thread-1: ${status}`
+        `gmail.threads.get failed (HTTP ${status}).`
       )
     }
   )

apps/sim/connectors/gmail/gmail.ts

@@ -3,9 +3,9 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
 import { isPlainRecord } from '@sim/utils/object'
 import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
 import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits'
-import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server'
 import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta'
+import { fetchGoogleApiWithRetry, GoogleApiError } from '@/connectors/google-workspace/api-errors'
 import {
   getGoogleWorkspaceDocument,
   InvalidGoogleWorkspaceCursor,
@@ -53,16 +53,6 @@ const CHANGED_THREAD_CONCURRENCY = 5
 /** Gmail's thread listing omits these unless `includeSpamTrash` is set; the feed must agree. */
 const HIDDEN_LABEL_IDS = new Set(['SPAM', 'TRASH'])
 
-class GmailApiError extends Error {
-  constructor(
-    message: string,
-    readonly status: number
-  ) {
-    super(`${message}: ${status}`)
-    this.name = 'GmailApiError'
-  }
-}
-
 interface GmailHeader {
   name: string
   value: string
@@ -229,26 +219,23 @@ async function getLabelIndex(
 
   let index: GmailLabelIndex | null = null
   try {
-    const response = await fetchWithRetry(`${GMAIL_API_BASE}/labels`, {
-      method: 'GET',
-      signal,
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        Accept: 'application/json',
-      },
-    })
+    const response = await fetchGoogleApiWithRetry(
+      'gmail.labels.list',
+      `${GMAIL_API_BASE}/labels`,
+      {
+        method: 'GET',
+        signal,
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          Accept: 'application/json',
+        },
+      }
+    )
 
-    if (response.status === 401) {
-      throw new GmailApiError('Failed to fetch Gmail labels', response.status)
-    }
-    if (response.ok) {
-      index = buildLabelIndex(await readLabels(response))
-    } else {
-      logger.warn('Failed to fetch Gmail labels', { status: response.status })
-    }
+    index = buildLabelIndex(await readLabels(response))
   } catch (error) {
     signal?.throwIfAborted()
-    if (error instanceof GmailApiError && error.status === 401) throw error
+    if (error instanceof GoogleApiError && error.status === 401) throw error
     logger.warn('Failed to fetch Gmail labels', { error: toError(error).message })
   }
 
@@ -387,17 +374,15 @@ async function readMessageBody(
   if (!body.attachmentId) return body.data ? decodeBase64Url(body.data, context) : ''
 
   const params = new URLSearchParams({ fields: 'data,size' })
-  const response = await fetchWithRetry(
+  const response = await fetchGoogleApiWithRetry(
+    'gmail.messages.attachments.get',
     `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}/attachments/${encodeURIComponent(body.attachmentId)}?${params}`,
     {
       method: 'GET',
       signal: context.signal,
       headers: { Authorization: `Bearer ${context.accessToken}`, Accept: 'application/json' },
     }
   )
-  if (!response.ok) {
-    throw new GmailApiError('Failed to fetch Gmail message body', response.status)
-  }
 
   let fetchedBody: unknown
   try {
@@ -591,18 +576,16 @@ async function fetchThread(
     params.set('fields', 'id,historyId,snippet,messages(id,labelIds,internalDate)')
   const url = `${GMAIL_API_BASE}/threads/${encodeURIComponent(threadId)}?${params}`
 
-  const response = await fetchWithRetry(url, {
-    method: 'GET',
-    signal,
-    headers: {
-      Authorization: `Bearer ${accessToken}`,
-      Accept: 'application/json',
-    },
-  })
-
-  if (!response.ok) {
-    if (response.status === 404) return null
-    throw new GmailApiError(`Failed to fetch thread ${threadId}`, response.status)
+  let response: Response
+  try {
+    response = await fetchGoogleApiWithRetry('gmail.threads.get', url, {
+      method: 'GET',
+      signal,
+      headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' },
+    })
+  } catch (error) {
+    if (error instanceof GoogleApiError && error.status === 404) return null
+    throw error
   }
 
   const thread = await readResponseJsonWithLimit(response, {
@@ -790,18 +773,21 @@ function threadInScope(thread: GmailThread, scope: GmailChangeScope): boolean {
 const gmailMailboxConnector: ConnectorConfig = {
   ...gmailConnectorMeta,
 
-  isCredentialInvalidError: (error) => error instanceof GmailApiError && error.status === 401,
+  isCredentialInvalidError: (error) => error instanceof GoogleApiError && error.status === 401,
 
   /** The mailbox's current history id; `users.history.list` replays everything after it. */
   getChangeCursor: async (accessToken, _sourceConfig, syncContext): Promise<string> => {
     if (syncContext?.mirrorsSourceAcls === true) {
       throw new Error('Company-wide Gmail indexing uses complete mailbox listings')
     }
-    const response = await fetchWithRetry(`${GMAIL_API_BASE}/profile?fields=historyId`, {
-      method: 'GET',
-      headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' },
-    })
-    if (!response.ok) throw new GmailApiError('Failed to read the Gmail profile', response.status)
+    const response = await fetchGoogleApiWithRetry(
+      'gmail.users.getProfile',
+      `${GMAIL_API_BASE}/profile?fields=historyId`,
+      {
+        method: 'GET',
+        headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' },
+      }
+    )
     const data: unknown = await response.json()
     if (
       !isPlainRecord(data) ||
@@ -851,14 +837,14 @@ const gmailMailboxConnector: ConnectorConfig = {
     for (const type of HISTORY_TYPES) params.append('historyTypes', type)
     if (pageToken) params.set('pageToken', pageToken)
 
-    const response = await fetchWithRetry(`${GMAIL_API_BASE}/history?${params.toString()}`, {
-      method: 'GET',
-      headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' },
-    })
-    if (!response.ok) {
-      logger.warn('Failed to list Gmail history', { status: response.status })
-      throw new GmailApiError('Failed to list Gmail history', response.status)
-    }
+    const response = await fetchGoogleApiWithRetry(
+      'gmail.history.list',
+      `${GMAIL_API_BASE}/history?${params.toString()}`,
+      {
+        method: 'GET',
+        headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' },
+      }
+    )
     const page = parseHistoryList(await response.json())
 
     const changes = await mapWithConcurrency(
@@ -881,7 +867,7 @@ const gmailMailboxConnector: ConnectorConfig = {
   /** Gmail answers 404 once `startHistoryId` falls outside the history it retains. */
   isChangeCursorInvalidError: (error) =>
     error instanceof InvalidGmailChangeCursorError ||
-    (error instanceof GmailApiError && error.status === 404),
+    (error instanceof GoogleApiError && error.status === 404),
 
   listDocuments: async (
     accessToken: string,
@@ -953,7 +939,7 @@ const gmailMailboxConnector: ConnectorConfig = {
       maxThreads,
     })
 
-    const response = await fetchWithRetry(url, {
+    const response = await fetchGoogleApiWithRetry('gmail.threads.list', url, {
       method: 'GET',
       signal,
       headers: {
@@ -962,11 +948,6 @@ const gmailMailboxConnector: ConnectorConfig = {
       },
     })
 
-    if (!response.ok) {
-      logger.error('Failed to list Gmail threads', { status: response.status })
-      throw new GmailApiError('Failed to list Gmail threads', response.status)
-    }
-
     /** Gmail can return 204 when an empty listing has no requested metadata fields. */
     const { threads, nextPageToken } = parseThreadList(
       response.status === 204
@@ -1102,7 +1083,8 @@ const gmailMailboxConnector: ConnectorConfig = {
 
     try {
       const profileUrl = `${GMAIL_API_BASE}/profile`
-      const profileResponse = await fetchWithRetry(
+      await fetchGoogleApiWithRetry(
+        'gmail.users.getProfile',
         profileUrl,
         {
           method: 'GET',
@@ -1115,10 +1097,6 @@ const gmailMailboxConnector: ConnectorConfig = {
         VALIDATE_RETRY_OPTIONS
       )
 
-      if (!profileResponse.ok) {
-        return { valid: false, error: `Failed to access Gmail: ${profileResponse.status}` }
-      }
-
       /**
        * Labels may arrive as ids (from the `gmail.labels` selector) or as names
        * (typed into the advanced input), so both forms are accepted here and the
@@ -1128,7 +1106,8 @@ const gmailMailboxConnector: ConnectorConfig = {
       let labelIndex = EMPTY_LABEL_INDEX
       if (configuredLabels.length > 0) {
         const labelsUrl = `${GMAIL_API_BASE}/labels`
-        const labelsResponse = await fetchWithRetry(
+        const labelsResponse = await fetchGoogleApiWithRetry(
+          'gmail.labels.list',
           labelsUrl,
           {
             method: 'GET',
@@ -1141,10 +1120,6 @@ const gmailMailboxConnector: ConnectorConfig = {
           VALIDATE_RETRY_OPTIONS
         )
 
-        if (!labelsResponse.ok) {
-          return { valid: false, error: 'Failed to fetch labels' }
-        }
-
         const labels = await readLabels(labelsResponse)
         labelIndex = buildLabelIndex(labels)
         const missing = configuredLabels.filter(
@@ -1171,7 +1146,8 @@ const gmailMailboxConnector: ConnectorConfig = {
       if (query?.trim()) {
         const searchQuery = buildSearchQuery(sourceConfig, labelIndex)
         const testUrl = `${GMAIL_API_BASE}/threads?q=${encodeURIComponent(searchQuery)}&maxResults=1`
-        const testResponse = await fetchWithRetry(
+        await fetchGoogleApiWithRetry(
+          'gmail.threads.list',
           testUrl,
           {
             method: 'GET',
@@ -1183,10 +1159,6 @@ const gmailMailboxConnector: ConnectorConfig = {
           },
           VALIDATE_RETRY_OPTIONS
         )
-
-        if (!testResponse.ok) {
-          return { valid: false, error: 'Invalid search query. Check Gmail search syntax.' }
-        }
       }
 
       return { valid: true }