fix(search): preserve Google sync diagnostics and retry transient failures (#7895)

* fix(search): preserve Google sync diagnostics and retry transient failures

* fix(search): sanitize in-process database failure logs

Author: waleedlatif1

apps/docs/content/docs/search/gmail.mdx

@@ -139,6 +139,8 @@ Updates, removals, and access refresh in the background. Empty mailboxes and fil
 
 ## Troubleshooting
 
+An individual thread failure does not mean the whole mailbox failed. Sim retries temporary server and rate-limit errors with bounded backoff. Error diagnostics record the Google API operation, HTTP status, and a recognized error reason when available.
+
 | What you see | What to do |
 | --- | --- |
 | A different email is requested | Use the Google account matching your verified Sim email. A separate personal account or alias does not satisfy the match. |

apps/docs/content/docs/search/google-calendar.mdx

@@ -135,6 +135,8 @@ Search schedules syncs hourly. Event edits, cancellations, access changes, inact
 
 ## Troubleshooting
 
+When a sync fails, **Sync history** includes the Google API operation, HTTP status, and a recognized reason when available. A `403` alone does not establish a missing scope: `rateLimitExceeded` and `userRateLimitExceeded` are retried with bounded backoff. A persistent access error requires checking the affected user’s Calendar access.
+
 | What you see | What to do |
 | --- | --- |
 | No events | Check the date range, search query, and calendar IDs. Use an empty calendar selection or `primary` for each person's own calendar. |

apps/docs/content/docs/search/google-drive.mdx

@@ -155,6 +155,8 @@ Search schedules syncs hourly. Central crawls revisit the selected users' files
 
 ## Troubleshooting
 
+**Directory permission sync failed** means Sim could not fully verify group membership. Check the Directory administrator’s access to the affected group and any nested groups; this is separate from file-download access. An incomplete membership read does not replace the last verified membership, which remains subject to freshness checks.
+
 | Problem | Next step |
 | --- | --- |
 | Google rejects authorization (`unauthorized_client`) | In **Manage Domain Wide Delegation**, verify the numeric **Client ID** matches `client_id` in the JSON key uploaded to Sim and all required scopes appear under **View details**. Check pending approval and allow time for recent changes to propagate. Changing the OAuth consent screen alone does not authorize delegation. |

apps/sim/background/knowledge-processing.test.ts

@@ -1,6 +1,7 @@
 /**
  * @vitest-environment node
  */
+import { DrizzleQueryError } from 'drizzle-orm/errors'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const {
@@ -566,6 +567,22 @@ describe('knowledge processing worker', () => {
     expect(mockTrigger).not.toHaveBeenCalled()
   })
 
+  it('keeps database failures retryable without sending SQL or parameters to Trigger', async () => {
+    const error = new DrizzleQueryError(
+      'insert private SQL',
+      ['private bound content'],
+      Object.assign(new Error('private database detail'), { code: '57014' })
+    )
+    mockProcessDocumentAsync.mockRejectedValueOnce(error)
+    const failure = await runDocumentProcessing(WORKSPACE_PAYLOAD).catch(
+      (caught: unknown) => caught
+    )
+    expect(failure).toBeInstanceOf(Error)
+    expect(failure).toMatchObject({ message: 'Database request failed (SQLSTATE 57014).' })
+    expect(failure).not.toHaveProperty('cause')
+    expect(JSON.stringify(failure)).not.toContain('private')
+  })
+
   it('retries failed provider continuation dispatch instead of reporting a successful deferral', async () => {
     const error = new Error('Trigger dispatch unavailable')
     mockTrigger.mockRejectedValue(error)

apps/sim/background/knowledge-processing.ts

@@ -7,6 +7,7 @@ import {
   isBYOKEmbeddingCredentialRejection,
   isEmbeddingQuotaExhaustion,
 } from '@/lib/embeddings'
+import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error'
 import {
   getOcrRequestRejection,
   isPermanentDocumentProcessingError,
@@ -194,7 +195,12 @@ export async function runDocumentProcessing(
         processingTime: Date.now() - startedAt,
       }
     }
-    logger.error(`[${requestId}] Failed to process document: ${docData.filename}`, error)
+    const diagnostic = getConnectorFailureDiagnostic(error)
+    logger.error(
+      `[${requestId}] Failed to process document: ${docData.filename}`,
+      diagnostic ?? error
+    )
+    if (diagnostic?.category === 'database') throw new Error(diagnostic.message)
     throw error
   }
 }

apps/sim/connectors/gmail/company-crawl.test.ts

@@ -9,8 +9,13 @@ const { fetchProvider, listUsers, getUser } = vi.hoisted(() => ({
   getUser: vi.fn(),
 }))
 
-vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} }))
-vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: fetchProvider }))
+vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({
+  fetchWithRetry: (
+    url: string,
+    init: RequestInit,
+    options?: import('@/lib/knowledge/documents/utils').RetryOptions
+  ) => (options?.fetcher ? options.fetcher(url, init, fetchProvider) : fetchProvider(url, init)),
+}))
 vi.mock('@/components/icons', () => ({ GmailIcon: () => null }))
 vi.mock('@/connectors/google-workspace/users', () => ({
   GOOGLE_WORKSPACE_USERS_PAGE_SIZE: 100,
@@ -180,8 +185,7 @@ describe('company-wide Gmail indexing', () => {
     ).resolves.toEqual({ valid: true })
     expect(fetchProvider).toHaveBeenCalledWith(
       expect.stringContaining('/profile'),
-      expect.objectContaining({ signal: controller.signal }),
-      expect.any(Object)
+      expect.objectContaining({ signal: controller.signal })
     )
   })
 

apps/sim/connectors/gmail/gmail.test.ts

@@ -5,9 +5,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() }))
 
-vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} }))
 vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({
-  fetchWithRetry: mockFetchWithRetry,
+  fetchWithRetry: (
+    url: string,
+    init: RequestInit,
+    options: {
+      fetcher?: (url: string, init: RequestInit, transport: typeof fetch) => Promise<Response>
+    }
+  ) =>
+    options.fetcher
+      ? options.fetcher(url, init, mockFetchWithRetry)
+      : mockFetchWithRetry(url, init),
 }))
 vi.mock('@/components/icons', () => ({ GmailIcon: () => null }))
 vi.mock('@/lib/knowledge/documents/service', () => ({
@@ -618,7 +626,9 @@ describe('Gmail separately stored message bodies', () => {
         .catch((caught: unknown) => caught)
 
       expect(error).toBeInstanceOf(Error)
-      expect(error).toMatchObject({ message: `Failed to fetch Gmail message body: ${status}` })
+      expect(error).toMatchObject({
+        message: `gmail.messages.attachments.get failed (HTTP ${status}).`,
+      })
       expect(gmailConnector.isCredentialInvalidError?.(error)).toBe(status === 401)
     }
   )
@@ -899,7 +909,7 @@ describe('Gmail thread revisions and deferred content', () => {
         .mockResolvedValueOnce(Response.json({ threads: [{ id: 'thread-1' }] }))
         .mockResolvedValueOnce(new Response(null, { status }))
       await expect(gmailConnector.listDocuments('token', {}, undefined, {})).rejects.toThrow(
-        `Failed to fetch thread thread-1: ${status}`
+        `gmail.threads.get failed (HTTP ${status}).`
       )
     }
   )