feat(knowledge): administrator access mode with mirrored source permissions

apps/docs/content/docs/platform/self-hosting/background-jobs.mdx

@@ -49,6 +49,7 @@ Point cron at an **internal** address where possible (the in-cluster Service, or
 | Workspace file search dispatch | `/api/cron/workspace-file-search-dispatch` | `*/1 * * * *` | Dispatches indexing work for workspace file search |
 | Connector sync | `/api/knowledge/connectors/sync` | `*/5 * * * *` | Knowledge base connector syncs |
 | Connector member sync | `/api/knowledge/connectors/member-sync` | `*/5 * * * *` | Per-member access sync for permission-aware connectors |
+| Connector directory sync | `/api/knowledge/connectors/directory-sync` | `*/5 * * * *` | Refreshes the directory groups administrator-mode connectors mirror, so a membership change takes effect without waiting for a content sync |
 | Workspace events poll | `/api/workspace-events/poll` | `*/15 * * * *` | Workspace event triggers |
 | Table row TTL cleanup | `/api/cron/cleanup-table-row-ttl` | `*/15 * * * *` | Deletes table rows whose TTL column has expired |
 | Data drains | `/api/cron/run-data-drains` | `0 * * * *` | Enterprise data drains |

apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts

@@ -0,0 +1,76 @@
+/**
+ * @vitest-environment node
+ */
+import { createMockRequest } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockVerifyCronAuth, mockConnectorRows, mockDispatch } = vi.hoisted(() => ({
+  mockVerifyCronAuth: vi.fn(() => null),
+  mockConnectorRows: vi.fn(),
+  mockDispatch: vi.fn(),
+}))
+
+vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: mockVerifyCronAuth }))
+vi.mock('@/lib/knowledge/connectors/directory-queue', () => ({
+  dispatchDirectorySync: mockDispatch,
+}))
+vi.mock('@sim/db', () => ({
+  db: {
+    select: () => ({
+      from: () => ({
+        innerJoin: () => ({
+          where: () => ({ orderBy: () => ({ limit: () => mockConnectorRows() }) }),
+        }),
+      }),
+    }),
+  },
+}))
+
+import { GET } from '@/app/api/knowledge/connectors/directory-sync/route'
+
+function connector(overrides: Record<string, unknown> = {}) {
+  return { id: 'connector-1', ...overrides }
+}
+
+async function run() {
+  const response = await GET(createMockRequest('GET'))
+  return response.json()
+}
+
+describe('connector directory sync scheduler', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockVerifyCronAuth.mockReturnValue(null)
+    mockDispatch.mockResolvedValue(undefined)
+  })
+
+  /**
+   * Every eligible connector is offered under one tick time; the tenant-level
+   * freshness check in the refresh, not the scheduler, decides which walk.
+   */
+  it('dispatches a refresh for every admin-mode connector under the same tick', async () => {
+    mockConnectorRows.mockResolvedValue([connector(), connector({ id: 'connector-2' })])
+
+    await expect(run()).resolves.toMatchObject({ considered: 2, dispatched: 2, failed: 0 })
+    expect(mockDispatch).toHaveBeenCalledTimes(2)
+    const [, first] = mockDispatch.mock.calls[0]
+    const [, second] = mockDispatch.mock.calls[1]
+    expect(first.tickAt).toBe(second.tickAt)
+  })
+
+  it('contains a dispatch failure to the connector that caused it', async () => {
+    mockConnectorRows.mockResolvedValue([connector(), connector({ id: 'connector-2' })])
+    mockDispatch.mockRejectedValueOnce(new Error('queue unreachable'))
+
+    await expect(run()).resolves.toMatchObject({ dispatched: 1, failed: 1 })
+  })
+
+  it('refuses an unauthenticated tick', async () => {
+    mockVerifyCronAuth.mockReturnValue(new Response('nope', { status: 401 }))
+
+    const response = await GET(createMockRequest('GET'))
+
+    expect(response.status).toBe(401)
+    expect(mockConnectorRows).not.toHaveBeenCalled()
+  })
+})

apps/sim/app/api/knowledge/connectors/directory-sync/route.ts

@@ -0,0 +1,83 @@
+import { db } from '@sim/db'
+import { knowledgeBase, knowledgeConnector } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { and, asc, eq, inArray, isNotNull, isNull } from 'drizzle-orm'
+import type { NextRequest } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
+import { generateRequestId } from '@/lib/core/utils/request'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { MIRRORING_ACCESS_MODES } from '@/lib/knowledge/connectors/access-modes'
+import { dispatchDirectorySync } from '@/lib/knowledge/connectors/directory-queue'
+import { RUNNABLE_CONNECTOR_STATUSES } from '@/lib/knowledge/connectors/sync-lock'
+
+export const dynamic = 'force-dynamic'
+
+const logger = createLogger('ConnectorDirectorySyncSchedulerAPI')
+
+/** Connectors offered per tick, and how many dispatches are in flight at once. */
+const MAX_DIRECTORIES_PER_TICK = 200
+const DISPATCH_CONCURRENCY = 8
+
+/**
+ * Refreshes the external directories that admin-mode connectors mirror.
+ *
+ * Group membership decides who can read an already-indexed document, so it has
+ * to move on its own clock: someone leaving a group should lose access in
+ * minutes, not on whatever schedule the corpus happens to be re-crawled on. The
+ * admin crawl refreshes the directory too — so a crawl can never publish grants
+ * against membership nobody has read — but that is a floor, not the cadence.
+ *
+ * Every eligible connector is offered each tick, and
+ * `syncExternalDirectoryGroups` decides whether its directory is actually due:
+ * a tenant is the credential's own site or domain, which the row does not
+ * carry, so connectors sharing one cost a refresh and a skip rather than a
+ * refresh each. The walk itself runs in the background, like every other
+ * connector job, because a large domain takes longer than a scheduler request
+ * lives.
+ */
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  const requestId = generateRequestId()
+  const tickAt = new Date()
+  logger.info(`[${requestId}] Connector directory sync scheduler triggered`)
+
+  const authError = verifyCronAuth(request, 'Connector directory sync scheduler')
+  if (authError) return authError
+
+  const connectors = await db
+    .select({ id: knowledgeConnector.id })
+    .from(knowledgeConnector)
+    .innerJoin(knowledgeBase, eq(knowledgeConnector.knowledgeBaseId, knowledgeBase.id))
+    .where(
+      and(
+        inArray(knowledgeConnector.accessMode, MIRRORING_ACCESS_MODES),
+        inArray(knowledgeConnector.status, RUNNABLE_CONNECTOR_STATUSES),
+        isNull(knowledgeConnector.archivedAt),
+        isNull(knowledgeConnector.deletedAt),
+        isNull(knowledgeBase.deletedAt),
+        isNotNull(knowledgeBase.workspaceId)
+      )
+    )
+    .orderBy(asc(knowledgeConnector.createdAt))
+    .limit(MAX_DIRECTORIES_PER_TICK)
+
+  let dispatched = 0
+  let failed = 0
+  await mapWithConcurrency(connectors, DISPATCH_CONCURRENCY, async ({ id: connectorId }) => {
+    try {
+      await dispatchDirectorySync(connectorId, { requestId, tickAt })
+      dispatched += 1
+    } catch (error) {
+      failed += 1
+      logger.error(`[${requestId}] Failed to dispatch a directory refresh`, {
+        connectorId,
+        error: getErrorMessage(error),
+      })
+    }
+  })
+
+  const summary = { considered: connectors.length, dispatched, failed }
+  logger.info(`[${requestId}] Connector directory sync scheduler finished`, summary)
+  return Response.json({ success: true, ...summary })
+})

apps/sim/app/api/knowledge/connectors/member-sync/route.ts

@@ -20,6 +20,7 @@ import {
   MAX_CONSECUTIVE_FAILURES,
   MEMBER_SYNC_STALE_LOCK_TTL_MS,
 } from '@/lib/knowledge/connectors/sync-limits'
+import { RUNNABLE_CONNECTOR_STATUSES } from '@/lib/knowledge/connectors/sync-lock'
 
 export const dynamic = 'force-dynamic'
 
@@ -168,7 +169,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
       .where(
         and(
           eq(knowledgeConnector.accessMode, 'members'),
-          inArray(knowledgeConnector.status, ['active', 'error']),
+          inArray(knowledgeConnector.status, RUNNABLE_CONNECTOR_STATUSES),
           inArray(knowledgeConnector.memberSyncStatus, QUEUEABLE_MEMBER_SYNC_STATUSES),
           lte(knowledgeConnector.nextMemberSyncAt, now),
           isNull(knowledgeConnector.archivedAt),

apps/sim/app/api/knowledge/connectors/sync/route.ts

@@ -8,6 +8,7 @@ import { resolveSystemBillingAttribution } from '@/lib/billing/core/billing-attr
 import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { CONTENT_ENGINE_ACCESS_MODES } from '@/lib/knowledge/connectors/access-modes'
 import { dispatchSync } from '@/lib/knowledge/connectors/queue'
 import {
   CONNECTOR_AUTO_DISABLED_ERROR,
@@ -16,6 +17,7 @@ import {
   CONNECTOR_SYNC_STALE_LOCK_TTL_MS,
   MAX_CONSECUTIVE_FAILURES,
 } from '@/lib/knowledge/connectors/sync-limits'
+import { RUNNABLE_CONNECTOR_STATUSES } from '@/lib/knowledge/connectors/sync-lock'
 
 export const dynamic = 'force-dynamic'
 
@@ -303,8 +305,8 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
       .innerJoin(knowledgeBase, eq(knowledgeConnector.knowledgeBaseId, knowledgeBase.id))
       .where(
         and(
-          inArray(knowledgeConnector.status, ['active', 'error']),
-          eq(knowledgeConnector.accessMode, 'workspace'),
+          inArray(knowledgeConnector.status, RUNNABLE_CONNECTOR_STATUSES),
+          inArray(knowledgeConnector.accessMode, CONTENT_ENGINE_ACCESS_MODES),
           lte(knowledgeConnector.nextSyncAt, now),
           isNull(knowledgeConnector.archivedAt),
           isNull(knowledgeConnector.deletedAt),

apps/sim/app/api/v1/admin/dashboard/actor.ts

@@ -1,16 +1,18 @@
 import { db } from '@sim/db'
-import { user } from '@sim/db/schema'
-import { eq, or } from 'drizzle-orm'
+import { foldedEmail, user } from '@sim/db/schema'
+import { normalizeEmail } from '@sim/utils/string'
+import { eq } from 'drizzle-orm'
 import type { NextRequest } from 'next/server'
 import type { AdminMutationActor } from '@/lib/admin/dashboard'
 
 export async function getAdminAuditActor(request: NextRequest): Promise<AdminMutationActor> {
-  const email = request.headers.get('x-admin-email')?.trim().toLowerCase()
+  const rawEmail = request.headers.get('x-admin-email')
+  const email = rawEmail ? normalizeEmail(rawEmail) : ''
   if (!email) return { id: null, name: 'Admin API', email: null }
   const [admin] = await db
     .select({ id: user.id, name: user.name, email: user.email })
     .from(user)
-    .where(or(eq(user.email, email), eq(user.normalizedEmail, email)))
+    .where(eq(foldedEmail(user.email), email))
     .limit(1)
   return admin ?? { id: null, name: 'Admin Panel', email }
 }

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx

@@ -43,7 +43,7 @@ import {
 import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge'
 import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields'
 import {
-  memberCapFieldIds,
+  derivedAclCapFieldIds,
   useConnectorMemberGroupOptions,
 } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options'
 import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider'
@@ -95,6 +95,7 @@ export function AddConnectorModal({
   const { ownerBilling, features } = useWorkspaceHostContext()
   const { canAdmin } = useUserPermissionsContext()
   const memberAccessAvailable = features?.knowledgeMemberAccess === true
+  const mirroredAccessAvailable = features?.knowledgeSourceMirroredAccess === true
   const { mutate: createConnector, isPending: isCreating } = useCreateConnector()
 
   const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling)
@@ -111,7 +112,7 @@ export function AddConnectorModal({
   const membersChoiceOpen =
     isMembersMode && groupOptions.needsChoice && !access.credentialGroupOptionId
   const hiddenCapFieldIds = useMemo(
-    () => memberCapFieldIds(connectorConfig, access.accessMode),
+    () => derivedAclCapFieldIds(connectorConfig, access.accessMode),
     [connectorConfig, access.accessMode]
   )
   /** True when the connector declares its key optional (public sources need none). */
@@ -126,28 +127,14 @@ export function AddConnectorModal({
   )
 
   const {
-    data: rawCredentials = [],
+    data: credentials = [],
     isLoading: credentialsLoading,
     refetch: refetchCredentials,
   } = useOAuthCredentials(connectorProviderId ?? undefined, {
     enabled: Boolean(connectorConfig) && !isApiKeyMode,
     workspaceId,
   })
 
-  /**
-   * The credential list also returns the provider's service accounts, but
-   * `ConnectorAuthConfig` has no service-account mode: the sync engine resolves
-   * connector tokens through `refreshAccessTokenIfNeeded`, which passes no scopes
-   * and drops the `cloudId`/`domain`/`authStyle` a service account resolves with.
-   * Offering them here would surface credentials no connector can authenticate
-   * with, so — like a workflow picker that has not opted in via
-   * `allowServiceAccounts` — list OAuth accounts only.
-   */
-  const credentials = useMemo(
-    () => rawCredentials.filter((cred) => cred.type !== 'service_account'),
-    [rawCredentials]
-  )
-
   useCredentialRefreshTriggers(refetchCredentials, connectorProviderId ?? '', workspaceId)
 
   const effectiveCredentialId =
@@ -261,7 +248,7 @@ export function AddConnectorModal({
                 credentialGroupId: access.credentialGroupId,
                 credentialGroupOptionId: access.credentialGroupOptionId,
               }
-            : { credentialId: effectiveCredentialId! }),
+            : { accessMode: access.accessMode, credentialId: effectiveCredentialId! }),
         sourceConfig: finalSourceConfig,
         syncIntervalMinutes: syncInterval,
       },
@@ -347,13 +334,15 @@ export function AddConnectorModal({
             </div>
           ) : connectorConfig ? (
             <>
-              {!isApiKeyMode && memberAccessAvailable && (
+              {!isApiKeyMode && (memberAccessAvailable || mirroredAccessAvailable) && (
                 <ConnectorAccessField
                   connectorConfig={connectorConfig}
                   value={access}
                   onChange={setAccess}
                   groupOptions={groupOptions}
                   canAdmin={canAdmin}
+                  allowMembers={memberAccessAvailable}
+                  allowAdmin={mirroredAccessAvailable}
                   disabled={isCreating}
                 />
               )}