refactor(knowledge): collapse the access-mode vocabulary and the duplication around it

/simplify and /cleanup over the branch. The mode literal 'admin' appeared in
eight modules deciding four different things; it is now one leaf predicate
each site reads.

- access-modes owns MIRRORING_ACCESS_MODES/mirrorsSourceAcls and aclIsDerived;
  the identity mapper, the second name for the same union, and the duplicate
  credential-backed predicate are gone, and the contract derives its enum from
  the leaf
- listing caps are stripped once per rule rather than at three depths: create,
  the mode switch, and a config edit all key off aclIsDerived, so the stored
  config agrees with what runs
- the mirroring assertion moved into the source-config validator the
  application layer already owns, so orchestration stays mode-agnostic
- one resolver for a connector's token user, shared by the engine and the
  directory refresh, replacing two copies that had already drifted
- Drive asks for permissions only on a run the engine says mirrors, drains its
  permission pages through the shared Google helper, and spreads the ACL
  context instead of relisting its fields
- Confluence memoises its three per-run lookups through one helper, keys pages
  by one map, drops the unreachable space-principal branch, and shares the
  cursor parser with the content listing
- nested Drive subgroups are read once per directory rather than once per
  parent; the scheduler dispatches with bounded concurrency; batch loops use
  chunkArray

Behaviour is unchanged except where the reviews found it wrong: a config edit
on a mirroring connector now strips caps and re-asserts the administrator, and
a crawl that is not mirroring no longer pulls a permission array per file.

Author: waleedlatif1

apps/sim/app/api/knowledge/connectors/directory-sync/route.ts

@@ -5,17 +5,20 @@ import { getErrorMessage } from '@sim/utils/errors'
 import { and, asc, eq, inArray, isNotNull, isNull } from 'drizzle-orm'
 import type { NextRequest } from 'next/server'
 import { verifyCronAuth } from '@/lib/auth/internal'
+import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { MIRRORING_ACCESS_MODES } from '@/lib/knowledge/connectors/access-modes'
 import { dispatchDirectorySync } from '@/lib/knowledge/connectors/directory-queue'
 import { RUNNABLE_CONNECTOR_STATUSES } from '@/lib/knowledge/connectors/sync-lock'
 
 export const dynamic = 'force-dynamic'
 
 const logger = createLogger('ConnectorDirectorySyncSchedulerAPI')
 
-/** Connectors offered per tick. */
+/** Connectors offered per tick, and how many dispatches are in flight at once. */
 const MAX_DIRECTORIES_PER_TICK = 200
+const DISPATCH_CONCURRENCY = 8
 
 /**
  * Refreshes the external directories that admin-mode connectors mirror.
@@ -48,7 +51,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
     .innerJoin(knowledgeBase, eq(knowledgeConnector.knowledgeBaseId, knowledgeBase.id))
     .where(
       and(
-        eq(knowledgeConnector.accessMode, 'admin'),
+        inArray(knowledgeConnector.accessMode, MIRRORING_ACCESS_MODES),
         inArray(knowledgeConnector.status, RUNNABLE_CONNECTOR_STATUSES),
         isNull(knowledgeConnector.archivedAt),
         isNull(knowledgeConnector.deletedAt),
@@ -61,7 +64,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
 
   let dispatched = 0
   let failed = 0
-  for (const { id: connectorId } of connectors) {
+  await mapWithConcurrency(connectors, DISPATCH_CONCURRENCY, async ({ id: connectorId }) => {
     try {
       await dispatchDirectorySync(connectorId, { requestId, tickAt })
       dispatched += 1
@@ -72,7 +75,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
         error: getErrorMessage(error),
       })
     }
-  }
+  })
 
   const summary = { considered: connectors.length, dispatched, failed }
   logger.info(`[${requestId}] Connector directory sync scheduler finished`, summary)

apps/sim/app/api/knowledge/connectors/sync/route.ts

@@ -306,7 +306,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
       .where(
         and(
           inArray(knowledgeConnector.status, RUNNABLE_CONNECTOR_STATUSES),
-          inArray(knowledgeConnector.accessMode, [...CONTENT_ENGINE_ACCESS_MODES]),
+          inArray(knowledgeConnector.accessMode, CONTENT_ENGINE_ACCESS_MODES),
           lte(knowledgeConnector.nextSyncAt, now),
           isNull(knowledgeConnector.archivedAt),
           isNull(knowledgeConnector.deletedAt),

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx

@@ -43,7 +43,7 @@ import {
 import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge'
 import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields'
 import {
-  memberCapFieldIds,
+  derivedAclCapFieldIds,
   useConnectorMemberGroupOptions,
 } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options'
 import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider'
@@ -112,7 +112,7 @@ export function AddConnectorModal({
   const membersChoiceOpen =
     isMembersMode && groupOptions.needsChoice && !access.credentialGroupOptionId
   const hiddenCapFieldIds = useMemo(
-    () => memberCapFieldIds(connectorConfig, access.accessMode),
+    () => derivedAclCapFieldIds(connectorConfig, access.accessMode),
     [connectorConfig, access.accessMode]
   )
   /** True when the connector declares its key optional (public sources need none). */

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx

@@ -42,21 +42,20 @@ interface ConnectorAccessFieldProps {
 
 /**
  * Each mode decides who can read the indexed documents, which the three labels
- * cannot say on their own — and getting it wrong is the kind of mistake that is
- * only noticed once the wrong person finds a document.
+ * cannot say on their own.
  */
-function accessHint(input: {
-  mode: ConnectorAccessMode
-  connectorConfig: ConnectorMeta
+function accessHint(
+  mode: ConnectorAccessMode,
+  sourceName: string,
   allowMembers: boolean
-}): string | undefined {
-  if (input.mode === 'members') {
-    return `Everyone in the workspace is invited by email to connect their ${input.connectorConfig.name} account when the first sync starts. Each member sees only the documents their own account can open; scheduled, API, and chat runs see workspace-visible documents only.`
+): string | undefined {
+  if (mode === 'members') {
+    return `Everyone in the workspace is invited by email to connect their ${sourceName} account when the first sync starts. Each member sees only the documents their own account can open; scheduled, API, and chat runs see workspace-visible documents only.`
   }
-  if (input.mode === 'admin') {
-    return `Indexed once as an administrator, keeping each document's own permissions. People see only what ${input.connectorConfig.name} already lets them open; scheduled, API, and chat runs see workspace-visible documents only.`
+  if (mode === 'admin') {
+    return `Indexed once as an administrator, keeping each document's own permissions. People see only what ${sourceName} already lets them open; scheduled, API, and chat runs see workspace-visible documents only.`
   }
-  return input.allowMembers ? undefined : 'Per-member access is turned off for this workspace.'
+  return allowMembers ? undefined : 'Per-member access is turned off for this workspace.'
 }
 
 /**
@@ -90,25 +89,26 @@ export function ConnectorAccessField({
   const showAdmin = allowAdmin && Boolean(connectorConfig.mirrorsSourceAcls)
   if (!membersSupported && !showAdmin) return null
 
+  /** One ordered list, rendered by both the read-only and the editable branch. */
+  const modes: { mode: ConnectorAccessMode; label: string; shown: boolean }[] = [
+    { mode: 'workspace', label: 'Workspace', shown: true },
+    { mode: 'members', label: 'Per member', shown: membersSupported },
+    { mode: 'admin', label: 'Mirror source', shown: showAdmin },
+  ]
+  const modeItems = (isDisabled: (mode: ConnectorAccessMode) => boolean) =>
+    modes
+      .filter((entry) => entry.shown)
+      .map((entry) => (
+        <ButtonGroupItem key={entry.mode} value={entry.mode} disabled={isDisabled(entry.mode)}>
+          {entry.label}
+        </ButtonGroupItem>
+      ))
+
   if (!canAdmin) {
     if (value.accessMode === 'workspace') return null
     return (
       <ChipModalField type='custom' title='Access'>
-        <ButtonGroup value={value.accessMode}>
-          <ButtonGroupItem value='workspace' disabled>
-            Workspace
-          </ButtonGroupItem>
-          {membersSupported ? (
-            <ButtonGroupItem value='members' disabled>
-              Per member
-            </ButtonGroupItem>
-          ) : null}
-          {showAdmin ? (
-            <ButtonGroupItem value='admin' disabled>
-              Mirror source
-            </ButtonGroupItem>
-          ) : null}
-        </ButtonGroup>
+        <ButtonGroup value={value.accessMode}>{modeItems(() => true)}</ButtonGroup>
       </ChipModalField>
     )
   }
@@ -125,7 +125,7 @@ export function ConnectorAccessField({
       type='custom'
       title='Access'
       error={error?.message}
-      hint={accessHint({ mode: value.accessMode, connectorConfig, allowMembers })}
+      hint={accessHint(value.accessMode, connectorConfig.name, allowMembers)}
     >
       <div className='flex flex-col gap-2'>
         <ButtonGroup
@@ -134,19 +134,7 @@ export function ConnectorAccessField({
             if (isConnectorAccessMode(mode)) onChange({ accessMode: mode })
           }}
         >
-          <ButtonGroupItem value='workspace' disabled={disabled}>
-            Workspace
-          </ButtonGroupItem>
-          {membersSupported ? (
-            <ButtonGroupItem value='members' disabled={disabled || !allowMembers}>
-              Per member
-            </ButtonGroupItem>
-          ) : null}
-          {showAdmin ? (
-            <ButtonGroupItem value='admin' disabled={disabled}>
-              Mirror source
-            </ButtonGroupItem>
-          ) : null}
+          {modeItems((mode) => disabled || (mode === 'members' && !allowMembers))}
         </ButtonGroup>
 
         {value.accessMode === 'members' && showPicker && (

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx

@@ -21,7 +21,7 @@ import { RefreshCw, SquareArrowUpRight } from '@sim/emcn/icons'
 import { createLogger } from '@sim/logger'
 import { useParams } from 'next/navigation'
 import type { ConnectorAccessMode } from '@/lib/api/contracts/knowledge/connectors'
-import { isCredentialBackedAccessMode } from '@/lib/knowledge/connectors/access-modes'
+import { isContentEngineAccessMode } from '@/lib/knowledge/connectors/access-modes'
 import { getProviderIdFromServiceId, type OAuthProvider } from '@/lib/oauth'
 import {
   ConnectorAccessField,
@@ -40,7 +40,7 @@ import type {
 } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields'
 import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields'
 import {
-  memberCapFieldIds,
+  derivedAclCapFieldIds,
   useConnectorMemberGroupOptions,
 } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options'
 import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider'
@@ -60,6 +60,21 @@ import { useOAuthCredentials } from '@/hooks/queries/oauth/oauth-credentials'
 
 const logger = createLogger('EditConnectorModal')
 
+/** What the apply button says, and what it warns will happen, per mode. */
+const SWITCH_LABEL: Record<ConnectorAccessMode, string> = {
+  workspace: 'Switch to workspace access',
+  members: 'Switch to per-member access',
+  admin: 'Switch to mirrored access',
+}
+
+const SWITCH_NOTICE: Record<ConnectorAccessMode, string> = {
+  workspace: 'Every workspace member can read every synced document once the next sync completes.',
+  members:
+    'Everyone in the workspace is invited to connect their account. Documents stay hidden until members connect and sync; listing caps are cleared.',
+  admin:
+    'Documents stay hidden until the next sync mirrors their permissions from the source; listing caps are cleared.',
+}
+
 /** Keys injected by the sync engine or modal state — not user-editable */
 const INTERNAL_CONFIG_KEYS = new Set(['tagSlotMapping', 'disabledTagIds', '_canonicalModes'])
 
@@ -247,9 +262,10 @@ export function EditConnectorModal({
   const { mutate: updateAccess, isPending: isSwitchingAccess } = useUpdateConnectorAccess()
   const isSaving = isSavingSettings || isSwitchingAccess
   /**
-   * The field shows where the flag is on. A connector already syncing per
-   * member keeps it where the flag has since been turned off, so an admin can
-   * still bring it back to workspace mode; per-member cannot be re-chosen.
+   * The field shows where either flag is on. A connector already in a
+   * non-workspace mode keeps it where its flag has since been turned off, so
+   * an admin can still bring it back to workspace mode; that mode cannot be
+   * re-chosen.
    */
   const memberAccessAvailable = features?.knowledgeMemberAccess === true
   const mirroredAccessAvailable = features?.knowledgeSourceMirroredAccess === true
@@ -268,7 +284,7 @@ export function EditConnectorModal({
   /** Leaving members mode for a mode that syncs with one credential needs that credential. */
   const needsWorkspaceCredential =
     accessDirty &&
-    isCredentialBackedAccessMode(access.accessMode) &&
+    isContentEngineAccessMode(access.accessMode) &&
     persistedAccess.accessMode === 'members'
   const accessComplete =
     !accessDirty ||
@@ -278,7 +294,7 @@ export function EditConnectorModal({
   /** A disabled member sync is re-enabled by applying the current binding again. */
   const canReenableMemberSync =
     !accessDirty && connector.accessMode === 'members' && connector.memberSyncStatus === 'disabled'
-  const hiddenCapFieldIds = memberCapFieldIds(connectorConfig, access.accessMode)
+  const hiddenCapFieldIds = derivedAclCapFieldIds(connectorConfig, access.accessMode)
 
   const persistedCanonicalModes = useMemo(
     () => readPersistedCanonicalModes(connector.sourceConfig),
@@ -606,11 +622,7 @@ function SettingsTab({
                       ? 'Switching…'
                       : isRebind
                         ? 'Change credential group'
-                        : access.accessMode === 'members'
-                          ? 'Switch to per-member access'
-                          : access.accessMode === 'admin'
-                            ? 'Switch to mirrored access'
-                            : 'Switch to workspace access'}
+                        : SWITCH_LABEL[access.accessMode]}
                   </Button>
                   <Button variant='default' size='sm' onClick={onResetAccess} disabled={isSaving}>
                     Cancel
@@ -619,11 +631,7 @@ function SettingsTab({
                 <p className='text-[var(--text-muted)] text-caption leading-snug'>
                   {isRebind
                     ? 'Members of the previous group lose access; members of the new group are invited to connect.'
-                    : access.accessMode === 'members'
-                      ? 'Everyone in the workspace is invited to connect their account. Documents stay hidden until members connect and sync; listing caps are cleared.'
-                      : access.accessMode === 'admin'
-                        ? 'Documents stay hidden until the next sync mirrors their permissions from the source; listing caps are cleared.'
-                        : 'Every workspace member can read every synced document once the next sync completes.'}
+                    : SWITCH_NOTICE[access.accessMode]}
                 </p>
               </div>
             ) : undefined

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options.ts

@@ -9,6 +9,7 @@ import {
   getCredentialGroupProviderId,
   isCredentialGroupProvider,
 } from '@/lib/credential-groups/providers'
+import { aclIsDerived } from '@/lib/knowledge/connectors/access-modes'
 import type { ConnectorMeta } from '@/connectors/types'
 import { useCredentialGroups } from '@/hooks/queries/credential-groups'
 
@@ -40,12 +41,12 @@ export function connectorMemberGroupProvider(
 }
 
 /** The config fields a per-member connector hides: its listing caps, which the server clears. */
-export function memberCapFieldIds(
+export function derivedAclCapFieldIds(
   connectorConfig: ConnectorMeta | null,
   accessMode: ConnectorAccessMode
 ): ReadonlySet<string> {
   return new Set(
-    accessMode === 'members' ? (connectorConfig?.permissionScopedListing?.capFieldIds ?? []) : []
+    aclIsDerived(accessMode) ? (connectorConfig?.permissionScopedListing?.capFieldIds ?? []) : []
   )
 }
 

apps/sim/connectors/confluence/confluence.test.ts

@@ -10,11 +10,11 @@ import {
   buildLastModifiedClause,
   confluenceConnector,
   escapeCql,
-  extractCursor,
   isCurrentContent,
   preserveConfluenceCallouts,
   readIncludedLabels,
 } from '@/connectors/confluence/confluence'
+import { extractCursor } from '@/connectors/confluence/cursor'
 import { htmlToPlainText } from '@/connectors/utils'
 
 describe('escapeCql', () => {