v0.8.49: knowledge search improvements, tables perf improvements

Author: waleedlatif1

apps/sim/app/api/knowledge/search/utils.test.ts

@@ -877,14 +877,4 @@ describe('Knowledge Search Utils', () => {
       )
     })
   })
-
-  describe('getDocumentMetadataByIds', () => {
-    it('should handle empty input gracefully', async () => {
-      const { getDocumentMetadataByIds } = await import('@/lib/knowledge/search/queries')
-
-      const result = await getDocumentMetadataByIds([])
-
-      expect(result).toEqual({})
-    })
-  })
 })

apps/sim/app/api/knowledge/utils.ts

@@ -8,6 +8,7 @@ interface KnowledgeBaseData {
   userId: string
   workspaceId?: string | null
   name: string
+  isSearchIndex: boolean
   description?: string | null
   tokenCount: number
   embeddingModel: string
@@ -22,7 +23,13 @@ export interface KnowledgeBaseAccessResult {
   hasAccess: true
   knowledgeBase: Pick<
     KnowledgeBaseData,
-    'id' | 'userId' | 'workspaceId' | 'name' | 'embeddingModel' | 'embeddingDimension'
+    | 'id'
+    | 'userId'
+    | 'workspaceId'
+    | 'name'
+    | 'isSearchIndex'
+    | 'embeddingModel'
+    | 'embeddingDimension'
   >
 }
 
@@ -52,6 +59,7 @@ async function resolveKnowledgeBaseAccess(
       userId: knowledgeBase.userId,
       workspaceId: knowledgeBase.workspaceId,
       name: knowledgeBase.name,
+      isSearchIndex: knowledgeBase.isSearchIndex,
       embeddingModel: knowledgeBase.embeddingModel,
       embeddingDimension: knowledgeBase.embeddingDimension,
     })

apps/sim/app/api/table/capability-gate.test.ts

@@ -20,14 +20,26 @@ import {
 import { NextRequest } from 'next/server'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockGetTableById, mockGetUserEntityPermissions, mockAddTableColumn, mockListTableViews } =
+const { mockGetTableById, mockCheckWorkspaceAccess, mockAddTableColumn, mockListTableViews } =
   vi.hoisted(() => ({
     mockGetTableById: vi.fn(),
-    mockGetUserEntityPermissions: vi.fn(),
+    mockCheckWorkspaceAccess: vi.fn(),
     mockAddTableColumn: vi.fn(),
     mockListTableViews: vi.fn(),
   }))
 
+/** The shape `checkAccess` reads: the viewer's permission plus the workspace it just loaded. */
+function workspaceAccess(permission: string | null, organizationId: string | null = 'org-1') {
+  return {
+    exists: true,
+    hasAccess: permission !== null,
+    canWrite: permission === 'admin' || permission === 'write',
+    canAdmin: permission === 'admin',
+    workspace: { id: 'ws-1', organizationId },
+    permission,
+  }
+}
+
 vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock)
 
 vi.mock('@/lib/table', () => ({
@@ -44,7 +56,7 @@ vi.mock('@/lib/table/events', () => ({ signalTableSchemaChanged: vi.fn() }))
 vi.mock('@/lib/table/orchestration', () => ({ performUpdateTableColumn: vi.fn() }))
 vi.mock('@/lib/table/wire', () => ({ normalizeColumn: (column: unknown) => column }))
 vi.mock('@/lib/workspaces/permissions/utils', () => ({
-  getUserEntityPermissions: mockGetUserEntityPermissions,
+  checkWorkspaceAccess: mockCheckWorkspaceAccess,
 }))
 vi.mock('@/lib/workspaces/utils', () => ({ getWorkspaceOrganizationId: vi.fn() }))
 
@@ -96,11 +108,42 @@ describe('tables.use gate on the raw /api/table routes', () => {
       authType: 'session',
     })
     mockGetTableById.mockResolvedValue(TABLE)
-    mockGetUserEntityPermissions.mockResolvedValue('admin')
+    mockCheckWorkspaceAccess.mockResolvedValue(workspaceAccess('admin'))
     mockAddTableColumn.mockResolvedValue({ schema: { columns: [{ name: 'expires_at' }] } })
     mockListTableViews.mockResolvedValue([])
   })
 
+  /**
+   * The capability resolver looks the workspace up itself when the organization is omitted, so a
+   * call site that already access-checked the workspace and drops the id pays a second read of a
+   * value it is holding — once on every raw table route. Asserted on the resolver rather than on
+   * a query count because that is where the omission would show.
+   */
+  it('hands the capability resolver the organization it just loaded, not undefined', async () => {
+    mockCheckWorkspaceAccess.mockResolvedValue(workspaceAccess('admin', 'org-42'))
+
+    await listViews()
+
+    expect(permissionGroupScopeMockFns.mockResolvePermissionGroupConfig).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.any(String),
+      'org-42'
+    )
+  })
+
+  /** A personal workspace has no organization; `null` is the answer, and still not a lookup. */
+  it('passes null for a workspace that belongs to no organization', async () => {
+    mockCheckWorkspaceAccess.mockResolvedValue(workspaceAccess('admin', null))
+
+    await listViews()
+
+    expect(permissionGroupScopeMockFns.mockResolvePermissionGroupConfig).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.any(String),
+      null
+    )
+  })
+
   describe('when the group withholds Tables', () => {
     beforeEach(() => {
       permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({
@@ -131,7 +174,7 @@ describe('tables.use gate on the raw /api/table routes', () => {
     })
 
     it('still conceals a table the caller cannot reach, rather than naming the capability', async () => {
-      mockGetUserEntityPermissions.mockResolvedValue(null)
+      mockCheckWorkspaceAccess.mockResolvedValue(workspaceAccess(null))
 
       const response = await listViews()
 

apps/sim/app/api/table/utils.ts

@@ -23,7 +23,7 @@ import { TableLockedError } from '@/lib/table/mutation-locks'
 import { isTablePredicate } from '@/lib/table/query-builder/converters'
 import { validateStoragePredicate } from '@/lib/table/query-builder/validate'
 import type { TableLockKind } from '@/lib/table/types'
-import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
+import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils'
 import { getWorkspaceOrganizationId } from '@/lib/workspaces/utils'
 
 /**
@@ -338,12 +338,14 @@ export async function checkAccess(
     return { ok: false, status: 404 }
   }
 
-  const permission = await getUserEntityPermissions(
-    roleSubjectUserId(principal),
-    'workspace',
-    table.workspaceId
-  )
-  if (!permissionSatisfies(permission, level)) {
+  /**
+   * Resolved through {@link checkWorkspaceAccess} rather than `getUserEntityPermissions`, which
+   * delegates to it and returns the permission alone. Same single resolution, but it also hands
+   * back the workspace this check just loaded — and with it the owning organization the
+   * capability gate below would otherwise look up for itself.
+   */
+  const access = await checkWorkspaceAccess(table.workspaceId, roleSubjectUserId(principal))
+  if (!permissionSatisfies(access.permission, level)) {
     return { ok: false, status: 403 }
   }
 
@@ -352,7 +354,18 @@ export async function checkAccess(
   if (
     governedUserId &&
     table.workspaceId &&
-    (await isWorkspaceCapabilityWithheld(governedUserId, table.workspaceId, 'tables.use'))
+    /**
+     * The organization is passed, not re-derived: omitting it makes the resolver load this very
+     * workspace a second time (see `getUserPermissionConfig`), which is one extra round trip on
+     * every raw table route. `access.workspace` is non-null on this line — a missing workspace
+     * resolves to a null permission, which the gate above already refused.
+     */
+    (await isWorkspaceCapabilityWithheld(
+      governedUserId,
+      table.workspaceId,
+      'tables.use',
+      access.workspace?.organizationId ?? null
+    ))
   ) {
     return { ok: false, status: 403, capability: 'tables.use' }
   }

apps/sim/app/api/v1/knowledge/search/route.test.ts

@@ -16,7 +16,6 @@ const {
   mockExecuteKnowledgeSearch,
   mockRetrievalStatus,
   mockGenerateSearchEmbedding,
-  mockGetDocumentMetadataByIds,
   mockGetDocumentTagDefinitions,
   mockAuthenticateRequest,
   mockValidateWorkspaceAccess,
@@ -28,7 +27,6 @@ const {
   mockExecuteKnowledgeSearch: vi.fn(),
   mockRetrievalStatus: vi.fn(() => ({ status: 'complete', timedOutLegs: [] })),
   mockGenerateSearchEmbedding: vi.fn(),
-  mockGetDocumentMetadataByIds: vi.fn(),
   mockGetDocumentTagDefinitions: vi.fn(),
   mockAuthenticateRequest: vi.fn(),
   mockValidateWorkspaceAccess: vi.fn(),
@@ -60,9 +58,7 @@ vi.mock('@/lib/knowledge/search/queries', () => ({
   retrieveKnowledgeSearch: async (params: { access: unknown }) => ({
     rows: await mockExecuteKnowledgeSearch(params),
     retrieval: mockRetrievalStatus(),
-    readAccess: params.access,
   }),
-  getDocumentMetadataByIds: mockGetDocumentMetadataByIds,
 }))
 
 vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock)
@@ -132,7 +128,6 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
       isBYOK: false,
     })
     mockExecuteKnowledgeSearch.mockResolvedValue([])
-    mockGetDocumentMetadataByIds.mockResolvedValue({})
     mockGetDocumentTagDefinitions.mockResolvedValue([])
     mockResolveBillingAttribution.mockImplementation(
       ({ actorUserId, workspaceId }: { actorUserId: string; workspaceId: string }) =>
@@ -164,7 +159,6 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
     )
     expect(mockExecuteKnowledgeSearch).toHaveBeenCalledOnce()
     expect(response.status).toBe(500)
-    expect(mockGetDocumentMetadataByIds).not.toHaveBeenCalled()
   })
 
   it('retains the reader provider for ranked results and returned document metadata', async () => {
@@ -193,17 +187,11 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
         accessProvider: provider,
       })
     )
-    expect(mockGetDocumentMetadataByIds).toHaveBeenCalledWith([], access)
   })
 
-  it.each([
-    ['query', false],
-    ['query', true],
-    ['filters', false],
-    ['filters', true],
-  ] as const)(
-    'omits newly denied content from %s results and counts when all denied is %s',
-    async (mode, allDenied) => {
+  it.each(['query', 'filters'] as const)(
+    'renders the source card each %s result row carries',
+    async (mode) => {
       const access = { kind: 'user' as const, userId: 'user-1', tokens: ['reader-token'] }
       const provider = {
         get: vi.fn().mockResolvedValue(access),
@@ -219,26 +207,17 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
         { tagSlot: 'tag1', displayName: 'category', fieldType: 'text' },
       ])
       mockExecuteKnowledgeSearch.mockResolvedValue([
-        {
-          documentId: 'revoked-document',
-          knowledgeBaseId: 'kb-1',
-          content: 'revoked page content',
-          tag1: 'revoked tag',
-          chunkIndex: 0,
-          distance: 0.1,
-        },
         {
           documentId: 'allowed-document',
           knowledgeBaseId: 'kb-1',
           content: 'allowed page content',
+          filename: 'Allowed page',
+          sourceUrl: null,
           tag1: 'docs',
           chunkIndex: 0,
           distance: 0.2,
         },
       ])
-      mockGetDocumentMetadataByIds.mockResolvedValue(
-        allDenied ? {} : { 'allowed-document': { filename: 'Allowed page', sourceUrl: null } }
-      )
       const response = await POST(
         createMockRequest('POST', {
           workspaceId: 'ws-1',
@@ -250,24 +229,19 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
       )
       const body = await response.json()
       expect(response.status).toBe(200)
-      expect(mockGetDocumentMetadataByIds).toHaveBeenCalledWith(
-        ['revoked-document', 'allowed-document'],
-        access
-      )
-      expect(body.data.results).toEqual(
-        allDenied
-          ? []
-          : [
-              expect.objectContaining({
-                documentId: 'allowed-document',
-                documentName: 'Allowed page',
-                content: 'allowed page content',
-                metadata: { category: 'docs' },
-              }),
-            ]
+      expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith(
+        expect.objectContaining({ access, accessProvider: provider })
       )
-      expect(body.data.totalResults).toBe(allDenied ? 0 : 1)
-      expect(JSON.stringify(body)).not.toContain('revoked')
+      expect(body.data.results).toEqual([
+        expect.objectContaining({
+          documentId: 'allowed-document',
+          documentName: 'Allowed page',
+          sourceUrl: null,
+          content: 'allowed page content',
+          metadata: { category: 'docs' },
+        }),
+      ])
+      expect(body.data.totalResults).toBe(1)
     }
   )
 
@@ -372,7 +346,7 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
     expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled()
   })
 
-  it('surfaces sourceUrl from document metadata in search results', async () => {
+  it('surfaces the sourceUrl a result row carries', async () => {
     mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
       hasAccess: true,
       knowledgeBase: baseKb('kb-confluence', 'text-embedding-3-small'),
@@ -382,16 +356,12 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
         documentId: 'doc-confluence',
         knowledgeBaseId: 'kb-confluence',
         content: 'page content',
+        filename: 'Runbook.md',
+        sourceUrl: 'https://example.atlassian.net/wiki/spaces/DOCS/pages/12345',
         chunkIndex: 0,
         distance: 0.1,
       },
     ])
-    mockGetDocumentMetadataByIds.mockResolvedValue({
-      'doc-confluence': {
-        filename: 'Runbook.md',
-        sourceUrl: 'https://example.atlassian.net/wiki/spaces/DOCS/pages/12345',
-      },
-    })
 
     const req = createMockRequest('POST', {
       workspaceId: 'ws-1',

apps/sim/app/api/v1/knowledge/search/route.ts

@@ -17,7 +17,6 @@ import {
 import { SearchDeadlineError } from '@/lib/knowledge/search/budget'
 import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults'
 import {
-  getDocumentMetadataByIds,
   type KnowledgeRetrievalResult,
   retrieveKnowledgeSearch,
   type SearchResult,
@@ -267,6 +266,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
         accessProvider,
         searchMode,
         boostRecency,
+        searchIndexOnly: accessibleKbs.every((kb) => kb.isSearchIndex),
         query,
         queryVector: {
           vector: JSON.stringify(queryEmbeddingResult.embedding),
@@ -316,14 +316,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     /** v1 cannot express an incomplete search, so a leg that ran out of time fails the request. */
     if (retrieved.retrieval.status === 'partial') throw new SearchDeadlineError()
     const results = retrieved.rows
-    const documentIds = results.map((r) => r.documentId)
-    const documentMetadataMap = await getDocumentMetadataByIds(documentIds, retrieved.readAccess)
-    const readableResults = results.filter((result) => documentMetadataMap[result.documentId])
 
     return NextResponse.json({
       success: true,
       data: {
-        results: readableResults.map((result) => {
+        results: results.map((result) => {
           const kbTagMap = tagDefinitionsMap[result.knowledgeBaseId] || {}
           const tags: Record<string, string | number | boolean | Date | null> = {}
 
@@ -335,11 +332,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
             }
           })
 
-          const docMeta = documentMetadataMap[result.documentId]
           return {
             documentId: result.documentId,
-            documentName: docMeta?.filename || undefined,
-            sourceUrl: docMeta?.sourceUrl ?? null,
+            documentName: result.filename || undefined,
+            sourceUrl: result.sourceUrl,
             content: result.content,
             chunkIndex: result.chunkIndex,
             metadata: tags,
@@ -349,7 +345,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
         query: query || '',
         knowledgeBaseIds: accessibleKbIds,
         topK,
-        totalResults: readableResults.length,
+        totalResults: results.length,
       },
     })
   } catch (error) {