improvement(knowledge): read the source card with the result and hold Tin readiness longer

Author: waleedlatif1

apps/sim/lib/knowledge/application/search.test.ts

@@ -21,7 +21,6 @@ const mocks = vi.hoisted(() => ({
   generateEmbedding: vi.fn(),
   executeSearch: vi.fn(),
   retrieval: vi.fn(),
-  getDocumentMetadata: vi.fn(),
   getTagDefinitions: vi.fn(),
   getTagDefinitionsBatch: vi.fn(),
   recordEmbeddingUsage: vi.fn(),
@@ -100,7 +99,6 @@ vi.mock('@/lib/knowledge/search/queries', () => ({
     retrieval: mocks.retrieval(),
     readAccess: (args[0] as { access: unknown }).access,
   }),
-  getDocumentMetadataByIds: mocks.getDocumentMetadata,
 }))
 
 vi.mock('@/lib/knowledge/tags/service', () => ({
@@ -170,6 +168,9 @@ describe('knowledge search application use case', () => {
         content: 'answer',
         chunkIndex: 0,
         distance: 0.2,
+        filename: 'guide.pdf',
+        sourceUrl: null,
+        connectorType: null,
         tag1: null,
         tag2: null,
         tag3: null,
@@ -189,29 +190,11 @@ describe('knowledge search application use case', () => {
         boolean3: null,
       },
     ])
-    mocks.getDocumentMetadata.mockResolvedValue({
-      'document-1': { filename: 'guide.pdf', sourceUrl: null },
-    })
     mocks.getTagDefinitions.mockResolvedValue([])
     mocks.recordEmbeddingUsage.mockResolvedValue(undefined)
     mocks.importProvenance.mockResolvedValue({ imported: true, documentMetadata: {} })
   })
 
-  it('drops passages whose access was revoked before the final document metadata read', async () => {
-    mocks.getDocumentMetadata.mockResolvedValue({})
-    const result = await searchKnowledge.execute({
-      principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' },
-      input: {
-        workspaceId: 'workspace-1',
-        knowledgeBaseIds: ['knowledge-1'],
-        query: 'orion',
-        topK: 20,
-      },
-    })
-    expect(result.results).toEqual([])
-    expect(result.totalResults).toBe(0)
-  })
-
   it.each([false, true])(
     'requires explicit partial-result support for empty incomplete searches (allowPartialResults=%s)',
     async (allowPartialResults) => {
@@ -512,7 +495,6 @@ describe('knowledge search application use case', () => {
       })
     ).rejects.toThrow('Search superseded during embedding')
     expect(mocks.executeSearch).not.toHaveBeenCalled()
-    expect(mocks.getDocumentMetadata).not.toHaveBeenCalled()
   })
 
   it('does not start reranking or metadata reads after retrieval is cancelled', async () => {
@@ -536,7 +518,6 @@ describe('knowledge search application use case', () => {
       })
     ).rejects.toThrow('Search superseded during retrieval')
     expect(mocks.rerank).not.toHaveBeenCalled()
-    expect(mocks.getDocumentMetadata).not.toHaveBeenCalled()
     expect(mocks.searched).not.toHaveBeenCalled()
   })
 
@@ -550,6 +531,9 @@ describe('knowledge search application use case', () => {
         content: 'first',
         chunkIndex: 0,
         distance: 0.1,
+        filename: 'thread',
+        sourceUrl: null,
+        connectorType: 'slack',
       },
       {
         id: 'chunk-2',
@@ -558,6 +542,9 @@ describe('knowledge search application use case', () => {
         content: 'second',
         chunkIndex: 1,
         distance: 0.2,
+        filename: 'thread',
+        sourceUrl: null,
+        connectorType: 'slack',
       },
       {
         id: 'chunk-3',
@@ -566,12 +553,11 @@ describe('knowledge search application use case', () => {
         content: 'third',
         chunkIndex: 0,
         distance: 0.3,
+        filename: 'page',
+        sourceUrl: null,
+        connectorType: 'gitlab',
       },
     ])
-    mocks.getDocumentMetadata.mockResolvedValue({
-      'document-1': { filename: 'thread', connectorType: 'slack' },
-      'document-2': { filename: 'page', connectorType: 'gitlab' },
-    })
     await searchKnowledge.execute({
       principal: { kind: 'personal_api_key', userId: 'user-1', keyId: 'key-1' },
       input: {
@@ -808,14 +794,20 @@ describe('knowledge search application use case', () => {
   it('keeps the source card metadata when a provenance registry is present', async () => {
     const registry = { markIncomplete: vi.fn() }
     const sourceModifiedAt = new Date('2026-08-20T12:00:00Z')
-    mocks.getDocumentMetadata.mockResolvedValueOnce({
-      'document-1': {
+    mocks.executeSearch.mockResolvedValueOnce([
+      {
+        id: 'embedding-1',
+        documentId: 'document-1',
+        knowledgeBaseId: 'knowledge-1',
+        content: 'answer',
+        chunkIndex: 0,
+        distance: 0.2,
+        sourceModifiedAt,
         filename: 'guide.pdf',
         sourceUrl: 'https://example.com/guide',
-        sourceModifiedAt,
         connectorType: 'google_drive',
       },
-    })
+    ])
 
     const result = await searchKnowledge.execute({
       principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' },
@@ -828,7 +820,6 @@ describe('knowledge search application use case', () => {
       },
     })
 
-    expect(mocks.getDocumentMetadata).toHaveBeenCalledWith(['document-1'], expect.anything())
     expect(result.results[0]).toMatchObject({
       documentName: 'guide.pdf',
       sourceUrl: 'https://example.com/guide',

apps/sim/lib/knowledge/application/search.ts

@@ -42,7 +42,6 @@ import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults'
 import { annotateSearchDiagnostics, measureSearchStage } from '@/lib/knowledge/search/diagnostics'
 import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters'
 import {
-  getDocumentMetadataByIds,
   type RetrievalStatus,
   retrieveKnowledgeSearch,
   type SearchResult,
@@ -634,49 +633,40 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({
       ])
     )
     /**
-     * Always read: the provenance snapshot vouches for the name, URL, and tags
-     * a model may see, but the source card's modified time and connector type
-     * are only carried here, under the same access predicate as the search.
+     * The provenance snapshot vouches for the name, URL, and tags a model may see; the source
+     * card's modified time and connector type ride on the hydrated row, read under the same
+     * predicate as the content.
      */
-    const basicDocumentMetadata = await measureSearchStage('metadata', () =>
-      getDocumentMetadataByIds(
-        rows.map((row) => row.documentId),
-        retrieved.readAccess
-      )
-    )
-    const results = rows
-      .filter((row) => basicDocumentMetadata[row.documentId])
-      .map((row): KnowledgeSearchItem => {
-        const metadata: Record<string, unknown> = {}
-        const tagMap = tagMaps.get(row.knowledgeBaseId)
-        const provenanceDocument = provenanceSnapshot?.documentMetadata[row.documentId]
-        const basicDocument = basicDocumentMetadata[row.documentId]
-        const document = provenanceDocument ?? basicDocument
-        for (const slot of ALL_TAG_SLOTS) {
-          const value =
-            provenanceDocument && slot.startsWith('tag')
-              ? provenanceDocument[
-                  slot as 'tag1' | 'tag2' | 'tag3' | 'tag4' | 'tag5' | 'tag6' | 'tag7'
-                ]
-              : row[slot]
-          if (value !== null && value !== undefined) metadata[tagMap?.get(slot) ?? slot] = value
-        }
-        const rerankerScore = rerankerScores.get(row.id)
-        return {
-          embeddingId: row.id,
-          knowledgeBaseId: row.knowledgeBaseId,
-          documentId: row.documentId,
-          documentName: document?.filename ?? null,
-          sourceUrl: document?.sourceUrl ?? null,
-          sourceModifiedAt: basicDocument?.sourceModifiedAt ?? null,
-          connectorType: basicDocument?.connectorType ?? null,
-          content: row.content,
-          chunkIndex: row.chunkIndex,
-          metadata,
-          similarity: hasQuery ? 1 - row.distance : 1,
-          ...(rerankerScore !== undefined ? { rerankerScore } : {}),
-        }
-      })
+    const results = rows.map((row): KnowledgeSearchItem => {
+      const metadata: Record<string, unknown> = {}
+      const tagMap = tagMaps.get(row.knowledgeBaseId)
+      const provenanceDocument = provenanceSnapshot?.documentMetadata[row.documentId]
+      const document = provenanceDocument ?? row
+      for (const slot of ALL_TAG_SLOTS) {
+        const value =
+          provenanceDocument && slot.startsWith('tag')
+            ? provenanceDocument[
+                slot as 'tag1' | 'tag2' | 'tag3' | 'tag4' | 'tag5' | 'tag6' | 'tag7'
+              ]
+            : row[slot]
+        if (value !== null && value !== undefined) metadata[tagMap?.get(slot) ?? slot] = value
+      }
+      const rerankerScore = rerankerScores.get(row.id)
+      return {
+        embeddingId: row.id,
+        knowledgeBaseId: row.knowledgeBaseId,
+        documentId: row.documentId,
+        documentName: document?.filename ?? null,
+        sourceUrl: document?.sourceUrl ?? null,
+        sourceModifiedAt: row.sourceModifiedAt ?? null,
+        connectorType: row.connectorType ?? null,
+        content: row.content,
+        chunkIndex: row.chunkIndex,
+        metadata,
+        similarity: hasQuery ? 1 - row.distance : 1,
+        ...(rerankerScore !== undefined ? { rerankerScore } : {}),
+      }
+    })
     if (registry && provenanceSnapshot) {
       for (const [documentId, document] of Object.entries(provenanceSnapshot.documentMetadata)) {
         const renderedMetadata = results

apps/sim/lib/knowledge/search/diagnostics.ts

@@ -38,7 +38,6 @@ export type SearchStage =
   | 'usage_recording'
   | 'overage_billing'
   | 'tag_definitions'
-  | 'metadata'
   | 'metadata.sql'
   | 'metadata_provenance'
   | 'activity_recording'

apps/sim/lib/knowledge/search/queries.test.ts

@@ -557,6 +557,10 @@ describe('workspace-scoped vector retrieval', () => {
     expect(fields).toContain(String(schemaMock.embeddingSearch.vector512))
     expect(fields).not.toContain(String(schemaMock.embedding.embedding))
     expect(JSON.stringify(dbChainMockFns.leftJoin.mock.calls)).toContain('embeddingSearch')
+    /** The source card's name, URL and connector type ride on the same read; no second pass. */
+    expect(fields).toContain(String(schemaMock.document.filename))
+    expect(fields).toContain(String(schemaMock.knowledgeConnector.connectorType))
+    expect(JSON.stringify(dbChainMockFns.leftJoin.mock.calls)).toContain('knowledgeConnector')
   })
 
   it('passes over a slice whose documents went away instead of ending the pool there', async () => {

apps/sim/lib/knowledge/search/queries.ts

@@ -65,7 +65,6 @@ import type { StructuredFilter } from '@/lib/knowledge/types'
 import {
   embeddingCandidateDimensions,
   embeddingCandidateDistance,
-  embeddingDistance,
 } from '@/lib/knowledge/vector-columns'
 
 const logger = createLogger('KnowledgeSearchQueries')
@@ -345,6 +344,10 @@ export interface SearchResult {
   knowledgeBaseId: string
   /** When the source last changed the document; NULL for uploads and sources that do not say. */
   sourceModifiedAt: Date | null
+  filename: string
+  sourceUrl: string | null
+  /** The connector type behind the document; NULL for an upload. */
+  connectorType: string | null
 }
 
 /**
@@ -440,6 +443,9 @@ const getSearchResultFields = (distanceExpr: SQL<number> | SQL.Aliased<number>)
   distance: distanceExpr,
   knowledgeBaseId: embedding.knowledgeBaseId,
   sourceModifiedAt: document.sourceModifiedAt,
+  filename: document.filename,
+  sourceUrl: document.sourceUrl,
+  connectorType: knowledgeConnector.connectorType,
 })
 
 /**
@@ -921,6 +927,7 @@ function hydrateSearchCandidates(
       .from(embedding)
       .innerJoin(document, eq(embedding.documentId, document.id))
       .leftJoin(embeddingSearch, eq(embeddingSearch.id, embedding.id))
+      .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId))
       .where(
         and(
           inArray(embedding.id, ids),
@@ -1018,6 +1025,7 @@ export async function handleTagOnlySearch(params: SearchParams): Promise<SearchR
         .select(getSearchResultFields(sql<number>`0`.as('distance')))
         .from(embedding)
         .innerJoin(document, eq(embedding.documentId, document.id))
+        .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId))
         .where(
           and(
             eq(embedding.knowledgeBaseId, kbId),
@@ -1036,6 +1044,7 @@ export async function handleTagOnlySearch(params: SearchParams): Promise<SearchR
     .select(getSearchResultFields(sql<number>`0`.as('distance')))
     .from(embedding)
     .innerJoin(document, eq(embedding.documentId, document.id))
+    .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId))
     .where(
       and(
         inArray(embedding.knowledgeBaseId, knowledgeBaseIds),
@@ -2320,15 +2329,21 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
     return []
   }
 
-  /** Hydration pass: full rows plus the cosine distance, bounded to the survivors. */
+  /** Hydration pass: full rows plus the projection's distance, bounded to the survivors. */
   const hydrated = await db
     .select(
       getSearchResultFields(
-        embeddingDistance(queryVector.dimensions, queryVector.vector).as('distance')
+        embeddingCandidateDistance(
+          queryVector.dimensions,
+          queryVector.vector,
+          queryVector.model
+        ).as('distance')
       )
     )
     .from(embedding)
     .innerJoin(document, eq(embedding.documentId, document.id))
+    .leftJoin(embeddingSearch, eq(embeddingSearch.id, embedding.id))
+    .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId))
     .where(and(inArray(embedding.id, topIds), ...getVisibilityConditions(access, params.filters)))
 
   const rowById = new Map(hydrated.map((row) => [row.id, row]))

apps/sim/lib/knowledge/search/tin-keyword.ts

@@ -11,9 +11,10 @@ const logger = createLogger('TinKeywordSearch')
 
 /**
  * How long a readiness answer holds. The index only becomes valid once the projection is fully
- * backfilled, and flipping the flag off takes effect within this window.
+ * backfilled, and never invalid again; the flag is checked on every search regardless, so a
+ * long hold costs nothing but the one read it saves each search.
  */
-const READINESS_TTL_MS = 60 * 1000
+const READINESS_TTL_MS = 10 * 60 * 1000
 const SEARCH_INDEX_TTL_MS = 10 * 60 * 1000
 
 /**