improvement(knowledge): rank organization keyword search with Tin (#8007)

A member who reaches more of an organization search index than an exact
ranking can afford searched keywords through the GIN projection, which
scores every chunk matching the term before access is checked, so a common
word cost seconds. Where the database provides the tin extension, a BM25
projection of search-index chunks is kept by embedding and knowledge-base
triggers, and the keyword leg ranks with Tin first and checks access only on
the top of that ranking, widening the window while too few ranked chunks are
readable and leaving a page to GIN if the widest window cannot fill it.

The query is analyzed by the same websearch_to_tsquery as the GIN path and
translated to TINQL; shapes TINQL cannot express keep GIN. The path is gated
by the knowledge-tin-keyword flag, a valid Tin index (built only after the
backfill completes), and every base being a search index. Script migration
0019 installs the projection only where tin is available and creatable, so
self-hosted databases keep an empty table and the GIN path.

Author: waleedlatif1

.github/workflows/test-build.yml

@@ -241,6 +241,7 @@ jobs:
           bunx vitest run
           script-migrations/0016_backfill_search_vectors.postgres.test.ts
           script-migrations/0018_repair_workspace_file_content_revision.postgres.test.ts
+          script-migrations/0019_tin_keyword_projection.postgres.test.ts
 
       - name: Verify Search progress, pagination, and outbox scheduling in PostgreSQL
         working-directory: apps/sim

apps/sim/lib/core/config/env.ts

@@ -629,6 +629,7 @@ export const env = createEnv({
     TABLE_ROW_TTL:                        z.boolean().optional(),
     CREDENTIAL_GROUPS:                    z.boolean().optional(),                 // Enable enterprise Credential Groups globally
     KNOWLEDGE_MEMBER_ACCESS:              z.boolean().optional(),                 // Enable per-member knowledge connectors and hybrid-by-default retrieval globally
+    KNOWLEDGE_TIN_KEYWORD:                z.boolean().optional(),                 // Rank large-scope keyword retrieval through the Tin text index where it exists
 
     // Organizations - for self-hosted deployments
     ORGANIZATIONS_ENABLED:                 z.boolean().optional(),                 // Enable organizations on self-hosted (bypasses plan requirements)

apps/sim/lib/core/config/feature-flags.test.ts

@@ -15,6 +15,7 @@ const { mockFetch, mockIsPlatformAdmin, envRef } = vi.hoisted(() => ({
     TABLE_ROW_TTL: undefined as boolean | undefined,
     CREDENTIAL_GROUPS: undefined as boolean | undefined,
     KNOWLEDGE_MEMBER_ACCESS: undefined as boolean | undefined,
+    KNOWLEDGE_TIN_KEYWORD: undefined as boolean | undefined,
     SLACK_SEARCH_SHARED_APP: undefined as boolean | undefined,
   },
 }))
@@ -126,6 +127,7 @@ describe('isFeatureEnabled', () => {
     setEnvFlags({ isAppConfigEnabled: false })
     envRef.CREDENTIAL_GROUPS = undefined
     envRef.KNOWLEDGE_MEMBER_ACCESS = undefined
+    envRef.KNOWLEDGE_TIN_KEYWORD = undefined
     envRef.SLACK_SEARCH_SHARED_APP = undefined
   })
 
@@ -162,6 +164,19 @@ describe('isFeatureEnabled', () => {
     })
   })
 
+  describe('knowledge-tin-keyword flag', () => {
+    it('is a global switch', async () => {
+      expect(await isFeatureEnabled('knowledge-tin-keyword')).toBe(false)
+      envRef.KNOWLEDGE_TIN_KEYWORD = true
+      expect(await isFeatureEnabled('knowledge-tin-keyword')).toBe(true)
+    })
+
+    it('follows an AppConfig global rule', async () => {
+      withAppConfig({ 'knowledge-tin-keyword': { enabled: true } })
+      expect(await isFeatureEnabled('knowledge-tin-keyword')).toBe(true)
+    })
+  })
+
   describe('knowledge-member-access flag', () => {
     it('uses a global fallback switch off AppConfig', async () => {
       expect(await isFeatureEnabled('knowledge-member-access')).toBe(false)

apps/sim/lib/core/config/feature-flags.ts

@@ -93,6 +93,13 @@ const FEATURE_FLAGS = {
       'KNOWLEDGE_MEMBER_ACCESS.',
     fallback: 'KNOWLEDGE_MEMBER_ACCESS',
   },
+  'knowledge-tin-keyword': {
+    description:
+      'Rank keyword retrieval for members whose permitted set is too large to enumerate through ' +
+      'the Tin text index instead of GIN. Has no effect where the Tin keyword index is absent or ' +
+      'invalid. Off-AppConfig falls back to KNOWLEDGE_TIN_KEYWORD.',
+    fallback: 'KNOWLEDGE_TIN_KEYWORD',
+  },
 } satisfies Record<string, FeatureFlagDefinition>
 
 /**

apps/sim/lib/knowledge/search/diagnostics.ts

@@ -51,6 +51,9 @@ export type SearchStage =
   | 'vector.exact_candidates'
   | 'vector.exact'
   | 'vector.candidate_search'
+  | 'keyword.tin'
+  | 'keyword.tin_readiness'
+  | 'keyword.tin_query'
   | 'source_overview'
   | 'source_overview.availability'
   | 'source_overview.providers'
@@ -98,6 +101,13 @@ export interface SearchDiagnosticMetadata {
   permittedDocuments?: 'bounded' | 'unbounded'
   /** Documents in a bounded permitted set. */
   permittedDocumentCount?: number
+  /**
+   * Which index ranked an unbounded keyword leg: `tin` ranks by BM25 and checks access on the top
+   * of that ranking; `gin` ranks every match. Absent when the leg ranked inside a bounded set.
+   */
+  keywordRanking?: 'tin' | 'gin'
+  /** Candidates Tin ranked before access was checked on the last keyword page. */
+  keywordTinWindow?: number
   vectorCandidateCount?: number
   vectorCandidateDimensions?: number
   resultCount?: number

apps/sim/lib/knowledge/search/queries.test.ts

@@ -10,6 +10,15 @@ import {
   schemaMock,
 } from '@sim/testing'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockResolveTinKeywordQuery } = vi.hoisted(() => ({
+  mockResolveTinKeywordQuery: vi.fn<() => Promise<string | null>>(async () => null),
+}))
+
+vi.mock('@/lib/knowledge/search/tin-keyword', () => ({
+  resolveTinKeywordQuery: mockResolveTinKeywordQuery,
+}))
+
 import {
   type KnowledgeAccessProvider,
   type UserAccessScope,
@@ -1489,6 +1498,106 @@ describe('permitted-document planner', () => {
     expect(JSON.stringify(keyword)).toContain('doc-a')
   })
 
+  describe('Tin keyword ranking for an unbounded caller', () => {
+    const unbounded: PermittedDocuments = { kind: 'unbounded' }
+    const keyword = (overrides: Partial<Parameters<typeof executeKeywordSearch>[0]> = {}) =>
+      executeKeywordSearch({
+        ...params,
+        topK: 1,
+        query: 'release',
+        queryVector: params.queryVector!,
+        permitted: unbounded,
+        ...overrides,
+      })
+    const tinStatements = () =>
+      statements().filter((query) => query.sql.includes('ranked_tin_chunks'))
+    const ginStatements = () =>
+      statements().filter((query) => query.sql.includes('WITH matched_keyword_chunks'))
+    let tinPages: Array<{ ranked: number; candidates: ReturnType<typeof hit>[] }>
+
+    beforeEach(() => {
+      mockResolveTinKeywordQuery.mockReset()
+      mockResolveTinKeywordQuery.mockResolvedValue('"releas"')
+      tinPages = []
+      dbChainMockFns.execute.mockImplementation(async (query) =>
+        render(query).sql.includes('ranked_tin_chunks')
+          ? [tinPages.shift() ?? { ranked: 0, candidates: [] }]
+          : []
+      )
+    })
+
+    it('ranks with Tin and checks access only on the top of that ranking', async () => {
+      tinPages = [{ ranked: 1500, candidates: [hit('a', null)] }]
+      queueTableRows(schemaMock.embedding, [{ ...hit('a', null), content: 'release notes' }])
+      const results = await keyword()
+      expect(results.map((row) => row.id)).toEqual(['a'])
+      expect(mockResolveTinKeywordQuery).toHaveBeenCalledWith(
+        ['org-index'],
+        'release',
+        'english',
+        params.budget
+      )
+      expect(ginStatements()).toHaveLength(0)
+      expect(JSON.stringify(tinStatements()[0])).toContain('2000')
+      /** `==>` binds tighter than `||`, so the concatenated query must be parenthesized. */
+      expect(tinStatements()[0].sql).toContain('==> (?)')
+    })
+
+    it('widens the ranked window while too few ranked chunks are readable', async () => {
+      tinPages = [
+        { ranked: 2000, candidates: [] },
+        { ranked: 4000, candidates: [hit('b', null)] },
+      ]
+      queueTableRows(schemaMock.embedding, [{ ...hit('b', null), content: 'release notes' }])
+      expect((await keyword()).map((row) => row.id)).toEqual(['b'])
+      const windows = tinStatements().map((query) => JSON.stringify(query))
+      expect(windows[0]).toContain('2000')
+      expect(windows[1]).toContain('10000')
+      expect(ginStatements()).toHaveLength(0)
+    })
+
+    it('stops widening once Tin ranked every match', async () => {
+      tinPages = [{ ranked: 12, candidates: [] }]
+      expect(await keyword()).toEqual([])
+      expect(tinStatements()).toHaveLength(1)
+      expect(ginStatements()).toHaveLength(0)
+    })
+
+    it('leaves the page to the GIN ranking when the widest window cannot fill it', async () => {
+      tinPages = [
+        { ranked: 2000, candidates: [] },
+        { ranked: 10_000, candidates: [] },
+        { ranked: 50_000, candidates: [] },
+      ]
+      await keyword()
+      expect(tinStatements()).toHaveLength(3)
+      expect(ginStatements()).toHaveLength(1)
+    })
+
+    it.each([
+      ['a bounded permitted set', { permitted: bounded({ id: 'doc-a', connectorId: null }) }],
+      [
+        'structured tag filters',
+        {
+          structuredFilters: [
+            { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'x' },
+          ] as StructuredFilter[],
+        },
+      ],
+    ])('keeps GIN ranking for %s', async (_case, overrides) => {
+      await keyword(overrides)
+      expect(mockResolveTinKeywordQuery).not.toHaveBeenCalled()
+      expect(tinStatements()).toHaveLength(0)
+    })
+
+    it('keeps GIN ranking when Tin is not ready or cannot express the query', async () => {
+      mockResolveTinKeywordQuery.mockResolvedValue(null)
+      await keyword()
+      expect(tinStatements()).toHaveLength(0)
+      expect(ginStatements()).toHaveLength(1)
+    })
+  })
+
   it('skips keyword SQL entirely when nothing is permitted', async () => {
     expect(
       await executeKeywordSearch({

apps/sim/lib/knowledge/search/queries.ts

@@ -3,6 +3,7 @@ import {
   document,
   embedding,
   embeddingKeywordSearch,
+  embeddingKeywordTin,
   embeddingSearch,
   knowledgeConnector,
 } from '@sim/db/schema'
@@ -35,6 +36,7 @@ import {
 import { workspaceSearchFilterConditions } from '@/lib/knowledge/search/filter-conditions'
 import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters'
 import { applyRecencyBoost, RRF_K } from '@/lib/knowledge/search/recency'
+import { resolveTinKeywordQuery } from '@/lib/knowledge/search/tin-keyword'
 import {
   coerceTagFilterValue,
   escapeLikePattern,
@@ -491,6 +493,13 @@ export function getStructuredTagFilters(filters: StructuredFilter[], embeddingTa
  */
 const FTS_CONFIG = 'english'
 
+/**
+ * Chunks Tin ranks before access is checked, widening while too few are readable to fill a page.
+ * A caller past the permitted-set limit reads a large share of the index, so the first window
+ * almost always fills; the widest bounds the work before the GIN ranking takes over.
+ */
+const TIN_KEYWORD_WINDOWS = [2000, 10_000, 50_000] as const
+
 /**
  * Row visibility predicates shared by every search leg: a chunk is only
  * retrievable when both it and its document are enabled, the document finished
@@ -1348,6 +1357,96 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
       ...tagFilterConditions,
     ]
     const candidateRank = sql<number>`ts_rank_cd(${embeddingKeywordSearch.contentTsv}, ${tsQuery})`
+    /**
+     * A caller reaching past the permitted-set limit reads much of the index, so ranking every
+     * match before checking access is the leg's whole cost for a common term. Where the Tin
+     * projection is complete, BM25 ranks inside the bases first and access is checked only on the
+     * top of that ranking.
+     */
+    const tinQuery =
+      params.permitted?.kind === 'unbounded' && tagFilterConditions.length === 0
+        ? await resolveTinKeywordQuery(knowledgeBaseIds, query, FTS_CONFIG, params.budget)
+        : null
+    annotateSearchDiagnostics({
+      ...(params.permitted?.kind === 'unbounded'
+        ? { keywordRanking: tinQuery ? 'tin' : 'gin' }
+        : {}),
+    })
+    const documentConditions = (excludedSources: readonly string[]) =>
+      and(
+        ...candidateDocumentConditions(
+          knowledgeBaseIds,
+          access,
+          params.filters,
+          knowledgeMetadataCandidateAccessCondition(access)
+        ),
+        excludeSearchSources(excludedSources)
+      )
+    /**
+     * One page from the top of Tin's ranking. The window of ranked chunks widens while too few of
+     * them are readable to fill the page; if the widest window still cannot, the page is left to
+     * the GIN ranking, which covers every match.
+     */
+    const selectTinPage = async (
+      scopedQuery: SQL,
+      limit: number,
+      offset: number,
+      excludedSources: readonly string[]
+    ): Promise<SearchReadCandidatePage | null> => {
+      for (const window of TIN_KEYWORD_WINDOWS) {
+        if (window < offset + limit) continue
+        const [page] = await runSearchQuery(params.budget, 'keyword.tin', (executor) =>
+          executor.execute<{ ranked: number; candidates: SearchReadCandidate[] }>(sql`
+            WITH ranked_tin_chunks AS MATERIALIZED (
+              SELECT ${embeddingKeywordTin.id} AS id, ${embeddingKeywordTin.documentId} AS document_id,
+                ${embeddingKeywordTin.enabled} AS enabled,
+                tin.full_score(${embeddingKeywordTin}.ctid) AS keyword_rank
+              FROM ${embeddingKeywordTin}
+              WHERE ${embeddingKeywordTin.content} ==> (${scopedQuery})
+              ORDER BY keyword_rank DESC
+              LIMIT ${window}
+            ), visible_keyword_documents AS MATERIALIZED (
+              SELECT ${document.id} AS id FROM ${document}
+              WHERE ${and(
+                sql`${document.id} = ANY (ARRAY(SELECT document_id FROM ranked_tin_chunks))`,
+                documentConditions(excludedSources)
+              )}
+            ), page AS (
+              SELECT ranked_tin_chunks.id, ${document.id} AS "documentId",
+                ${document.connectorId} AS "connectorId",
+                ${SEARCH_READ_CANDIDATE_FIELDS.liveAuthorizationSource} AS "liveAuthorizationSource",
+                ranked_tin_chunks.keyword_rank
+              FROM ranked_tin_chunks INNER JOIN ${document}
+                ON ${document.id} = ranked_tin_chunks.document_id
+              WHERE ranked_tin_chunks.enabled
+                AND ranked_tin_chunks.document_id IN (SELECT id FROM visible_keyword_documents)
+              ORDER BY ranked_tin_chunks.keyword_rank DESC, ranked_tin_chunks.id
+              LIMIT ${limit} OFFSET ${offset}
+            )
+            SELECT (SELECT count(*)::int FROM ranked_tin_chunks) AS ranked,
+              coalesce((
+                SELECT json_agg(json_build_object(
+                  'id', page.id, 'documentId', page."documentId", 'connectorId', page."connectorId",
+                  'liveAuthorizationSource', page."liveAuthorizationSource"
+                ) ORDER BY page.keyword_rank DESC, page.id)
+                FROM page
+              ), '[]'::json) AS candidates
+          `)
+        )
+        annotateSearchDiagnostics({ keywordTinWindow: window })
+        if (page.candidates.length === limit || page.ranked < window) {
+          return { candidates: page.candidates, nextOffset: offset + page.candidates.length }
+        }
+      }
+      return null
+    }
+    /** Parenthesized where used: `==>` binds tighter than `||`. */
+    const tinScope = tinQuery
+      ? sql`'(' || ${sql.join(
+          knowledgeBaseIds.map((id) => sql`knowledge_tin_base_token(${id}) || '^0'`),
+          sql` || ' OR ' || `
+        )} || ') AND (' || ${tinQuery} || ')'`
+      : undefined
     /** Keep readable identities and rank scalars separate so sorts never carry full text-search vectors. */
     return selectAuthorizedSearchResults({
       leg: 'keyword',
@@ -1368,6 +1467,11 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
             ? permittedDocumentIds(params.permitted.documents, excludedSources)
             : undefined
         if (permittedIds?.length === 0) return { candidates: [], nextOffset: offset }
+        if (tinScope && !permittedIds) {
+          const tinPage = await selectTinPage(tinScope, limit, offset, excludedSources)
+          if (tinPage) return tinPage
+          annotateSearchDiagnostics({ keywordRanking: 'gin' })
+        }
         const baseScope = and(
           inArray(embeddingKeywordSearch.knowledgeBaseId, knowledgeBaseIds),
           eq(embeddingKeywordSearch.enabled, true)
@@ -1411,13 +1515,7 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
               SELECT ${document.id} AS id FROM ${document}
               WHERE ${and(
                 sql`${document.id} = ANY (ARRAY(SELECT document_id FROM matched_keyword_chunks))`,
-                ...candidateDocumentConditions(
-                  knowledgeBaseIds,
-                  access,
-                  params.filters,
-                  knowledgeMetadataCandidateAccessCondition(access)
-                ),
-                excludeSearchSources(excludedSources)
+                documentConditions(excludedSources)
               )}
             ), ranked_keyword_candidates AS MATERIALIZED (
               SELECT matched_keyword_chunks.id, matched_keyword_chunks.document_id,

apps/sim/lib/knowledge/search/tin-keyword-readiness.test.ts

@@ -0,0 +1,31 @@
+/**
+ * @vitest-environment node
+ */
+import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing'
+import { expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/core/config/feature-flags', () => ({
+  isFeatureEnabled: vi.fn(async () => true),
+}))
+
+import { resolveTinKeywordQuery } from '@/lib/knowledge/search/tin-keyword'
+
+/** Its own file, so the process-wide readiness cache starts empty. */
+it('stays on the GIN projection while the Tin index is incomplete, and remembers that', async () => {
+  resetDbChainMock()
+  let indexValid = false
+  dbChainMockFns.execute.mockImplementation(async (query) => {
+    const text = JSON.stringify(query)
+    if (text.includes('indisvalid')) return [{ valid: indexValid }]
+    if (text.includes('websearch_to_tsquery')) return [{ rendered: "'releas'" }]
+    return []
+  })
+  queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-index', isSearchIndex: true }])
+  expect(await resolveTinKeywordQuery(['kb-index'], 'release', 'english', undefined)).toBeNull()
+  indexValid = true
+  expect(await resolveTinKeywordQuery(['kb-index'], 'release', 'english', undefined)).toBeNull()
+  const readinessReads = dbChainMockFns.execute.mock.calls.filter(([query]) =>
+    JSON.stringify(query).includes('indisvalid')
+  )
+  expect(readinessReads).toHaveLength(1)
+})