diff --git a/components/search-dialog.tsx b/components/search-dialog.tsx index c5a3ef07f..4b57cd89e 100644 --- a/components/search-dialog.tsx +++ b/components/search-dialog.tsx @@ -22,9 +22,10 @@ import { * * The per-locale Orama indexes are built at build time by * app/api/search/[lang]/route.ts; this component downloads only the current - * reader's language (via `from`) and runs every query client-side. It mirrors - * fumadocs' DefaultSearchDialog layout (so the UX is unchanged) but swaps the - * fetch client for the static Orama client with a locale-aware `initOrama`. + * reader's language and runs every query client-side. It mirrors fumadocs' + * DefaultSearchDialog layout (so the UX is unchanged) but swaps the fetch client + * for a static one with a locale-aware `initOrama` and page-level ranking + * (lib/search-client.ts, lib/search-ranking.ts). * * `initOrama` must reproduce the SAME tokenizer the route used to build each * locale's index, otherwise the query is tokenized differently from the stored @@ -36,10 +37,9 @@ import { export default function StaticSearchDialog(props: SharedProps) { const { locale } = useI18n() const lang = locale ?? i18n.defaultLanguage - // Created inline (like fumadocs' DefaultSearchDialog); useDocsSearch tracks - // re-queries via the client's `deps` ([tag, locale]), and the downloaded index - // is cached module-side keyed by URL, so this stays cheap across renders. - // `from` is the per-locale endpoint so we only download this reader's language. + // Created inline (like fumadocs' DefaultSearchDialog); useDocsSearch re-queries + // when the client's `deps` ([locale]) change, and the downloaded index is + // cached module-side per locale, so this stays cheap across renders. const client = createDocsSearchClient(lang) const { search, setSearch, query } = useDocsSearch({ client }) diff --git a/lib/search-client.ts b/lib/search-client.ts index 0cd6d26cf..78f110745 100644 --- a/lib/search-client.ts +++ b/lib/search-client.ts @@ -1,6 +1,7 @@ -import { create, type AnyOrama } from '@orama/orama' -import { oramaStaticClient } from 'fumadocs-core/search/client/orama-static' +import { create, load, type AnyOrama } from '@orama/orama' +import type { SearchClient } from 'fumadocs-core/search/client' import { SEARCH_LANGUAGE_BY_LOCALE } from '@/lib/search-languages' +import { searchRanked } from '@/lib/search-ranking' /** * Build the same locale-specific tokenizer used by the static search route. @@ -37,10 +38,39 @@ export async function initOrama(locale?: string): Promise { }) } -export function createDocsSearchClient(locale: string) { - return oramaStaticClient({ - from: `/api/search/${encodeURIComponent(locale)}`, - initOrama, - locale, - }) +// One download per locale per page load, shared by every client. A failed +// download is evicted so the next query can retry. +const dbCache = new Map>() + +function getLocaleDB(locale: string): Promise { + let cached = dbCache.get(locale) + if (!cached) { + cached = (async () => { + const res = await fetch(`/api/search/${encodeURIComponent(locale)}`) + if (!res.ok) throw new Error(`failed to fetch search index for "${locale}": ${res.status}`) + const exported = (await res.json()) as { data: Record } + const data = exported.data[locale] + if (!data) return undefined + const db = await initOrama(locale) + load(db, data as Parameters[1]) + return db + })() + cached.catch(() => dbCache.delete(locale)) + dbCache.set(locale, cached) + } + return cached +} + +/** + * Static search over the per-locale index, ranked by page (see + * lib/search-ranking.ts) instead of fumadocs' per-fragment ordering. + */ +export function createDocsSearchClient(locale: string): SearchClient { + return { + deps: [locale], + async search(query) { + const db = await getLocaleDB(locale) + return db ? searchRanked(db, query) : [] + }, + } } diff --git a/lib/search-ranking.test.ts b/lib/search-ranking.test.ts new file mode 100644 index 000000000..d4a0ed8ca --- /dev/null +++ b/lib/search-ranking.test.ts @@ -0,0 +1,118 @@ +import { create, insertMultiple } from '@orama/orama' +import { describe, expect, it } from 'vitest' +import { rankPages, scoreTitle, searchRanked } from './search-ranking' + +const SCHEMA = { + id: 'string', + page_id: 'string', + type: 'string', + content: 'string', + url: 'string', + breadcrumbs: 'string[]', +} as const + +// Mirrors the shape fumadocs exports for the advanced index: one `page` doc per +// page (content = title, id = page_id) plus heading/text fragments. +function page(url: string, title: string, fragments: string[]) { + return [ + { id: url, page_id: url, type: 'page', content: title, url, breadcrumbs: [] }, + ...fragments.map((content, i) => ({ + id: `${url}-${i}`, + page_id: url, + type: 'text', + content, + url: `${url}#${i}`, + breadcrumbs: [], + })), + ] +} + +async function createDB() { + const db = create({ schema: SCHEMA }) + await insertMultiple(db, [ + ...page('/docs/remote', 'Overview', [ + 'Development oriented docker compose', + 'For IaaS we recommend Docker Compose', + 'Deployment oriented docker compose', + 'There are two docker compose files', + 'Why Docker Compose? Docker guide', + 'deploy-compose includes Nginx, internal to Docker', + ]), + ...page('/docs/local/docker', 'Docker', ['Install LibreChat with docker compose']), + ...page('/docs/remote/docker_linux', 'Docker (Remote Linux)', ['docker.io and docker-ce']), + ...page('/docs/configuration/cdn', 'File Storage & CDN', ['Store uploaded files']), + ...page('/docs/features/rag_api', 'RAG API', ['Chat with files using the RAG API']), + ]) + return db +} + +const pageUrls = (results: { type: string; url: string }[]) => + results.filter((r) => r.type === 'page').map((r) => r.url) + +describe('scoreTitle', () => { + it('ranks exact, prefix, and all-terms title matches in that order', () => { + expect(scoreTitle('Docker', 'docker')).toBeGreaterThan(scoreTitle('Docker Override', 'docker')) + expect(scoreTitle('Docker Override', 'docker')).toBeGreaterThan( + scoreTitle('Remote Docker Guide', 'docker'), + ) + expect(scoreTitle('Remote Docker Guide', 'docker')).toBeGreaterThan( + scoreTitle('Overview', 'docker'), + ) + }) + + it('matches query terms on word prefixes only', () => { + expect(scoreTitle('File Storage & CDN', 'rag')).toBe(0) + expect(scoreTitle('RAG API', 'rag')).toBeGreaterThan(0) + }) + + it('falls back to substring matching for CJK titles', () => { + expect(scoreTitle('Docker 部署指南', '部署')).toBeGreaterThan(0) + }) +}) + +describe('rankPages', () => { + it('never lets body hits lift a weaker title match above a stronger one', () => { + const doc = (page_id: string, type: 'page' | 'text', content: string) => ({ + id: `${page_id}-${content}`, + page_id, + type, + content, + url: page_id, + }) + const pages = { + '/override': doc('/override', 'page', 'Docker Override'), + '/guide': doc('/guide', 'page', 'Remote Docker Guide'), + } + const hits = [ + { score: 1, document: doc('/override', 'text', 'docker') }, + ...Array.from({ length: 10 }, (_, i) => ({ + score: 100, + document: doc('/guide', 'text', `docker ${i}`), + })), + ] + const ranked = rankPages(hits, 'docker', (id) => pages[id as keyof typeof pages]) + expect(ranked.map((r) => r.page.url)).toEqual(['/override', '/guide']) + }) +}) + +describe('searchRanked', () => { + it('puts the page titled with the query above pages that only repeat it in the body', async () => { + const results = await searchRanked(await createDB(), 'docker') + expect(pageUrls(results).slice(0, 2)).toEqual([ + '/docs/local/docker', + '/docs/remote/docker_linux', + ]) + }) + + it('caps fragments per page', async () => { + const results = await searchRanked(await createDB(), 'docker', { maxFragmentsPerPage: 2 }) + const overviewFragments = results.filter( + (r) => r.type !== 'page' && r.url.startsWith('/docs/remote#'), + ) + expect(overviewFragments).toHaveLength(2) + }) + + it('returns nothing for a blank query', async () => { + expect(await searchRanked(await createDB(), ' ')).toEqual([]) + }) +}) diff --git a/lib/search-ranking.ts b/lib/search-ranking.ts new file mode 100644 index 000000000..586143b2b --- /dev/null +++ b/lib/search-ranking.ts @@ -0,0 +1,174 @@ +import { getByID, search, type AnyOrama } from '@orama/orama' +import { createContentHighlighter, type SortedResult } from 'fumadocs-core/search' + +/** + * Page-level ranking for the static (advanced) Orama index. + * + * fumadocs' built-in `searchAdvanced` scores every heading and paragraph as an + * independent document with BM25 and then orders pages by their first hit. Short + * paragraphs that repeat a term therefore outrank the page that is actually about + * it: searching "docker" put the "Overview" page (seven short paragraphs that + * mention docker) above the page titled "Docker". It also shows up to eight + * fragments per page, so one page fills the whole dialog. + * + * Here Orama only finds the candidates; each page is then scored by how well its + * title matches the query, with body relevance as the tie-breaker, and keeps a + * few fragments only. + */ + +type IndexedDoc = { + id: string + page_id: string + type: SortedResult['type'] + content: string + url: string + breadcrumbs?: string[] +} + +export type RankOptions = { + /** Pages to return. */ + maxPages?: number + /** Heading/text fragments shown under each page. */ + maxFragmentsPerPage?: number +} + +const CANDIDATE_LIMIT = 500 +const TYPE_WEIGHT: Record = { page: 1, heading: 1.5, text: 1 } +const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u + +function normalize(value: string): string { + return value.toLowerCase().replaceAll(/\s+/g, ' ').trim() +} + +function splitWords(value: string): string[] { + return normalize(value) + .split(/[^\p{L}\p{N}]+/u) + .filter(Boolean) +} + +/** + * Title relevance: exact title, then a title that starts with the query, then a + * title containing every query term as a word prefix ("rag" matches "RAG API", + * not "Storage"). CJK titles have no word breaks, so they fall back to substring. + */ +export function scoreTitle(title: string, query: string): number { + const t = normalize(title) + const q = normalize(query) + if (!q) return 0 + if (t === q) return 100 + if (t.startsWith(q)) return 60 + + const terms = splitWords(q) + if (terms.length === 0) return 0 + const words = splitWords(t) + const matched = terms.filter((term) => + CJK.test(term) ? t.includes(term) : words.some((word) => word.startsWith(term)), + ).length + if (matched === terms.length) return 40 + return 20 * (matched / terms.length) +} + +type Fragment = { doc: IndexedDoc; score: number } +type PageGroup = { pageId: string; best: number; count: number; fragments: Fragment[] } + +export type RankedPage = { + page: IndexedDoc + titleScore: number + bodyScore: number + fragments: IndexedDoc[] +} + +/** + * Groups Orama hits by page and orders pages by title match first, then by + * their best weighted body hit, then by how many fragments matched. The title + * tier is compared on its own so body hits can never lift a weaker title match + * above a stronger one. + */ +export function rankPages( + hits: { score: number; document: IndexedDoc }[], + query: string, + getPage: (pageId: string) => IndexedDoc | undefined, + { maxPages = 10, maxFragmentsPerPage = 3 }: RankOptions = {}, +): RankedPage[] { + const groups = new Map() + let maxScore = 0 + + for (const hit of hits) { + const doc = hit.document + const weighted = hit.score * TYPE_WEIGHT[doc.type] + maxScore = Math.max(maxScore, weighted) + + let group = groups.get(doc.page_id) + if (!group) { + group = { pageId: doc.page_id, best: 0, count: 0, fragments: [] } + groups.set(doc.page_id, group) + } + group.best = Math.max(group.best, weighted) + group.count++ + if (doc.type !== 'page') group.fragments.push({ doc, score: weighted }) + } + + const ranked: RankedPage[] = [] + for (const group of groups.values()) { + const page = getPage(group.pageId) + if (!page) continue + const body = maxScore > 0 ? (group.best / maxScore) * 30 : 0 + const coverage = (Math.min(group.count, 10) / 10) * 10 + ranked.push({ + page, + titleScore: scoreTitle(page.content, query), + bodyScore: body + coverage, + fragments: group.fragments + .sort((a, b) => b.score - a.score) + .slice(0, maxFragmentsPerPage) + .map((f) => f.doc), + }) + } + + return ranked + .sort((a, b) => b.titleScore - a.titleScore || b.bodyScore - a.bodyScore) + .slice(0, maxPages) +} + +/** Runs a query against an exported fumadocs advanced index. */ +export async function searchRanked( + db: AnyOrama, + query: string, + options?: RankOptions, +): Promise { + if (!query.trim()) return [] + + const result = await search(db, { + term: query, + limit: CANDIDATE_LIMIT, + properties: ['content'], + }) + const pages = rankPages( + result.hits as unknown as { score: number; document: IndexedDoc }[], + query, + (pageId) => getByID(db, pageId) as IndexedDoc | undefined, + options, + ) + + const highlighter = createContentHighlighter(query) + const list: SortedResult[] = [] + for (const { page, fragments } of pages) { + list.push({ + id: page.id, + type: 'page', + content: highlighter.highlightMarkdown(page.content), + breadcrumbs: page.breadcrumbs, + url: page.url, + }) + for (const doc of fragments) { + list.push({ + id: doc.id.toString(), + type: doc.type, + content: highlighter.highlightMarkdown(doc.content), + breadcrumbs: doc.breadcrumbs, + url: doc.url, + }) + } + } + return list +} diff --git a/scripts/search-eval.ts b/scripts/search-eval.ts new file mode 100644 index 000000000..6b9e95285 --- /dev/null +++ b/scripts/search-eval.ts @@ -0,0 +1,87 @@ +/** + * Search relevance check against a real exported index. + * + * Usage: + * pnpm tsx scripts/search-eval.ts https://www.librechat.ai/api/search/en + * pnpm tsx scripts/search-eval.ts .next/server/app/api/search/en.body + * + * Each case lists the pages that must appear in the top 3 results (any one of + * them passes). Exits 1 when a case fails, so it can gate ranking changes. + */ +import { readFile } from 'node:fs/promises' +import { create, load } from '@orama/orama' +import { searchRanked } from '../lib/search-ranking' + +const TOP_N = 3 + +const CASES: { query: string; expect: string[] }[] = [ + { query: 'docker', expect: ['/docs/local/docker'] }, + { query: 'docker override', expect: ['/docs/configuration/docker_override'] }, + { query: 'ollama', expect: ['/docs/configuration/librechat_yaml/ai_endpoints/ollama'] }, + { query: 'mcp', expect: ['/docs/features/mcp'] }, + { query: 'rag api', expect: ['/docs/features/rag_api', '/docs/configuration/rag_api'] }, + { query: 'meilisearch', expect: ['/docs/configuration/meilisearch'] }, + { query: 'helm', expect: ['/docs/local/helm_chart'] }, + { query: 'nginx', expect: ['/docs/remote/nginx'] }, + { query: 'ldap', expect: ['/docs/configuration/authentication/ldap'] }, + { query: 'keycloak', expect: ['/docs/configuration/authentication/OAuth2-OIDC/keycloak'] }, + { query: 'azure openai', expect: ['/docs/configuration/librechat_yaml/ai_endpoints/azure'] }, + { query: 'bedrock', expect: ['/docs/configuration/pre_configured_ai/bedrock'] }, + { query: 'redis', expect: ['/docs/configuration/redis'] }, + { query: 'web search', expect: ['/docs/features/web_search'] }, + { query: 'memory', expect: ['/docs/features/memory'] }, + { query: 'agents', expect: ['/docs/features/agents'] }, + { query: 'code interpreter', expect: ['/docs/features/code_interpreter'] }, + { query: 'environment variables', expect: ['/docs/configuration/dotenv'] }, + { query: 'custom endpoints', expect: ['/docs/quick_start/custom_endpoints'] }, + { query: 'openrouter', expect: ['/docs/configuration/librechat_yaml/ai_endpoints/openrouter'] }, + { + query: 'model specs', + expect: ['/docs/configuration/librechat_yaml/object_structure/model_specs'], + }, + { query: 'railway', expect: ['/docs/remote/railway'] }, + { query: 'mongodb atlas', expect: ['/docs/configuration/mongodb/mongodb_atlas'] }, + { query: 'artifacts', expect: ['/docs/features/artifacts'] }, + { query: 'ocr', expect: ['/docs/features/ocr'] }, +] + +async function readIndex(source: string): Promise { + if (/^https?:\/\//.test(source)) { + const res = await fetch(source) + if (!res.ok) throw new Error(`${source} returned ${res.status}`) + return res.json() + } + return JSON.parse(await readFile(source, 'utf8')) +} + +async function main() { + const source = process.argv[2] + if (!source) { + console.error('usage: tsx scripts/search-eval.ts ') + process.exit(2) + } + + const exported = (await readIndex(source)) as { type: string; data: Record } + const raw = exported.type === 'i18n' ? exported.data.en : exported + const db = create({ schema: { _: 'string' }, language: 'english' }) + load(db, raw as Parameters[1]) + + let failed = 0 + for (const { query, expect } of CASES) { + const pages = (await searchRanked(db, query)) + .filter((r) => r.type === 'page') + .slice(0, TOP_N) + .map((r) => r.url) + const ok = pages.some((url) => expect.includes(url)) + if (!ok) failed++ + console.log(`${ok ? 'PASS' : 'FAIL'} ${query.padEnd(22)} ${pages.join(' ')}`) + } + + console.log(`\n${CASES.length - failed}/${CASES.length} passed`) + if (failed > 0) process.exit(1) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +})