From 140b740b7405d58bf65b1dde75d1d104f8394c50 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:27:34 +0100 Subject: [PATCH 1/2] fix: rank docs search results by page instead of by fragment Orama scored each heading and paragraph independently and fumadocs ordered pages by their first hit, so short paragraphs repeating a term outranked the page titled with it (searching "docker" put "Overview" first) and one page could fill the dialog with eight fragments. Rank whole pages: title match first (exact, prefix, all terms), body relevance as tie-breaker, at most three fragments per page. Add a 25-query relevance check (scripts/search-eval.ts) that runs against any exported index; it passes 25/25 against production versus 13/25 before. --- components/search-dialog.tsx | 14 +-- lib/search-client.ts | 46 ++++++++-- lib/search-ranking.test.ts | 93 ++++++++++++++++++++ lib/search-ranking.ts | 164 +++++++++++++++++++++++++++++++++++ scripts/search-eval.ts | 87 +++++++++++++++++++ 5 files changed, 389 insertions(+), 15 deletions(-) create mode 100644 lib/search-ranking.test.ts create mode 100644 lib/search-ranking.ts create mode 100644 scripts/search-eval.ts diff --git a/components/search-dialog.tsx b/components/search-dialog.tsx index c5a3ef07fd..4b57cd89ec 100644 --- a/components/search-dialog.tsx +++ b/components/search-dialog.tsx @@ -22,9 +22,10 @@ import { * * The per-locale Orama indexes are built at build time by * app/api/search/[lang]/route.ts; this component downloads only the current - * reader's language (via `from`) and runs every query client-side. It mirrors - * fumadocs' DefaultSearchDialog layout (so the UX is unchanged) but swaps the - * fetch client for the static Orama client with a locale-aware `initOrama`. + * reader's language and runs every query client-side. It mirrors fumadocs' + * DefaultSearchDialog layout (so the UX is unchanged) but swaps the fetch client + * for a static one with a locale-aware `initOrama` and page-level ranking + * (lib/search-client.ts, lib/search-ranking.ts). * * `initOrama` must reproduce the SAME tokenizer the route used to build each * locale's index, otherwise the query is tokenized differently from the stored @@ -36,10 +37,9 @@ import { export default function StaticSearchDialog(props: SharedProps) { const { locale } = useI18n() const lang = locale ?? i18n.defaultLanguage - // Created inline (like fumadocs' DefaultSearchDialog); useDocsSearch tracks - // re-queries via the client's `deps` ([tag, locale]), and the downloaded index - // is cached module-side keyed by URL, so this stays cheap across renders. - // `from` is the per-locale endpoint so we only download this reader's language. + // Created inline (like fumadocs' DefaultSearchDialog); useDocsSearch re-queries + // when the client's `deps` ([locale]) change, and the downloaded index is + // cached module-side per locale, so this stays cheap across renders. const client = createDocsSearchClient(lang) const { search, setSearch, query } = useDocsSearch({ client }) diff --git a/lib/search-client.ts b/lib/search-client.ts index 0cd6d26cff..78f1107454 100644 --- a/lib/search-client.ts +++ b/lib/search-client.ts @@ -1,6 +1,7 @@ -import { create, type AnyOrama } from '@orama/orama' -import { oramaStaticClient } from 'fumadocs-core/search/client/orama-static' +import { create, load, type AnyOrama } from '@orama/orama' +import type { SearchClient } from 'fumadocs-core/search/client' import { SEARCH_LANGUAGE_BY_LOCALE } from '@/lib/search-languages' +import { searchRanked } from '@/lib/search-ranking' /** * Build the same locale-specific tokenizer used by the static search route. @@ -37,10 +38,39 @@ export async function initOrama(locale?: string): Promise { }) } -export function createDocsSearchClient(locale: string) { - return oramaStaticClient({ - from: `/api/search/${encodeURIComponent(locale)}`, - initOrama, - locale, - }) +// One download per locale per page load, shared by every client. A failed +// download is evicted so the next query can retry. +const dbCache = new Map>() + +function getLocaleDB(locale: string): Promise { + let cached = dbCache.get(locale) + if (!cached) { + cached = (async () => { + const res = await fetch(`/api/search/${encodeURIComponent(locale)}`) + if (!res.ok) throw new Error(`failed to fetch search index for "${locale}": ${res.status}`) + const exported = (await res.json()) as { data: Record } + const data = exported.data[locale] + if (!data) return undefined + const db = await initOrama(locale) + load(db, data as Parameters[1]) + return db + })() + cached.catch(() => dbCache.delete(locale)) + dbCache.set(locale, cached) + } + return cached +} + +/** + * Static search over the per-locale index, ranked by page (see + * lib/search-ranking.ts) instead of fumadocs' per-fragment ordering. + */ +export function createDocsSearchClient(locale: string): SearchClient { + return { + deps: [locale], + async search(query) { + const db = await getLocaleDB(locale) + return db ? searchRanked(db, query) : [] + }, + } } diff --git a/lib/search-ranking.test.ts b/lib/search-ranking.test.ts new file mode 100644 index 0000000000..3dbb26b6d9 --- /dev/null +++ b/lib/search-ranking.test.ts @@ -0,0 +1,93 @@ +import { create, insertMultiple } from '@orama/orama' +import { describe, expect, it } from 'vitest' +import { scoreTitle, searchRanked } from './search-ranking' + +const SCHEMA = { + id: 'string', + page_id: 'string', + type: 'string', + content: 'string', + url: 'string', + breadcrumbs: 'string[]', +} as const + +// Mirrors the shape fumadocs exports for the advanced index: one `page` doc per +// page (content = title, id = page_id) plus heading/text fragments. +function page(url: string, title: string, fragments: string[]) { + return [ + { id: url, page_id: url, type: 'page', content: title, url, breadcrumbs: [] }, + ...fragments.map((content, i) => ({ + id: `${url}-${i}`, + page_id: url, + type: 'text', + content, + url: `${url}#${i}`, + breadcrumbs: [], + })), + ] +} + +async function createDB() { + const db = create({ schema: SCHEMA }) + await insertMultiple(db, [ + ...page('/docs/remote', 'Overview', [ + 'Development oriented docker compose', + 'For IaaS we recommend Docker Compose', + 'Deployment oriented docker compose', + 'There are two docker compose files', + 'Why Docker Compose? Docker guide', + 'deploy-compose includes Nginx, internal to Docker', + ]), + ...page('/docs/local/docker', 'Docker', ['Install LibreChat with docker compose']), + ...page('/docs/remote/docker_linux', 'Docker (Remote Linux)', ['docker.io and docker-ce']), + ...page('/docs/configuration/cdn', 'File Storage & CDN', ['Store uploaded files']), + ...page('/docs/features/rag_api', 'RAG API', ['Chat with files using the RAG API']), + ]) + return db +} + +const pageUrls = (results: { type: string; url: string }[]) => + results.filter((r) => r.type === 'page').map((r) => r.url) + +describe('scoreTitle', () => { + it('ranks exact, prefix, and all-terms title matches in that order', () => { + expect(scoreTitle('Docker', 'docker')).toBeGreaterThan(scoreTitle('Docker Override', 'docker')) + expect(scoreTitle('Docker Override', 'docker')).toBeGreaterThan( + scoreTitle('Remote Docker Guide', 'docker'), + ) + expect(scoreTitle('Remote Docker Guide', 'docker')).toBeGreaterThan( + scoreTitle('Overview', 'docker'), + ) + }) + + it('matches query terms on word prefixes only', () => { + expect(scoreTitle('File Storage & CDN', 'rag')).toBe(0) + expect(scoreTitle('RAG API', 'rag')).toBeGreaterThan(0) + }) + + it('falls back to substring matching for CJK titles', () => { + expect(scoreTitle('Docker 部署指南', '部署')).toBeGreaterThan(0) + }) +}) + +describe('searchRanked', () => { + it('puts the page titled with the query above pages that only repeat it in the body', async () => { + const results = await searchRanked(await createDB(), 'docker') + expect(pageUrls(results).slice(0, 2)).toEqual([ + '/docs/local/docker', + '/docs/remote/docker_linux', + ]) + }) + + it('caps fragments per page', async () => { + const results = await searchRanked(await createDB(), 'docker', { maxFragmentsPerPage: 2 }) + const overviewFragments = results.filter( + (r) => r.type !== 'page' && r.url.startsWith('/docs/remote#'), + ) + expect(overviewFragments).toHaveLength(2) + }) + + it('returns nothing for a blank query', async () => { + expect(await searchRanked(await createDB(), ' ')).toEqual([]) + }) +}) diff --git a/lib/search-ranking.ts b/lib/search-ranking.ts new file mode 100644 index 0000000000..4a8c261e90 --- /dev/null +++ b/lib/search-ranking.ts @@ -0,0 +1,164 @@ +import { getByID, search, type AnyOrama } from '@orama/orama' +import { createContentHighlighter, type SortedResult } from 'fumadocs-core/search' + +/** + * Page-level ranking for the static (advanced) Orama index. + * + * fumadocs' built-in `searchAdvanced` scores every heading and paragraph as an + * independent document with BM25 and then orders pages by their first hit. Short + * paragraphs that repeat a term therefore outrank the page that is actually about + * it: searching "docker" put the "Overview" page (seven short paragraphs that + * mention docker) above the page titled "Docker". It also shows up to eight + * fragments per page, so one page fills the whole dialog. + * + * Here Orama only finds the candidates; each page is then scored by how well its + * title matches the query, with body relevance as the tie-breaker, and keeps a + * few fragments only. + */ + +type IndexedDoc = { + id: string + page_id: string + type: SortedResult['type'] + content: string + url: string + breadcrumbs?: string[] +} + +export type RankOptions = { + /** Pages to return. */ + maxPages?: number + /** Heading/text fragments shown under each page. */ + maxFragmentsPerPage?: number +} + +const CANDIDATE_LIMIT = 500 +const TYPE_WEIGHT: Record = { page: 1, heading: 1.5, text: 1 } +const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u + +function normalize(value: string): string { + return value.toLowerCase().replaceAll(/\s+/g, ' ').trim() +} + +function splitWords(value: string): string[] { + return normalize(value) + .split(/[^\p{L}\p{N}]+/u) + .filter(Boolean) +} + +/** + * Title relevance: exact title, then a title that starts with the query, then a + * title containing every query term as a word prefix ("rag" matches "RAG API", + * not "Storage"). CJK titles have no word breaks, so they fall back to substring. + */ +export function scoreTitle(title: string, query: string): number { + const t = normalize(title) + const q = normalize(query) + if (!q) return 0 + if (t === q) return 100 + if (t.startsWith(q)) return 60 + + const terms = splitWords(q) + if (terms.length === 0) return 0 + const words = splitWords(t) + const matched = terms.filter((term) => + CJK.test(term) ? t.includes(term) : words.some((word) => word.startsWith(term)), + ).length + if (matched === terms.length) return 40 + return 20 * (matched / terms.length) +} + +type Fragment = { doc: IndexedDoc; score: number } +type PageGroup = { pageId: string; best: number; count: number; fragments: Fragment[] } + +export type RankedPage = { page: IndexedDoc; score: number; fragments: IndexedDoc[] } + +/** + * Groups Orama hits by page and orders pages by title match first, then by + * their best weighted body hit, then by how many fragments matched. + */ +export function rankPages( + hits: { score: number; document: IndexedDoc }[], + query: string, + getPage: (pageId: string) => IndexedDoc | undefined, + { maxPages = 10, maxFragmentsPerPage = 3 }: RankOptions = {}, +): RankedPage[] { + const groups = new Map() + let maxScore = 0 + + for (const hit of hits) { + const doc = hit.document + const weighted = hit.score * TYPE_WEIGHT[doc.type] + maxScore = Math.max(maxScore, weighted) + + let group = groups.get(doc.page_id) + if (!group) { + group = { pageId: doc.page_id, best: 0, count: 0, fragments: [] } + groups.set(doc.page_id, group) + } + group.best = Math.max(group.best, weighted) + group.count++ + if (doc.type !== 'page') group.fragments.push({ doc, score: weighted }) + } + + const ranked: RankedPage[] = [] + for (const group of groups.values()) { + const page = getPage(group.pageId) + if (!page) continue + const body = maxScore > 0 ? (group.best / maxScore) * 30 : 0 + const coverage = (Math.min(group.count, 10) / 10) * 10 + ranked.push({ + page, + score: scoreTitle(page.content, query) + body + coverage, + fragments: group.fragments + .sort((a, b) => b.score - a.score) + .slice(0, maxFragmentsPerPage) + .map((f) => f.doc), + }) + } + + return ranked.sort((a, b) => b.score - a.score).slice(0, maxPages) +} + +/** Runs a query against an exported fumadocs advanced index. */ +export async function searchRanked( + db: AnyOrama, + query: string, + options?: RankOptions, +): Promise { + if (!query.trim()) return [] + + const result = await search(db, { + term: query, + limit: CANDIDATE_LIMIT, + properties: ['content'], + }) + const pages = rankPages( + result.hits as unknown as { score: number; document: IndexedDoc }[], + query, + (pageId) => getByID(db, pageId) as IndexedDoc | undefined, + options, + ) + + const highlighter = createContentHighlighter(query) + const list: SortedResult[] = [] + for (const { page, fragments } of pages) { + list.push({ + id: page.id, + type: 'page', + content: highlighter.highlightMarkdown(page.content), + breadcrumbs: page.breadcrumbs, + url: page.url, + }) + for (const doc of fragments) { + list.push({ + id: doc.id.toString(), + type: doc.type, + content: highlighter.highlightMarkdown(doc.content), + breadcrumbs: doc.breadcrumbs, + url: doc.url, + }) + } + } + return list +} diff --git a/scripts/search-eval.ts b/scripts/search-eval.ts new file mode 100644 index 0000000000..6b9e95285a --- /dev/null +++ b/scripts/search-eval.ts @@ -0,0 +1,87 @@ +/** + * Search relevance check against a real exported index. + * + * Usage: + * pnpm tsx scripts/search-eval.ts https://www.librechat.ai/api/search/en + * pnpm tsx scripts/search-eval.ts .next/server/app/api/search/en.body + * + * Each case lists the pages that must appear in the top 3 results (any one of + * them passes). Exits 1 when a case fails, so it can gate ranking changes. + */ +import { readFile } from 'node:fs/promises' +import { create, load } from '@orama/orama' +import { searchRanked } from '../lib/search-ranking' + +const TOP_N = 3 + +const CASES: { query: string; expect: string[] }[] = [ + { query: 'docker', expect: ['/docs/local/docker'] }, + { query: 'docker override', expect: ['/docs/configuration/docker_override'] }, + { query: 'ollama', expect: ['/docs/configuration/librechat_yaml/ai_endpoints/ollama'] }, + { query: 'mcp', expect: ['/docs/features/mcp'] }, + { query: 'rag api', expect: ['/docs/features/rag_api', '/docs/configuration/rag_api'] }, + { query: 'meilisearch', expect: ['/docs/configuration/meilisearch'] }, + { query: 'helm', expect: ['/docs/local/helm_chart'] }, + { query: 'nginx', expect: ['/docs/remote/nginx'] }, + { query: 'ldap', expect: ['/docs/configuration/authentication/ldap'] }, + { query: 'keycloak', expect: ['/docs/configuration/authentication/OAuth2-OIDC/keycloak'] }, + { query: 'azure openai', expect: ['/docs/configuration/librechat_yaml/ai_endpoints/azure'] }, + { query: 'bedrock', expect: ['/docs/configuration/pre_configured_ai/bedrock'] }, + { query: 'redis', expect: ['/docs/configuration/redis'] }, + { query: 'web search', expect: ['/docs/features/web_search'] }, + { query: 'memory', expect: ['/docs/features/memory'] }, + { query: 'agents', expect: ['/docs/features/agents'] }, + { query: 'code interpreter', expect: ['/docs/features/code_interpreter'] }, + { query: 'environment variables', expect: ['/docs/configuration/dotenv'] }, + { query: 'custom endpoints', expect: ['/docs/quick_start/custom_endpoints'] }, + { query: 'openrouter', expect: ['/docs/configuration/librechat_yaml/ai_endpoints/openrouter'] }, + { + query: 'model specs', + expect: ['/docs/configuration/librechat_yaml/object_structure/model_specs'], + }, + { query: 'railway', expect: ['/docs/remote/railway'] }, + { query: 'mongodb atlas', expect: ['/docs/configuration/mongodb/mongodb_atlas'] }, + { query: 'artifacts', expect: ['/docs/features/artifacts'] }, + { query: 'ocr', expect: ['/docs/features/ocr'] }, +] + +async function readIndex(source: string): Promise { + if (/^https?:\/\//.test(source)) { + const res = await fetch(source) + if (!res.ok) throw new Error(`${source} returned ${res.status}`) + return res.json() + } + return JSON.parse(await readFile(source, 'utf8')) +} + +async function main() { + const source = process.argv[2] + if (!source) { + console.error('usage: tsx scripts/search-eval.ts ') + process.exit(2) + } + + const exported = (await readIndex(source)) as { type: string; data: Record } + const raw = exported.type === 'i18n' ? exported.data.en : exported + const db = create({ schema: { _: 'string' }, language: 'english' }) + load(db, raw as Parameters[1]) + + let failed = 0 + for (const { query, expect } of CASES) { + const pages = (await searchRanked(db, query)) + .filter((r) => r.type === 'page') + .slice(0, TOP_N) + .map((r) => r.url) + const ok = pages.some((url) => expect.includes(url)) + if (!ok) failed++ + console.log(`${ok ? 'PASS' : 'FAIL'} ${query.padEnd(22)} ${pages.join(' ')}`) + } + + console.log(`\n${CASES.length - failed}/${CASES.length} passed`) + if (failed > 0) process.exit(1) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) From c5503d1c4546be006d3c354ba50d74ef4adce61e Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:31:57 +0100 Subject: [PATCH 2/2] fix: compare search title tiers before body relevance Body relevance and coverage could add up to 40 points, more than the gap between title tiers, so a page whose title only contained the query could outrank one whose title started with it on the strength of repeated fragments. Sort by title tier first and use body relevance only to break ties within a tier. --- lib/search-ranking.test.ts | 27 ++++++++++++++++++++++++++- lib/search-ranking.ts | 18 ++++++++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/lib/search-ranking.test.ts b/lib/search-ranking.test.ts index 3dbb26b6d9..d4a0ed8ca4 100644 --- a/lib/search-ranking.test.ts +++ b/lib/search-ranking.test.ts @@ -1,6 +1,6 @@ import { create, insertMultiple } from '@orama/orama' import { describe, expect, it } from 'vitest' -import { scoreTitle, searchRanked } from './search-ranking' +import { rankPages, scoreTitle, searchRanked } from './search-ranking' const SCHEMA = { id: 'string', @@ -70,6 +70,31 @@ describe('scoreTitle', () => { }) }) +describe('rankPages', () => { + it('never lets body hits lift a weaker title match above a stronger one', () => { + const doc = (page_id: string, type: 'page' | 'text', content: string) => ({ + id: `${page_id}-${content}`, + page_id, + type, + content, + url: page_id, + }) + const pages = { + '/override': doc('/override', 'page', 'Docker Override'), + '/guide': doc('/guide', 'page', 'Remote Docker Guide'), + } + const hits = [ + { score: 1, document: doc('/override', 'text', 'docker') }, + ...Array.from({ length: 10 }, (_, i) => ({ + score: 100, + document: doc('/guide', 'text', `docker ${i}`), + })), + ] + const ranked = rankPages(hits, 'docker', (id) => pages[id as keyof typeof pages]) + expect(ranked.map((r) => r.page.url)).toEqual(['/override', '/guide']) + }) +}) + describe('searchRanked', () => { it('puts the page titled with the query above pages that only repeat it in the body', async () => { const results = await searchRanked(await createDB(), 'docker') diff --git a/lib/search-ranking.ts b/lib/search-ranking.ts index 4a8c261e90..586143b2b5 100644 --- a/lib/search-ranking.ts +++ b/lib/search-ranking.ts @@ -71,11 +71,18 @@ export function scoreTitle(title: string, query: string): number { type Fragment = { doc: IndexedDoc; score: number } type PageGroup = { pageId: string; best: number; count: number; fragments: Fragment[] } -export type RankedPage = { page: IndexedDoc; score: number; fragments: IndexedDoc[] } +export type RankedPage = { + page: IndexedDoc + titleScore: number + bodyScore: number + fragments: IndexedDoc[] +} /** * Groups Orama hits by page and orders pages by title match first, then by - * their best weighted body hit, then by how many fragments matched. + * their best weighted body hit, then by how many fragments matched. The title + * tier is compared on its own so body hits can never lift a weaker title match + * above a stronger one. */ export function rankPages( hits: { score: number; document: IndexedDoc }[], @@ -109,7 +116,8 @@ export function rankPages( const coverage = (Math.min(group.count, 10) / 10) * 10 ranked.push({ page, - score: scoreTitle(page.content, query) + body + coverage, + titleScore: scoreTitle(page.content, query), + bodyScore: body + coverage, fragments: group.fragments .sort((a, b) => b.score - a.score) .slice(0, maxFragmentsPerPage) @@ -117,7 +125,9 @@ export function rankPages( }) } - return ranked.sort((a, b) => b.score - a.score).slice(0, maxPages) + return ranked + .sort((a, b) => b.titleScore - a.titleScore || b.bodyScore - a.bodyScore) + .slice(0, maxPages) } /** Runs a query against an exported fumadocs advanced index. */