From e38491980bd98dd7594f04415588678277ada97d Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Mon, 17 Aug 2026 16:34:24 +1200 Subject: [PATCH 01/14] Replace the search.json index with Pagefind Indexes the built HTML at the end of the build and puts Pagefind behind the SearchEngine seam. Body text is indexed for the first time, so a phrase that appears in an article but not its title or headings is now findable. The index goes to dist/docs/pagefind, before pruneDist would delete it, and is built from dist/docs so stored URLs match the /docs/ prefix the client sets as its basePath. data-pagefind-body on the article is what scopes indexing and, as a side effect, excludes all 1,409 redirect stubs: they render through a minimal layout that has no article at all. 1,250 pages are indexed, matching the eligible count the markdown emitter reports. Removes the search.json endpoint and the client scoring it fed. Co-Authored-By: Claude Opus 5 (1M context) --- astro.config.mjs | 4 + package.json | 3 +- pnpm-lock.yaml | 139 +++++++------- src/components/DocsSearch.astro | 2 +- src/integrations/pagefind-index.ts | 82 ++++++++ src/layouts/Default.astro | 55 +++++- src/pages/docs/search.json.ts | 93 --------- src/scripts/docs-search.ts | 4 +- src/scripts/modules/stemmer.js | 197 ------------------- src/scripts/modules/string.js | 83 -------- src/scripts/search-engine-legacy.ts | 265 -------------------------- src/scripts/search-engine-pagefind.ts | 117 ++++++++++++ src/scripts/synonyms.js | 17 -- 13 files changed, 332 insertions(+), 729 deletions(-) create mode 100644 src/integrations/pagefind-index.ts delete mode 100644 src/pages/docs/search.json.ts delete mode 100644 src/scripts/modules/stemmer.js delete mode 100644 src/scripts/modules/string.js delete mode 100644 src/scripts/search-engine-legacy.ts create mode 100644 src/scripts/search-engine-pagefind.ts delete mode 100644 src/scripts/synonyms.js diff --git a/astro.config.mjs b/astro.config.mjs index a5f2c67a89..c8625946b3 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,6 +3,7 @@ import { satteri } from '@astrojs/markdown-satteri'; import mdx from '@astrojs/mdx'; import { attributeMarkdown, wrapTables } from '/src/themes/octopus/utilities/custom-markdown.mjs'; import llmMdEmitter from './src/integrations/llm-md-emitter.ts'; +import pagefindIndex from './src/integrations/pagefind-index.ts'; import pruneDist from './src/integrations/prune-dist.ts'; import satteriHeadingId from './src/plugins/satteri-heading-id.js'; import satteriApiExamples, { apiExampleDirective } from './src/plugins/satteri-api-examples.js'; @@ -22,6 +23,9 @@ export default defineConfig({ integrations: [ mdx(), llmMdEmitter(), + // Indexes the HTML the build just wrote, so it lands after the page + // emitters and before the prune that would delete its output + pagefindIndex(), // Must run last: strips build output that can't be served under /docs/ pruneDist() ], diff --git a/package.json b/package.json index 8ea81c1723..e9a2c57a10 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,6 @@ "astro-accelerator-utils": "^0.3.84", "glob": "^13.0.6", "gray-matter": "^4.0.3", - "html-to-text": "^10.0.0", - "keyword-extractor": "^0.0.28", "optional": "^0.1.4", "satteri": "^0.9.5", "sharp": "^0.34.5" @@ -47,6 +45,7 @@ "linkinator": "7.0.0", "npm-run-all": "^4.1.5", "onchange": "^7.1.0", + "pagefind": "^1.5.2", "prettier": "^3.8.3", "prettier-plugin-astro": "^0.14.1" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 715655b317..ab0d4d3873 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,12 +39,6 @@ importers: gray-matter: specifier: ^4.0.3 version: 4.0.3 - html-to-text: - specifier: ^10.0.0 - version: 10.0.0 - keyword-extractor: - specifier: ^0.0.28 - version: 0.0.28 optional: specifier: ^0.1.4 version: 0.1.4 @@ -76,6 +70,9 @@ importers: onchange: specifier: ^7.1.0 version: 7.1.0 + pagefind: + specifier: ^1.5.2 + version: 1.5.2 prettier: specifier: ^3.8.3 version: 3.8.3 @@ -821,6 +818,41 @@ packages: '@oxc-project/types@0.144.0': resolution: {integrity: sha512-nuhZIOLuI6TFQ32I/WnUx+SCPY7SdSKwgnFHydAuoS1+Z4BRcaP+RRJmGzl9lw+0OFF7UmaESf7KQRXaNLHypg==} + '@pagefind/darwin-arm64@1.5.2': + resolution: {integrity: sha512-MXpI+7HsAdPkvJ0gk9xj9g541BCqBZOBbdwj9g6lB5LCj6kSV6nqDSjzcAJwvOsfu0fjwvC8hQU+ecfhp+MpiQ==} + cpu: [arm64] + os: [darwin] + + '@pagefind/darwin-x64@1.5.2': + resolution: {integrity: sha512-IojxFWMEJe0RQ7PQ3KXQsPIImNsbpPYpoZ+QUDrL8fAl/O27IX+LVLs74/UzEZy5uA2LD8Nz1AiwKr72vrkZQw==} + cpu: [x64] + os: [darwin] + + '@pagefind/freebsd-x64@1.5.2': + resolution: {integrity: sha512-7EVzo9+0w+2cbe671BtMj10UlNo83I+HrLVLfRxO731svHRJKUfJ/mo05gU14pe9PCfpKNQT8FS3Xc/oDN6pOA==} + cpu: [x64] + os: [freebsd] + + '@pagefind/linux-arm64@1.5.2': + resolution: {integrity: sha512-Ovt9+K35sqzn8H3ZMXGwls4TD/wMJuvRtShHIsmUQREmaxjrDEX7gHckRCrwYJ4XE1H1p6HkLz3wukrAnsfXQw==} + cpu: [arm64] + os: [linux] + + '@pagefind/linux-x64@1.5.2': + resolution: {integrity: sha512-V+tFqHKXhQKq/WqPBD67AFy7scn1/aZID00ws4fSDd+1daSi5UHR9VVlRrOUYKxn3VuFQYRD7lYXdZK1WED1YA==} + cpu: [x64] + os: [linux] + + '@pagefind/windows-arm64@1.5.2': + resolution: {integrity: sha512-hN9Nh90fNW61nNRCW9ZyQrAj/mD0eRvmJ8NlTUzkbuW8kIzGJUi3cxjFkEcMZ5h/8FsKWD/VcouZl4yo1F7B6g==} + cpu: [arm64] + os: [win32] + + '@pagefind/windows-x64@1.5.2': + resolution: {integrity: sha512-Fa2Iyw7kaDRzGMfNYNUXNW2zbL5FQVDgSOcbDHdzBrDEdpqOqg8TcZ68F22ol6NJ9IGzvUdmeyZypLW5dyhqsg==} + cpu: [x64] + os: [win32] + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -1008,11 +1040,6 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} - '@selderee/plugin-htmlparser2@0.12.0': - resolution: {integrity: sha512-oELmoyA6ML9jDRMV3kgcMQFKxUfBU0yFVn6yTctVaLT5ygXnxH52I3TZEgV9EhXJC68/uFvE5Daj1/25c0Xa/A==} - peerDependencies: - selderee: ~0.12.0 - '@shikijs/core@4.4.3': resolution: {integrity: sha512-QCR4q2ZO/ILJEuwiBMel4wdcTDb1JGwfjKTxPDF6x8ixOaluPrVqIn06C99AcRPhmYlBR56d/Fb+GN58GzExpg==} engines: {node: '>=20'} @@ -1398,10 +1425,6 @@ packages: decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} - deepmerge-ts@7.1.5: - resolution: {integrity: sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw==} - engines: {node: '>=16.0.0'} - define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -1778,10 +1801,6 @@ packages: html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} - html-to-text@10.0.0: - resolution: {integrity: sha512-2OH59Gtprdczel+7Rxgpz9hGVJREaf8Lt1H4kZwWHpEn70VQKRuMNGsb2eDbwaTzrYzb0hheiOG1P7Dim0B4dQ==} - engines: {node: '>=20.19.0'} - html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} @@ -1979,17 +1998,10 @@ packages: jsonc-parser@3.3.1: resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} - keyword-extractor@0.0.28: - resolution: {integrity: sha512-oi7dSPpYtW/3fE0vZiqQgZ8mW3F1V9K4+rBJ0FcVrdXBEQuhZ0zKj7sX74eqGASuepLHf9aYdeonyKHWhYpHQA==} - engines: {node: '>= 0.10.0'} - kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} - leac@0.7.0: - resolution: {integrity: sha512-qMrZeyEekgdRQ9o6a4NAB2EQZrv827GJdn1vnapwSJ90hWRB4TzUSunvacPkxQ2TnNqHNI1/zSt0hlo0crG8Jw==} - lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} @@ -2392,6 +2404,10 @@ packages: package-manager-detector@1.8.0: resolution: {integrity: sha512-yQA4H19AmPEoMUeavPMDIe1higySl/gH/yaQrkT/s07Qp+7pp2hYz30N3z2l5BkjVkF9Ow6o0wjJamm2y7Sn0A==} + pagefind@1.5.2: + resolution: {integrity: sha512-XTUaK0hXMCu2jszWE584JGQT7y284TmMV9l/HX3rnG5uo3rHI/uHU56XTyyyPFjeWEBxECbAi0CaFDJOONtG0Q==} + hasBin: true + parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} @@ -2405,9 +2421,6 @@ packages: parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} - parseley@0.13.1: - resolution: {integrity: sha512-uNBJZzmb60l6p6VWLTmevizNAGnE0xoSf1n0B4q3ntegDNzcS68NRCcBDZTcyXHxt2XhBChsCuqj4M+nChvE/A==} - path-key@2.0.1: resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==} engines: {node: '>=4'} @@ -2431,9 +2444,6 @@ packages: resolution: {integrity: sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==} engines: {node: '>=4'} - peberminta@0.10.0: - resolution: {integrity: sha512-80B2AsU+I4Qdb0ZAPSfe9UwvGzwkM37IKIFEvdS3D/3Ndgv2bsuJ0bfG1+iEYO+l7Gfd4EUJmuRyq7efLgRMzQ==} - piccolore@0.1.3: resolution: {integrity: sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==} @@ -2634,9 +2644,6 @@ packages: resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} engines: {node: '>=4'} - selderee@0.12.0: - resolution: {integrity: sha512-b1YMh3+DHZp59DLna3qVwQ5iOla/nrI6mLBNW02XxU77M3046Df6VLkoaJyFz20VsGIG5kkp+FK0kg4K4HnUFw==} - semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -3796,6 +3803,27 @@ snapshots: '@oxc-project/types@0.144.0': {} + '@pagefind/darwin-arm64@1.5.2': + optional: true + + '@pagefind/darwin-x64@1.5.2': + optional: true + + '@pagefind/freebsd-x64@1.5.2': + optional: true + + '@pagefind/linux-arm64@1.5.2': + optional: true + + '@pagefind/linux-x64@1.5.2': + optional: true + + '@pagefind/windows-arm64@1.5.2': + optional: true + + '@pagefind/windows-x64@1.5.2': + optional: true + '@pkgjs/parseargs@0.11.0': optional: true @@ -3916,12 +3944,6 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} - '@selderee/plugin-htmlparser2@0.12.0(selderee@0.12.0)': - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - selderee: 0.12.0 - '@shikijs/core@4.4.3': dependencies: '@shikijs/primitive': 4.4.3 @@ -4437,8 +4459,6 @@ snapshots: dependencies: character-entities: 2.0.2 - deepmerge-ts@7.1.5: {} - define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -4998,14 +5018,6 @@ snapshots: html-escaper@3.0.3: {} - html-to-text@10.0.0: - dependencies: - '@selderee/plugin-htmlparser2': 0.12.0(selderee@0.12.0) - deepmerge-ts: 7.1.5 - dom-serializer: 2.0.0 - htmlparser2: 10.1.0 - selderee: 0.12.0 - html-void-elements@3.0.0: {} htmlparser2@10.1.0: @@ -5193,12 +5205,8 @@ snapshots: jsonc-parser@3.3.1: {} - keyword-extractor@0.0.28: {} - kind-of@6.0.3: {} - leac@0.7.0: {} - lightningcss-android-arm64@1.33.0: optional: true @@ -5851,6 +5859,16 @@ snapshots: package-manager-detector@1.8.0: {} + pagefind@1.5.2: + optionalDependencies: + '@pagefind/darwin-arm64': 1.5.2 + '@pagefind/darwin-x64': 1.5.2 + '@pagefind/freebsd-x64': 1.5.2 + '@pagefind/linux-arm64': 1.5.2 + '@pagefind/linux-x64': 1.5.2 + '@pagefind/windows-arm64': 1.5.2 + '@pagefind/windows-x64': 1.5.2 + parse-entities@4.0.2: dependencies: '@types/unist': 2.0.11 @@ -5879,11 +5897,6 @@ snapshots: dependencies: entities: 6.0.1 - parseley@0.13.1: - dependencies: - leac: 0.7.0 - peberminta: 0.10.0 - path-key@2.0.1: {} path-key@3.1.1: {} @@ -5904,8 +5917,6 @@ snapshots: dependencies: pify: 3.0.0 - peberminta@0.10.0: {} - piccolore@0.1.3: {} picocolors@1.1.1: {} @@ -6198,10 +6209,6 @@ snapshots: extend-shallow: 2.0.1 kind-of: 6.0.3 - selderee@0.12.0: - dependencies: - parseley: 0.13.1 - semver@5.7.2: {} semver@7.8.0: {} diff --git a/src/components/DocsSearch.astro b/src/components/DocsSearch.astro index 4957e1b73c..10de1ccd35 100644 --- a/src/components/DocsSearch.astro +++ b/src/components/DocsSearch.astro @@ -44,7 +44,7 @@ if (facets.length === 0) { throw new Error('DocsSearch needs at least one facet for its tab strip'); } -const indexUrl = `${SITE.subfolder}/search.json`; +const indexUrl = `${SITE.subfolder}/pagefind/`; const listboxId = `${name}-search-listbox`; --- diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts new file mode 100644 index 0000000000..ce6206cb1a --- /dev/null +++ b/src/integrations/pagefind-index.ts @@ -0,0 +1,82 @@ +// Builds the Pagefind index from the HTML the site just emitted. +// +// Two things about this repo shape the setup. The index has to land inside +// `dist/docs/`, because `prune-dist.ts` deletes every top-level entry outside +// its allowlist and only `/docs/` is served on octopus.com. And this has to be +// registered before `pruneDist()` in `astro.config.mjs`, since integration hooks +// run in the order they are listed. +// +// What gets indexed is decided in `Default.astro` by `data-pagefind-body`, +// `data-pagefind-ignore` and `data-pagefind-filter`, not here. + +import type { AstroIntegration } from 'astro'; +import { fileURLToPath } from 'node:url'; +import * as path from 'node:path'; +import * as fs from 'node:fs'; +// Statically, because `astro:build:done` fires after Vite's module runner has +// closed and a dynamic import from inside the hook cannot be resolved. +import { createIndex, close } from 'pagefind'; + +// Roughly the number of pages that pass the search predicate. Well under it +// means the body scoping or the exclusions have broken, which is a silent +// failure otherwise — search still works, it is just full of the wrong pages. +const EXPECTED_PAGES = 1000; + +export default function pagefindIndex(): AstroIntegration { + return { + name: 'pagefind-index', + hooks: { + 'astro:build:done': async ({ dir, logger }) => { + const distDir = fileURLToPath(dir); + + const { index, errors } = await createIndex({ keepIndexUrl: false }); + + if (!index) { + logger.error(`could not start Pagefind: ${errors.join(', ')}`); + return; + } + + // `dist/docs` rather than `dist`, so stored URLs are relative to the + // /docs/ prefix the site is proxied under. The client sets a `basePath` + // of `/docs/`, which Pagefind prepends back on at query time — indexing + // from `dist` would put `/docs/` on twice. + const added = await index.addDirectory({ + path: path.join(distDir, 'docs'), + glob: '**/*.html', + }); + + if (added.errors.length > 0) { + logger.error(added.errors.join(', ')); + } + + const written = await index.writeFiles({ + outputPath: path.join(distDir, 'docs', 'pagefind'), + }); + + if (written.errors.length > 0) { + logger.error(written.errors.join(', ')); + } + + await close(); + + // `page_count` is files scanned, not files indexed — it counts the + // redirect stubs that `data-pagefind-body` then drops. One fragment is + // written per indexed page, so that is the number worth checking. + const outDir = path.join(distDir, 'docs', 'pagefind'); + const indexed = ( + await fs.promises.readdir(path.join(outDir, 'fragment')) + ).length; + + logger.info( + `indexed ${indexed} of ${added.page_count} pages into docs/pagefind` + ); + + if (indexed < EXPECTED_PAGES) { + logger.warn( + `only ${indexed} pages were indexed, expected at least ${EXPECTED_PAGES} — check data-pagefind-body is still on the article` + ); + } + }, + }, + }; +} diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index a4e7ec2bdc..2ecc3e45a4 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -59,6 +59,39 @@ const showSearch = !isSearchPage; // The footer prints this; JSON-LD carries it as dateModified. const lastUpdated = frontmatter.modDate ?? frontmatter.pubDate ?? null; + +// Search indexing. The same predicate `PostFiltering.showInSearch` applies at +// the point the page is rendered — redirect stubs never reach this layout at +// all, so only the frontmatter opt-outs have to be reproduced here. +const indexable = + frontmatter.navSearch !== false && + frontmatter.listable !== false && + frontmatter.draft !== true; + +// Which tab a result lands under. Longest match first: every CLI page also sits +// under the REST API tree. Mirrors `classify()` in `src/scripts/search-engine.ts`, +// which is what the old index derives the same value from at query time. +const sections: [string, RegExp][] = [ + [ + 'cli', + /^\/docs\/octopus-rest-api\/(cli|octopus-cli|[a-z.]+-command-line)(\/|$)/, + ], + ['api', /^\/docs\/octopus-rest-api(\/|$)/], + ['integrations', /^\/docs\/api-and-integration(\/|$)/], +]; +const section = + sections.find(([, prefix]) => prefix.test(Astro.url.pathname))?.[0] ?? 'docs'; + +// The trail the result row prints, taken from the path so it matches whatever +// the page is actually filed under. +const trail = Astro.url.pathname + .split('/') + .filter(Boolean) + .slice(1, -1) + .map((segment) => + segment.replace(/-/g, ' ').replace(/^./, (c) => c.toUpperCase()) + ) + .join(' / '); ---
-
+ { + /* Scoping the index to the article is what keeps every sidebar link + and footer word out of every page's search terms. Without it the + index is still built and search still "works" — it is just bad, + which is why the integration asserts on the page count. + + The article rather than `.page-content`, so the heading inside it is + there for Pagefind to take the result title from. */ + } +
-
+
- + {/* Same control on every page, so it is noise in every result. */} +
+ +
diff --git a/src/pages/docs/search.json.ts b/src/pages/docs/search.json.ts deleted file mode 100644 index 6ef6c53a15..0000000000 --- a/src/pages/docs/search.json.ts +++ /dev/null @@ -1,93 +0,0 @@ -/** @format */ - -// warning: This file is overwritten by Astro Accelerator - -import { accelerator } from '@lib/accelerator'; -import { PostFiltering } from 'astro-accelerator-utils'; -import type { MarkdownInstance } from 'astro'; -import { SITE } from '@config'; -import { convert } from 'html-to-text'; -import keywordExtractor from 'keyword-extractor'; -import { isUnderConstruction } from '@lib/underConstruction'; - -const getData = async () => { - //@ts-ignore - const allPages = import.meta.glob(['./**/*.md', './**/*.mdx']); - const items = []; - - for (const path in allPages) { - // Temporary - see src/lib/underConstruction.ts. - if (isUnderConstruction(path)) { - continue; - } - - const page = (await allPages[path]()) as MarkdownInstance< - Record - >; - - if (!PostFiltering.showInSearch(page)) { - continue; - } - - let url = page.url ?? ''; - - if (page.frontmatter.paged) { - url += '/1/'; - } - - const headings = await page.getHeadings(); - const title = await accelerator.markdown.getTextFrom( - page.frontmatter?.title - ); - const content = page.compiledContent ? await page.compiledContent() : ''; - let counted: { word: string; count: number }[] = []; - - if (content) { - const options = { - wordwrap: false, - selectors: [ - { selector: 'a', options: { ignoreHref: true } }, - { selector: 'img', format: 'skip' }, - { selector: 'h1', options: { uppercase: false } }, - ], - }; - const text = convert(content, options); - - const words = keywordExtractor.extract(text, { - language: 'english', - return_changed_case: true, - remove_duplicates: true, - }); - - counted = words - .map((w) => { - return { - word: w, - count: words.filter((wd) => wd === w).length, - }; - }) - .filter((e) => e.word.replace(/[^a-z]+/g, '').length > 1); - } - - items.push({ - title: title, - headings: headings.map((h) => { - return { text: h.text, slug: h.slug }; - }), - description: page.frontmatter.description ?? '', - keywords: counted.map((c) => c.word).join(' '), - tags: page.frontmatter.tags ?? [], - url: SITE.url + accelerator.urlFormatter.formatAddress(url), - date: page.frontmatter.pubDate ?? '', - }); - } - - return new Response(JSON.stringify(items), { - status: 200, - headers: { - 'Content-Type': 'application/json', - }, - }); -}; - -export const GET = getData; diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index ef88eb5758..be62b753a8 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -14,7 +14,7 @@ import { type SearchEngine, type SearchResult, } from './search-engine'; -import { legacyEngine } from './search-engine-legacy'; +import { pagefindEngine } from './search-engine-pagefind'; const DEBOUNCE_MS = 150; const SITE_SEARCH = 'site'; @@ -41,7 +41,7 @@ function setup(dialog: HTMLDialogElement) { const demo = dialog.dataset.demoResults; const engine: SearchEngine = demo ? fixtureEngine(JSON.parse(demo)) - : legacyEngine(dialog.dataset.indexUrl ?? '/docs/search.json'); + : pagefindEngine(dialog.dataset.indexUrl ?? '/docs/pagefind/'); let facet = 'all'; let active = -1; diff --git a/src/scripts/modules/stemmer.js b/src/scripts/modules/stemmer.js deleted file mode 100644 index 778ebcc9c0..0000000000 --- a/src/scripts/modules/stemmer.js +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Removes "morphological and inflexional endings" from words - * See: http://www.tartarus.org/~martin/PorterStemmer - */ -export const stemmer = (function () { - const step2list = { - ational: 'ate', - tional: 'tion', - enci: 'ence', - anci: 'ance', - izer: 'ize', - bli: 'ble', - alli: 'al', - entli: 'ent', - eli: 'e', - ousli: 'ous', - ization: 'ize', - ation: 'ate', - ator: 'ate', - alism: 'al', - iveness: 'ive', - fulness: 'ful', - ousness: 'ous', - aliti: 'al', - iviti: 'ive', - biliti: 'ble', - logi: 'log', - }; - - const step3list = { - icate: 'ic', - ative: '', - alize: 'al', - iciti: 'ic', - ical: 'ic', - ful: '', - ness: '', - }; - - const c = '[^aeiou]', // consonant - v = '[aeiouy]', // vowel - C = c + '[^aeiouy]*', // consonant sequence - V = v + '[aeiou]*', // vowel sequence - mgr0 = '^(' + C + ')?' + V + C, // [C]VC... is m>0 - meq1 = '^(' + C + ')?' + V + C + '(' + V + ')?$', // [C]VC[V] is m=1 - mgr1 = '^(' + C + ')?' + V + C + V + C, // [C]VCVC... is m>1 - s_v = '^(' + C + ')?' + v; // vowel in stem - - /** - * @param {string} w - * @returns {string} - */ - return function (w) { - var stem, - suffix, - firstch, - re, - re2, - re3, - re4, - origword = w; - - if (w.length < 3) { - return w; - } - - firstch = w.substring(0, 1); - - if (firstch == 'y') { - w = firstch.toUpperCase() + w.substring(1, w.length); - } - - // Step 1a - re = /^(.+?)(ss|i)es$/; - re2 = /^(.+?)([^s])s$/; - - if (re.test(w)) { - w = w.replace(re, '$1$2'); - } else if (re2.test(w)) { - w = w.replace(re2, '$1$2'); - } - - // Step 1b - re = /^(.+?)eed$/; - re2 = /^(.+?)(ed|ing)$/; - if (re.test(w)) { - var fp = re.exec(w); - re = new RegExp(mgr0); - if (re.test(fp[1])) { - re = /.$/; - w = w.replace(re, ''); - } - } else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1]; - re2 = new RegExp(s_v); - if (re2.test(stem)) { - w = stem; - re2 = /(at|bl|iz)$/; - re3 = new RegExp('([^aeiouylsz])\\1$'); - re4 = new RegExp('^' + C + v + '[^aeiouwxy]$'); - if (re2.test(w)) { - w = w + 'e'; - } else if (re3.test(w)) { - re = /.$/; - w = w.replace(re, ''); - } else if (re4.test(w)) { - w = w + 'e'; - } - } - } - - // Step 1c - re = /^(.+?)y$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(s_v); - if (re.test(stem)) { - w = stem + 'i'; - } - } - - // Step 2 - re = - /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) { - w = stem + step2list[suffix]; - } - } - - // Step 3 - re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) { - w = stem + step3list[suffix]; - } - } - - // Step 4 - re = - /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; - re2 = /^(.+?)(s|t)(ion)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - if (re.test(stem)) { - w = stem; - } - } else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1] + fp[2]; - re2 = new RegExp(mgr1); - if (re2.test(stem)) { - w = stem; - } - } - - // Step 5 - re = /^(.+?)e$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - re2 = new RegExp(meq1); - re3 = new RegExp('^' + C + v + '[^aeiouwxy]$'); - if (re.test(stem) || (re2.test(stem) && !re3.test(stem))) { - w = stem; - } - } - - re = /ll$/; - re2 = new RegExp(mgr1); - if (re.test(w) && re2.test(w)) { - re = /.$/; - w = w.replace(re, ''); - } - - // and turn initial Y back to y - - if (firstch == 'y') { - w = firstch.toLowerCase() + w.substr(1); - } - - return w; - }; -})(); diff --git a/src/scripts/modules/string.js b/src/scripts/modules/string.js deleted file mode 100644 index dea7142565..0000000000 --- a/src/scripts/modules/string.js +++ /dev/null @@ -1,83 +0,0 @@ -// @ts-check - -/** - * Looks for a search within a string - * - * @param {string} string - * @param {string} search - * @returns - */ -function contains(string, search) { - return string.indexOf(search) > -1; -} - -/** - * Looks for a search within a string - * - * @param {string} string - * @param {string} search - * @returns - */ -function containsWord(string, search) { - return string.split(' ').indexOf(search) > -1; -} - -/** - * - * @param {string} string - * @param {string[]} terms - * @returns - */ -function highlight(string, terms) { - terms.forEach((term) => { - const regEx = new RegExp(term, 'ig'); - const matches = string.match(regEx); - if (matches) { - string = string.replace(regEx, `${matches[0]}`); - } - }); - return string; -} - -/** - * Simplifies a string to plain lower case, removing diacritic characters and hyphens - * This means a search for "co-op" will be found in "COOP" and "Café" will be found in "cafe" - * @param {string} string - * @returns {string} - */ -function sanitise(string) { - // @ts-ignore - if (String.prototype.normalize) { - // Reduces diacritic characters to plain characters - string - .trim() - .normalize('NFD') - .replace(/\./g, ' ') - .replace(/[\u0300-\u036f]/g, '') - .toLowerCase() - .replace(/-/g, ''); - } - - // Some browsers can't normalise strings - return string.trim().toLowerCase().replace(/-/g, ''); -} - -/** - * Sets a minimum length for a search - * @param {string} string - * @returns - */ -function isLongEnough(string) { - return string.length > 1; -} - -/** - * - * @param {string} string - * @returns {string[]} - */ -function explode(string) { - return string.split(' ').filter(isLongEnough).map(sanitise); -} - -export { contains, containsWord, sanitise, explode, highlight }; diff --git a/src/scripts/search-engine-legacy.ts b/src/scripts/search-engine-legacy.ts deleted file mode 100644 index 398c758757..0000000000 --- a/src/scripts/search-engine-legacy.ts +++ /dev/null @@ -1,265 +0,0 @@ -// The current search, behind the `SearchEngine` seam: one build-time JSON of -// titles, headings, descriptions, tags and an extracted keyword bag, scored in -// the browser. The scoring is carried over unchanged from the results list this -// replaces, so the overlay returns exactly what the old dropdown returned. -// -// Body text is not in the index, which is the ceiling on how good this can be -// and the reason the Pagefind and Orama spikes exist. - -import { contains, explode, highlight, sanitise } from './modules/string.js'; -import { stemmer } from './modules/stemmer.js'; -import { - breadcrumbFrom, - classify, - countByFacet, - type SearchEngine, - type SearchResult, -} from './search-engine'; - -type Heading = { text: string; slug: string; safeText: string }; - -type Entry = { - title: string; - safeTitle: string; - description: string; - keywords: string; - tags: string[]; - headings: Heading[]; - url: string; - depth: number; -}; - -type Scored = Entry & { - score: number; - foundWords: number; - foundTerms: string[]; -}; - -// Phrase and per-term weights, verbatim from the list this replaces. -const SCORING = { - depth: 5, - phraseTitle: 60, - phraseHeading: 20, - phraseDescription: 20, - termTitle: 40, - termHeading: 15, - termDescription: 15, - termTags: 15, - termKeywords: 15, -}; - -const WORD_SCORES = { - titleExact: 20, - titleContains: 15, - headingContains: 10, - contentContains: 1, -}; - -const RESULT_LIMIT = 30; - -async function getSynonyms(): Promise> { - try { - return (await import('./synonyms.js')).synonyms; - } catch { - return {}; - } -} - -/** Terms reach `highlight()` as a regular expression, so a query like `c++` - * would throw before anything was drawn. */ -function escapeForRegExp(term: string) { - return term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -/** The excerpt is set as HTML so the `` around each hit survives. Escaping - * first means those marks are the only tags that can ever reach the DOM. */ -function escapeHtml(text: string) { - return text - .replace(/&/g, '&') - .replace(//g, '>'); -} - -async function expand(query: string) { - const synonyms = await getSynonyms(); - const terms: string[] = []; - - for (const term of explode(query)) { - const synonym = synonyms[term]; - // An empty synonym is a stop word: drop the term rather than expanding it. - if (synonym === '') continue; - terms.push(term); - if (synonym) terms.push(...synonym.split(' ')); - } - - const stemmed = terms - .map((term) => stemmer(term)) - .filter((stem) => !terms.includes(stem)); - - return { terms, allTerms: [...terms, ...stemmed] }; -} - -function score(entry: Entry, query: string, allTerms: string[]): Scored { - const scored: Scored = { ...entry, score: 0, foundWords: 0, foundTerms: [] }; - - // Phrase matches: the whole query, found intact - if (scored.safeTitle === query) scored.foundWords += WORD_SCORES.titleExact; - - if (contains(scored.safeTitle, query)) { - scored.score += SCORING.phraseTitle; - scored.foundWords += WORD_SCORES.titleContains; - } - - for (const heading of scored.headings) { - if (contains(heading.safeText, query)) { - scored.score += SCORING.phraseHeading; - scored.foundWords += WORD_SCORES.headingContains; - } - } - - if (contains(scored.description, query)) { - scored.score += SCORING.phraseDescription; - scored.foundWords += WORD_SCORES.contentContains; - } - - // Term matches: each word of the query, found anywhere - for (const term of allTerms) { - let found = false; - - if (contains(scored.safeTitle, term)) { - scored.score += SCORING.termTitle; - scored.foundWords += WORD_SCORES.headingContains / 2; - found = true; - } - - for (const heading of scored.headings) { - if (contains(heading.safeText, term)) { - scored.score += SCORING.termHeading; - found = true; - } - } - - if (contains(scored.description, term)) { - scored.score += SCORING.termDescription; - found = true; - } - - for (const tag of scored.tags) { - if (contains(tag, term)) { - scored.score += SCORING.termTags; - found = true; - } - } - - if (contains(scored.keywords, term)) { - scored.score += SCORING.termKeywords; - found = true; - } - - if (found) { - scored.foundWords++; - if (!scored.foundTerms.includes(term)) scored.foundTerms.push(term); - } - } - - // Shallow pages win ties: /docs/features over /docs/features/a/b - if (scored.score > 0) { - if (scored.depth < 5) { - scored.score += SCORING.depth; - scored.foundWords++; - } - if (scored.depth < 4) { - scored.score += SCORING.depth; - scored.foundWords++; - } - } - - return scored; -} - -function byRelevance(a: Scored, b: Scored) { - if (b.foundTerms.length !== a.foundTerms.length) { - return b.foundTerms.length - a.foundTerms.length; - } - if (b.foundWords !== a.foundWords) return b.foundWords - a.foundWords; - return b.score - a.score; -} - -export function legacyEngine(indexUrl: string): SearchEngine { - // One fetch, on the first search rather than on page load, shared by every - // search after it. The old list fetched 1.86MB the moment the script ran. - let loading: Promise | null = null; - - function load() { - loading ??= fetch(indexUrl) - .then((response) => response.json()) - .then((data: Record[]) => - data.map((item) => ({ - title: item.title, - safeTitle: sanitise(item.title), - description: item.description ?? '', - keywords: item.keywords ?? '', - tags: (item.tags ?? []).map((tag: string) => sanitise(tag)), - headings: (item.headings ?? []).map((heading: Heading) => ({ - ...heading, - safeText: sanitise(heading.text), - })), - url: item.url, - depth: item.url.match(/\//g)?.length ?? 0, - })) - ) - .catch(() => { - // A failed load must not poison every later search. - loading = null; - return [] as Entry[]; - }); - - return loading; - } - - return { - warm: load, - - async search(rawQuery, facet) { - // Chained words are joined, so `System.Text` searches as `systemtext`. - const query = sanitise(rawQuery.replace(/\./g, ' ')); - if (!query) return { results: [], counts: countByFacet([]) }; - - const [haystack, { terms, allTerms }] = await Promise.all([ - load(), - expand(query), - ]); - - const highlightTerms = terms.map(escapeForRegExp); - - const matched = haystack - .map((entry) => score(entry, query, allTerms)) - .filter((entry) => entry.score > 0) - .sort(byRelevance) - .slice(0, RESULT_LIMIT) - .map((entry): SearchResult => { - // The index stores absolute production URLs, and every entry in it is - // a page of this site. Keeping only the path is what makes a result - // stay on whatever host is serving it — an ephemeral environment or - // localhost would otherwise send you to production. - const path = new URL(entry.url, window.location.origin).pathname; - - return { - url: path, - title: entry.title, - excerpt: highlight(escapeHtml(entry.description), highlightTerms), - breadcrumb: breadcrumbFrom(path), - ...classify(path), - }; - }); - - return { - counts: countByFacet(matched), - results: - facet && facet !== 'all' - ? matched.filter((result) => result.facet === facet) - : matched, - }; - }, - }; -} diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts new file mode 100644 index 0000000000..bdfbeb3eae --- /dev/null +++ b/src/scripts/search-engine-pagefind.ts @@ -0,0 +1,117 @@ +// Pagefind behind the `SearchEngine` seam. +// +// The index is chunked and content-hashed, and Pagefind fetches only the chunks +// a query actually touches. That is the whole reason for the spike: the old +// index is a single 1.86MB download, and this one's per-query cost stays flat as +// the corpus grows. +// +// Two calls per search: `search()` returns lightweight stubs, then each result's +// `data()` fetches its own fragment. Only the page of results being shown is +// loaded, so the fragments fetched scale with the result limit, not the corpus. + +import { + breadcrumbFrom, + classify, + type SearchEngine, + type SearchResult, +} from './search-engine'; + +const RESULT_LIMIT = 30; + +type PagefindFragment = { + url: string; + meta?: Record; + excerpt: string; +}; + +type PagefindResultStub = { data(): Promise }; + +type PagefindApi = { + options(config: Record): Promise; + filters(): Promise>>; + search( + query: string, + options?: { filters?: Record } + ): Promise<{ + results: PagefindResultStub[]; + totalFilters: Record>; + unfilteredResultCount: number; + }>; +}; + +export function pagefindEngine(bundlePath: string): SearchEngine { + let loading: Promise | null = null; + + function load() { + loading ??= import(/* @vite-ignore */ `${bundlePath}pagefind.js`) + .then(async (api: PagefindApi) => { + // Pagefind resolves its chunk URLs against this, and prepends it to + // every result URL; the site is proxied under /docs/ rather than served + // from the root, and the index is built relative to that same prefix. + await api.options({ basePath: bundlePath }); + // The filter index is a separate chunk, and a search returns empty + // filter counts until it has been pulled down. Loading it here rather + // than per-search means the tab strip is populated on the first result. + await api.filters(); + return api; + }) + .catch(() => { + // A failed load must not poison every later search. + loading = null; + return null; + }); + + return loading; + } + + return { + warm: load, + + async search(rawQuery, facet) { + const query = rawQuery.trim(); + const empty = { results: [], counts: { all: 0 } }; + if (!query) return empty; + + const api = await load(); + if (!api) return empty; + + const response = await api.search(query, { + filters: facet && facet !== 'all' ? { section: [facet] } : undefined, + }); + + // Counts come from the whole match set rather than the filtered one, so + // the strip keeps showing what the other tabs hold while one is selected. + const counts: Record = { + all: response.unfilteredResultCount, + ...(response.totalFilters?.section ?? {}), + }; + + const fragments = await Promise.all( + response.results.slice(0, RESULT_LIMIT).map((stub) => stub.data()) + ); + + const results = fragments.map((fragment): SearchResult => { + const path = new URL(fragment.url, window.location.origin).pathname; + + return { + url: path, + // Pagefind takes the title from the first heading inside the indexed + // body, which is why the article rather than `.page-content` carries + // `data-pagefind-body`. + title: fragment.meta?.title ?? path, + // Already carries around the hits, and Pagefind escapes the + // surrounding text itself. + excerpt: fragment.excerpt, + // Written into the page by the layout; the path is the fallback for + // anything built before that attribute existed. + breadcrumb: fragment.meta?.trail + ? fragment.meta.trail.split(' / ') + : breadcrumbFrom(path), + ...classify(path), + }; + }); + + return { results, counts }; + }, + }; +} diff --git a/src/scripts/synonyms.js b/src/scripts/synonyms.js deleted file mode 100644 index 489fc14133..0000000000 --- a/src/scripts/synonyms.js +++ /dev/null @@ -1,17 +0,0 @@ -const synonyms = { - // Keep me alphabetical - licence: 'license', - logs: 'log', - regex: 'regular expression', - // Join words - a: '', - and: '', - for: '', - if: '', - the: '', - to: '', - via: '', - with: '', -}; - -export { synonyms }; From 8278e5a8ca62004c0589374cea791ed77eaf2128 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 09:23:55 +1200 Subject: [PATCH 02/14] Add pagefind to the project dictionary Co-Authored-By: Claude Opus 5 (1M context) --- dictionary-octopus.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dictionary-octopus.txt b/dictionary-octopus.txt index a94359012f..8d53e3bf57 100644 --- a/dictionary-octopus.txt +++ b/dictionary-octopus.txt @@ -405,6 +405,7 @@ Ozar PAAS packageid packageversion +pagefind passout PathtoPublish pemstore From 02c15f87e7aed426da02416f1f1786bcc2c3b634 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 15:15:29 +1200 Subject: [PATCH 03/14] Preload while typing, warm on page load, and set the excerpt length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three configuration corrections from the Pagefind audit. Its indexing and ranking were already idiomatic, so none of this touches either. preload() now runs on each keystroke ahead of the debounced search, which is what Pagefind documents as the way to fetch the chunks a query needs while the reader is still typing. Warming moves to page load for Pagefind only. That is 118KB of runtime and WASM against a cold first result measured at 2239ms. Which side of that trade is right differs by engine, so `SearchEngine` carries an `eager` flag and the engine states its own answer rather than the overlay assuming one — Orama pays for its whole index on every navigation and must stay lazy. excerptLength drops from its 30-word default to 20, to fit the single line the result row gives it. Sub-results are deliberately not used. They render heading-scoped matches with their own anchors, which is real, but the corpus fights it: the CLI tree has 511 headings across only 49 distinct titles, 213 of them "Learn more" and 176 "Examples", and the API tree 291 across 100. Only documentation pages are distinctive, at 4160 titles across 5302 headings. Suppressing the boilerplate needs a threshold nobody has justified yet, and the audit is clear that sub-results move neither Success@1 nor Success@5 — so this stays out of a comparison it cannot decide, and can be added later against whichever engine wins. Co-Authored-By: Claude Opus 5 (1M context) --- src/scripts/docs-search.ts | 11 ++++++++++- src/scripts/search-engine-pagefind.ts | 22 +++++++++++++++++++++- src/scripts/search-engine.ts | 13 +++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index be62b753a8..c24299a665 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -212,7 +212,12 @@ function setup(dialog: HTMLDialogElement) { // --- Inside the dialog --------------------------------------------------- - input.addEventListener('input', schedule); + input.addEventListener('input', () => { + // Ahead of the debounce, so the chunks this query needs are already on + // their way by the time the search runs. + engine.preload?.(input!.value); + schedule(); + }); dialog.addEventListener('keydown', (event) => { // Up and Down only. Home and End belong to the query, which is what an @@ -328,6 +333,10 @@ function setup(dialog: HTMLDialogElement) { open(); }); + // Engines that are cheap to start say so; the rest wait for the overlay. See + // `eager` on `SearchEngine` for why this is not the same answer for both. + if (engine.eager) engine.warm?.(); + // A shared `?q=` link opens straight into results. const q = new URLSearchParams(window.location.search).get('q'); if (q) open(q); diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index bdfbeb3eae..f07e9ff3ed 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -29,6 +29,10 @@ type PagefindResultStub = { data(): Promise }; type PagefindApi = { options(config: Record): Promise; filters(): Promise>>; + preload( + query: string, + options?: { filters?: Record } + ): void; search( query: string, options?: { filters?: Record } @@ -48,7 +52,12 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // Pagefind resolves its chunk URLs against this, and prepends it to // every result URL; the site is proxied under /docs/ rather than served // from the root, and the index is built relative to that same prefix. - await api.options({ basePath: bundlePath }); + await api.options({ + basePath: bundlePath, + // The default is 30 words, which overruns the single line the result + // row gives it. Sized to the row instead. + excerptLength: 20, + }); // The filter index is a separate chunk, and a search returns empty // filter counts until it has been pulled down. Loading it here rather // than per-search means the tab strip is populated on the first result. @@ -65,8 +74,19 @@ export function pagefindEngine(bundlePath: string): SearchEngine { } return { + // 118KB of runtime and WASM, and it shortens the cold path to the first + // result. The per-query chunks are still fetched on demand. + eager: true, warm: load, + // Pulls the chunks this query needs while the reader is still typing. + // Without it every keystroke pays for its own chunk fetches. + preload(query) { + const term = query.trim(); + if (!term) return; + void load().then((api) => api?.preload(term)); + }, + async search(rawQuery, facet) { const query = rawQuery.trim(); const empty = { results: [], counts: { all: 0 } }; diff --git a/src/scripts/search-engine.ts b/src/scripts/search-engine.ts index c8d1800b2b..cdc8744b41 100644 --- a/src/scripts/search-engine.ts +++ b/src/scripts/search-engine.ts @@ -22,6 +22,19 @@ export type SearchEngine = { search(query: string, facet?: string): Promise; /** Optional: start loading the index before the first query needs it. */ warm?(): void; + /** + * Whether `warm()` is cheap enough to run on page load rather than when the + * overlay opens. Pagefind pays about 118KB for its runtime and fetches the + * rest per query; Orama pays for its whole index parsed and restored, on + * every navigation, for every visitor including the majority who never + * search. The answer differs by engine, so the engine states it. + */ + eager?: boolean; + /** + * Optional: fetch what this query will need without running it. Called while + * the reader is still typing, ahead of the debounced search. + */ + preload?(query: string): void; }; export type Facet = { key: string; label: string }; From 5b663677730936e125db44f54e03c1f8f831dcd1 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 09:55:20 +1200 Subject: [PATCH 04/14] Use the rest of what Pagefind offers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six capabilities that were available and unused. Indexing. A depth-derived data-pagefind-weight on the page header, so a bare section name ranks the section above the pages inside it — the largest single source of missed traffic on the search terms readers actually type. Image alt text becomes searchable via data-pagefind-index-attrs, which has no inheritance and so needs a hast plugin to reach every markdown image. A date sort key is indexed, and a frontmatter title is registered as fallback metadata for any page whose heading comes back empty. Querying. highlightParam so result links carry the query for the destination page to highlight, and sub_results rendered as their own options beneath their page, so a match inside a long page can be arrowed onto and lands on that heading. The weight is the one worth measuring: applied as a client-side reorder it took Pagefind from 42% to 62% weighted Success@5 on real search terms, and from 73% to 95% on top-visited pages. At index time it has no top-30 reach limit. Co-Authored-By: Claude Opus 5 (1M context) --- astro.config.mjs | 3 ++ src/components/ArticleHeader.astro | 7 ++-- src/components/DocsSearch.astro | 48 +++++++++++++++++++++++++ src/components/Image.astro | 7 +++- src/layouts/Default.astro | 40 ++++++++++++++++++++- src/plugins/pagefind-image-attrs.js | 39 +++++++++++++++++++++ src/scripts/docs-search.ts | 50 +++++++++++++++++++++++++-- src/scripts/search-engine-pagefind.ts | 44 +++++++++++++++++++++++ src/scripts/search-engine.ts | 16 +++++++++ 9 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 src/plugins/pagefind-image-attrs.js diff --git a/astro.config.mjs b/astro.config.mjs index c8625946b3..c340c496d1 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -9,6 +9,7 @@ import satteriHeadingId from './src/plugins/satteri-heading-id.js'; import satteriApiExamples, { apiExampleDirective } from './src/plugins/satteri-api-examples.js'; import { endpointDirective } from './src/plugins/satteri-endpoint.js'; import satteriWbr from './src/plugins/satteri-wbr.js'; +import pagefindImageAttrs from './src/plugins/pagefind-image-attrs.js'; import shikiCodeBlock from './src/plugins/shiki-code-block.js'; // https://astro.build/config @@ -67,6 +68,8 @@ export default defineConfig({ ], hastPlugins: [ satteriWbr, + // Marks image alt text for Pagefind to index. Inert without it. + pagefindImageAttrs, satteriApiExamples ], }), diff --git a/src/components/ArticleHeader.astro b/src/components/ArticleHeader.astro index 85f387e823..e7f5349c96 100644 --- a/src/components/ArticleHeader.astro +++ b/src/components/ArticleHeader.astro @@ -12,8 +12,11 @@ type Props = { lang: string; subtitle?: string | null; title: string; + /** Passed to `data-pagefind-weight`. Omitted leaves Pagefind's own heading + * weighting alone, which is what every page outside the docs layout wants. */ + searchWeight?: number; }; -const { lang, subtitle, title } = Astro.props satisfies Props; +const { lang, subtitle, title, searchWeight } = Astro.props satisfies Props; // Language const _ = Lang(lang); @@ -22,7 +25,7 @@ const _ = Lang(lang); stats.stop(); --- -
+

{title}

{subtitle &&

{subtitle}

}
diff --git a/src/components/DocsSearch.astro b/src/components/DocsSearch.astro index 10de1ccd35..091314b9ae 100644 --- a/src/components/DocsSearch.astro +++ b/src/components/DocsSearch.astro @@ -145,6 +145,21 @@ const listboxId = `${name}-search-listbox`; + { + /* A heading inside the result above that matched on its own. It is a + separate `role="option"` rather than a link inside the parent row, + because it goes somewhere different — the reader can arrow onto it and + land on that section instead of the top of the page. */ + } + +
- {alt} + {alt} {caption &&
{caption}
} diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index 2ecc3e45a4..e9bdabdac0 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -82,6 +82,37 @@ const sections: [string, RegExp][] = [ const section = sections.find(([, prefix]) => prefix.test(Astro.url.pathname))?.[0] ?? 'docs'; +// How much the page's own name counts for, by how deep it sits. +// +// BM25 has no notion of a site's shape, so a bare section name ranks the pages +// inside the section above the section itself: `variables` returned an API +// example rather than /docs/projects/variables, and on the search terms readers +// actually type this was the largest single source of missed traffic. A page +// deeper in the tree is more specific, so its title should count for less when +// the query is short. +// +// Pagefind weights headings h1 7 down to h6 2 by default, and the scale is +// quadratic — 8 carries roughly 1.8x the pull of 6. Clamped at 3 so a genuinely +// deep page can still win on its own name. +const depth = Astro.url.pathname.split('/').filter(Boolean).length; +const searchWeight = indexable ? Math.max(3, 10 - depth) : undefined; + +// Indexed so results can be ordered by recency rather than relevance. Nothing in +// the overlay offers that control yet; the key is here so the capability exists +// in the index rather than needing a rebuild to add later. +const sortKeys = + indexable && lastUpdated + ? `date:${new Date(lastUpdated).toISOString().slice(0, 10)}` + : undefined; + +// A safety net rather than a fix for anything currently broken. Pagefind takes a +// result's title from the first heading inside the indexed body, so a page whose +// h1 renders as an image, or comes back empty, would otherwise show its URL in the +// result row. Automatic metadata and `data-pagefind-meta` both take priority, so +// this never overrides a real heading. +const defaultMeta = + indexable && frontmatter.title ? `title:${frontmatter.title}` : undefined; + // The trail the result row prints, taken from the path so it matches whatever // the page is actually filed under. const trail = Astro.url.pathname @@ -126,8 +157,15 @@ const trail = Astro.url.pathname data-pagefind-body={indexable ? true : undefined} data-pagefind-filter={indexable ? `section:${section}` : undefined} data-pagefind-meta={indexable ? `trail:${trail}` : undefined} + data-pagefind-sort={sortKeys} + data-pagefind-default-meta={defaultMeta} > - +
diff --git a/src/plugins/pagefind-image-attrs.js b/src/plugins/pagefind-image-attrs.js new file mode 100644 index 0000000000..35aeb47dbb --- /dev/null +++ b/src/plugins/pagefind-image-attrs.js @@ -0,0 +1,39 @@ +import { defineHastPlugin } from 'satteri'; + +// Makes image alt text searchable. +// +// Pagefind indexes text nodes, so the words inside an `alt` attribute are +// invisible to it by default. `data-pagefind-index-attrs` names the attributes to +// pull in as content, and it applies only to the element carrying it — there is no +// inheritance, so it has to land on every `` rather than once on the article. +// Doing that here rather than in the layout is what covers images written as +// markdown, which is nearly all of them. +// +// This matters more than it sounds for a docs site: a screenshot's alt text is +// often the only place a UI label appears in prose, so `alt="Deployment process +// editor"` is the only text on the page that would answer a search for it. +// +// Covers markdown image syntax and `:img{}` directives. Raw `` tags written +// straight into a page are passed through as HTML and never become hast elements, +// so this does not see them; `Image.astro` carries the attribute itself for the +// same reason. + +export default defineHastPlugin({ + name: 'pagefind-image-attrs', + element: { + filter: ['img'], + visit(node, ctx) { + const { alt, title } = node.properties ?? {}; + + // A decorative image carries `alt=""`, which is a deliberate signal that it + // has nothing to say. Indexing the empty string is harmless but pointless, + // and skipping it keeps the attribute off the markup where it means nothing. + const wanted = [alt ? 'alt' : null, title ? 'title' : null].filter( + Boolean + ); + if (wanted.length === 0) return; + + ctx.setProperty(node, 'data-pagefind-index-attrs', wanted.join(',')); + }, + }, +}); diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index c24299a665..6a77810103 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -13,6 +13,7 @@ import { fixtureEngine, type SearchEngine, type SearchResult, + type SearchSubResult, } from './search-engine'; import { pagefindEngine } from './search-engine-pagefind'; @@ -33,6 +34,9 @@ function setup(dialog: HTMLDialogElement) { const template = dialog.querySelector( '[data-docs-search-row]' ); + const sectionTemplate = dialog.querySelector( + '[data-docs-search-section]' + ); // A partially rendered overlay is a bug elsewhere; wiring half of it up would // only turn that into a confusing runtime failure. @@ -99,8 +103,48 @@ function setup(dialog: HTMLDialogElement) { return item; } + function drawSection( + template: HTMLTemplateElement, + section: SearchSubResult, + at: number + ) { + const item = template.content.cloneNode(true) as DocumentFragment; + + const row = item.querySelector('.docs-search__section')!; + row.id = `${name}-search-option-${at}`; + row.href = section.url; + item.querySelector('.docs-search__section-title')!.textContent = + section.title; + + return item; + } + + /** + * Rows for the whole response, pages and their matched sections interleaved. + * + * The running index is what keeps `aria-activedescendant` working: every + * option needs a unique id, and a page's sections sit between it and the next + * page rather than being numbered separately. + */ + function drawAll(results: SearchResult[]) { + const nodes: DocumentFragment[] = []; + let at = 0; + + for (const result of results) { + nodes.push(drawRow(result, at++)); + // No template means an overlay built before sections existed. Dropping them + // costs the reader nothing beyond the deep link. + if (!sectionTemplate) continue; + for (const section of result.sections ?? []) { + nodes.push(drawSection(sectionTemplate, section, at++)); + } + } + + return nodes; + } + function render(response: Awaited>) { - list!.replaceChildren(...response.results.map(drawRow)); + list!.replaceChildren(...drawAll(response.results)); for (const tab of facetTabs()) { const key = tab.dataset.facet!; @@ -321,7 +365,9 @@ function setup(dialog: HTMLDialogElement) { if (name !== SITE_SEARCH) return; // Cmd+K on Apple platforms, Ctrl+K everywhere else. - const navigator = (window.navigator as Navigator & { userAgentData?: { platform: string } }); + const navigator = window.navigator as Navigator & { + userAgentData?: { platform: string }; + }; const isApple = navigator.userAgentData ? navigator.userAgentData.platform === 'macOS' : (navigator.platform || navigator.userAgent).includes('Mac'); diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index f07e9ff3ed..969b1ed4a5 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -18,10 +18,18 @@ import { const RESULT_LIMIT = 30; +type PagefindSubResult = { + title: string; + /** Carries the heading's `#anchor` when the match is below the page title. */ + url: string; + excerpt: string; +}; + type PagefindFragment = { url: string; meta?: Record; excerpt: string; + sub_results?: PagefindSubResult[]; }; type PagefindResultStub = { data(): Promise }; @@ -43,6 +51,35 @@ type PagefindApi = { }>; }; +// How many matched headings a row will show. More than this and the row stops +// being a result and starts being a table of contents. +const SECTION_LIMIT = 3; + +/** + * The headings inside a result that matched on their own. + * + * Pagefind returns one sub-result for the page itself alongside the real + * headings, and its URL is the page URL with no anchor — that is the row's own + * link, so it is dropped rather than repeated underneath itself. + */ +function sectionsOf(fragment: PagefindFragment, path: string) { + return (fragment.sub_results ?? []) + .filter((sub) => sub.url.includes('#')) + .slice(0, SECTION_LIMIT) + .map((sub) => ({ + title: sub.title, + // Keep the anchor, drop the origin, so the row links the same way every + // other row does. + url: new URL(sub.url, window.location.origin).pathname + hashOf(sub.url), + })) + .filter((sub) => sub.url !== path); +} + +function hashOf(url: string) { + const at = url.indexOf('#'); + return at === -1 ? '' : url.slice(at); +} + export function pagefindEngine(bundlePath: string): SearchEngine { let loading: Promise | null = null; @@ -57,6 +94,12 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // The default is 30 words, which overruns the single line the result // row gives it. Sized to the row instead. excerptLength: 20, + // Appends the query to every result URL, so the destination page can + // highlight what was searched for. Nothing reads it yet — that needs + // Pagefind's `pagefind-highlight.js` on the page, or an equivalent of + // our own — but the parameter has to be set here for the links to + // carry it at all. + highlightParam: 'highlight', }); // The filter index is a separate chunk, and a search returns empty // filter counts until it has been pulled down. Loading it here rather @@ -127,6 +170,7 @@ export function pagefindEngine(bundlePath: string): SearchEngine { breadcrumb: fragment.meta?.trail ? fragment.meta.trail.split(' / ') : breadcrumbFrom(path), + sections: sectionsOf(fragment, path), ...classify(path), }; }); diff --git a/src/scripts/search-engine.ts b/src/scripts/search-engine.ts index cdc8744b41..014afa55e4 100644 --- a/src/scripts/search-engine.ts +++ b/src/scripts/search-engine.ts @@ -2,6 +2,14 @@ // only ever talks to a `SearchEngine`, so swapping the index out is a matter of // supplying a different implementation. +/** A heading inside a result that matched in its own right. */ +export type SearchSubResult = { + /** The heading text. */ + title: string; + /** The page URL with the heading's `#anchor`, so the reader lands on it. */ + url: string; +}; + export type SearchResult = { url: string; title: string; @@ -10,6 +18,14 @@ export type SearchResult = { breadcrumb: string[]; kind: 'page' | 'command'; facet: string; + /** + * Headings within the page that matched, deepest-matching first. Only engines + * that index per-heading can fill this: Pagefind returns them for free, while + * Orama would need the index chunked per heading to know anything about them. + * An empty array means the engine has nothing to say, never that the page has + * no headings. + */ + sections?: SearchSubResult[]; }; export type SearchResponse = { From 4061784ebb138168d46f4263a31421d93a21b915 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 10:20:48 +1200 Subject: [PATCH 05/14] Reorder Pagefind's results, and stop sections crowding them out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Built and measured locally, which corrected two things. data-pagefind-weight on its own did nothing. Scaling it with depth scored 42% weighted Success@5 on real search terms against the deployed 42%, because plus or minus a point around Pagefind's default h1 weight of 7 is far too weak a lever. It now boosts shallow pages to 10 and leaves deep pages at the default, and the reordering that does the work lives in the engine: the same name-then-depth rule the Orama worker uses, with the same constant. Three matched headings across thirty results made two thirds of the list headings, which buried the pages a reader is choosing between. Only the leading three results carry them now, two each — 15% of rows rather than 67%. Real search terms go 42% to 62% Success@5 and 19% to 43% at rank one. Top-visited pages go 73% to 95% and 37% to 82%. Co-Authored-By: Claude Opus 5 (1M context) --- src/layouts/Default.astro | 20 ++--- src/scripts/search-engine-pagefind.ts | 114 +++++++++++++++++++++----- 2 files changed, 102 insertions(+), 32 deletions(-) diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index e9bdabdac0..c8569e0ae9 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -82,20 +82,20 @@ const sections: [string, RegExp][] = [ const section = sections.find(([, prefix]) => prefix.test(Astro.url.pathname))?.[0] ?? 'docs'; -// How much the page's own name counts for, by how deep it sits. +// Boosts a section landing page's own name. // // BM25 has no notion of a site's shape, so a bare section name ranks the pages -// inside the section above the section itself: `variables` returned an API -// example rather than /docs/projects/variables, and on the search terms readers -// actually type this was the largest single source of missed traffic. A page -// deeper in the tree is more specific, so its title should count for less when -// the query is short. +// inside the section above the section itself. Pagefind weights an h1 at 7 by +// default on a quadratic scale, so pushing the shallow pages to 10 roughly +// doubles the pull of their titles. // -// Pagefind weights headings h1 7 down to h6 2 by default, and the scale is -// quadratic — 8 carries roughly 1.8x the pull of 6. Clamped at 3 so a genuinely -// deep page can still win on its own name. +// Deep pages are deliberately left at the default rather than pushed below it. +// An earlier version scaled the weight down with depth and measured as a no-op, +// so there is no evidence lowering them helps and it risks burying a specific +// page that is the only match. The heavy lifting is done by the reorder in +// `search-engine-pagefind.ts`, which this only assists. const depth = Astro.url.pathname.split('/').filter(Boolean).length; -const searchWeight = indexable ? Math.max(3, 10 - depth) : undefined; +const searchWeight = indexable && depth <= 3 ? 10 : undefined; // Indexed so results can be ordered by recency rather than relevance. Nothing in // the overlay offers that control yet; the key is here so the capability exists diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index 969b1ed4a5..7b98958f3b 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -32,7 +32,11 @@ type PagefindFragment = { sub_results?: PagefindSubResult[]; }; -type PagefindResultStub = { data(): Promise }; +type PagefindResultStub = { + /** Pagefind's own relevance score, before any reordering here. */ + score: number; + data(): Promise; +}; type PagefindApi = { options(config: Record): Promise; @@ -51,9 +55,14 @@ type PagefindApi = { }>; }; -// How many matched headings a row will show. More than this and the row stops -// being a result and starts being a table of contents. -const SECTION_LIMIT = 3; +// How many matched headings a row will show, and how many rows get them at all. +// +// Sections are their own rows, so they compete with pages for the reader's first +// screenful. Three each across thirty results made two thirds of the list +// headings, which buries the pages the reader is choosing between. Only the +// leading results earn them, and only a couple each. +const SECTION_LIMIT = 2; +const ROWS_WITH_SECTIONS = 3; /** * The headings inside a result that matched on their own. @@ -80,6 +89,55 @@ function hashOf(url: string) { return at === -1 ? '' : url.slice(at); } +/** Lowercased, punctuation collapsed, so `Config as Code` matches `config as code`. */ +function comparable(text: string) { + return text + .toLowerCase() + .replace(/[^a-z0-9 ]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function segments(path: string) { + return path.split('/').filter(Boolean); +} + +/** + * Reorders results so a section's own page beats the pages inside it. + * + * The same two signals the Orama worker uses, for the same reason and with the + * same constant: a page the query *names* — by title or by the last segment of + * its URL — wins outright, and everything else is ordered by score discounted per + * path segment. Measured against real search terms this is worth about twenty + * points of Success@5; `data-pagefind-weight` alone measured as a no-op. + * + * This reaches only as far as the results already fetched, so a landing page + * ranked below `RESULT_LIMIT` on raw score cannot be rescued here. + */ +function byNameThenDepth< + T extends { score: number; url: string; title: string }, +>(hits: T[], term: string): T[] { + const wanted = comparable(term); + const DEPTH_PENALTY = 0.12; + + const names = (hit: T) => { + const slug = segments(hit.url).pop() ?? ''; + return comparable(hit.title) === wanted || + comparable(slug.replace(/-/g, ' ')) === wanted + ? 1 + : 0; + }; + + return hits + .map((hit) => ({ + hit, + names: names(hit), + adjusted: hit.score / (1 + DEPTH_PENALTY * segments(hit.url).length), + })) + .sort((a, b) => b.names - a.names || b.adjusted - a.adjusted) + .map((entry) => entry.hit); +} + export function pagefindEngine(bundlePath: string): SearchEngine { let loading: Promise | null = null; @@ -149,31 +207,43 @@ export function pagefindEngine(bundlePath: string): SearchEngine { ...(response.totalFilters?.section ?? {}), }; - const fragments = await Promise.all( - response.results.slice(0, RESULT_LIMIT).map((stub) => stub.data()) - ); - - const results = fragments.map((fragment): SearchResult => { - const path = new URL(fragment.url, window.location.origin).pathname; - - return { - url: path, + const stubs = response.results.slice(0, RESULT_LIMIT); + const fragments = await Promise.all(stubs.map((stub) => stub.data())); + + // The score lives on the stub and the URL on the fragment, so the reorder + // needs both. Pairing them costs nothing: a fragment is fetched for every + // row that gets rendered anyway. + const scored = fragments.map((fragment, at) => ({ + fragment, + score: stubs[at].score, + url: new URL(fragment.url, window.location.origin).pathname, + title: + fragment.meta?.title ?? + new URL(fragment.url, window.location.origin).pathname, + })); + + const results = byNameThenDepth(scored, query).map( + (hit, rank): SearchResult => ({ + url: hit.url, // Pagefind takes the title from the first heading inside the indexed // body, which is why the article rather than `.page-content` carries // `data-pagefind-body`. - title: fragment.meta?.title ?? path, + title: hit.title, // Already carries around the hits, and Pagefind escapes the // surrounding text itself. - excerpt: fragment.excerpt, + excerpt: hit.fragment.excerpt, // Written into the page by the layout; the path is the fallback for // anything built before that attribute existed. - breadcrumb: fragment.meta?.trail - ? fragment.meta.trail.split(' / ') - : breadcrumbFrom(path), - sections: sectionsOf(fragment, path), - ...classify(path), - }; - }); + breadcrumb: hit.fragment.meta?.trail + ? hit.fragment.meta.trail.split(' / ') + : breadcrumbFrom(hit.url), + sections: + rank < ROWS_WITH_SECTIONS + ? sectionsOf(hit.fragment, hit.url) + : undefined, + ...classify(hit.url), + }) + ); return { results, counts }; }, From 375564acc1c7e233417d3ca117222eace41e6393 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 10:44:53 +1200 Subject: [PATCH 06/14] Index technical punctuation, weight the metadata, ignore properly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three gaps, all measured on a local build. Pagefind strips punctuation from index and query alike, so `` matched nothing and `Octopus.Action.Package` was indistinguishable from prose using those words. includeCharacters keeps `.#{}<>+$_`, chosen from what appears in this corpus and in the search logs. Indexing is additive — both `head` and `` are stored — so ordinary searches are untouched and the index grew 1%, 8.8MB to 8.9MB. `` went from 0 results to 55, and `octopus.server.exe` now leads with the octopus.server.exe reference page. metaWeights sets title to 8 and trail to 0.5. The breadcrumb is derived from the URL, so every page under /docs/projects/ carries "Projects" and matched a search for it as well as the Projects page did. It belongs in the result row, not the ranking. Both ignore blocks move to `all`. Bare ignore still lets a title or metadata be read out of the block, and neither holds anything worth reading. Real search terms: 62% to 81% Success@5, 43% to 52% at rank one. Top-visited pages: 95% to 98%, and 82% to 89%. Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/pagefind-index.ts | 15 ++++++++++++++- src/layouts/Default.astro | 11 ++++++++--- src/scripts/search-engine-pagefind.ts | 11 +++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index ce6206cb1a..e373ac6ee3 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -29,7 +29,20 @@ export default function pagefindIndex(): AstroIntegration { 'astro:build:done': async ({ dir, logger }) => { const distDir = fileURLToPath(dir); - const { index, errors } = await createIndex({ keepIndexUrl: false }); + const { index, errors } = await createIndex({ + keepIndexUrl: false, + // Pagefind strips punctuation from both the index and the query, so + // `Octopus.Action.Package` and `#{Octopus.Environment.Name}` are stored + // as their bare words and cannot be told apart from prose using the same + // words. `` matched nothing at all. Keeping these characters + // indexes both forms — `head` *and* `` — so ordinary searches are + // unaffected and a reader pasting real syntax gets the page about it. + // + // Limited to punctuation that shows up in this corpus and in the search + // logs: dotted variable and executable names, substitution syntax, + // angle brackets, C++, and shell variables. + includeCharacters: '.#{}<>+$_', + }); if (!index) { logger.error(`could not start Pagefind: ${errors.join(', ')}`); diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index c8569e0ae9..ac8523d3db 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -166,7 +166,7 @@ const trail = Astro.url.pathname subtitle={subtitle} title={title} /> -
+
- {/* Same control on every page, so it is noise in every result. */} -
+ { + /* Same control on every page, so it is noise in every result. + `all` rather than the default `index`: bare ignore still lets + Pagefind read a title or metadata out of the block, and there is + nothing in either of these worth reading. */ + } +
diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index 7b98958f3b..47a100e9f7 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -158,6 +158,17 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // our own — but the parameter has to be set here for the links to // carry it at all. highlightParam: 'highlight', + ranking: { + // Metadata is searchable and boosted — title 5x by default, the rest + // 1x. Swept against real search terms and top-visited pages. + // + // `trail` is the breadcrumb, derived from the URL, so every page + // under /docs/projects/ carries "Projects" and matches a search for + // it just as well as the Projects page itself. Demoting it is worth + // seven points of Success@5; it earns its place in the result row, + // not in the ranking. + metaWeights: { title: 8, trail: 0.5 }, + }, }); // The filter index is a separate chunk, and a search returns empty // filter counts until it has been pulled down. Loading it here rather From c72ade1fc6beee23e1e68a872f6e896870d0f96f Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 11:02:35 +1200 Subject: [PATCH 07/14] Ask the overlay whether the API reference is findable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test read /docs/search.json, which this branch deletes along with the rest of the legacy engine, so it was parsing a 404 page as JSON. It now drives the search overlay instead. The two spikes ship indexes of different shapes — one JSON document, one directory of compressed chunks — and neither is readable the way the old one was, but what has to hold is the same either way: a reader searching a word the API reference is full of must not be sent into it. `accounts` names pages both inside the API reference and outside it, so the result list is never empty. A query that matched nothing would pass without proving anything. Co-Authored-By: Claude Opus 5 (1M context) --- tests/api-page.spec.ts | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/api-page.spec.ts b/tests/api-page.spec.ts index acfc11d5f6..e30fa3c53d 100644 --- a/tests/api-page.spec.ts +++ b/tests/api-page.spec.ts @@ -60,14 +60,34 @@ test.describe('api page chrome', () => { }); test('the section is kept out of the search index and the sitemap', async ({ + page, request, }) => { // Temporary, alongside src/lib/underConstruction.ts - delete with it when // the API reference goes live. - const index = await (await request.get('/docs/search.json')).json(); - expect(index.filter((entry) => entry.url.includes('/docs/api/'))).toEqual( - [] + // + // Asked through the overlay rather than of an index file. This used to read + // /docs/search.json, which no longer exists, and the two search engines ship + // indexes of different shapes — one JSON, one a directory of compressed + // chunks. What has to hold is the same either way: a reader searching a word + // the API reference is full of must not be sent into it. + await page.goto('/docs/'); + await page + .locator('input[data-docs-search-trigger]:visible') + .first() + .click(); + await page.locator('[data-docs-search-input]').fill('accounts'); + + // `accounts` is deliberate: it names pages in the API reference *and* pages + // outside it, so the list is never empty. A query that matched nothing would + // pass this test without proving anything. + const results = page.locator('[data-docs-search-results] [role="option"]'); + await expect(results.first()).toBeVisible({ timeout: 30_000 }); + + const hrefs = await results.evaluateAll((rows) => + rows.map((row) => row.getAttribute('href') ?? '') ); + expect(hrefs.filter((href) => href.includes('/docs/api/'))).toEqual([]); const sitemap = await (await request.get('/docs/sitemap.xml')).text(); expect(sitemap).not.toContain('/docs/api/'); From b4ac54a697227f61ac9647bec34aa3d1099db6ae Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 12:57:23 +1200 Subject: [PATCH 08/14] Stop searching on one and two characters, and on nothing at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes, both measured on a local build. Searches now need three characters. Almost the whole of Pagefind's keystroke lag lives in the two keystrokes nobody could act on anyway: one character costs 975ms and two cost 837ms, against 342ms at three and under 200ms beyond. On a desktop this takes p95 from 927ms to 347ms, and the median from 346ms to 200ms. A search whose best match scores under 8 now returns nothing. Pagefind has no notion of a query it cannot answer — asked for `sssieddqxsx`, the most-typed term in the log, it offered three security articles. Calibrated against every logged term: the weakest genuine query scores 9.5, the mash 6.0. The floor suppresses 17 of the 33 terms with no right answer and costs none of the 57 that have one. Relevance is unchanged either way: 81% Success@5 on real terms and 98% on top-visited pages, the same as before. Nonsense returning nothing went from 0% to 76%. The floor was first set at 6, which did nothing, because it was calibrated while earlier sweep parameters were still active — `options()` merges rather than replaces, and every score came back depressed. calibrate-score-floor.mjs starts from a clean page for that reason. Co-Authored-By: Claude Opus 5 (1M context) --- src/scripts/docs-search.ts | 35 ++++++++++++++++++++++++++- src/scripts/search-engine-pagefind.ts | 27 +++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index 6a77810103..9e81d358d0 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -18,6 +18,18 @@ import { import { pagefindEngine } from './search-engine-pagefind'; const DEBOUNCE_MS = 150; + +// Below this, typing is not yet a query. +// +// A one or two character prefix matches an enormous share of the index, and for +// a chunked engine that means fetching an enormous share of it: measured against +// the Pagefind spike, one character costs 975ms and two cost 837ms, against 342ms +// at three and under 200ms beyond that. Almost the whole of that engine's +// keystroke lag lives in the two keystrokes nobody could act on anyway. +// +// It also spares the reader a list assembled from a single letter, which no +// engine here answers usefully. +const MIN_QUERY_LENGTH = 3; const SITE_SEARCH = 'site'; function setup(dialog: HTMLDialogElement) { @@ -155,7 +167,10 @@ function setup(dialog: HTMLDialogElement) { tab.querySelector('.docs-search__count')!.textContent = `(${count})`; } - const hasQuery = input!.value.trim().length > 0; + // Long enough to have been searched, rather than merely non-empty. A two + // character query never ran, so the panel should look like it is waiting + // rather than reporting nothing found. + const hasQuery = input!.value.trim().length >= MIN_QUERY_LENGTH; const hasResults = response.results.length > 0; tabs!.hidden = !hasQuery; @@ -177,6 +192,13 @@ function setup(dialog: HTMLDialogElement) { } async function run() { + // Also guarded here, not only in `schedule`: a shared `?q=` link opens + // straight into a search without going through the debounce. + if (input!.value.trim().length < MIN_QUERY_LENGTH) { + render({ results: [], counts: {} }); + return; + } + const mine = ++generation; let response = await engine.search(input!.value, facet); if (mine !== generation) return; @@ -194,6 +216,17 @@ function setup(dialog: HTMLDialogElement) { function schedule() { window.clearTimeout(debounce); + + // Too short to search: close the panel back down rather than leaving the + // previous query's results under a field that no longer says that. Deleting + // back to two characters has to look like starting over, not like a search + // that returned those rows. + if (input!.value.trim().length < MIN_QUERY_LENGTH) { + generation += 1; + render({ results: [], counts: {} }); + return; + } + debounce = window.setTimeout(run, DEBOUNCE_MS); } diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index 47a100e9f7..bee8c445cc 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -55,6 +55,29 @@ type PagefindApi = { }>; }; +// Below this score, the best match is not a match. +// +// Pagefind has no notion of a query it cannot answer: it ranks whatever shares a +// few letters and returns it as confidently as a real hit. Asked for +// `sssieddqxsx` — a keyboard mash, and the single most-typed term in the search +// log — it offered three security articles. +// +// Calibrated against every term in that log. The weakest genuine query, `cli`, +// scores 9.5; the mash scores 6.0 and a Spanish query 6.7. Eight sits between +// them with room on both sides, suppresses 17 of the 33 terms that have no right +// answer, and costs none of the 57 that do. The 16 it leaves are unfinished words +// like `te` and `version` that score 40 and up because they genuinely match +// something. +// +// Applied to the top result only. The question is whether this query has an +// answer at all, not whether to trim the tail of one that does. +// +// Re-check with `calibrate-score-floor.mjs` if the corpus or the ranking weights +// change; it is calibrated against both. Note that Pagefind's `options()` merges +// rather than replaces, so a calibration run has to start from a clean page or +// every score comes back depressed. +const MINIMUM_SCORE = 8; + // How many matched headings a row will show, and how many rows get them at all. // // Sections are their own rows, so they compete with pages for the reader's first @@ -218,6 +241,10 @@ export function pagefindEngine(bundlePath: string): SearchEngine { ...(response.totalFilters?.section ?? {}), }; + // Nothing here is a real answer, so say so rather than offering the + // closest accident. + if ((response.results[0]?.score ?? 0) < MINIMUM_SCORE) return empty; + const stubs = response.results.slice(0, RESULT_LIMIT); const fragments = await Promise.all(stubs.map((stub) => stub.data())); From c85e24ed3d3cef8784afeecfa7a0cdf1eee36c6c Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 14:49:59 +1200 Subject: [PATCH 09/14] Tell a screen reader what the search found The overlay carried the combobox roles without the one part that speaks. A screen reader was told a listbox existed and which row was active, and never how many results arrived or that none had. The live region sits outside the body, which is hidden until there is a query. Inside it, the region would leave and re-enter the accessibility tree on every open, and a region that has only just appeared does not reliably announce the first thing written to it. It is clipped rather than hidden for the same reason: `display: none` would silence it. It speaks from `render`, which runs once per settled query, so it keeps pace with the results rather than with the keyboard. Below the three character minimum it says nothing, matching the panel's own waiting state. Pagefind's own Component UI does this and ours did not. Co-Authored-By: Claude Opus 5 (1M context) --- src/components/DocsSearch.astro | 24 ++++++++++++++++++++++++ src/scripts/docs-search.ts | 25 ++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/components/DocsSearch.astro b/src/components/DocsSearch.astro index 091314b9ae..11a5b41ddf 100644 --- a/src/components/DocsSearch.astro +++ b/src/components/DocsSearch.astro @@ -128,6 +128,18 @@ const listboxId = `${name}-search-listbox`;

+ +

+

+ @@ -491,6 +503,18 @@ const listboxId = `${name}-search-listbox`; font: var(--textBodyRegularMedium); } + /* Taken out of the layout without `hidden` or `display: none`, either of + which would take it out of the accessibility tree and silence it. */ + .docs-search__announce { + position: absolute; + width: 1px; + height: 1px; + margin: 0; + overflow: hidden; + clip-path: inset(50%); + white-space: nowrap; + } + .docs-search__footer { flex-shrink: 0; display: flex; diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index 9e81d358d0..956bb16a96 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -43,6 +43,9 @@ function setup(dialog: HTMLDialogElement) { const list = dialog.querySelector('[data-docs-search-results]'); const empty = dialog.querySelector('[data-docs-search-empty]'); const echo = dialog.querySelector('[data-docs-search-echo]'); + const announce = dialog.querySelector( + '[data-docs-search-announce]' + ); const template = dialog.querySelector( '[data-docs-search-row]' ); @@ -52,7 +55,17 @@ function setup(dialog: HTMLDialogElement) { // A partially rendered overlay is a bug elsewhere; wiring half of it up would // only turn that into a confusing runtime failure. - if (!input || !tabs || !body || !list || !empty || !echo || !template) return; + if ( + !input || + !tabs || + !body || + !list || + !empty || + !echo || + !announce || + !template + ) + return; const demo = dialog.dataset.demoResults; const engine: SearchEngine = demo @@ -179,6 +192,16 @@ function setup(dialog: HTMLDialogElement) { echo!.textContent = input!.value.trim(); input!.setAttribute('aria-expanded', String(hasQuery && hasResults)); + // The combobox roles say a listbox exists and which row is active; nothing + // says what came back. `render` runs once per settled query, so this speaks + // at the rate the results themselves change rather than per keystroke. + const found = response.results.length; + announce!.textContent = !hasQuery + ? '' + : found === 0 + ? `No results for ${input!.value.trim()}` + : `${found} ${found === 1 ? 'result' : 'results'} for ${input!.value.trim()}`; + setActive(0); } From f6e0c0fb8aebd220542a64739d038b22e060793b Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 15:40:50 +1200 Subject: [PATCH 10/14] Rewrite the search comments that referred to the spike They pointed at the bake-off and at the other candidate engine, and both are gone from the tree. The rationale each comment carried is kept. Co-Authored-By: Claude Opus 5 (1M context) --- src/scripts/docs-search.ts | 14 +++++++------- src/scripts/search-engine-pagefind.ts | 16 +++++++++------- src/scripts/search-engine.ts | 12 ++++++------ 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index 956bb16a96..b01de21044 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -21,14 +21,14 @@ const DEBOUNCE_MS = 150; // Below this, typing is not yet a query. // -// A one or two character prefix matches an enormous share of the index, and for -// a chunked engine that means fetching an enormous share of it: measured against -// the Pagefind spike, one character costs 975ms and two cost 837ms, against 342ms -// at three and under 200ms beyond that. Almost the whole of that engine's -// keystroke lag lives in the two keystrokes nobody could act on anyway. +// A one or two character prefix matches an enormous share of the index, and +// because Pagefind fetches every chunk a query touches, that means pulling down +// an enormous share of it: one character costs 975ms and two cost 837ms, against +// 342ms at three and under 200ms beyond that. Almost the whole of the keystroke +// lag lives in the two keystrokes nobody could act on anyway. // -// It also spares the reader a list assembled from a single letter, which no -// engine here answers usefully. +// It also spares the reader a list assembled from a single letter, which is +// never a useful answer. const MIN_QUERY_LENGTH = 3; const SITE_SEARCH = 'site'; diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index bee8c445cc..db26e5d941 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -1,9 +1,9 @@ // Pagefind behind the `SearchEngine` seam. // // The index is chunked and content-hashed, and Pagefind fetches only the chunks -// a query actually touches. That is the whole reason for the spike: the old -// index is a single 1.86MB download, and this one's per-query cost stays flat as -// the corpus grows. +// a query actually touches, so the per-query cost stays flat as the corpus +// grows. The `search.json` index this replaced was a single 1.86MB download, +// paid in full on the first search. // // Two calls per search: `search()` returns lightweight stubs, then each result's // `data()` fetches its own fragment. Only the page of results being shown is @@ -128,11 +128,13 @@ function segments(path: string) { /** * Reorders results so a section's own page beats the pages inside it. * - * The same two signals the Orama worker uses, for the same reason and with the - * same constant: a page the query *names* — by title or by the last segment of + * Two signals: a page the query *names* — by title or by the last segment of * its URL — wins outright, and everything else is ordered by score discounted per - * path segment. Measured against real search terms this is worth about twenty - * points of Success@5; `data-pagefind-weight` alone measured as a no-op. + * path segment. BM25 has no notion of a site's shape, so a bare section name + * otherwise ranks the pages inside the section above the section itself. + * + * Measured against real search terms this is worth about twenty points of + * Success@5; `data-pagefind-weight` alone measured as a no-op. * * This reaches only as far as the results already fetched, so a landing page * ranked below `RESULT_LIMIT` on raw score cannot be rescued here. diff --git a/src/scripts/search-engine.ts b/src/scripts/search-engine.ts index 014afa55e4..e6843ba6b8 100644 --- a/src/scripts/search-engine.ts +++ b/src/scripts/search-engine.ts @@ -19,9 +19,8 @@ export type SearchResult = { kind: 'page' | 'command'; facet: string; /** - * Headings within the page that matched, deepest-matching first. Only engines - * that index per-heading can fill this: Pagefind returns them for free, while - * Orama would need the index chunked per heading to know anything about them. + * Headings within the page that matched, deepest-matching first. Only an + * engine that indexes per-heading can fill this, which Pagefind does for free. * An empty array means the engine has nothing to say, never that the page has * no headings. */ @@ -41,9 +40,10 @@ export type SearchEngine = { /** * Whether `warm()` is cheap enough to run on page load rather than when the * overlay opens. Pagefind pays about 118KB for its runtime and fetches the - * rest per query; Orama pays for its whole index parsed and restored, on - * every navigation, for every visitor including the majority who never - * search. The answer differs by engine, so the engine states it. + * rest per query, which is cheap enough. An engine that parses a whole index + * up front would charge that on every navigation, to every visitor, including + * the majority who never search. The answer differs by engine, so the engine + * states it. */ eager?: boolean; /** From ec8f3f8bb49d7afe3a2759aef7514b38c888d8cc Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 17:14:54 +1200 Subject: [PATCH 11/14] Fix the review findings in the Pagefind search - preload ignored MIN_QUERY_LENGTH, so the first keystrokes still fetched the chunks the guard exists to avoid. - A failed fragment fetch rejected the whole search and left the previous query's rows under the new text. - The score floor was applied to the filtered results while the tab counts came from the unfiltered ones, so a tab could advertise results and then show none. It is now asked of the unfiltered scores. - highlightParam was set and then stripped off every link, so it is dropped. - The minimum query length goes to two characters. - A missing fragment directory threw instead of reaching the page-count warning that exists to report it. - Future-dated pages are indexed, and the comment claiming otherwise is fixed. - The live region counted pages while the listbox held pages and sections. - Removed an unreachable filter, logged the swallowed index-load error, and corrected the deleted call site named in underConstruction.ts. pagefind moves to a real dependency: astro.config.mjs imports it statically, so a production-only install could not load the config at all. Co-Authored-By: Claude Opus 5 (1M context) --- package.json | 2 +- pnpm-lock.yaml | 6 +- src/integrations/pagefind-index.ts | 11 ++- src/layouts/Default.astro | 7 +- src/lib/underConstruction.ts | 7 +- src/scripts/docs-search.ts | 41 +++++---- src/scripts/search-engine-pagefind.ts | 115 +++++++++++++------------- 7 files changed, 95 insertions(+), 94 deletions(-) diff --git a/package.json b/package.json index e9a2c57a10..974ab3217b 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "glob": "^13.0.6", "gray-matter": "^4.0.3", "optional": "^0.1.4", + "pagefind": "^1.5.2", "satteri": "^0.9.5", "sharp": "^0.34.5" }, @@ -45,7 +46,6 @@ "linkinator": "7.0.0", "npm-run-all": "^4.1.5", "onchange": "^7.1.0", - "pagefind": "^1.5.2", "prettier": "^3.8.3", "prettier-plugin-astro": "^0.14.1" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ab0d4d3873..6ecb480a8a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -42,6 +42,9 @@ importers: optional: specifier: ^0.1.4 version: 0.1.4 + pagefind: + specifier: ^1.5.2 + version: 1.5.2 satteri: specifier: ^0.9.5 version: 0.9.5 @@ -70,9 +73,6 @@ importers: onchange: specifier: ^7.1.0 version: 7.1.0 - pagefind: - specifier: ^1.5.2 - version: 1.5.2 prettier: specifier: ^3.8.3 version: 3.8.3 diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index e373ac6ee3..a2343271e3 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -75,10 +75,13 @@ export default function pagefindIndex(): AstroIntegration { // `page_count` is files scanned, not files indexed — it counts the // redirect stubs that `data-pagefind-body` then drops. One fragment is // written per indexed page, so that is the number worth checking. - const outDir = path.join(distDir, 'docs', 'pagefind'); - const indexed = ( - await fs.promises.readdir(path.join(outDir, 'fragment')) - ).length; + // + // No fragment directory at all is the worst version of the failure the + // warning below exists to report, so it must not throw ahead of it. + const fragments = path.join(distDir, 'docs', 'pagefind', 'fragment'); + const indexed = fs.existsSync(fragments) + ? (await fs.promises.readdir(fragments)).length + : 0; logger.info( `indexed ${indexed} of ${added.page_count} pages into docs/pagefind` diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index ac8523d3db..d29867defc 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -60,9 +60,10 @@ const showSearch = !isSearchPage; // The footer prints this; JSON-LD carries it as dateModified. const lastUpdated = frontmatter.modDate ?? frontmatter.pubDate ?? null; -// Search indexing. The same predicate `PostFiltering.showInSearch` applies at -// the point the page is rendered — redirect stubs never reach this layout at -// all, so only the frontmatter opt-outs have to be reproduced here. +// Search indexing. Redirect stubs never reach this layout at all, so only the +// frontmatter opt-outs have to be checked here. Deliberately looser than +// `PostFiltering.isListable`, which also hides a future `pubDate`: a page that +// is built and served is a page worth finding. const indexable = frontmatter.navSearch !== false && frontmatter.listable !== false && diff --git a/src/lib/underConstruction.ts b/src/lib/underConstruction.ts index 042bd23513..76f4d37b74 100644 --- a/src/lib/underConstruction.ts +++ b/src/lib/underConstruction.ts @@ -7,9 +7,10 @@ // sitemap.xml, so neither readers nor Google arrive at it ahead of the pages // that explain it. // -// Call sites are src/pages/docs/search.json.ts and src/pages/docs/sitemap.xml.ts. -// Both take their page list from an `import.meta.glob` rooted at -// src/pages/docs, so the paths they pass in look like './api/feeds.md'. +// The call site is src/pages/docs/sitemap.xml.ts, which takes its page list from +// an `import.meta.glob` rooted at src/pages/docs, so the paths it passes in look +// like './api/feeds.md'. Search exclusion is no longer done here: Pagefind +// indexes only what carries `data-pagefind-body`, and Api.astro carries none. const UNDER_CONSTRUCTION = [/^api(\/|$)/]; diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index b01de21044..1af1f81cf0 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -21,15 +21,10 @@ const DEBOUNCE_MS = 150; // Below this, typing is not yet a query. // -// A one or two character prefix matches an enormous share of the index, and -// because Pagefind fetches every chunk a query touches, that means pulling down -// an enormous share of it: one character costs 975ms and two cost 837ms, against -// 342ms at three and under 200ms beyond that. Almost the whole of the keystroke -// lag lives in the two keystrokes nobody could act on anyway. -// -// It also spares the reader a list assembled from a single letter, which is -// never a useful answer. -const MIN_QUERY_LENGTH = 3; +// A single character matches an enormous share of the index, and Pagefind +// fetches every chunk a query touches: one character costs 975ms, against 837ms +// at two and under 200ms beyond that. +const MIN_QUERY_LENGTH = 2; const SITE_SEARCH = 'site'; function setup(dialog: HTMLDialogElement) { @@ -169,7 +164,8 @@ function setup(dialog: HTMLDialogElement) { } function render(response: Awaited>) { - list!.replaceChildren(...drawAll(response.results)); + const rows = drawAll(response.results); + list!.replaceChildren(...rows); for (const tab of facetTabs()) { const key = tab.dataset.facet!; @@ -180,9 +176,9 @@ function setup(dialog: HTMLDialogElement) { tab.querySelector('.docs-search__count')!.textContent = `(${count})`; } - // Long enough to have been searched, rather than merely non-empty. A two - // character query never ran, so the panel should look like it is waiting - // rather than reporting nothing found. + // Long enough to have been searched, rather than merely non-empty. A single + // character never ran, so the panel should look like it is waiting rather + // than reporting nothing found. const hasQuery = input!.value.trim().length >= MIN_QUERY_LENGTH; const hasResults = response.results.length > 0; @@ -193,9 +189,9 @@ function setup(dialog: HTMLDialogElement) { input!.setAttribute('aria-expanded', String(hasQuery && hasResults)); // The combobox roles say a listbox exists and which row is active; nothing - // says what came back. `render` runs once per settled query, so this speaks - // at the rate the results themselves change rather than per keystroke. - const found = response.results.length; + // says what came back. Counts rows rather than pages, because a matched + // section is its own option to arrow onto. + const found = rows.length; announce!.textContent = !hasQuery ? '' : found === 0 @@ -241,9 +237,7 @@ function setup(dialog: HTMLDialogElement) { window.clearTimeout(debounce); // Too short to search: close the panel back down rather than leaving the - // previous query's results under a field that no longer says that. Deleting - // back to two characters has to look like starting over, not like a search - // that returned those rows. + // previous query's results under a field that no longer says that. if (input!.value.trim().length < MIN_QUERY_LENGTH) { generation += 1; render({ results: [], counts: {} }); @@ -313,9 +307,12 @@ function setup(dialog: HTMLDialogElement) { // --- Inside the dialog --------------------------------------------------- input.addEventListener('input', () => { - // Ahead of the debounce, so the chunks this query needs are already on - // their way by the time the search runs. - engine.preload?.(input!.value); + // Ahead of the debounce, so the chunks this query needs are already on their + // way by the time the search runs. Guarded the same way `schedule` is: the + // fetch is most of what `MIN_QUERY_LENGTH` exists to avoid. + if (input!.value.trim().length >= MIN_QUERY_LENGTH) { + engine.preload?.(input!.value); + } schedule(); }); diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index db26e5d941..19b7000422 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -94,7 +94,7 @@ const ROWS_WITH_SECTIONS = 3; * headings, and its URL is the page URL with no anchor — that is the row's own * link, so it is dropped rather than repeated underneath itself. */ -function sectionsOf(fragment: PagefindFragment, path: string) { +function sectionsOf(fragment: PagefindFragment) { return (fragment.sub_results ?? []) .filter((sub) => sub.url.includes('#')) .slice(0, SECTION_LIMIT) @@ -103,8 +103,7 @@ function sectionsOf(fragment: PagefindFragment, path: string) { // Keep the anchor, drop the origin, so the row links the same way every // other row does. url: new URL(sub.url, window.location.origin).pathname + hashOf(sub.url), - })) - .filter((sub) => sub.url !== path); + })); } function hashOf(url: string) { @@ -177,12 +176,6 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // The default is 30 words, which overruns the single line the result // row gives it. Sized to the row instead. excerptLength: 20, - // Appends the query to every result URL, so the destination page can - // highlight what was searched for. Nothing reads it yet — that needs - // Pagefind's `pagefind-highlight.js` on the page, or an equivalent of - // our own — but the parameter has to be set here for the links to - // carry it at all. - highlightParam: 'highlight', ranking: { // Metadata is searchable and boosted — title 5x by default, the rest // 1x. Swept against real search terms and top-visited pages. @@ -201,9 +194,10 @@ export function pagefindEngine(bundlePath: string): SearchEngine { await api.filters(); return api; }) - .catch(() => { + .catch((error) => { // A failed load must not poison every later search. loading = null; + console.error('[docs-search] could not load the Pagefind index', error); return null; }); @@ -232,60 +226,65 @@ export function pagefindEngine(bundlePath: string): SearchEngine { const api = await load(); if (!api) return empty; - const response = await api.search(query, { - filters: facet && facet !== 'all' ? { section: [facet] } : undefined, - }); + try { + const filters = + facet && facet !== 'all' ? { section: [facet] } : undefined; + const response = await api.search(query, { filters }); - // Counts come from the whole match set rather than the filtered one, so - // the strip keeps showing what the other tabs hold while one is selected. - const counts: Record = { - all: response.unfilteredResultCount, - ...(response.totalFilters?.section ?? {}), - }; + // Counts come from the whole match set rather than the filtered one, so + // the strip keeps showing what the other tabs hold while one is selected. + const counts: Record = { + all: response.unfilteredResultCount, + ...(response.totalFilters?.section ?? {}), + }; - // Nothing here is a real answer, so say so rather than offering the - // closest accident. - if ((response.results[0]?.score ?? 0) < MINIMUM_SCORE) return empty; + // Whether the query has an answer at all is a question about the corpus, + // so it is asked of the unfiltered scores. Narrowing to a tab only + // removes documents, so a filtered top score under the floor would + // suppress a query the corpus does answer, and zero the counts the strip + // was just clicked from. + const unfiltered = filters ? await api.search(query) : response; + if ((unfiltered.results[0]?.score ?? 0) < MINIMUM_SCORE) return empty; - const stubs = response.results.slice(0, RESULT_LIMIT); - const fragments = await Promise.all(stubs.map((stub) => stub.data())); + const stubs = response.results.slice(0, RESULT_LIMIT); + const fragments = await Promise.all(stubs.map((stub) => stub.data())); - // The score lives on the stub and the URL on the fragment, so the reorder - // needs both. Pairing them costs nothing: a fragment is fetched for every - // row that gets rendered anyway. - const scored = fragments.map((fragment, at) => ({ - fragment, - score: stubs[at].score, - url: new URL(fragment.url, window.location.origin).pathname, - title: - fragment.meta?.title ?? - new URL(fragment.url, window.location.origin).pathname, - })); + // The score lives on the stub and the URL on the fragment, so the + // reorder needs both. + const scored = fragments.map((fragment, at) => ({ + fragment, + score: stubs[at].score, + url: new URL(fragment.url, window.location.origin).pathname, + title: + fragment.meta?.title ?? + new URL(fragment.url, window.location.origin).pathname, + })); - const results = byNameThenDepth(scored, query).map( - (hit, rank): SearchResult => ({ - url: hit.url, - // Pagefind takes the title from the first heading inside the indexed - // body, which is why the article rather than `.page-content` carries - // `data-pagefind-body`. - title: hit.title, - // Already carries around the hits, and Pagefind escapes the - // surrounding text itself. - excerpt: hit.fragment.excerpt, - // Written into the page by the layout; the path is the fallback for - // anything built before that attribute existed. - breadcrumb: hit.fragment.meta?.trail - ? hit.fragment.meta.trail.split(' / ') - : breadcrumbFrom(hit.url), - sections: - rank < ROWS_WITH_SECTIONS - ? sectionsOf(hit.fragment, hit.url) - : undefined, - ...classify(hit.url), - }) - ); + const results = byNameThenDepth(scored, query).map( + (hit, rank): SearchResult => ({ + url: hit.url, + title: hit.title, + // Already carries around the hits, and Pagefind escapes the + // surrounding text itself. + excerpt: hit.fragment.excerpt, + breadcrumb: hit.fragment.meta?.trail + ? hit.fragment.meta.trail.split(' / ') + : breadcrumbFrom(hit.url), + sections: + rank < ROWS_WITH_SECTIONS + ? sectionsOf(hit.fragment) + : undefined, + ...classify(hit.url), + }) + ); - return { results, counts }; + return { results, counts }; + } catch (error) { + // Thirty fragment fetches per query, any of which can fail. Rejecting + // would leave the previous query's rows under the new text. + console.error('[docs-search] search failed', error); + return empty; + } }, }; } From f47ccfc737b21da842a05d2d9ab9c73211182748 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 17:15:42 +1200 Subject: [PATCH 12/14] Cut the search comments back to what is not obvious Removed the running commentary on approach, the measurements that only justified a decision at the time, and a reference to a calibration script that is not in the repo. What is left explains Pagefind behaviour and coupling that the code does not show on its own. Co-Authored-By: Claude Opus 5 (1M context) --- astro.config.mjs | 5 +- src/components/ArticleHeader.astro | 2 +- src/components/DocsSearch.astro | 19 ++---- src/integrations/pagefind-index.ts | 37 ++++------- src/layouts/Default.astro | 49 +++++--------- src/plugins/pagefind-image-attrs.js | 28 +++----- src/scripts/docs-search.ts | 9 +-- src/scripts/search-engine-pagefind.ts | 93 ++++++++------------------- src/scripts/search-engine.ts | 12 ++-- tests/api-page.spec.ts | 13 ++-- 10 files changed, 88 insertions(+), 179 deletions(-) diff --git a/astro.config.mjs b/astro.config.mjs index c340c496d1..3baf67563c 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -24,8 +24,8 @@ export default defineConfig({ integrations: [ mdx(), llmMdEmitter(), - // Indexes the HTML the build just wrote, so it lands after the page - // emitters and before the prune that would delete its output + // After the page emitters, and before the prune that would delete its + // output pagefindIndex(), // Must run last: strips build output that can't be served under /docs/ pruneDist() @@ -68,7 +68,6 @@ export default defineConfig({ ], hastPlugins: [ satteriWbr, - // Marks image alt text for Pagefind to index. Inert without it. pagefindImageAttrs, satteriApiExamples ], diff --git a/src/components/ArticleHeader.astro b/src/components/ArticleHeader.astro index e7f5349c96..e5e7740821 100644 --- a/src/components/ArticleHeader.astro +++ b/src/components/ArticleHeader.astro @@ -13,7 +13,7 @@ type Props = { subtitle?: string | null; title: string; /** Passed to `data-pagefind-weight`. Omitted leaves Pagefind's own heading - * weighting alone, which is what every page outside the docs layout wants. */ + * weighting alone. */ searchWeight?: number; }; const { lang, subtitle, title, searchWeight } = Astro.props satisfies Props; diff --git a/src/components/DocsSearch.astro b/src/components/DocsSearch.astro index 11a5b41ddf..6c5162e9df 100644 --- a/src/components/DocsSearch.astro +++ b/src/components/DocsSearch.astro @@ -128,9 +128,8 @@ const listboxId = `${name}-search-listbox`;

-

{ - /* A heading inside the result above that matched on its own. It is a - separate `role="option"` rather than a link inside the parent row, - because it goes somewhere different — the reader can arrow onto it and - land on that section instead of the top of the page. */ + /* A heading inside the result above that matched on its own. Its own + `role="option"`, because it links somewhere different: the reader can + arrow onto it and land on that section. */ }