diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f16da9eb8..f28ba79d88 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -230,6 +230,18 @@ jobs: assert_header "http://localhost:8080/cli/llms.txt" "Content-Type" "text/markdown" assert_header "http://localhost:8080/cli/llms-full.txt" "Content-Type" "text/markdown" + echo "🧪 Checking doubled section prefix redirect..." + # Regression tests for the llms-txt plugin workaround in nginx.conf. + # Doubled prefixes redirect to the canonical URL and keep the query + # string. Remove together with the workaround. + assert_status "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start" "301" + assert_header "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start" "Location" "^Location: http://[^/]*/sdk/js/docs/introduction/quick-start$" + assert_header "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start.md?x=1" "Location" "^Location: http://[^/]*/sdk/js/docs/introduction/quick-start[.]md[?]x=1$" + assert_header "http://localhost:8080/sdk/python/sdk/python/docs/changelog" "Location" "^Location: http://[^/]*/sdk/python/docs/changelog$" + assert_header "http://localhost:8080/api/client/js/api/client/js/docs/changelog" "Location" "^Location: http://[^/]*/api/client/js/docs/changelog$" + assert_header "http://localhost:8080/api/client/python/api/client/python/docs/changelog" "Location" "^Location: http://[^/]*/api/client/python/docs/changelog$" + assert_header "http://localhost:8080/cli/cli/docs/changelog" "Location" "^Location: http://[^/]*/cli/docs/changelog$" + echo "✅ All Nginx header checks passed." - name: Verify per-page .md navigation headers diff --git a/nginx.conf b/nginx.conf index 1996d680fa..857dea168f 100644 --- a/nginx.conf +++ b/nginx.conf @@ -63,6 +63,17 @@ server { rewrite ^(.+)/$ $1$is_args$args? redirect; } + # @signalwire/docusaurus-plugin-llms-txt 1.2.2 repeats the section prefix in the + # .md files and llms*.txt it generates for sites served under a sub-path, so links + # such as /sdk/python/sdk/python/docs/... 404. The five sub-sites still pin ^1.2.2. + # This redirect makes those links resolve in one hop. Keep it after the sub-sites + # stop generating them: the doubled URLs stay in search indexes and agent caches. + # It only collapses a repeated same-section prefix. The plugin also prefixes + # cross-section links (/sdk/js/cli/docs/...), which this cannot detect. + location ~ "^/(sdk/js|sdk/python|api/client/js|api/client/python|cli)/\1(/.*)?$" { + return 301 /$1$2$is_args$args; + } + location / { set $rewrite_condition "$serve_markdown$has_no_extension"; set $proxy_path $request_uri; diff --git a/scripts/checkLlmsSize.mjs b/scripts/checkLlmsSize.mjs index fad4bc3f03..1ff1c8ef97 100644 --- a/scripts/checkLlmsSize.mjs +++ b/scripts/checkLlmsSize.mjs @@ -27,6 +27,20 @@ if (llmsChars === null || llmsFullChars === null) { process.exit(1); } +// Guards the substitution in joinLlmsFiles.mjs. The sub-sites generate links with their +// section prefix repeated; if the substitution stops matching, the joined files silently +// fill up with URLs that only resolve through the redirect in nginx.conf. +const DOUBLED_SECTION_PREFIX = + /https:\/\/docs\.apify\.com\/(sdk\/js|sdk\/python|api\/client\/js|api\/client\/python|cli)\/\1\//g; + +for (const filePath of [llmsPath, llmsFullPath]) { + const matches = (await fs.readFile(filePath, 'utf8')).match(DOUBLED_SECTION_PREFIX); + if (matches) { + console.error(`\nERROR: ${filePath} has ${matches.length} links with a doubled section prefix`); + process.exitCode = 1; + } +} + console.log(`llms.txt: ${llmsChars.toLocaleString()} characters`); console.log(`llms-full.txt: ${llmsFullChars.toLocaleString()} characters`); diff --git a/scripts/joinLlmsFiles.mjs b/scripts/joinLlmsFiles.mjs index 34130cb539..f43699c2a6 100644 --- a/scripts/joinLlmsFiles.mjs +++ b/scripts/joinLlmsFiles.mjs @@ -253,9 +253,18 @@ async function joinFiles() { console.log('Wrote llms-full.txt to build/'); } +// WORKAROUND: @signalwire/docusaurus-plugin-llms-txt 1.2.2 doubles the section prefix in +// cross-links on sites served under a sub-path, so the SDK/client/CLI sections we fetch above +// arrive with links such as /sdk/python/sdk/python/docs/... Fixed upstream in 2.0.0-alpha.6; +// those sites still pin ^1.2.2. Remove once they stop generating the doubled prefix. +const DOUBLED_SECTION_PREFIX = + /(https:\/\/docs\.apify\.com\/(sdk\/js|sdk\/python|api\/client\/js|api\/client\/python|cli))\/\2\//g; + async function sanitizeFile(filePath) { const content = await fs.readFile(filePath, 'utf8'); - const sanitizedContent = content.replace(/<[^>]*>/g, ''); // Remove HTML tags + const sanitizedContent = content + .replace(/<[^>]*>/g, '') // Remove HTML tags + .replace(DOUBLED_SECTION_PREFIX, '$1/'); await fs.writeFile(filePath, sanitizedContent, 'utf8'); console.log(`Sanitized ${filePath}`); }