From 44d70793881ec928d82a6d5426a64d5eb2af4681 Mon Sep 17 00:00:00 2001 From: Thomas Jung Date: Thu, 3 Sep 2026 11:12:36 -0400 Subject: [PATCH] fix(sitemap): guard catalog-only rebuilds from wiping /tutorials/ URLs /sitemap.xml is served from the HANA page-sitemap.xml blob and republished by publish-content.ts on every non-slug rebuild (discoverPageFiles, gated `if (!opts.slug)`). catalog-only skips "Fetch tutorials", so Hugo bakes a sitemap with NO /tutorials/ URLs and republishing it WIPES the live sitemap down to ~180 links (reported 2026-09-03 by the Intelligent Search Data Crawling team: crawled docs ~1700 -> ~1500). Same wipe class as the /browse/ and /authors/ catalog-only wipes. - scripts/seed-sitemap-from-deployed.ts: catalog-only preserve step. Runs AFTER the Hugo build (the sitemap is a build output, not an input) and overwrites the tutorial-less sitemap with the one the approuter is currently serving. Fails the run if the deployed sitemap is unreadable or names zero tutorials (ALLOW_EMPTY_SITEMAP=1 escape hatch for seed envs). - scripts/check-sitemap-tutorials.cjs: fail-closed guard. Runs in full + catalog-only before publish; fails if the baked sitemap has zero /tutorials/ (the exact wipe signature). - rebuild-content.yml: wire both steps in after "Build Hugo site". - seo-files.test.js: smoke assertion that /sitemap.xml contains /tutorials/. - unit tests cover both tutorial- counters (shared logic, run through both impls so they can't diverge). --- .github/workflows/rebuild-content.yml | 30 ++++ package.json | 1 + .../__tests__/check-sitemap-tutorials.test.ts | 72 ++++++++++ scripts/check-sitemap-tutorials.cjs | 129 ++++++++++++++++++ scripts/seed-sitemap-from-deployed.ts | 100 ++++++++++++++ test/smoke/seo-files.test.js | 6 + 6 files changed, 338 insertions(+) create mode 100644 scripts/__tests__/check-sitemap-tutorials.test.ts create mode 100644 scripts/check-sitemap-tutorials.cjs create mode 100644 scripts/seed-sitemap-from-deployed.ts diff --git a/.github/workflows/rebuild-content.yml b/.github/workflows/rebuild-content.yml index e841a9ad6..108cd223e 100644 --- a/.github/workflows/rebuild-content.yml +++ b/.github/workflows/rebuild-content.yml @@ -650,6 +650,36 @@ jobs: - name: Build Hugo site run: /tmp/hugo --source hugo --minify + # Preserve /sitemap.xml across catalog-only rebuilds (same wipe class as + # /browse/ + /authors/ above; reported 2026-09-03 by the Intelligent Search + # Data Crawling team). /sitemap.xml is served from the HANA `page-sitemap.xml` + # blob and publish-content.ts republishes it on every non-slug rebuild. + # catalog-only skips "Fetch tutorials", so the Hugo build just above bakes a + # sitemap with NO /tutorials/ URLs — publishing that would wipe the live + # sitemap to ~180 links. The sitemap can't be regenerated here (it needs the + # ~1.4k tutorial pages the skipped GitHub fetch provides), so — UNLIKE the + # browse/author preserve steps, which re-hydrate build INPUTS before the + # build — this runs AFTER the build and overwrites the freshly-baked sitemap + # with the one the approuter is CURRENTLY serving. Fails the run (non-zero) + # rather than ship a wiped sitemap if the deployed one is unreadable or names + # zero tutorials. Not needed in `full` (regenerates it) or `slug-targeted` + # (page-sitemap.xml is not republished). + - name: Preserve sitemap.xml for catalog-only rebuild + if: ${{ steps.mode.outputs.effective_mode == 'catalog-only' }} + env: + APPROUTER_URL: ${{ steps.url.outputs.approuter_url }} + run: npx tsx scripts/seed-sitemap-from-deployed.ts + + # Fail-closed net for the sitemap wipe class. Runs in the modes that + # republish the page-sitemap.xml blob (full + catalog-only; slug-targeted + # does not touch it). Fails BEFORE the HANA publish if the sitemap about to + # be published names zero /tutorials/ URLs — whatever the cause (broken + # preserve step, a future fetch-skipping mode, a bad Hugo build). A red run + # is recoverable (re-run mode=full); a silently wiped sitemap is the incident. + - name: Guard - sitemap includes tutorial URLs + if: ${{ steps.mode.outputs.effective_mode != 'slug-targeted' }} + run: node scripts/check-sitemap-tutorials.cjs --hugo-dir hugo/public + # [#1657] Restore CDS syntax highlighting to the runtime-push path. Hugo's # render-codeblock.html maps ```cds → the Chroma SQL lexer as a PLACEHOLDER # (Chroma has no CDS lexer) and tags the wrapper `data-lang=cds`; the SQL diff --git a/package.json b/package.json index 835766644..1009283c5 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "build:analytics-explorer": "npm --prefix app/analytics-explorer install && npm --prefix app/analytics-explorer run build", "check-explore-bundle-manifest": "node scripts/check-explore-bundle-manifest.cjs", "check-verb-shelves": "node scripts/check-verb-shelves.cjs", + "check-sitemap-tutorials": "node scripts/check-sitemap-tutorials.cjs", "build:hugo": "npm run check-explore-bundle-manifest && npm run check-verb-shelves && hugo --source hugo --minify", "build:page-fallback": "node scripts/build-page-fallback.cjs", "build:whats-new-snapshot": "node scripts/build-whats-new-snapshot.cjs", diff --git a/scripts/__tests__/check-sitemap-tutorials.test.ts b/scripts/__tests__/check-sitemap-tutorials.test.ts new file mode 100644 index 000000000..04d5b4c86 --- /dev/null +++ b/scripts/__tests__/check-sitemap-tutorials.test.ts @@ -0,0 +1,72 @@ +// scripts/__tests__/check-sitemap-tutorials.test.ts +// +// Unit tests for the build-time sitemap guard (scripts/check-sitemap-tutorials.cjs) +// and the shared tutorial- counter used by the catalog-only preserve step +// (scripts/seed-sitemap-from-deployed.ts). Both defend the 2026-09-03 sitemap-wipe +// class: a catalog-only rebuild republished the page-sitemap.xml blob with zero +// /tutorials/ URLs, dropping the live sitemap from ~1.6k links to ~180. +// +// We import the pure cores and assert against synthetic sitemap XML — same +// approach as check-verb-shelves.test.ts. The CLI wiring (file read + exit code) +// is exercised by the live rebuild-content.yml guard step. + +import { describe, it, expect } from 'vitest'; +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { countTutorialLocs } = require('../check-sitemap-tutorials.cjs'); +import { countTutorialLocs as countTutorialLocsTs } from '../seed-sitemap-from-deployed'; + +const urlset = (locs: string[]) => + `\n\n` + + locs.map((l) => ` ${l}2026-09-03T13:31:00Z`).join('\n') + + `\n\n`; + +const HOST = 'https://developers.sap.com'; + +// The two implementations are deliberately duplicated (a .cjs guard + a .ts +// preserve script that can't cleanly import the .cjs under tsx). Run every case +// through BOTH so they can never silently diverge. +const impls: Array<[string, (xml: string) => number]> = [ + ['check-sitemap-tutorials.cjs', countTutorialLocs], + ['seed-sitemap-from-deployed.ts', countTutorialLocsTs], +]; + +for (const [name, count] of impls) { + describe(`countTutorialLocs (${name})`, () => { + it('counts absolute tutorial URLs', () => { + const xml = urlset([ + `${HOST}/`, + `${HOST}/tutorials/abap-custom-ui-trust-cf/`, + `${HOST}/tutorials/hana-cloud-mission/`, + `${HOST}/topics/cap/`, + ]); + expect(count(xml)).toBe(2); + }); + + it('returns 0 for the wipe signature: no tutorial URLs', () => { + const xml = urlset([`${HOST}/`, `${HOST}/browse/`, `${HOST}/authors/tomjung/`, `${HOST}/topics/`]); + expect(count(xml)).toBe(0); + }); + + it('does NOT count the bare /tutorials/ section index', () => { + const xml = urlset([`${HOST}/tutorials/`, `${HOST}/tutorials/real-slug/`]); + expect(count(xml)).toBe(1); + }); + + it('does NOT count /tutorials- paths', () => { + const xml = urlset([`${HOST}/tutorial-navigator/`, `${HOST}/tutorials-qa/foo/`]); + expect(count(xml)).toBe(0); + }); + + it('matches root-relative tutorial locs defensively', () => { + const xml = urlset([`/tutorials/some-slug/`, `/browse/`]); + expect(count(xml)).toBe(1); + }); + + it('handles empty / non-string input', () => { + expect(count('')).toBe(0); + // @ts-expect-error deliberate wrong type + expect(count(null)).toBe(0); + expect(count('')).toBe(0); + }); + }); +} diff --git a/scripts/check-sitemap-tutorials.cjs b/scripts/check-sitemap-tutorials.cjs new file mode 100644 index 000000000..612c08afe --- /dev/null +++ b/scripts/check-sitemap-tutorials.cjs @@ -0,0 +1,129 @@ +// scripts/check-sitemap-tutorials.cjs +// +// Build-time guard for the runtime-push (rebuild-content.yml) path. Fails the +// run if the baked hugo/public/sitemap.xml contains ZERO /tutorials/ +// entries — the exact signature of a sitemap wiped down to only its +// non-tutorial pages. +// +// Root cause it prevents (reported 2026-09-03 by the Intelligent Search Data +// Crawling team — crawled docs dropped ~1700 → ~1500, live sitemap held only +// ~180 links): +// /sitemap.xml is served from HANA as the `page-sitemap.xml` blob (see +// srv/lib/page-key-map.js IN_SCOPE_PAGES). publish-content.ts merges that +// page-* blob into the publish set via discoverPageFiles() — but ONLY when +// NOT slug-scoped (`if (!opts.slug)`). So: +// - slug-targeted → page-sitemap.xml NOT republished (server carries it +// forward). SAFE. +// - full → "Fetch tutorials" runs, Hugo bakes the full ~1.4k +// tutorial , republishes the good sitemap. SAFE. +// - catalog-only → "Fetch tutorials" is SKIPPED, so hugo/content/tutorials +// is EMPTY, Hugo bakes a sitemap with NO /tutorials/ URLs, +// and that tutorial-less blob is republished — WIPING the +// live sitemap to ~180 URLs. THIS is the wipe class. +// Same family as the /browse/ (2026-08-07) and /authors/ (#1659 Phase C) +// catalog-only wipes. The durable fix pairs this guard with +// scripts/seed-sitemap-from-deployed.ts, which re-hydrates the deployed +// sitemap in catalog-only BEFORE this guard runs so a legit catalog-only +// rebuild carries the tutorial s forward and passes. +// +// This guard is the fail-closed net: whatever the cause (broken preserve step, +// a future mode that skips fetch, a bad Hugo build), if the sitemap about to be +// published names zero tutorials it FAILS the run rather than let publish-content +// clobber the good page-sitemap.xml blob. A hard red run is visible and +// recoverable (re-run mode=full); a silently wiped sitemap is the incident. +// +// Why a standalone .cjs (not a prebuild hook): the project's global npm config +// sets ignore-scripts=true, which blocks all pre/post lifecycle hooks. Invoked +// as an explicit workflow step. Mirrors scripts/check-verb-shelves.cjs. +// +// Exit codes: +// 0 sitemap has >= MIN_SITEMAP_TUTORIALS tutorial entries. +// 1 sitemap file missing/unreadable, OR fewer than the minimum tutorial URLs +// (default 1). Set ALLOW_EMPTY_SITEMAP=1 to allow zero (genuinely empty +// seed envs), mirroring seed-browse's ALLOW_EMPTY_BROWSE escape hatch. +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); + +// --------------------------------------------------------------------------- +// Pure core (unit-tested in scripts/__tests__/check-sitemap-tutorials.test.ts). +// Count entries whose URL path is under /tutorials/. Host-agnostic: matches +// both absolute (https://developers.sap.com/tutorials//) and, defensively, +// root-relative (/tutorials//) forms. The bare section index /tutorials/ +// (no trailing slug) is excluded from the sitemap by the template, so any match +// here is a real tutorial page. +// --------------------------------------------------------------------------- +function countTutorialLocs(xml) { + if (typeof xml !== 'string' || xml.length === 0) return 0; + let count = 0; + const re = /\s*([^<]+?)\s*<\/loc>/gi; + let m; + while ((m = re.exec(xml)) !== null) { + let loc = m[1]; + // Strip scheme+host so we compare on the path only. + const pathOnly = loc.replace(/^https?:\/\/[^/]+/i, ''); + // A real tutorial URL has a slug after /tutorials/ — require a non-empty + // path segment so the bare /tutorials/ section index (if ever emitted) and + // /tutorials- siblings don't count. + if (/^\/tutorials\/[^/][^<]*$/i.test(pathOnly)) count += 1; + } + return count; +} + +module.exports = { countTutorialLocs }; + +// --------------------------------------------------------------------------- +// CLI (only when run directly, so the pure core imports cleanly under Vitest). +// --------------------------------------------------------------------------- +function fail(lines) { + console.error(''); + console.error('[rebuild] sitemap-tutorials guard FAILED:'); + for (const l of lines) console.error(` ${l}`); + console.error(''); + console.error(' /sitemap.xml is served from the HANA `page-sitemap.xml` blob and is'); + console.error(' republished on every full/catalog-only rebuild. A sitemap with no'); + console.error(' /tutorials/ URLs means the Hugo build had no tutorial content — a'); + console.error(' catalog-only rebuild that skipped "Fetch tutorials" and whose'); + console.error(' seed-sitemap-from-deployed.ts preserve step did not carry the'); + console.error(' deployed sitemap forward. Fix: re-run the rebuild with mode=full to'); + console.error(' regenerate the full sitemap, then catalog-only will preserve it.'); + console.error(''); + process.exit(1); +} + +function argOf(flag, fallback) { + const i = process.argv.indexOf(flag); + return i !== -1 && process.argv[i + 1] ? process.argv[i + 1] : fallback; +} + +function main() { + const hugoDir = argOf('--hugo-dir', 'hugo/public'); + const sitemapPath = path.resolve(process.cwd(), hugoDir, 'sitemap.xml'); + const allowEmpty = process.env.ALLOW_EMPTY_SITEMAP === '1'; + const min = allowEmpty ? 0 : Math.max(1, parseInt(process.env.MIN_SITEMAP_TUTORIALS || '1', 10)); + + if (!fs.existsSync(sitemapPath)) { + fail([`sitemap not found at ${path.relative(process.cwd(), sitemapPath)}.`, + 'The Hugo build did not emit a sitemap — check hugo.toml `home` outputs include \'sitemap\'.']); + } + + let xml; + try { + xml = fs.readFileSync(sitemapPath, 'utf-8'); + } catch (err) { + fail([`could not read ${sitemapPath}: ${err.message}`]); + } + + const tutorialCount = countTutorialLocs(xml); + const totalLocs = (xml.match(//gi) || []).length; + + if (tutorialCount < min) { + fail([`sitemap has ${tutorialCount} /tutorials/ URL(s) (require >= ${min}); ${totalLocs} total.`, + 'This is the wipe signature — the live sitemap would drop to non-tutorial pages only.']); + } + + console.log(`[rebuild] sitemap-tutorials guard OK — ${tutorialCount} /tutorials/ URLs of ${totalLocs} total .`); +} + +if (require.main === module) main(); diff --git a/scripts/seed-sitemap-from-deployed.ts b/scripts/seed-sitemap-from-deployed.ts new file mode 100644 index 000000000..65e1b0f3f --- /dev/null +++ b/scripts/seed-sitemap-from-deployed.ts @@ -0,0 +1,100 @@ +import { writeFileSync, mkdirSync, existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +// scripts/seed-sitemap-from-deployed.ts +// +// Preserves /sitemap.xml across catalog-only rebuilds — same wipe class as the +// /browse/ (seed-browse-from-deployed.ts) and /authors/ (seed-authors-from- +// deployed.ts) preserve steps. Reported 2026-09-03 by the Intelligent Search +// Data Crawling team (crawled docs ~1700 → ~1500; live sitemap held ~180 links). +// +// THE BUG +// /sitemap.xml is served from HANA as the `page-sitemap.xml` blob (srv/lib/ +// page-key-map.js IN_SCOPE_PAGES). scripts/publish-content.ts merges that +// page-* blob into the publish set via discoverPageFiles() whenever the run is +// NOT slug-scoped. The rebuild-content workflow skips "Fetch tutorials" in +// catalog-only mode, so hugo/content/tutorials is empty, the Hugo build bakes a +// sitemap with NO /tutorials/ URLs, and publishing that tutorial-less blob +// WIPES the live sitemap down to its non-tutorial pages (~180 URLs). +// +// WHY NOT just regenerate the sitemap in catalog-only? +// The sitemap is generated by Hugo iterating site.Pages — it needs the ~1.4k +// tutorial content pages present at build time. Those come ONLY from the +// GitHub tutorial-markdown fetch that catalog-only deliberately skips (it's the +// fast admin-edit path). Running the fetch would defeat the mode. +// +// THE FIX (this script) +// In catalog-only mode, AFTER the Hugo build (unlike seed-browse/seed-authors, +// which re-hydrate build INPUTS before the build), overwrite the freshly-baked +// (tutorial-less) hugo/public/sitemap.xml with the sitemap the approuter is +// CURRENTLY serving at /sitemap.xml. That deployed sitemap already carries the +// full tutorial from the last full rebuild, so the publish step +// carries it forward verbatim instead of shipping a wiped one. The +// check-sitemap-tutorials.cjs guard then runs as the fail-closed net. +// +// FAIL-SAFE +// If the deployed sitemap can't be fetched, isn't a , or names zero +// /tutorials/ URLs, this script EXITS NON-ZERO without writing — deliberately +// FAILING the catalog-only rebuild rather than shipping a wiped sitemap. A hard +// failure is visible and recoverable (re-run mode=full); a silent wipe is the +// exact incident this fixes. The one acceptable "empty" case (a fresh/seed env +// with no tutorials yet) is handled by ALLOW_EMPTY_SITEMAP=1. + +const APPROUTER_URL = (process.env.APPROUTER_URL || '').replace(/\/+$/, ''); +const OUT_PATH = join('hugo', 'public', 'sitemap.xml'); +const ALLOW_EMPTY = process.env.ALLOW_EMPTY_SITEMAP === '1'; + +// Count entries under /tutorials/. Kept in sync with +// scripts/check-sitemap-tutorials.cjs countTutorialLocs (this file can't import +// the .cjs cleanly under tsx without pulling its CLI, so the tiny matcher is +// duplicated; both are covered by unit tests). Exported for the unit test. +export function countTutorialLocs(xml: string): number { + if (typeof xml !== 'string' || xml.length === 0) return 0; + let count = 0; + const re = /\s*([^<]+?)\s*<\/loc>/gi; + let m: RegExpExecArray | null; + while ((m = re.exec(xml)) !== null) { + const pathOnly = m[1].replace(/^https?:\/\/[^/]+/i, ''); + if (/^\/tutorials\/[^/][^<]*$/i.test(pathOnly)) count += 1; + } + return count; +} + +function die(msg: string): never { + console.error(`[seed-sitemap] FAILED: ${msg}`); + console.error('[seed-sitemap] Refusing to proceed — a catalog-only rebuild must not ship a sitemap with no /tutorials/ URLs.'); + console.error('[seed-sitemap] Recover by re-running the rebuild with mode=full, or set ALLOW_EMPTY_SITEMAP=1 if the catalog is genuinely empty.'); + process.exit(1); +} + +async function main() { + if (!APPROUTER_URL) die('APPROUTER_URL is not set'); + + let xml: string; + try { + const res = await fetch(`${APPROUTER_URL}/sitemap.xml`, { redirect: 'follow' }); + if (!res.ok) die(`GET ${APPROUTER_URL}/sitemap.xml returned ${res.status}`); + xml = await res.text(); + } catch (err) { + die(`could not fetch ${APPROUTER_URL}/sitemap.xml — ${err instanceof Error ? err.message : err}`); + } + + if (!xml.includes(' document (got an error page or redirect body?)'); + } + + const tutorialCount = countTutorialLocs(xml); + if (tutorialCount === 0 && !ALLOW_EMPTY) { + die('deployed /sitemap.xml has zero /tutorials/ URLs. This env has no populated sitemap to preserve — a previous catalog-only rebuild may have already wiped it. Re-run with mode=full to regenerate.'); + } + + if (!existsSync(dirname(OUT_PATH))) mkdirSync(dirname(OUT_PATH), { recursive: true }); + writeFileSync(OUT_PATH, xml, 'utf-8'); + console.log(`[seed-sitemap] wrote ${xml.length} bytes to ${OUT_PATH} (preserved from deployed ${APPROUTER_URL}/sitemap.xml, ${tutorialCount} /tutorials/ URLs).`); +} + +// Only run main() when invoked directly (not when imported by the unit test). +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(e => { console.error(e); process.exit(1); }); +} diff --git a/test/smoke/seo-files.test.js b/test/smoke/seo-files.test.js index 26200dbdf..d8219f319 100644 --- a/test/smoke/seo-files.test.js +++ b/test/smoke/seo-files.test.js @@ -19,6 +19,12 @@ describe('SEO files', () => { expect(text).toContain('https:\/\/developers\.sap\.com\//); expect(text).toMatch(//); + // Guards the 2026-09-03 sitemap-wipe class (reported by the Intelligent Search + // Data Crawling team): a catalog-only rebuild that skipped "Fetch tutorials" + // republished the page-sitemap.xml blob with NO /tutorials/ URLs, dropping the + // live sitemap from ~1.6k links to ~180. The sitemap MUST name tutorial pages. + expect(text, 'sitemap must contain /tutorials/ URLs — a sitemap without them is the wipe signature') + .toMatch(/https:\/\/developers\.sap\.com\/tutorials\/[^<]+<\/loc>/); }); it('301-redirects legacy AEM sitemap URLs to /sitemap.xml', async () => {