fix(sitemap): guard catalog-only rebuilds from wiping /tutorials/ URLs

.github/workflows/rebuild-content.yml

@@ -650,6 +650,36 @@ jobs:
       - name: Build Hugo site
         run: /tmp/hugo --source hugo --minify
 
+      # Preserve /sitemap.xml across catalog-only rebuilds (same wipe class as
+      # /browse/ + /authors/ above; reported 2026-09-03 by the Intelligent Search
+      # Data Crawling team). /sitemap.xml is served from the HANA `page-sitemap.xml`
+      # blob and publish-content.ts republishes it on every non-slug rebuild.
+      # catalog-only skips "Fetch tutorials", so the Hugo build just above bakes a
+      # sitemap with NO /tutorials/ URLs — publishing that would wipe the live
+      # sitemap to ~180 links. The sitemap can't be regenerated here (it needs the
+      # ~1.4k tutorial pages the skipped GitHub fetch provides), so — UNLIKE the
+      # browse/author preserve steps, which re-hydrate build INPUTS before the
+      # build — this runs AFTER the build and overwrites the freshly-baked sitemap
+      # with the one the approuter is CURRENTLY serving. Fails the run (non-zero)
+      # rather than ship a wiped sitemap if the deployed one is unreadable or names
+      # zero tutorials. Not needed in `full` (regenerates it) or `slug-targeted`
+      # (page-sitemap.xml is not republished).
+      - name: Preserve sitemap.xml for catalog-only rebuild
+        if: ${{ steps.mode.outputs.effective_mode == 'catalog-only' }}
+        env:
+          APPROUTER_URL: ${{ steps.url.outputs.approuter_url }}
+        run: npx tsx scripts/seed-sitemap-from-deployed.ts
+
+      # Fail-closed net for the sitemap wipe class. Runs in the modes that
+      # republish the page-sitemap.xml blob (full + catalog-only; slug-targeted
+      # does not touch it). Fails BEFORE the HANA publish if the sitemap about to
+      # be published names zero /tutorials/ URLs — whatever the cause (broken
+      # preserve step, a future fetch-skipping mode, a bad Hugo build). A red run
+      # is recoverable (re-run mode=full); a silently wiped sitemap is the incident.
+      - name: Guard - sitemap includes tutorial URLs
+        if: ${{ steps.mode.outputs.effective_mode != 'slug-targeted' }}
+        run: node scripts/check-sitemap-tutorials.cjs --hugo-dir hugo/public
+
       # [#1657] Restore CDS syntax highlighting to the runtime-push path. Hugo's
       # render-codeblock.html maps ```cds → the Chroma SQL lexer as a PLACEHOLDER
       # (Chroma has no CDS lexer) and tags the wrapper `data-lang=cds`; the SQL

package.json

@@ -79,6 +79,7 @@
     "build:analytics-explorer": "npm --prefix app/analytics-explorer install && npm --prefix app/analytics-explorer run build",
     "check-explore-bundle-manifest": "node scripts/check-explore-bundle-manifest.cjs",
     "check-verb-shelves": "node scripts/check-verb-shelves.cjs",
+    "check-sitemap-tutorials": "node scripts/check-sitemap-tutorials.cjs",
     "build:hugo": "npm run check-explore-bundle-manifest && npm run check-verb-shelves && hugo --source hugo --minify",
     "build:page-fallback": "node scripts/build-page-fallback.cjs",
     "build:whats-new-snapshot": "node scripts/build-whats-new-snapshot.cjs",

scripts/__tests__/check-sitemap-tutorials.test.ts

@@ -0,0 +1,72 @@
+// scripts/__tests__/check-sitemap-tutorials.test.ts
+//
+// Unit tests for the build-time sitemap guard (scripts/check-sitemap-tutorials.cjs)
+// and the shared tutorial-<loc> counter used by the catalog-only preserve step
+// (scripts/seed-sitemap-from-deployed.ts). Both defend the 2026-09-03 sitemap-wipe
+// class: a catalog-only rebuild republished the page-sitemap.xml blob with zero
+// /tutorials/ URLs, dropping the live sitemap from ~1.6k links to ~180.
+//
+// We import the pure cores and assert against synthetic sitemap XML — same
+// approach as check-verb-shelves.test.ts. The CLI wiring (file read + exit code)
+// is exercised by the live rebuild-content.yml guard step.
+
+import { describe, it, expect } from 'vitest';
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { countTutorialLocs } = require('../check-sitemap-tutorials.cjs');
+import { countTutorialLocs as countTutorialLocsTs } from '../seed-sitemap-from-deployed';
+
+const urlset = (locs: string[]) =>
+  `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n` +
+  locs.map((l) => `  <url><loc>${l}</loc><lastmod>2026-09-03T13:31:00Z</lastmod></url>`).join('\n') +
+  `\n</urlset>\n`;
+
+const HOST = 'https://developers.sap.com';
+
+// The two implementations are deliberately duplicated (a .cjs guard + a .ts
+// preserve script that can't cleanly import the .cjs under tsx). Run every case
+// through BOTH so they can never silently diverge.
+const impls: Array<[string, (xml: string) => number]> = [
+  ['check-sitemap-tutorials.cjs', countTutorialLocs],
+  ['seed-sitemap-from-deployed.ts', countTutorialLocsTs],
+];
+
+for (const [name, count] of impls) {
+  describe(`countTutorialLocs (${name})`, () => {
+    it('counts absolute tutorial URLs', () => {
+      const xml = urlset([
+        `${HOST}/`,
+        `${HOST}/tutorials/abap-custom-ui-trust-cf/`,
+        `${HOST}/tutorials/hana-cloud-mission/`,
+        `${HOST}/topics/cap/`,
+      ]);
+      expect(count(xml)).toBe(2);
+    });
+
+    it('returns 0 for the wipe signature: no tutorial URLs', () => {
+      const xml = urlset([`${HOST}/`, `${HOST}/browse/`, `${HOST}/authors/tomjung/`, `${HOST}/topics/`]);
+      expect(count(xml)).toBe(0);
+    });
+
+    it('does NOT count the bare /tutorials/ section index', () => {
+      const xml = urlset([`${HOST}/tutorials/`, `${HOST}/tutorials/real-slug/`]);
+      expect(count(xml)).toBe(1);
+    });
+
+    it('does NOT count /tutorials-<sibling> paths', () => {
+      const xml = urlset([`${HOST}/tutorial-navigator/`, `${HOST}/tutorials-qa/foo/`]);
+      expect(count(xml)).toBe(0);
+    });
+
+    it('matches root-relative tutorial locs defensively', () => {
+      const xml = urlset([`/tutorials/some-slug/`, `/browse/`]);
+      expect(count(xml)).toBe(1);
+    });
+
+    it('handles empty / non-string input', () => {
+      expect(count('')).toBe(0);
+      // @ts-expect-error deliberate wrong type
+      expect(count(null)).toBe(0);
+      expect(count('<urlset></urlset>')).toBe(0);
+    });
+  });
+}

scripts/check-sitemap-tutorials.cjs

@@ -0,0 +1,129 @@
+// scripts/check-sitemap-tutorials.cjs
+//
+// Build-time guard for the runtime-push (rebuild-content.yml) path. Fails the
+// run if the baked hugo/public/sitemap.xml contains ZERO /tutorials/ <loc>
+// entries — the exact signature of a sitemap wiped down to only its
+// non-tutorial pages.
+//
+// Root cause it prevents (reported 2026-09-03 by the Intelligent Search Data
+// Crawling team — crawled docs dropped ~1700 → ~1500, live sitemap held only
+// ~180 links):
+//   /sitemap.xml is served from HANA as the `page-sitemap.xml` blob (see
+//   srv/lib/page-key-map.js IN_SCOPE_PAGES). publish-content.ts merges that
+//   page-* blob into the publish set via discoverPageFiles() — but ONLY when
+//   NOT slug-scoped (`if (!opts.slug)`). So:
+//     - slug-targeted  → page-sitemap.xml NOT republished (server carries it
+//                        forward). SAFE.
+//     - full           → "Fetch tutorials" runs, Hugo bakes the full ~1.4k
+//                        tutorial <urlset>, republishes the good sitemap. SAFE.
+//     - catalog-only   → "Fetch tutorials" is SKIPPED, so hugo/content/tutorials
+//                        is EMPTY, Hugo bakes a sitemap with NO /tutorials/ URLs,
+//                        and that tutorial-less blob is republished — WIPING the
+//                        live sitemap to ~180 URLs. THIS is the wipe class.
+//   Same family as the /browse/ (2026-08-07) and /authors/ (#1659 Phase C)
+//   catalog-only wipes. The durable fix pairs this guard with
+//   scripts/seed-sitemap-from-deployed.ts, which re-hydrates the deployed
+//   sitemap in catalog-only BEFORE this guard runs so a legit catalog-only
+//   rebuild carries the tutorial <loc>s forward and passes.
+//
+// This guard is the fail-closed net: whatever the cause (broken preserve step,
+// a future mode that skips fetch, a bad Hugo build), if the sitemap about to be
+// published names zero tutorials it FAILS the run rather than let publish-content
+// clobber the good page-sitemap.xml blob. A hard red run is visible and
+// recoverable (re-run mode=full); a silently wiped sitemap is the incident.
+//
+// Why a standalone .cjs (not a prebuild hook): the project's global npm config
+// sets ignore-scripts=true, which blocks all pre/post lifecycle hooks. Invoked
+// as an explicit workflow step. Mirrors scripts/check-verb-shelves.cjs.
+//
+// Exit codes:
+//   0  sitemap has >= MIN_SITEMAP_TUTORIALS tutorial <loc> entries.
+//   1  sitemap file missing/unreadable, OR fewer than the minimum tutorial URLs
+//      (default 1). Set ALLOW_EMPTY_SITEMAP=1 to allow zero (genuinely empty
+//      seed envs), mirroring seed-browse's ALLOW_EMPTY_BROWSE escape hatch.
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+// ---------------------------------------------------------------------------
+// Pure core (unit-tested in scripts/__tests__/check-sitemap-tutorials.test.ts).
+// Count <loc> entries whose URL path is under /tutorials/. Host-agnostic: matches
+// both absolute (https://developers.sap.com/tutorials/<slug>/) and, defensively,
+// root-relative (/tutorials/<slug>/) forms. The bare section index /tutorials/
+// (no trailing slug) is excluded from the sitemap by the template, so any match
+// here is a real tutorial page.
+// ---------------------------------------------------------------------------
+function countTutorialLocs(xml) {
+  if (typeof xml !== 'string' || xml.length === 0) return 0;
+  let count = 0;
+  const re = /<loc>\s*([^<]+?)\s*<\/loc>/gi;
+  let m;
+  while ((m = re.exec(xml)) !== null) {
+    let loc = m[1];
+    // Strip scheme+host so we compare on the path only.
+    const pathOnly = loc.replace(/^https?:\/\/[^/]+/i, '');
+    // A real tutorial URL has a slug after /tutorials/ — require a non-empty
+    // path segment so the bare /tutorials/ section index (if ever emitted) and
+    // /tutorials-<something> siblings don't count.
+    if (/^\/tutorials\/[^/][^<]*$/i.test(pathOnly)) count += 1;
+  }
+  return count;
+}
+
+module.exports = { countTutorialLocs };
+
+// ---------------------------------------------------------------------------
+// CLI (only when run directly, so the pure core imports cleanly under Vitest).
+// ---------------------------------------------------------------------------
+function fail(lines) {
+  console.error('');
+  console.error('[rebuild] sitemap-tutorials guard FAILED:');
+  for (const l of lines) console.error(`          ${l}`);
+  console.error('');
+  console.error('          /sitemap.xml is served from the HANA `page-sitemap.xml` blob and is');
+  console.error('          republished on every full/catalog-only rebuild. A sitemap with no');
+  console.error('          /tutorials/ URLs means the Hugo build had no tutorial content — a');
+  console.error('          catalog-only rebuild that skipped "Fetch tutorials" and whose');
+  console.error('          seed-sitemap-from-deployed.ts preserve step did not carry the');
+  console.error('          deployed sitemap forward. Fix: re-run the rebuild with mode=full to');
+  console.error('          regenerate the full sitemap, then catalog-only will preserve it.');
+  console.error('');
+  process.exit(1);
+}
+
+function argOf(flag, fallback) {
+  const i = process.argv.indexOf(flag);
+  return i !== -1 && process.argv[i + 1] ? process.argv[i + 1] : fallback;
+}
+
+function main() {
+  const hugoDir = argOf('--hugo-dir', 'hugo/public');
+  const sitemapPath = path.resolve(process.cwd(), hugoDir, 'sitemap.xml');
+  const allowEmpty = process.env.ALLOW_EMPTY_SITEMAP === '1';
+  const min = allowEmpty ? 0 : Math.max(1, parseInt(process.env.MIN_SITEMAP_TUTORIALS || '1', 10));
+
+  if (!fs.existsSync(sitemapPath)) {
+    fail([`sitemap not found at ${path.relative(process.cwd(), sitemapPath)}.`,
+          'The Hugo build did not emit a sitemap — check hugo.toml `home` outputs include \'sitemap\'.']);
+  }
+
+  let xml;
+  try {
+    xml = fs.readFileSync(sitemapPath, 'utf-8');
+  } catch (err) {
+    fail([`could not read ${sitemapPath}: ${err.message}`]);
+  }
+
+  const tutorialCount = countTutorialLocs(xml);
+  const totalLocs = (xml.match(/<loc>/gi) || []).length;
+
+  if (tutorialCount < min) {
+    fail([`sitemap has ${tutorialCount} /tutorials/ URL(s) (require >= ${min}); ${totalLocs} <loc> total.`,
+          'This is the wipe signature — the live sitemap would drop to non-tutorial pages only.']);
+  }
+
+  console.log(`[rebuild] sitemap-tutorials guard OK — ${tutorialCount} /tutorials/ URLs of ${totalLocs} total <loc>.`);
+}
+
+if (require.main === module) main();

scripts/seed-sitemap-from-deployed.ts

@@ -0,0 +1,100 @@
+import { writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+// scripts/seed-sitemap-from-deployed.ts
+//
+// Preserves /sitemap.xml across catalog-only rebuilds — same wipe class as the
+// /browse/ (seed-browse-from-deployed.ts) and /authors/ (seed-authors-from-
+// deployed.ts) preserve steps. Reported 2026-09-03 by the Intelligent Search
+// Data Crawling team (crawled docs ~1700 → ~1500; live sitemap held ~180 links).
+//
+// THE BUG
+//   /sitemap.xml is served from HANA as the `page-sitemap.xml` blob (srv/lib/
+//   page-key-map.js IN_SCOPE_PAGES). scripts/publish-content.ts merges that
+//   page-* blob into the publish set via discoverPageFiles() whenever the run is
+//   NOT slug-scoped. The rebuild-content workflow skips "Fetch tutorials" in
+//   catalog-only mode, so hugo/content/tutorials is empty, the Hugo build bakes a
+//   sitemap with NO /tutorials/ URLs, and publishing that tutorial-less blob
+//   WIPES the live sitemap down to its non-tutorial pages (~180 URLs).
+//
+// WHY NOT just regenerate the sitemap in catalog-only?
+//   The sitemap is generated by Hugo iterating site.Pages — it needs the ~1.4k
+//   tutorial content pages present at build time. Those come ONLY from the
+//   GitHub tutorial-markdown fetch that catalog-only deliberately skips (it's the
+//   fast admin-edit path). Running the fetch would defeat the mode.
+//
+// THE FIX (this script)
+//   In catalog-only mode, AFTER the Hugo build (unlike seed-browse/seed-authors,
+//   which re-hydrate build INPUTS before the build), overwrite the freshly-baked
+//   (tutorial-less) hugo/public/sitemap.xml with the sitemap the approuter is
+//   CURRENTLY serving at /sitemap.xml. That deployed sitemap already carries the
+//   full tutorial <urlset> from the last full rebuild, so the publish step
+//   carries it forward verbatim instead of shipping a wiped one. The
+//   check-sitemap-tutorials.cjs guard then runs as the fail-closed net.
+//
+// FAIL-SAFE
+//   If the deployed sitemap can't be fetched, isn't a <urlset>, or names zero
+//   /tutorials/ URLs, this script EXITS NON-ZERO without writing — deliberately
+//   FAILING the catalog-only rebuild rather than shipping a wiped sitemap. A hard
+//   failure is visible and recoverable (re-run mode=full); a silent wipe is the
+//   exact incident this fixes. The one acceptable "empty" case (a fresh/seed env
+//   with no tutorials yet) is handled by ALLOW_EMPTY_SITEMAP=1.
+
+const APPROUTER_URL = (process.env.APPROUTER_URL || '').replace(/\/+$/, '');
+const OUT_PATH = join('hugo', 'public', 'sitemap.xml');
+const ALLOW_EMPTY = process.env.ALLOW_EMPTY_SITEMAP === '1';
+
+// Count <loc> entries under /tutorials/<slug>. Kept in sync with
+// scripts/check-sitemap-tutorials.cjs countTutorialLocs (this file can't import
+// the .cjs cleanly under tsx without pulling its CLI, so the tiny matcher is
+// duplicated; both are covered by unit tests). Exported for the unit test.
+export function countTutorialLocs(xml: string): number {
+  if (typeof xml !== 'string' || xml.length === 0) return 0;
+  let count = 0;
+  const re = /<loc>\s*([^<]+?)\s*<\/loc>/gi;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(xml)) !== null) {
+    const pathOnly = m[1].replace(/^https?:\/\/[^/]+/i, '');
+    if (/^\/tutorials\/[^/][^<]*$/i.test(pathOnly)) count += 1;
+  }
+  return count;
+}
+
+function die(msg: string): never {
+  console.error(`[seed-sitemap] FAILED: ${msg}`);
+  console.error('[seed-sitemap] Refusing to proceed — a catalog-only rebuild must not ship a sitemap with no /tutorials/ URLs.');
+  console.error('[seed-sitemap] Recover by re-running the rebuild with mode=full, or set ALLOW_EMPTY_SITEMAP=1 if the catalog is genuinely empty.');
+  process.exit(1);
+}
+
+async function main() {
+  if (!APPROUTER_URL) die('APPROUTER_URL is not set');
+
+  let xml: string;
+  try {
+    const res = await fetch(`${APPROUTER_URL}/sitemap.xml`, { redirect: 'follow' });
+    if (!res.ok) die(`GET ${APPROUTER_URL}/sitemap.xml returned ${res.status}`);
+    xml = await res.text();
+  } catch (err) {
+    die(`could not fetch ${APPROUTER_URL}/sitemap.xml — ${err instanceof Error ? err.message : err}`);
+  }
+
+  if (!xml.includes('<urlset')) {
+    die('deployed /sitemap.xml is not a <urlset> document (got an error page or redirect body?)');
+  }
+
+  const tutorialCount = countTutorialLocs(xml);
+  if (tutorialCount === 0 && !ALLOW_EMPTY) {
+    die('deployed /sitemap.xml has zero /tutorials/ URLs. This env has no populated sitemap to preserve — a previous catalog-only rebuild may have already wiped it. Re-run with mode=full to regenerate.');
+  }
+
+  if (!existsSync(dirname(OUT_PATH))) mkdirSync(dirname(OUT_PATH), { recursive: true });
+  writeFileSync(OUT_PATH, xml, 'utf-8');
+  console.log(`[seed-sitemap] wrote ${xml.length} bytes to ${OUT_PATH} (preserved from deployed ${APPROUTER_URL}/sitemap.xml, ${tutorialCount} /tutorials/ URLs).`);
+}
+
+// Only run main() when invoked directly (not when imported by the unit test).
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().catch(e => { console.error(e); process.exit(1); });
+}

test/smoke/seo-files.test.js

@@ -19,6 +19,12 @@ describe('SEO files', () => {
     expect(text).toContain('<urlset');
     expect(text).toMatch(/<loc>https:\/\/developers\.sap\.com\//);
     expect(text).toMatch(/<lastmod>/);
+    // Guards the 2026-09-03 sitemap-wipe class (reported by the Intelligent Search
+    // Data Crawling team): a catalog-only rebuild that skipped "Fetch tutorials"
+    // republished the page-sitemap.xml blob with NO /tutorials/ URLs, dropping the
+    // live sitemap from ~1.6k links to ~180. The sitemap MUST name tutorial pages.
+    expect(text, 'sitemap must contain /tutorials/ URLs — a sitemap without them is the wipe signature')
+      .toMatch(/<loc>https:\/\/developers\.sap\.com\/tutorials\/[^<]+<\/loc>/);
   });
 
   it('301-redirects legacy AEM sitemap URLs to /sitemap.xml', async () => {