diff --git a/.deploy/mta.yaml b/.deploy/mta.yaml index 1463ec2ad..dba6a1444 100644 --- a/.deploy/mta.yaml +++ b/.deploy/mta.yaml @@ -10,7 +10,7 @@ ID: tutorials-ims # Bump this on each release you deploy — it's the version shown by `cf mtas` # and in the mtar filename (tutorials-ims_.mtar). Deploy is manual: # `cd .deploy && mbt build && cf deploy mta_archives/tutorials-ims_.mtar -e ../deploy/.mtaext -f`. -version: 1.20.1 +version: 1.21.0 # Top-level parameters (overridable per-env via deploy/.mtaext). parameters: @@ -172,7 +172,7 @@ modules: - cp -r ../../hugo/assets ./hugo/assets - cp -r ../../hugo/data ./hugo/data - cp -r ../../hugo/i18n ./hugo/i18n - - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/island-manifest.json srv/lib/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" + - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/island-manifest.json srv/lib/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" - bash -c "node -e \"const p=require('./package.json'); p.dependencies=Object.assign(p.dependencies||{},{cheerio:'^1.2.0','@sap-ai-sdk/foundation-models':'^2.10.0'}); require('fs').writeFileSync('./package.json', JSON.stringify(p,null,2));\"" properties: EXPOSE_CAP_UI: false diff --git a/.github/workflows/notify-qa.yml.template b/.github/workflows/notify-qa.yml.template index 6556984e6..7c05c2063 100644 --- a/.github/workflows/notify-qa.yml.template +++ b/.github/workflows/notify-qa.yml.template @@ -2,6 +2,16 @@ name: Notify tutorials-qa on: push: + # -Contribution repos are a MIXTURE of default branches: most default to + # `main`, but Tutorials-Contribution still defaults to `master`. List BOTH + # so this fires ONLY on the repo's default branch — and NEVER on feature + # branches. Without this filter a push to any branch (e.g. the devtoberfest + # validation-tutorial bot's `validation-tutorial/*` branches) dispatches a + # QA rebuild for a slug that exists only on that branch; QA discovery reads + # the default branch, so the slug is a phantom and the rebuild hard-fails + # "unknown slug in filter" (tutorials-ims#2097). Mirrors the branch filter + # in docs/authors/tutorial-repo-dispatch.yml (the PROD notify template). + branches: [master, main] paths: - 'tutorials/**' @@ -17,8 +27,15 @@ jobs: changed=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} \ | awk -F/ '/^tutorials\//{print $2}' | sort -u) count=$(echo "$changed" | wc -l) + # Only emit a slug when EXACTLY one tutorial changed AND it matches the + # strict slug charset (whole-value POSIX case match); anything else → + # empty slug → rebuild-content-qa.yml runs a full rebuild. Mirrors + # docs/authors/tutorial-repo-dispatch.yml. if [ "$count" = "1" ] && [ -n "$changed" ]; then - echo "slug=$changed" >> "$GITHUB_OUTPUT" + case "$changed" in + *[!a-z0-9-]* | -* ) echo "slug=" >> "$GITHUB_OUTPUT" ;; + *) echo "slug=$changed" >> "$GITHUB_OUTPUT" ;; + esac else echo "slug=" >> "$GITHUB_OUTPUT" fi diff --git a/CLAUDE.md b/CLAUDE.md index 659deac10..f9197520d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -108,62 +108,28 @@ Subsystem one-liners: - **Admin-UI changes need a FULL deploy + are bundle-gated** — the admin apps (`app/admin/*` + `app/admin-shell/`) are raw-copied into the approuter's `static/admin-ui/` by the MTA's approuter builder during `mbt build`. A `--skip-build` deploy (reuses a stale mtar), a module-scoped `cf deploy -m tutorials-srv`, or an mtar packaged before the change landed will silently ship a **stale admin UI** even though the fix is on `main` (this is why PR #1331/#1345's value-help fix looked "not deployed" on DEV). `npm run deploy` now runs **Step 3.5** (`scripts/check-shipped-admin-bundle.cjs`) which cracks the mtar and diffs the shipped admin component files against source, failing the deploy on drift. Rule: deploy admin-UI changes with a full `npm run deploy -- --env ` (NO `--skip-build`, NO `-m` scoping), and never bypass Step 3.5. -## Top Gotchas (rest in [tutorials-ims-gotchas.md](docs/developers/reference/tutorials-ims-gotchas.md)) - -- **Fresh worktree setup needs `npm run setup` after `npm install`** — global npmrc has `ignore-scripts=true`. Without it, `hugo-apps/node_modules` won't be populated and `better-sqlite3`'s native binding won't build. Symptoms: hugo-apps tests fail resolving `@mediapipe/tasks-vision`, `npm test` hangs. - -- **`ignore-scripts=true` silences `postbuild:apps` — build artifacts wired into it are NOT produced by local `npm run build:all`** — the global npmrc `ignore-scripts=true` (see above) means npm **lifecycle hooks never fire**. The `postbuild:apps` hook is where the #1604 island-fingerprint step (`build:island-manifest`, which writes `hugo/data/island_manifest.json`) and 8 static guards live. During a local `build:all`, none of them run. Symptom class: fresh JS/CSS **compiles** (Vite emits `navigator-.js`) but is **never referenced** — `hugo/layouts/partials/island-src.html` falls back to the unhashed `/js/.js`, Hugo bakes the stale path, and the approuter ships old bundles sitting next to the new ones. **Merged fixes look "not deployed" even though the deploy succeeded.** CI dodges this because `deploy.yml`/`unit-tests.yml` run `npm run postbuild:apps` as an **explicit step** (see deploy.yml:217-223 comment). Fix (2026-08-10): `build:all` now calls `npm run build:island-manifest` **explicitly** (not via the hook), and `scripts/deploy-mta.cjs` Step 2.5 fails the deploy if `hugo/public/index.html` bakes only unhashed island paths while a Vite manifest exists. Rule: any build **artifact** (not just a guard) needed for a correct ship must be an explicit step in `build:all`, never left to a `post*`/`pre*` lifecycle hook. - -- **`build:page-fallback` is an explicit `build:all` step (NOT a lifecycle hook)** — `scripts/build-page-fallback.cjs` copies in-scope page snapshots from `hugo/public` into `srv/page-fallback/.` after `build:hugo` runs. Because `ignore-scripts=true` silences all `pre*`/`post*` hooks (see above), it is wired as an explicit `npm run build:page-fallback` in the `build:all` chain, positioned right after `build:hugo`. If you add a new in-scope page to `IN_SCOPE_PAGES` in `srv/lib/page-key-map.js`, the snapshot is picked up automatically on the next full build. Snapshots are gitignored (`srv/page-fallback/*` except `.gitkeep`); the directory is committed empty and populated at build time. - -- **`hugo/content/tutorials/` is entirely generated** — Never edit; overwritten by `npm run fetch-tutorials`. Edit `scripts/parsers/` or source tutorials in `sap-tutorials` org. - -- **Group/Mission completions are rollup-derived (issue #1934)** — the CAP rewrite never carried over the legacy IMS TUTORIAL→GROUP→MISSION rollup, so GROUP/MISSION `TaskRecords` stopped being created at the 2026-08-10 cutover. `srv/lib/completion-rollup.js` recomputes parent group(s)/mission(s) after any TUTORIAL/PUZZLE/CHECKPOINT/PETOBERFEST completion — called from `_updateTutorialProgress`, `resetTutorialProgress`, the CHECKPOINT edge of `createTaskRecord` (developer-service), `puzzle-service`, and `petoberfest-upload`. Slot model: alt-groups (`#172`) collapse to one slot where **any branch satisfies**; a nested GROUP slot needs all its tutorials. Records key on `(user_ID, taskLegacyId=.legacyId, taskType)` and are upserted (SELECT-then-UPDATE-or-INSERT) with `stampSubmissionId` so they carry the NGDS dedup key. NGDS auto-send fires on the → COMPLETED edge (GROUP/MISSION are the only NGDS-eligible rollup types). The orchestrator never throws into the completion tx. Backfill (post-cutover only): `scripts/backfill-group-mission-completions.mjs` (bulk, `--dry-run`/`--since`/`--user`, no NGDS send) then `scripts/backfill-ngds-send.mjs` (rate-limited, resumable via `ImsConfig 'ngds.backfill.cursor'`, honors env=prod + kill-switch + epoch + canonical-sapId; receiver dedups on `submissionIdCompleted`). Pre-cutover completions are intentionally NOT re-minted (legacy IMS credited them; the NGDS epoch guard suppresses them). `completion-rollup.js` is NOT a `content-store.js` dependency → no `srv-qa` `cp` entry needed. - -- **Never run `publish-content` from a workstation** — Use `gh workflow run rebuild-content.yml`. Workstation publishes skip CI validation; the server-side no-revert guard catches the worst stale-cache regressions but not everything. - -- **Tutorial slugs are lowercase canonical** — Hugo emits lowercase; read path 301-redirects mixed-case. Write path lowercases via `tutorialsTableInfo` helper. Never compare slugs to publish payload without `.toLowerCase()`. Mismatches manifest as "0 steps" on group SSR. - -- **`TutorialMeta` and `Tutorials/Missions/Groups` slugs are unique** — `@assert.unique.slug` + `@assert.unique.tutorial`. New write paths MUST upsert on slug (SELECT-then-UPDATE-or-INSERT). Canonical pattern at `srv/lib/content-publish-session.js:285` / `:349`. Hybrid tests guard. - -- **Never SELECT a HANA BLOB alongside metadata in a single CDS QL query** — LOB locators expire before consumption when mixed with non-BLOB columns. Use raw `db.run()` for BLOB retrieval on HANA (`srv/lib/content-store.js` + `srv/lib/embedding-query.js`). CDS QL works on SQLite for unit tests. - -- **`CONTENT_API_KEY` env var required** for `POST /content/publish` and `/content/rollback`. Missing → 401. Set locally when testing publish. - -- **Content served from mutable `ContentCurrent` (Option B, #2017 / Workstream D), not the old `ContentFiles` snapshot-per-version** — three env flags gate the migration, all default OFF, flip in order: `CONTENT_DELTA_WRITE_ENABLED` (publish dual-writes changed slugs → `ContentCurrent` + append-only `ContentHistory`, fail-safe) → **seed** via a full force rebuild (`-f mode=full -f force-publish=true` dual-writes all slugs; no separate migration) → `CONTENT_DELTA_READ_ENABLED` (serve/readers hit `ContentCurrent`, per-slug fallback to `ContentFiles`) → `CONTENT_DELTA_SKIP_CARRYFORWARD` (publish skips `carryForwardUnchanged` → **O(changed) publish**; rollback then **replays `ContentHistory`** into `ContentCurrent`, not clear+fallback). Measured DEV: publish commit ~62s→973ms (PROD carry-forward was ~95s @ 11k files). Serve source header `X-Content-Source: db-current` (ContentCurrent) vs `db` (legacy). Revert = flip `SKIP_CARRYFORWARD` off + `cf restart`. Flags in `srv/lib/feature-flags/registry.js`; readers via `resolveContentBlob` (`srv/lib/content-store.js`). LOB reads stay raw `db.run()`. Cutover cleanup (delete `carryForwardUnchanged` + `ContentFiles`) deferred to a release after PROD soak. - -- **GitHub Actions secret is `DISPATCH_TOKEN`, not `GITHUB_DISPATCH_TOKEN`** — GH reserves the `GITHUB_` prefix. The runtime env var is `GITHUB_DISPATCH_TOKEN`, read by `srv/lib/rebuild-trigger.js`. - -- **`rebuild-content.yml` auto-infers `mode=slug-targeted`** when a `slug` input is set — don't pass `-f mode=slug-targeted`. Wall-clock: catalog-only ~5min, slug-targeted ~2min, full ~10min. - -- **Alert saves do NOT trigger rebuilds** — Alerts are runtime-served. Cache-bust on save is the only freshness mechanism; up-to-60s admin-to-visitor delay expected. - -- **NGDS auto-send is PROD-only + DB-gated (double gate)** — automatic push of task completions to NGDS (badging/gamification) fires from `srv/lib/ngds-autosend.js#maybeAutoSendCompletion`, called at the two completion transition points in `srv/developer-service.js` (`_updateTutorialProgress` + `createTaskRecord`). Sends ONLY when BOTH gates pass: (1) CF `space_name==='prod'` (`resolveDeployEnvironment`, not spoofable headers) AND (2) `ImsConfig` key `ngds.autosend.enabled==='true'` (admin kill-switch via `AdminService.toggleNgdsAutoSend`; 60s flag cache, busted on toggle). Edge-only (fires on the transition → COMPLETED, never on repeat saves) and task-type-allowlisted to TUTORIAL/GROUP/MISSION (legacy parity — PUZZLE/PETOBERFEST/STEP never sent). Fails CLOSED (config read error → disabled) and never throws into the completion tx (a send fault queues in `NGDSFailedMessages` for the 2h retry job). Bulk recompute (raw HANA MERGE) + migration (raw SQL) bypass the service layer, so they cannot flood NGDS. Payload shape itself is the legacy `MessageModel` (#1473). Default OFF in every env; enable in PROD via the admin toggle. Status: `AdminService.getNgdsAutoSendConfig()` returns `{enabled, environment, effective}`. - -- **`@cap-js/ai` plugin (issue #959, PR 2 of 2)** — adopted for RPT-1 recommendations on `@Common.ValueList` fields. Auto-hooks every such field in Fiori draft-enabled admin UIs. Local `cds watch` uses `AICore-mocked` (no recommendations, zero AI Core quota); hybrid/production use `AICore-btp` against the `aicore` VCAP binding. Per-field opt-out: `@UI.RecommendationState: 0`. Reference: [docs/developers/reference/cap-ai-plugin.md](docs/developers/reference/cap-ai-plugin.md). - -- **`KG_PAGERANK_ENABLED` env var (issue #916)** — when `'true'`, `rankNeighborhood` in `srv/knowledge-graph-service.js` multiplicatively blends per-tutorial PageRank (`weight *= 1 + α × normPR`) into all three tutorial-targeted arms (`prerequisitesOf`, `sharedConcepts`, `whatToLearnNext`) and sorts `teaches` by concept-side PageRank. Default off. Scores recomputed nightly at 03:53 UTC by `srv/jobs/kg-pagerank-job.js` — PageRank runs in **Node.js** (not HANA GraphScript — that engine ships no PageRank primitive) over `KG_PG_VERTICES_V` + `KG_PG_EDGES_V`, materialized into `ConceptRank`/`TutorialRank` sidecars. Fail-opens on every fault path (missing sidecars, HANA hiccup, empty maps → multiplier collapses to 1.0). Toggle: `cf set-env tutorials-srv KG_PAGERANK_ENABLED true && cf restart tutorials-srv`. Blend strength via `KG_PAGERANK_ALPHA` (default `1.0` → weights grow at most 2×). - -- **`KG_WCC_ISOLATION_THRESHOLD` env var (issue #918)** — nightly `srv/jobs/kg-wcc-job.js` runs at 04:07 UTC and materializes rows into `KgIsolation` for concept + tutorial vertices whose weakly-connected-component size ≤ threshold. Default `1`; `0` empties the table on the next run (effectively disables the "Isolated" red-badge column on the admin Concepts + Tutorials LRs). Compute is Node.js union-find over `KG_PG_VERTICES_V` + `KG_PG_EDGES_V` — same reason as #916 that HANA GraphScript ships no WCC primitive (SCC yes, WCC no; enumerated by the #916 Task 0 probe). Fail-quiet at read time: the `after('READ')` decorators on `KnowledgeGraphService.Concepts` and `AdminService.Tutorials` catch any SELECT throw and leave `isolated` unset — Fiori renders `null` boolean as no badge. Toggle: `cf set-env tutorials-srv KG_WCC_ISOLATION_THRESHOLD 2 && cf restart tutorials-srv` (or `0` to disable). - -- **`KG_ONDEMAND_ENABLED` env var / `KnowledgeGraphSettings.onDemandExtractionEnabled` (issue #948)** — when `true`, `expandSearchConcepts` fire-and-forgets an enqueue on zero-seed queries; a new 2-minute cron `kg-ondemand-drain` cosine-ranks the corpus and extracts concepts from top-K tutorials via `extractConceptsFromTutorial`. Coalesces near-duplicate queries; per-user (default 3/hr) and global (default 20/hr) rate-limit caps. Default OFF. Env knobs: `KG_ONDEMAND_USER_MAX_PER_HOUR`, `KG_ONDEMAND_GLOBAL_MAX_PER_HOUR`, `KG_ONDEMAND_DRAIN_BATCH` (default 3), `KG_ONDEMAND_TUTORIALS_PER_REQ` (default 5), `KG_ONDEMAND_MAX_ATTEMPTS` (default 3). Admin surface: `/admin-ui/#kgOnDemand`. Drain uses try/finally to recover stuck RUNNING rows on UPDATE failure. Toggle: flip `KnowledgeGraphSettings.onDemandExtractionEnabled=true` at `/admin-ui/#kg-settings` (or `cf set-env tutorials-srv KG_ONDEMAND_ENABLED true && cf restart tutorials-srv`). - -- **KG community detection (issue #917)** — Louvain community detection over `KG_PG_WORKSPACE` runs nightly at 03:57 UTC (`srv/jobs/kg-communities-job.js`) via HANA GraphScript `Communities_Louvain` in `db/src/procedures/KG_LOUVAIN_GRAPH.hdbprocedure`. Memberships materialize into the `KgCommunity` sidecar (`db/knowledge-graph-communities.cds`). Admin surface: `/admin-ui/#kgCommunities` renders a FE List Report (aggregated summary) + Object Page over `AdminService.KgCommunities` and `AdminService.KgCommunityMembers`. `promoteCommunityToMission(communityId, missionSlug, title)` action (SuperAdmin-gated) drafts a `Missions` row + `CompletionPaths` + `CompletionPathItems` sorted `Tutorials.title ASC`, with `Missions.sourceKgCommunityId` set so already-promoted communities can be filtered out. Nightly job fail-opens; empty sidecar renders as FE "No data", never a 500. **No env flag** — tile is always visible to XSUAA `Tutorial.Author` scope (Task 10 skipped: no shell-config precedent). **DEV-only in v1**; PROD rollout deferred. Metrics: `kg_communities_{duration_ms,count,max_size,failures}`. - -- **HCQL protocol adapter (issue #995)** — CAP 10 beta feature. `@hcql` annotation on 9 read-heavy services (AdminService, AuthorService, AnalyticsService, ExportsService, ConsolidationService, KnowledgeGraphService, HomepageService, SearchService, DeveloperService) in `srv/hcql-enablement.cds` exposes each service at its existing OData URL to also accept CQN `SELECT` bodies (HCQL and OData share URLs; dispatch is by request-body shape). Auth inherited from existing `@readonly`/`@requires`. Writes intentionally unsupported (beta not stable cross-runtime). **Runtime hazard:** CAP 10.0.3 exits the process on malformed CQN — do not expose to untrusted clients until CAP hardens the adapter. Kill switch: delete `srv/hcql-enablement.cds`, `cds build --production`, redeploy. Full reference: [docs/developers/reference/hcql-support.md](docs/developers/reference/hcql-support.md). - -- **`KG_RETIRE_ORPHANS_ENABLED` / `KG_RETIRE_ORPHANS_AGE_DAYS` (issue #1115)** — nightly `srv/jobs/kg-retire-orphans-job.js` at 04:37 UTC flips `Concepts.status` ACTIVE→RETIRED for concepts with zero links across all 10 link tables and `firstSeenAt` older than `KG_RETIRE_ORPHANS_AGE_DAYS` (default 14). RETIRED falls out of every read path (all filter `status='ACTIVE'` positively). First-run retirement **ramps** rather than purging instantly (~1 concept on the first run, climbing toward the full zero-link backlog as each concept ages past the grace window) — because on current data all zero-link orphans were minted within the last ~7-22 days. Reversible: `cf set-env tutorials-srv KG_RETIRE_ORPHANS_ENABLED false` (off) or bulk `UPDATE Concepts SET status='ACTIVE' WHERE status='RETIRED'` (data revert). Companion fix: on-demand extraction (#948) is now **link-only** — it attaches existing concepts (0.7 floor) but never mints, and a re-proposed retired slug is reactivated in-tx by `kg-merge-on-write.js` (`retiredBySlug` + `action:'reactivated'`). - -- **KG community peers + community labeling (issue #1126)** — `communityPeersEnabled` on `ChatSettings` (default `false`) gates the `findCommunityPeers` Joule tool (`srv/lib/kg/joule-tool-community-peers.js`). When enabled, the tool accepts a `tutorial_slug`, looks up the anchor's `communityFingerprint` in `KgCommunity`, and returns sibling tutorials from the same Louvain cluster plus the LLM-generated cluster label from `KgCommunityLabel`. Nightly `kg-community-labels` job (`srv/jobs/kg-community-label-job.js`) runs at 04:12 UTC (after Louvain at 03:57) and LLM-names each community with ≥ 2 tutorials. Identity key is `communityFingerprint` (String(64)); skip-key is `memberSlugsHash` (SHA-256 of sorted slugs) — stable member sets incur zero LLM calls. Daily budget is `communityLabelLlmBudgetPerDay` on `ChatSettings` (default 50); a fresh install ramps the backlog over several nights. Fail-open per community; overall throw → scheduler chassis logs `FAILED`. Toggle: `communityPeersEnabled` is a `ChatSettings` column (NOT an env var — nothing reads a `KG_COMMUNITY_PEERS_ENABLED`), enabled by an Admin via `PATCH /admin/ChatSettings()` on the AdminService singleton — the same entity the `/admin-ui/#joule` Joule Settings page edits (note that page's fixed field list does not yet surface this flag, so a direct PATCH is the enablement path until it is added). **DEV-only until PROD Louvain data verifies.** Metrics: `kg_community_label_{duration_ms,labeled,skipped,failures}`. - -- **`KG_COMMUNITY_WEIGHT` env var (issue #1171)** — when `> 0`, `SearchService.before('READ')` appends a SECOND additive rank term `+ KG_COMMUNITY_WEIGHT * (case slug when '' then 1.0 else 0 end)` alongside the existing concept-overlap `KG_WEIGHT` (#945). Peers are tutorials sharing a Louvain `communityFingerprint` (#917/#1126) with the top-`COMMUNITY_TOP_K` (5) concept-overlap hits. Default `0` (OFF) → `buildCommunityRankFragment` in `srv/lib/search-kg-signal.js` short-circuits before any DB fetch and the rank SQL is byte-identical to the #945 formula. **Only fires when `ChatSettings.searchKgRerankEnabled=true`** — the community term is computed inside the same `before('READ')` gate as the KG signal, so setting `KG_COMMUNITY_WEIGHT > 0` while KG rerank is off has no effect. Fail-open (any DB throw → term collapses to `''`). Membership fetched packet-safe (≤5 fingerprints `.in()`, members capped 200, filtered in Node — `cqn-where-in-hana-packet-cap`). Regression harness + churn report at `test/harness/community-rank-churn*`; do NOT enable in any env before the ON-vs-OFF churn is hand-reviewed. Toggle: `cf set-env tutorials-srv KG_COMMUNITY_WEIGHT 1.5 && cf restart tutorials-srv` (with `searchKgRerankEnabled=true`). - -- **`KG_COMMUNITY_COVERAGE_NUDGE_THRESHOLD` env var (issue #1172)** — the `after('READ','KgCommunities')` decorator in `srv/admin-service.js` computes, per community at read time, mission-coverage % + dominant published mission + orphan-tutorial count (helper: `srv/lib/kg-community-coverage.js`) and populates virtual fields on `AdminService.KgCommunities`. Coverage is **published-missions-only** and the % denominator is **tutorial members only** (concept/tag-only communities render N/A, not 0%). `coverageHigh` (`>= threshold`, default **70**) is the single server-computed flag driving both the LR criticality badge and the FE promote-time `MessageBox.warning` ("~X% already in — extend instead?") in `app/admin/kgCommunities/webapp/ext/KgCommunityActionsController.controller.js`. **Fail-quiet** in its own try/catch (separate from `topConceptSlugs`): any throw → warn-log, fields unset, no badge, never a 500 (mirrors #918). No new job/table/migration — computed live. Packet-safe: the covered-slug `.in()` is chunked at 500 ([[cqn-where-in-hana-packet-cap]]). SuperAdmin gate on `promoteCommunityToMission` unchanged; the nudge is advisory. Override: `cf set-env tutorials-srv KG_COMMUNITY_COVERAGE_NUDGE_THRESHOLD 80 && cf restart tutorials-srv`. DEV-only until the #1126 PROD Louvain rollout lands (no `KgCommunity` data in PROD → empty LR, no nudges). -- **Cluster-level Q&A in Joule (issue #1173)** — `describeCommunity` Joule tool (`srv/lib/kg/joule-tool-describe-community.js`) answers "what's the AI cluster?" / "everything around RAP" by resolving a free-text topic to a labeled Louvain community. **LLM-side matching:** `communityCatalogLayer` in `srv/lib/chat-context.js` injects the labeled-cluster catalog (from `KgCommunityLabel`, cached ~5min, cap 40) into the learner system prompt **only when `communityPeersEnabled` is true**; the model passes the chosen label as `matched_label`, and `matchLabel` (`srv/lib/kg/community-label-match.js`, pure) does case-insensitive exact match + token-overlap fallback + ambiguity detection. Reuses the existing `communityPeersEnabled` flag (NO new flag/schema), the `community-peers-cards` SSE frame + `renderCommunityPeersCards` render path, and the extracted `resolveCommunityMembers` helper (`srv/lib/kg/community-members.js`, also used by `findCommunityPeers`). Fail-open throughout (never 500). **Gotcha:** `buildSystemPromptLines` in `chat-orchestrator.js` is DEAD at runtime — `buildSystemPrompt` (chat-context.js) never calls it; the live guidance ships via `communityCatalogLayer`. DEV-only until PROD Louvain data verifies (same posture as #1126). -- **`test:e2e` is post-deploy only, not on PRs** — the five Playwright admin-UI specs (issue #1338) run in the `e2e` CI job after `smoke-test`. They self-skip when `SMOKE_BASE_URL`/`PLAYWRIGHT_BASE_URL` is absent so `npm test` is never affected. Auth uses `SMOKE_TECH_USER`/`SMOKE_TECH_PASSWORD` (same secrets as smoke) via Basic auth against the approuter — verified against current main; the XSUAA routes short-circuit their IDP redirect for a provisioned tech user. Runbook: `test/e2e/README.md`. Selector gotcha: served tutorials render `
`+`

`, NOT `
` — verified against DEV DOM on 2026-07-27. - -- **cds-caching CDS-DB store + metrics (issue #1179 → re-enabled #1182 → metrics disabled #1215 → metrics re-enabled #1222)** — the shared `caching` service (`cds.requires.caching`) uses `store: "cds"` in the `[hybrid]`/`[production]` profiles (base stays `store: "memory"` for local `cds watch` + unit tests). Gives multi-instance CF coherence (a `deleteByTag` bust on one instance invalidates all). **`metrics.enabled` is ON again (#1222)** — #1215 disabled it because on cds-caching ≤2.0.1 it threw `Wrong input for INT type` on every HANA flush (the plugin read back the hourly row via a flattened table-name SELECT → UPPERCASE column keys → `existingHourly.hits` is `undefined` → `undefined + n = NaN` → hdb INT-bind throw), so counters never accumulated (22 hourly rows stuck at 0). **cds-caching 2.0.2 fixes it** ([mikezaschka/cds-caching#27](https://github.com/mikezaschka/cds-caching/issues/27)): readback via the resolved CSN entity (`SELECT.one.from(Metrics)`) + `Number(existingHourly.) || 0` coercion in `StatisticsPersistenceManager.js`. Re-enable is **config-only** (one-way): the plugin auto-persists `METRICSENABLED=1` on connect when config says `enabled:true` (`CachingService.js:133-135`) — no manual SQL to turn it on (the #1215 *disable* needed a `UPDATE ...SET METRICSENABLED=0` because nothing writes it back to 0). After deploy, clear the stale zeroed rows once per env: `DELETE FROM "PLUGIN_CDS_CACHING_METRICS" WHERE "cache"='caching'` (+ `KEYMETRICS`). Guards: `test/unit/caching-metrics-enabled.test.js` (renamed from `-disabled`) + `test/hybrid/caching-cds-store-boot.test.js` now assert metrics ON. **#1179 crash history:** the first srv deploy carrying `store: "cds"` crash-looped on CF (`Duplicate definition of artifact`) and was reverted (PR #1207); #1182 re-enables it with the resolve-guard fix below. **CF resolve-guard fix (#1182), two required halves:** (1) the srv nodejs `build.task` `model` list also includes `cds-caching/db/cache-store` + `cds-caching/db/statistics`, baking the four `plugin.cds_caching.*` entities into `srv/csn.json` (+4 defs, 0 views lost); (2) `srv/lib/strip-precompiled-plugin-roots.js` (called at top of `srv/server.js`, after `cds.plugins`, before model resolve) strips the plugin's runtime `env.roots` push when a precompiled `srv/csn.json` is present — otherwise those 2 extra roots tip CF's `resolve.many(env.roots)` past `length===1`, re-merging every `requires[].model` onto the precompiled csn → crash. Gated on `srv/csn.json` existence: strips in CF production, no-op in hybrid `cds watch` (compiles from source) and dev/unit (`store:memory` pushes nothing). Baking alone or stripping alone is insufficient — both are load-bearing. Reproducible locally from `gen/srv` (not the source tree — that always collapses to the single csn). **Metrics HDI gotcha:** this project's **explicit `build.tasks` list in `.cdsrc.json` suppresses cds's auto-registration of the plugin's build task** — so the four tables (`CacheStore` + `Caches`/`Metrics`/`KeyMetrics`) do NOT emit unless you add them by hand: `{ "for": "cds-caching" }` (emits `CacheStore.hdbtable`) + `{ "for": "hana", "src": "db", "dest": "db", "options": { "model": ["cds-caching/db/statistics"] } }` (the 3 metrics tables — model must be `statistics`-only; adding `db` to a `db`-dest task's model drops all ~247 service `.hdbview`s). QA container (`tutorials-hana-qa`) intentionally gets none — srv-qa doesn't wire caching. Metrics OData management API deliberately NOT registered (write actions). Test harness: unit workers get a stable memory-store config via `cds_requires_caching_*` env vars in `vitest.config.ts` (NOT a setupFiles that imports `@sap/cds` — that installs getter-only `SELECT`/`INSERT` globals and breaks tests assigning `globalThis.SELECT`) to close the fork-pool boot race (#1177). Full reference: [docs/developers/reference/cds-caching-store.md](docs/developers/reference/cds-caching-store.md). - -- **User-facing UI changes want a committed e2e spec** — a per-PR unit test can't catch a cross-PR seam (that's how #1366's value-help widening shipped dead behind #1353's custom dialog; see #1371). Changing `app/admin/**`, `app/**/webapp/**`, `hugo/layouts/**`, or `hugo-apps/**` triggers an *advisory* PR nudge (`.github/workflows/e2e-coverage-nudge.yml`) to add/update a `test/e2e/` spec. It never blocks merge; the existing post-DEV-deploy `e2e` CI job is where coverage is actually exercised. Pattern: [docs/developers/reference/e2e-coverage-pattern.md](docs/developers/reference/e2e-coverage-pattern.md). - -- **Freshness detector grounding needs the corpus-embedding backfill** — the `checkFreshness`/`freshness-scan` engine cosine-searches `ApiDocs`/`Samples` embeddings. Those columns are populated by `srv/jobs/freshness-corpus-embedding-job.js` (nightly `17 3` + on-demand `runJob`). Until it runs in an env, grounding returns nothing and every API-obsolescence claim degrades to `confidence: Low` (fail-open, by design). LLM calls use the SAP AI SDK directly (`@sap-ai-sdk/orchestration`, forced tool-call), NOT `@cap-js/ai`; unit tests inject `globalThis.__FRESHNESS_TEST_IMPL__`. Bulk scan gated by `FRESHNESS_SCAN_ENABLED` (default OFF). **Tutorial markdown is sourced from `ContentFiles.sourceContent` via `getTutorialSource(slug)` in `srv/lib/content-store.js` — NOT from `Steps.description`** (Steps are never populated with step markdown; reading Steps would yield nothing). Findings carry a **global `codeBlockIndex`** across the whole-tutorial markdown — per-step attribution is deferred because the persisted source is not split per step. +## Top Gotchas + +The load-bearing few. **Full detail for every relocated item → [tutorials-ims-gotchas.md](docs/developers/reference/tutorials-ims-gotchas.md)** ("Top Gotchas — full detail" section); items with their own reference doc link straight to it. + +- **Fresh worktree: run `npm run setup` after `npm install`** — global npmrc `ignore-scripts=true` skips native builds; without it `hugo-apps/node_modules` is empty and `better-sqlite3` won't build (tests hang / fail resolving `@mediapipe/tasks-vision`). +- **`ignore-scripts=true` silences all `pre*`/`post*` hooks** — local `build:all` does NOT fire `postbuild:apps`; island-manifest + `build:page-fallback` are wired as explicit `build:all` steps. Rely on the hook and merged JS/CSS ships dead (unhashed island paths) → "not deployed" despite a green deploy. → gotchas.md "Build artifacts & lifecycle hooks". +- **`hugo/content/tutorials/` is entirely generated** — never edit; `fetch-tutorials` overwrites. Edit `scripts/parsers/` or source repos. +- **Group/Mission completions are rollup-derived (#1934)** — `srv/lib/completion-rollup.js` recomputes parents after any TUTORIAL/PUZZLE/CHECKPOINT/PETOBERFEST completion; upserts on `(user_ID, taskLegacyId, taskType)`; NOT a `content-store.js` dep. → gotchas.md "Completions rollup". +- **Never run `publish-content` from a workstation** — use `gh workflow run rebuild-content.yml`; workstation publishes skip CI validation. +- **Tutorial slugs are lowercase canonical** — always `.toLowerCase()` before comparing to publish payload; mismatch = "0 steps" on group SSR. +- **`TutorialMeta` + `Tutorials/Missions/Groups` slugs are unique** — new write paths MUST upsert on slug (SELECT-then-UPDATE-or-INSERT); pattern at `srv/lib/content-publish-session.js:285`/`:349`. +- **Never SELECT a HANA BLOB alongside metadata in one CDS QL query** — LOB locators expire; use raw `db.run()` (`srv/lib/content-store.js`, `srv/lib/embedding-query.js`). CDS QL is fine on SQLite unit tests. +- **`CONTENT_API_KEY` env var required** for `POST /content/publish` + `/content/rollback` (missing → 401). +- **Content served from mutable `ContentCurrent` (Option B, #2017)** — 3 env flags gate it (all default OFF, flip in order); rollback replays `ContentHistory`. Serve header `X-Content-Source: db-current` vs `db`. → gotchas.md "Content model — mutable ContentCurrent". +- **GitHub Actions secret is `DISPATCH_TOKEN`** (GH reserves `GITHUB_`) — runtime var `GITHUB_DISPATCH_TOKEN`, read by `srv/lib/rebuild-trigger.js`. +- **`rebuild-content.yml` auto-infers `mode=slug-targeted`** when `slug` is set — don't pass mode. Wall-clock: catalog ~5m, slug ~2m, full ~10m. +- **Alert saves do NOT trigger rebuilds** — runtime-served; cache-bust on save only, up-to-60s delay. +- **NGDS auto-send is PROD-only + DB-gated (double gate)** — fires only when CF `space_name==='prod'` AND `ImsConfig ngds.autosend.enabled==='true'`; edge-only, fails closed, never throws into the completion tx, allowlisted to TUTORIAL/GROUP/MISSION. → gotchas.md "NGDS auto-send". +- **`@cap-js/ai` for RPT-1 ValueList recommendations (#959)** — `AICore-mocked` locally, `AICore-btp` in hybrid/prod; per-field opt-out `@UI.RecommendationState: 0`. Ref: [cap-ai-plugin.md](docs/developers/reference/cap-ai-plugin.md). +- **Knowledge-graph feature flags (`KG_*`), all default OFF + DEV-only, all fail-open** — PageRank #916, WCC isolation #918, on-demand extraction #948, Louvain communities #917, orphan retirement #1115, community peers/labels #1126, community search weight #1171, coverage nudge #1172, cluster Q&A #1173. Toggles, nightly jobs, and fail-open specifics → gotchas.md "Knowledge graph feature flags". +- **HCQL protocol adapter (#995, CAP 10 beta)** — `@hcql` on 9 read services accepts CQN `SELECT` bodies at existing OData URLs. **CAP 10.0.3 exits the process on malformed CQN — do not expose to untrusted clients.** Kill: delete `srv/hcql-enablement.cds` + rebuild. Ref: [hcql-support.md](docs/developers/reference/hcql-support.md). +- **cds-caching CDS-DB store + metrics ON (#1222)** — `store:"cds"` in hybrid/prod, `memory` in base/unit. CF resolve-guard needs BOTH the baked csn entities AND `srv/lib/strip-precompiled-plugin-roots.js` (both load-bearing). Ref: [cds-caching-store.md](docs/developers/reference/cds-caching-store.md). +- **User-facing UI changes want a committed e2e spec** — advisory PR nudge on `app/**`/`hugo/**` changes; real coverage runs in the post-DEV-deploy `e2e` job. Ref: [e2e-coverage-pattern.md](docs/developers/reference/e2e-coverage-pattern.md). +- **`test:e2e` is post-deploy only, not on PRs** — self-skips without `SMOKE_BASE_URL`. Served tutorials render `
`+`

`, NOT `
`. Runbook: `test/e2e/README.md`. +- **Freshness detector grounding needs the corpus-embedding backfill** — until `srv/jobs/freshness-corpus-embedding-job.js` runs, every API-obsolescence claim degrades to `confidence: Low`. Tutorial source from `ContentFiles.sourceContent` via `getTutorialSource(slug)`, NOT `Steps.description`. → gotchas.md "Freshness detector". diff --git a/app/admin-annotations.cds b/app/admin-annotations.cds index caf86fb23..7d5a38e86 100644 --- a/app/admin-annotations.cds +++ b/app/admin-annotations.cds @@ -714,12 +714,14 @@ annotate AdminService.Tutorials with { annotate AdminService.TutorialContributors with { name @Common.Label: 'Name'; + login @Common.Label: 'GitHub'; email @Common.Label: 'Email'; role @Common.Label: 'Role'; }; annotate AdminService.TutorialContributors with @UI.LineItem: [ { Value: name }, + { $Type: 'UI.DataFieldWithUrl', Value: login, Url: profileUrl, Label: 'GitHub' }, { Value: email }, { Value: role } ]; @@ -881,6 +883,19 @@ annotate AdminService.ValidateAnswerSpecs with @UI: { } }; +// TutorialValidationRules — all validation rules for a tutorial (step, question, +// type, rule, AI-grading flag, correct answer). Joined via `validationRules` association. +annotate AdminService.TutorialValidationRules with @( + UI.LineItem: [ + { Value: stepNumber, Label: 'Step' }, + { Value: questionText, Label: 'Question' }, + { Value: questionType, Label: 'Type' }, + { Value: ruleType, Label: 'Rule' }, + { Value: aiGrading, Label: 'AI-Graded' }, + { Value: correctAnswer, Label: 'Correct Answer' } + ] +); + // CodeCheckSpecs — per-step code-check specs (goal + reference solution). // Joined via `codeCheckSpecs` association. annotate AdminService.CodeCheckSpecs with { @@ -954,14 +969,23 @@ annotate AdminService.Tutorials with @UI: { { $Type: 'UI.ReferenceFacet', Label: 'Contributors', ID: 'ContributorsFacet', Target: 'contributors/@UI.LineItem' }, { $Type: 'UI.ReferenceFacet', Label: 'Completion Stats', ID: 'CompletionStatsFacet', Target: 'completionStats/@UI.FieldGroup#Stats' }, - { $Type: 'UI.ReferenceFacet', Label: 'Validation Questions', ID: 'ValidationSpecsFacet', + { $Type: 'UI.ReferenceFacet', Label: 'AI-Graded Validation', ID: 'ValidationSpecsFacet', Target: 'validationSpecs/@UI.LineItem' }, + { $Type: 'UI.ReferenceFacet', Label: 'All Validation Rules', ID: 'AllValidationRulesFacet', + Target: 'validationRules/@UI.LineItem' }, { $Type: 'UI.ReferenceFacet', Label: 'Code-Check Specs', ID: 'CodeCheckSpecsFacet', Target: 'codeCheckSpecs/@UI.LineItem' }, { $Type: 'UI.ReferenceFacet', Label: 'AI-Author Requests', ID: 'AiRequestsFacet', Target: 'aiRequests/@UI.LineItem' }, + { $Type: 'UI.ReferenceFacet', Label: 'Freshness Reports', ID: 'FreshnessReportsFacet', Target: 'freshnessReports/@UI.PresentationVariant' }, { $Type: 'UI.ReferenceFacet', ID: 'FreshnessFacet', Label: 'Freshness', Target: 'freshnessFindings/@UI.LineItem' }, + { $Type: 'UI.ReferenceFacet', Label: 'Images', ID: 'MediaImagesFacet', Target: 'images/@UI.LineItem' }, + { $Type: 'UI.ReferenceFacet', Label: 'Assets', ID: 'MediaAssetsFacet', Target: 'assets/@UI.LineItem' }, + { $Type: 'UI.ReferenceFacet', Label: 'Knowledge Graph', ID: 'KgFieldsFacet', Target: '@UI.FieldGroup#KnowledgeGraph' }, + { $Type: 'UI.ReferenceFacet', Label: 'Concepts Taught', ID: 'ConceptsTaughtFacet', Target: 'conceptLinks/@UI.LineItem' }, + { $Type: 'UI.ReferenceFacet', Label: 'Co-Completed', ID: 'CoCompletionsFacet', Target: 'coCompletions/@UI.LineItem' }, + { $Type: 'UI.ReferenceFacet', Label: 'Community', ID: 'KgCommunityFacet', Target: 'communityMembership/@UI.LineItem' }, { $Type: 'UI.CollectionFacet', ID: 'Feedback', Label: 'Feedback', Facets: [ { $Type: 'UI.ReferenceFacet', ID: 'FeedbackSummary', Target: 'feedbackSummary/@UI.FieldGroup#FeedbackSummary', @@ -4348,6 +4372,20 @@ annotate AdminService.TopicClustersAdmin with @( // --- Tutorial Freshness Detector (spec 2026-08-22-tutorial-freshness-detector) --- // Surfaces per-finding analysis rows on the Tutorials Object Page and wires the +// Freshness Reports — report-level header (spec 2026-08-31 task-5). +// PresentationVariant sorts newest-first so the latest run appears at the top. +annotate AdminService.FreshnessReport with @( + UI.LineItem: [ + { Value: runAt, Label: 'Run At' }, + { Value: status, Label: 'Status' }, + { Value: model, Label: 'Model' }, + { Value: cost, Label: 'Cost' }, + { Value: openHighCount, Label: 'Open High' }, + { Value: error, Label: 'Error' } + ], + UI.PresentationVariant: { SortOrder: [{ Property: runAt, Descending: true }], Visualizations: ['@UI.LineItem'] } +); + // Set Disposition action. Criticality paths delegate to the virtual // `confidenceCriticality` field (computed by after('READ','FreshnessFinding') // in admin-service.js). @@ -4378,3 +4416,75 @@ annotate AdminService.FreshnessFinding with { suggestedFix @UI.MultiLineText; evidence @UI.MultiLineText; }; + +// --- Media facets: TutorialImages + TutorialAssets (Task 4) --- +annotate AdminService.TutorialImages with @( + UI.LineItem: [ + { Value: thumbUrl, Label: 'Preview' }, + { $Type: 'UI.DataFieldWithUrl', Value: viewUrl, Url: viewUrl, Label: 'Image (served)' }, + { $Type: 'UI.DataFieldWithUrl', Value: sourceUrl, Url: sourceUrl, Label: 'Source (GitHub)' }, + { Value: mimeType, Label: 'Type' }, + { Value: byteSize, Label: 'Bytes' }, + { Value: contentHash, Label: 'Hash' }, + { Value: channel, Label: 'Channel' } + ] +) { + // Render thumbUrl inline as an image thumbnail rather than as raw text. + thumbUrl @UI.IsImageURL; +}; +annotate AdminService.TutorialAssets with @( + UI.LineItem: [ + { Value: filename, Label: 'File' }, + { $Type: 'UI.DataFieldWithUrl', Value: sourceUrl, Url: sourceUrl, Label: 'Source (GitHub)' }, + { Value: mimeType, Label: 'Type' }, + { Value: byteSize, Label: 'Bytes' }, + { Value: contentHash, Label: 'Hash' } + ] +); + +// --- Knowledge Graph facets (task-3) --- +// TutorialConceptLinks LineItem: concept FK, predicate (teaches|extends), confidence score. +annotate AdminService.TutorialConceptLinks with @( + UI.LineItem: [ + { Value: concept_ID, Label: 'Concept' }, + { Value: predicate, Label: 'Relation' }, + { Value: confidence, Label: 'Confidence' } + ] +) { + // Show the human-readable concept name instead of the raw GUID FK. + // Annotate the association (not concept_ID) so the compiler propagates the + // text to the generated foreign key — mirrors the primaryTagRef precedent. + concept @Common.Text: concept.name @Common.TextArrangement: #TextOnly; +}; + +// CoCompletions LineItem: target tutorial slug + co-completion score. +annotate AdminService.CoCompletions with @( + UI.LineItem: [ + { Value: targetSlug, Label: 'Also Completed' }, + { Value: score, Label: 'Score' } + ] +); + +// KgCommunityMembers LineItem: community (with LLM label) + member slug + type. +annotate AdminService.KgCommunityMembers with @( + UI.LineItem: [ + { Value: communityId, Label: 'Community' }, + { Value: slug, Label: 'Member' }, + { Value: vertexType, Label: 'Type' } + ] +) { + // Prefix the numeric id with the LLM-generated cluster label when present + // (#1126); falls back to the bare id where labeling hasn't run yet. + communityId @Common.Text: labelInfo.label @Common.TextArrangement: #TextFirst; +}; + +// FieldGroup for PageRank score — shown in KgFieldsFacet on Tutorials OP. +// rank is a to-one Association (slug-joined) added in task-1. +annotate AdminService.Tutorials with @( + UI.FieldGroup #KnowledgeGraph: { + Label: 'Knowledge Graph', + Data: [ + { $Type: 'UI.DataField', Value: rank.score, Label: 'PageRank Score' } + ] + } +); diff --git a/approuter/lib/search-redirect.js b/approuter/lib/search-redirect.js new file mode 100644 index 000000000..ec41881da --- /dev/null +++ b/approuter/lib/search-redirect.js @@ -0,0 +1,82 @@ +// approuter/lib/search-redirect.js +// +// 301-redirects the legacy `/search` entry point to this platform's canonical +// tutorial finder at /tutorial-navigator/, preserving any query string. +// +// Background: +// The old site exposed a top-level /search page. That has been replaced by +// /tutorial-navigator/ (an SSR page served from HANA via CAP). External links +// and bookmarks to /search (and /search?q=…) must land on the navigator, with +// the user's query carried across so the navigator can pre-seed its filter. +// +// Only the EXACT /search entry point redirects here — deeper /search/ +// URLs (e.g. the search JSON API) still proxy to srv-api via xs-app.json. +// +// Why middleware, not an xs-app.json route: +// @sap/approuter route config only rewrites+proxies or serves a localDir — it +// has no native 3xx-redirect verb. A prior attempt added `"status": 301` to an +// xs-app.json route; @sap/approuter v16 rejects that unknown property at boot +// (`xs-app.json/routes//status: Additional properties not allowed`), which +// crash-loops the approuter. A middleware in insertMiddleware.first is the same +// deterministic pattern used by sitemap-index-redirect.js / security-txt.js and +// runs BEFORE xs-app.json route matching, so /search/ (which this does +// NOT match) still proxies to srv-api untouched. +// +// Pure matcher + handler, both exported for the unit test. Approuter-native CJS +// (like sitemap-index-redirect.js) — NOT copied from srv/lib. + +'use strict' + +// The exact legacy search entry point at the site root: /search or /search/, +// optionally followed by a query string. Deliberately does NOT match deeper +// /search/ URLs so those keep proxying to srv-api. +const SEARCH_ENTRY_RE = /^\/search\/?$/ + +const NAVIGATOR_PATH = '/tutorial-navigator/' + +/** + * Map the legacy /search entry point to /tutorial-navigator/, preserving the + * query string. + * + * /search → /tutorial-navigator/ + * /search/ → /tutorial-navigator/ + * /search?q=cap → /tutorial-navigator/?q=cap + * /search/?q=cap → /tutorial-navigator/?q=cap + * + * Returns null for anything else — notably /search/ (the search API), + * which must keep proxying to srv-api. + * + * @param {string} url - path-or-path+query URL, e.g. '/search?q=cap' + * @returns {string | null} the navigator target for the /search entry point, else null + */ +function matchSearchUrl(url) { + if (typeof url !== 'string' || url.length === 0) return null + const qIdx = url.indexOf('?') + const pathname = qIdx === -1 ? url : url.slice(0, qIdx) + if (!SEARCH_ENTRY_RE.test(pathname)) return null + const query = qIdx === -1 ? '' : url.slice(qIdx) // includes the leading '?' + return NAVIGATOR_PATH + query +} + +// Express-style middleware. Mount at path '/' in insertMiddleware.first, BEFORE +// the static/proxy handlers so /search is answered here and never falls through +// to the xs-app.json /search/ proxy route. +function searchRedirectHandler(req, res, next) { + if (req.method !== 'GET' && req.method !== 'HEAD') return next() + + const target = matchSearchUrl(req.url || '') + if (!target) return next() + + res.writeHead(301, { + Location: target, + 'Cache-Control': 'public, max-age=86400', + }) + res.end() +} + +module.exports = { + searchRedirectHandler, + // exported for the unit test + matchSearchUrl, + NAVIGATOR_PATH, +} diff --git a/approuter/server.js b/approuter/server.js index 53dddde65..ecd775f22 100644 --- a/approuter/server.js +++ b/approuter/server.js @@ -25,6 +25,7 @@ const { mcpManifestHandler } = require('./lib/well-known-mcp-manifest') const { mcpAuthChallengeHandler } = require('./lib/mcp-auth-challenge') const { securityTxtHandler } = require('./lib/security-txt') const { sitemapIndexRedirectHandler } = require('./lib/sitemap-index-redirect') +const { searchRedirectHandler } = require('./lib/search-redirect') const shouldProcessImage = require('./lib/img-cdn-should-process') const { buildImageOriginUrl } = require('./lib/img-cdn-origin') const { ImgCache } = require('./lib/img-cdn-cache') @@ -597,6 +598,7 @@ ar.start({ { path: '/', handler: mcpManifestHandler }, { path: '/', handler: securityTxtHandler }, { path: '/', handler: sitemapIndexRedirectHandler }, + { path: '/', handler: searchRedirectHandler }, { path: '/', handler: mcpAuthChallengeHandler }, { path: '/', handler: devtoberfestCspHandler }, { path: '/', handler: imgCdnHandler }, diff --git a/approuter/xs-app.json b/approuter/xs-app.json index 0ed2cb08a..a802fa8ea 100644 --- a/approuter/xs-app.json +++ b/approuter/xs-app.json @@ -396,7 +396,7 @@ "authenticationType": "none" }, { - "source": "^/build/(breadcrumb-context|catalog|co-completions|concepts|homepage-shelves|kg-stats|mission|my-progress|navigator|repo-catalog|slug-mapping|tag-labels|topics-gallery)(/.*)?(\\?.*)?$", + "source": "^/build/(breadcrumb-context|catalog|co-completions|concepts|homepage-shelves|kg-stats|mission|my-progress|navigator|repo-catalog|slug-mapping|tag-labels|topics-gallery|topics-tree|topics)(/.*)?(\\?.*)?$", "target": "/build/$1$2$3", "destination": "srv-api", "authenticationType": "none" @@ -582,7 +582,8 @@ "cacheControl": "public, max-age=3600" }, { "source": "^/browse/?(\\?.*)?$", "target": "/content/pages/browse/$1", "destination": "srv-api", "authenticationType": "none" }, - { "source": "^/topics/?(\\?.*)?$", "target": "/content/pages/topics/$1", "destination": "srv-api", "authenticationType": "none" }, + { "source": "^/topics/?(\\?.*)?$", "target": "/content/topics-index$1", "destination": "srv-api", "authenticationType": "none" }, + { "source": "^/topics/(.*)$", "target": "/content/topics/$1", "destination": "srv-api", "authenticationType": "none" }, { "source": "^/tutorial-navigator/?(\\?.*)?$", "target": "/content/pages/tutorial-navigator/$1", "destination": "srv-api", "authenticationType": "none" }, { "source": "^/developer-advocates/?(\\?.*)?$", "target": "/content/pages/developer-advocates/$1", "destination": "srv-api", "authenticationType": "none" }, { "source": "^/developer-advocates/([^/?]+)/?(\\?.*)?$", "target": "/content/developer-advocates/$1", "destination": "srv-api", "authenticationType": "none" }, diff --git a/db/last-dev/csn.json b/db/last-dev/csn.json index c574809bc..ea9751704 100644 --- a/db/last-dev/csn.json +++ b/db/last-dev/csn.json @@ -1075,6 +1075,21 @@ "length": 50, "@cds.persistence.name": "ROLE" }, + "login": { + "type": "cds.String", + "length": 255, + "@cds.persistence.name": "LOGIN" + }, + "avatarUrl": { + "type": "cds.String", + "length": 1024, + "@cds.persistence.name": "AVATARURL" + }, + "profileUrl": { + "type": "cds.String", + "length": 1024, + "@cds.persistence.name": "PROFILEURL" + }, "user_ID": { "type": "cds.String", "length": 36, @@ -1457,6 +1472,66 @@ }, "@cds.persistence.name": "COM_SAP_DEVELOPERS_IMS_DEVELOPERENVIRONMENTLINKS" }, + "com.sap.developers.ims.TutorialValidationRules": { + "kind": "entity", + "@cds.persistence.journal": true, + "elements": { + "tutorial_ID": { + "type": "cds.String", + "length": 36, + "@odata.foreignKey4": "tutorial", + "key": true, + "@cds.persistence.name": "TUTORIAL_ID" + }, + "stepNumber": { + "key": true, + "type": "cds.Integer", + "@cds.persistence.name": "STEPNUMBER" + }, + "questionId": { + "key": true, + "type": "cds.String", + "length": 100, + "@cds.persistence.name": "QUESTIONID" + }, + "questionText": { + "type": "cds.String", + "length": 2000, + "@cds.persistence.name": "QUESTIONTEXT" + }, + "ruleType": { + "type": "cds.String", + "length": 50, + "@cds.persistence.name": "RULETYPE" + }, + "questionType": { + "type": "cds.String", + "length": 20, + "@cds.persistence.name": "QUESTIONTYPE" + }, + "choiceMode": { + "type": "cds.String", + "length": 20, + "@cds.persistence.name": "CHOICEMODE" + }, + "options": { + "type": "cds.LargeString", + "@cds.persistence.name": "OPTIONS" + }, + "correctAnswer": { + "type": "cds.LargeString", + "@cds.persistence.name": "CORRECTANSWER" + }, + "aiGrading": { + "type": "cds.Boolean", + "default": { + "val": false + }, + "@cds.persistence.name": "AIGRADING" + } + }, + "@cds.persistence.name": "COM_SAP_DEVELOPERS_IMS_TUTORIALVALIDATIONRULES" + }, "com.sap.developers.ims.Puzzles": { "kind": "entity", "@assert.unique.slug": [ diff --git a/db/persistence.cds b/db/persistence.cds index 97d8c91c8..1ba6e23f8 100644 --- a/db/persistence.cds +++ b/db/persistence.cds @@ -50,3 +50,4 @@ annotate ims.CatGameAwards with @cds.persistence.journal; // #2042 Hit-the-Cat // #805 — Observability annotate ims.MetricSnapshots with @cds.persistence.journal; annotate ims.PublishTimings with @cds.persistence.journal; +annotate ims.TutorialValidationRules with @cds.persistence.journal; diff --git a/db/schema.cds b/db/schema.cds index 07ffce7df..e1d48af79 100644 --- a/db/schema.cds +++ b/db/schema.cds @@ -453,6 +453,9 @@ entity TutorialContributors : cuid, LegacyKeyed { name : String(255); email : String(255); role : String(50); + login : String(255); // GitHub handle + avatarUrl : String(1024); // https://github.com/.png + profileUrl : String(1024); // https://github.com/ user : Association to Users; } @@ -872,6 +875,23 @@ entity ValidateAnswerSpecs : managed { aiGrading : Boolean default false; } +// Full parsed rules.vr rule set for a tutorial, persisted at publish time. +// Unlike ValidateAnswerSpecs (AI-graded only), this holds ALL rule types so +// the admin Validation Questions facet can display the complete rule set. +// options and correctAnswer are JSON-serialised; null when not applicable. +entity TutorialValidationRules { + key tutorial : Association to Tutorials; + key stepNumber : Integer; + key questionId : String(100); + questionText : String(2000); + ruleType : String(50); // single-choice | multiple-choice | regex | exact-match | ... + questionType : String(20); // MCQ | TEXT + choiceMode : String(20); // single | multiple | null + options : LargeString; // JSON array of option strings (MCQ) or null + correctAnswer: LargeString; // reference answer (client-graded) or null when aiGrading + aiGrading : Boolean default false; +} + // Every learner submission. Drives offline grader-quality evaluation. // 'verdict' allows 'error' as a server-side outcome value (the LLM JSON // schema only emits 'pass' | 'partial' | 'fail'). diff --git a/db/src/ADVOCATE_VALUE_HELP_V1.hdbview b/db/src/ADVOCATE_VALUE_HELP_V1.hdbview new file mode 100644 index 000000000..787d75aca --- /dev/null +++ b/db/src/ADVOCATE_VALUE_HELP_V1.hdbview @@ -0,0 +1,21 @@ +-- Cross-container value-help view for the Devtoberfest planner's Session.host +-- field (#2102). Exposes ACTIVE developer advocates as a suggestion list: +-- a display NAME (firstName + ' ' + lastName), the name parts for OrderBy, the +-- advocate TITLE, and the linked user's EMAIL for disambiguation. +-- +-- All output columns are aliased UPPERCASE so the consumer synonym resolves +-- (camelCase aliases pass this build but fail the consumer resolve with +-- "invalid column name" — cross-container column-case rule). +-- +-- Granted via tutorial_reader / tutorial_reader# (one role, many views). +VIEW "ADVOCATE_VALUE_HELP_V1" AS + SELECT A."ID" AS "ID", + A."FIRSTNAME" || ' ' || A."LASTNAME" AS "NAME", + A."FIRSTNAME" AS "FIRSTNAME", + A."LASTNAME" AS "LASTNAME", + A."TITLE" AS "TITLE", + U."EMAIL" AS "EMAIL" + FROM "COM_SAP_DEVELOPERS_IMS_ADVOCATES" A + LEFT OUTER JOIN "COM_SAP_DEVELOPERS_IMS_USERS" U + ON A."USER_ID" = U."ID" + WHERE A."ISACTIVE" = TRUE diff --git a/db/src/com.sap.developers.ims.TutorialContributors.hdbmigrationtable b/db/src/com.sap.developers.ims.TutorialContributors.hdbmigrationtable index 4d27bb3be..8d7511e59 100644 --- a/db/src/com.sap.developers.ims.TutorialContributors.hdbmigrationtable +++ b/db/src/com.sap.developers.ims.TutorialContributors.hdbmigrationtable @@ -1,4 +1,4 @@ -== version=2 +== version=3 COLUMN TABLE com_sap_developers_ims_TutorialContributors ( ID NVARCHAR(36) NOT NULL, legacyId INTEGER, @@ -6,10 +6,17 @@ COLUMN TABLE com_sap_developers_ims_TutorialContributors ( name NVARCHAR(255), email NVARCHAR(255), role NVARCHAR(50), + login NVARCHAR(255), + avatarUrl NVARCHAR(1024), + profileUrl NVARCHAR(1024), user_ID NVARCHAR(36), PRIMARY KEY(ID) ) +== migration=3 +-- generated by cds-compiler version 7.0.1 +ALTER TABLE com_sap_developers_ims_TutorialContributors ADD (login NVARCHAR(255), avatarUrl NVARCHAR(1024), profileUrl NVARCHAR(1024)); + == migration=2 -- generated by cds-compiler version 6.9.0 ALTER TABLE com_sap_developers_ims_TutorialContributors ADD (user_ID NVARCHAR(36)); diff --git a/db/src/com.sap.developers.ims.TutorialValidationRules.hdbmigrationtable b/db/src/com.sap.developers.ims.TutorialValidationRules.hdbmigrationtable new file mode 100644 index 000000000..99ba6938b --- /dev/null +++ b/db/src/com.sap.developers.ims.TutorialValidationRules.hdbmigrationtable @@ -0,0 +1,14 @@ +== version=1 +COLUMN TABLE com_sap_developers_ims_TutorialValidationRules ( + tutorial_ID NVARCHAR(36) NOT NULL, + stepNumber INTEGER NOT NULL, + questionId NVARCHAR(100) NOT NULL, + questionText NVARCHAR(2000), + ruleType NVARCHAR(50), + questionType NVARCHAR(20), + choiceMode NVARCHAR(20), + options NCLOB, + correctAnswer NCLOB, + aiGrading BOOLEAN DEFAULT FALSE, + PRIMARY KEY(tutorial_ID, stepNumber, questionId) +) diff --git a/db/src/tutorial_reader.hdbrole b/db/src/tutorial_reader.hdbrole index a2f1b25b1..1fe21dd21 100644 --- a/db/src/tutorial_reader.hdbrole +++ b/db/src/tutorial_reader.hdbrole @@ -3,7 +3,8 @@ "name": "tutorial_reader", "object_privileges": [ { "name": "TUTORIAL_VALUE_HELP_V1", "type": "VIEW", "privileges": [ "SELECT" ] }, - { "name": "TASK_VALUE_HELP_V1", "type": "VIEW", "privileges": [ "SELECT" ] } + { "name": "TASK_VALUE_HELP_V1", "type": "VIEW", "privileges": [ "SELECT" ] }, + { "name": "ADVOCATE_VALUE_HELP_V1", "type": "VIEW", "privileges": [ "SELECT" ] } ] } } diff --git a/db/src/tutorial_reader_grantable.hdbrole b/db/src/tutorial_reader_grantable.hdbrole index 8cac93aad..e950dbe82 100644 --- a/db/src/tutorial_reader_grantable.hdbrole +++ b/db/src/tutorial_reader_grantable.hdbrole @@ -3,7 +3,8 @@ "name": "tutorial_reader#", "object_privileges": [ { "name": "TUTORIAL_VALUE_HELP_V1", "type": "VIEW", "privileges_with_grant_option": [ "SELECT" ] }, - { "name": "TASK_VALUE_HELP_V1", "type": "VIEW", "privileges_with_grant_option": [ "SELECT" ] } + { "name": "TASK_VALUE_HELP_V1", "type": "VIEW", "privileges_with_grant_option": [ "SELECT" ] }, + { "name": "ADVOCATE_VALUE_HELP_V1", "type": "VIEW", "privileges_with_grant_option": [ "SELECT" ] } ] } } diff --git a/db/tutorial-assets.cds b/db/tutorial-assets.cds index d030f5a54..a07a94de7 100644 --- a/db/tutorial-assets.cds +++ b/db/tutorial-assets.cds @@ -11,6 +11,7 @@ entity TutorialAssets { channel : String(8); // 'prod' | 'qa' contentHash : String(64); // sha-256 of stored bytes mimeType : String(128); + byteSize : Integer64; // original byte length captured at ingest filename : String(255); // for Content-Disposition content : Composition of many Attachments; } diff --git a/db/tutorial-images.cds b/db/tutorial-images.cds index 91a2f799d..903da7199 100644 --- a/db/tutorial-images.cds +++ b/db/tutorial-images.cds @@ -12,5 +12,6 @@ entity TutorialImages { channel : String(8); // 'prod' | 'qa' contentHash : String(64); // sha-256 of the stored original mimeType : String(128); + byteSize : Integer64; // original byte length captured at ingest content : Composition of many Attachments; } diff --git a/docs/developers/reference/tutorials-ims-gotchas.md b/docs/developers/reference/tutorials-ims-gotchas.md index 9fcb47ddc..bb4a08286 100644 --- a/docs/developers/reference/tutorials-ims-gotchas.md +++ b/docs/developers/reference/tutorials-ims-gotchas.md @@ -102,3 +102,48 @@ Cross-references: ## Tutorial Navigator - **Navigator "Featured" rail is curated via `/admin-ui/#/operations` → Featured Tasks** — draft CRUD (pick items by title via `FeaturedTaskCandidates` value-help, unique per item, order defaults to next integer); SSR from `browse.json`'s `featured[]` array (mission-curated or first-6-missions fallback when empty); live-rehydrated from `GET /build/featured` (ETag/304, 60s server cache, mixed tutorial/mission/group types); cache busts automatically on `FeaturedTasks` save/delete via `resetFeaturedCache()`. + +--- + +# Top Gotchas — full detail (relocated from CLAUDE.md) + +These paragraphs used to live inline in `CLAUDE.md`'s "Top Gotchas" section. They were moved here verbatim to keep `CLAUDE.md` lean; the headline + load-bearing one-liner + a link back to this section remain in `CLAUDE.md`. + +## Build artifacts & lifecycle hooks + +- **`ignore-scripts=true` silences `postbuild:apps` — build artifacts wired into it are NOT produced by local `npm run build:all`** — the global npmrc `ignore-scripts=true` means npm **lifecycle hooks never fire**. The `postbuild:apps` hook is where the #1604 island-fingerprint step (`build:island-manifest`, which writes `hugo/data/island_manifest.json`) and 8 static guards live. During a local `build:all`, none of them run. Symptom class: fresh JS/CSS **compiles** (Vite emits `navigator-.js`) but is **never referenced** — `hugo/layouts/partials/island-src.html` falls back to the unhashed `/js/.js`, Hugo bakes the stale path, and the approuter ships old bundles sitting next to the new ones. **Merged fixes look "not deployed" even though the deploy succeeded.** CI dodges this because `deploy.yml`/`unit-tests.yml` run `npm run postbuild:apps` as an **explicit step** (see deploy.yml:217-223 comment). Fix (2026-08-10): `build:all` now calls `npm run build:island-manifest` **explicitly** (not via the hook), and `scripts/deploy-mta.cjs` Step 2.5 fails the deploy if `hugo/public/index.html` bakes only unhashed island paths while a Vite manifest exists. Rule: any build **artifact** (not just a guard) needed for a correct ship must be an explicit step in `build:all`, never left to a `post*`/`pre*` lifecycle hook. +- **`build:page-fallback` is an explicit `build:all` step (NOT a lifecycle hook)** — `scripts/build-page-fallback.cjs` copies in-scope page snapshots from `hugo/public` into `srv/page-fallback/.` after `build:hugo` runs. Because `ignore-scripts=true` silences all `pre*`/`post*` hooks, it is wired as an explicit `npm run build:page-fallback` in the `build:all` chain, positioned right after `build:hugo`. If you add a new in-scope page to `IN_SCOPE_PAGES` in `srv/lib/page-key-map.js`, the snapshot is picked up automatically on the next full build. Snapshots are gitignored (`srv/page-fallback/*` except `.gitkeep`); the directory is committed empty and populated at build time. + +## Completions rollup (issue #1934) + +- **Group/Mission completions are rollup-derived** — the CAP rewrite never carried over the legacy IMS TUTORIAL→GROUP→MISSION rollup, so GROUP/MISSION `TaskRecords` stopped being created at the 2026-08-10 cutover. `srv/lib/completion-rollup.js` recomputes parent group(s)/mission(s) after any TUTORIAL/PUZZLE/CHECKPOINT/PETOBERFEST completion — called from `_updateTutorialProgress`, `resetTutorialProgress`, the CHECKPOINT edge of `createTaskRecord` (developer-service), `puzzle-service`, and `petoberfest-upload`. Slot model: alt-groups (`#172`) collapse to one slot where **any branch satisfies**; a nested GROUP slot needs all its tutorials. Records key on `(user_ID, taskLegacyId=.legacyId, taskType)` and are upserted (SELECT-then-UPDATE-or-INSERT) with `stampSubmissionId` so they carry the NGDS dedup key. NGDS auto-send fires on the → COMPLETED edge (GROUP/MISSION are the only NGDS-eligible rollup types). The orchestrator never throws into the completion tx. Backfill (post-cutover only): `scripts/backfill-group-mission-completions.mjs` (bulk, `--dry-run`/`--since`/`--user`, no NGDS send) then `scripts/backfill-ngds-send.mjs` (rate-limited, resumable via `ImsConfig 'ngds.backfill.cursor'`, honors env=prod + kill-switch + epoch + canonical-sapId; receiver dedups on `submissionIdCompleted`). Pre-cutover completions are intentionally NOT re-minted (legacy IMS credited them; the NGDS epoch guard suppresses them). `completion-rollup.js` is NOT a `content-store.js` dependency → no `srv-qa` `cp` entry needed. + +## Content model — mutable ContentCurrent (Option B, #2017 / Workstream D) + +- **Content served from mutable `ContentCurrent`, not the old `ContentFiles` snapshot-per-version** — three env flags gate the migration, all default OFF, flip in order: `CONTENT_DELTA_WRITE_ENABLED` (publish dual-writes changed slugs → `ContentCurrent` + append-only `ContentHistory`, fail-safe) → **seed** via a full force rebuild (`-f mode=full -f force-publish=true` dual-writes all slugs; no separate migration) → `CONTENT_DELTA_READ_ENABLED` (serve/readers hit `ContentCurrent`, per-slug fallback to `ContentFiles`) → `CONTENT_DELTA_SKIP_CARRYFORWARD` (publish skips `carryForwardUnchanged` → **O(changed) publish**; rollback then **replays `ContentHistory`** into `ContentCurrent`, not clear+fallback). Measured DEV: publish commit ~62s→973ms (PROD carry-forward was ~95s @ 11k files). Serve source header `X-Content-Source: db-current` (ContentCurrent) vs `db` (legacy). Revert = flip `SKIP_CARRYFORWARD` off + `cf restart`. Flags in `srv/lib/feature-flags/registry.js`; readers via `resolveContentBlob` (`srv/lib/content-store.js`). LOB reads stay raw `db.run()`. Cutover cleanup (delete `carryForwardUnchanged` + `ContentFiles`) deferred to a release after PROD soak. + +## NGDS auto-send + +- **NGDS auto-send is PROD-only + DB-gated (double gate)** — automatic push of task completions to NGDS (badging/gamification) fires from `srv/lib/ngds-autosend.js#maybeAutoSendCompletion`, called at the two completion transition points in `srv/developer-service.js` (`_updateTutorialProgress` + `createTaskRecord`). Sends ONLY when BOTH gates pass: (1) CF `space_name==='prod'` (`resolveDeployEnvironment`, not spoofable headers) AND (2) `ImsConfig` key `ngds.autosend.enabled==='true'` (admin kill-switch via `AdminService.toggleNgdsAutoSend`; 60s flag cache, busted on toggle). Edge-only (fires on the transition → COMPLETED, never on repeat saves) and task-type-allowlisted to TUTORIAL/GROUP/MISSION (legacy parity — PUZZLE/PETOBERFEST/STEP never sent). Fails CLOSED (config read error → disabled) and never throws into the completion tx (a send fault queues in `NGDSFailedMessages` for the 2h retry job). Bulk recompute (raw HANA MERGE) + migration (raw SQL) bypass the service layer, so they cannot flood NGDS. Payload shape itself is the legacy `MessageModel` (#1473). Default OFF in every env; enable in PROD via the admin toggle. Status: `AdminService.getNgdsAutoSendConfig()` returns `{enabled, environment, effective}`. + +## Knowledge graph feature flags + +All default OFF and DEV-only unless noted. Toggles fail-open on every fault path. + +- **`KG_PAGERANK_ENABLED` (issue #916)** — when `'true'`, `rankNeighborhood` in `srv/knowledge-graph-service.js` multiplicatively blends per-tutorial PageRank (`weight *= 1 + α × normPR`) into all three tutorial-targeted arms (`prerequisitesOf`, `sharedConcepts`, `whatToLearnNext`) and sorts `teaches` by concept-side PageRank. Scores recomputed nightly at 03:53 UTC by `srv/jobs/kg-pagerank-job.js` — PageRank runs in **Node.js** (not HANA GraphScript — that engine ships no PageRank primitive) over `KG_PG_VERTICES_V` + `KG_PG_EDGES_V`, materialized into `ConceptRank`/`TutorialRank` sidecars. Fail-opens on every fault path (missing sidecars, HANA hiccup, empty maps → multiplier collapses to 1.0). Toggle: `cf set-env tutorials-srv KG_PAGERANK_ENABLED true && cf restart tutorials-srv`. Blend strength via `KG_PAGERANK_ALPHA` (default `1.0` → weights grow at most 2×). +- **`KG_WCC_ISOLATION_THRESHOLD` (issue #918)** — nightly `srv/jobs/kg-wcc-job.js` runs at 04:07 UTC and materializes rows into `KgIsolation` for concept + tutorial vertices whose weakly-connected-component size ≤ threshold. Default `1`; `0` empties the table on the next run (effectively disables the "Isolated" red-badge column on the admin Concepts + Tutorials LRs). Compute is Node.js union-find over `KG_PG_VERTICES_V` + `KG_PG_EDGES_V` — same reason as #916 that HANA GraphScript ships no WCC primitive (SCC yes, WCC no). Fail-quiet at read time: the `after('READ')` decorators on `KnowledgeGraphService.Concepts` and `AdminService.Tutorials` catch any SELECT throw and leave `isolated` unset — Fiori renders `null` boolean as no badge. Toggle: `cf set-env tutorials-srv KG_WCC_ISOLATION_THRESHOLD 2 && cf restart tutorials-srv` (or `0` to disable). +- **`KG_ONDEMAND_ENABLED` / `KnowledgeGraphSettings.onDemandExtractionEnabled` (issue #948)** — when `true`, `expandSearchConcepts` fire-and-forgets an enqueue on zero-seed queries; a new 2-minute cron `kg-ondemand-drain` cosine-ranks the corpus and extracts concepts from top-K tutorials via `extractConceptsFromTutorial`. Coalesces near-duplicate queries; per-user (default 3/hr) and global (default 20/hr) rate-limit caps. Env knobs: `KG_ONDEMAND_USER_MAX_PER_HOUR`, `KG_ONDEMAND_GLOBAL_MAX_PER_HOUR`, `KG_ONDEMAND_DRAIN_BATCH` (default 3), `KG_ONDEMAND_TUTORIALS_PER_REQ` (default 5), `KG_ONDEMAND_MAX_ATTEMPTS` (default 3). Admin surface: `/admin-ui/#kgOnDemand`. Drain uses try/finally to recover stuck RUNNING rows on UPDATE failure. On-demand extraction is now **link-only** (#1115) — it attaches existing concepts (0.7 floor) but never mints. Toggle: flip `onDemandExtractionEnabled=true` at `/admin-ui/#kg-settings` (or `cf set-env tutorials-srv KG_ONDEMAND_ENABLED true && cf restart tutorials-srv`). +- **KG community detection (issue #917)** — Louvain community detection over `KG_PG_WORKSPACE` runs nightly at 03:57 UTC (`srv/jobs/kg-communities-job.js`) via HANA GraphScript `Communities_Louvain` in `db/src/procedures/KG_LOUVAIN_GRAPH.hdbprocedure`. Memberships materialize into the `KgCommunity` sidecar (`db/knowledge-graph-communities.cds`). Admin surface: `/admin-ui/#kgCommunities` renders a FE List Report (aggregated summary) + Object Page over `AdminService.KgCommunities` and `AdminService.KgCommunityMembers`. `promoteCommunityToMission(communityId, missionSlug, title)` action (SuperAdmin-gated) drafts a `Missions` row + `CompletionPaths` + `CompletionPathItems` sorted `Tutorials.title ASC`, with `Missions.sourceKgCommunityId` set so already-promoted communities can be filtered out. Nightly job fail-opens; empty sidecar renders as FE "No data", never a 500. **No env flag** — tile is always visible to XSUAA `Tutorial.Author` scope. **DEV-only in v1**; PROD rollout deferred. Metrics: `kg_communities_{duration_ms,count,max_size,failures}`. +- **`KG_RETIRE_ORPHANS_ENABLED` / `KG_RETIRE_ORPHANS_AGE_DAYS` (issue #1115)** — nightly `srv/jobs/kg-retire-orphans-job.js` at 04:37 UTC flips `Concepts.status` ACTIVE→RETIRED for concepts with zero links across all 10 link tables and `firstSeenAt` older than `KG_RETIRE_ORPHANS_AGE_DAYS` (default 14). RETIRED falls out of every read path (all filter `status='ACTIVE'` positively). First-run retirement **ramps** rather than purging instantly. Reversible: `cf set-env tutorials-srv KG_RETIRE_ORPHANS_ENABLED false` (off) or bulk `UPDATE Concepts SET status='ACTIVE' WHERE status='RETIRED'` (data revert). A re-proposed retired slug is reactivated in-tx by `kg-merge-on-write.js` (`retiredBySlug` + `action:'reactivated'`). +- **KG community peers + community labeling (issue #1126)** — `communityPeersEnabled` on `ChatSettings` (default `false`) gates the `findCommunityPeers` Joule tool (`srv/lib/kg/joule-tool-community-peers.js`). When enabled, the tool accepts a `tutorial_slug`, looks up the anchor's `communityFingerprint` in `KgCommunity`, and returns sibling tutorials from the same Louvain cluster plus the LLM-generated cluster label from `KgCommunityLabel`. Nightly `kg-community-labels` job (`srv/jobs/kg-community-label-job.js`) runs at 04:12 UTC (after Louvain at 03:57) and LLM-names each community with ≥ 2 tutorials. Identity key is `communityFingerprint` (String(64)); skip-key is `memberSlugsHash` (SHA-256 of sorted slugs) — stable member sets incur zero LLM calls. Daily budget is `communityLabelLlmBudgetPerDay` on `ChatSettings` (default 50). Fail-open per community. Toggle: `communityPeersEnabled` is a `ChatSettings` column (NOT an env var), enabled by an Admin via `PATCH /admin/ChatSettings()` on the AdminService singleton (the `/admin-ui/#joule` Joule Settings page edits the same entity but does not yet surface this flag). **DEV-only until PROD Louvain data verifies.** Metrics: `kg_community_label_{duration_ms,labeled,skipped,failures}`. +- **`KG_COMMUNITY_WEIGHT` (issue #1171)** — when `> 0`, `SearchService.before('READ')` appends a SECOND additive rank term `+ KG_COMMUNITY_WEIGHT * (case slug when '' then 1.0 else 0 end)` alongside the existing concept-overlap `KG_WEIGHT` (#945). Peers are tutorials sharing a Louvain `communityFingerprint` (#917/#1126) with the top-`COMMUNITY_TOP_K` (5) concept-overlap hits. Default `0` (OFF) → `buildCommunityRankFragment` in `srv/lib/search-kg-signal.js` short-circuits before any DB fetch and the rank SQL is byte-identical to the #945 formula. **Only fires when `ChatSettings.searchKgRerankEnabled=true`**. Fail-open (any DB throw → term collapses to `''`). Membership fetched packet-safe (≤5 fingerprints `.in()`, members capped 200, filtered in Node). Regression harness + churn report at `test/harness/community-rank-churn*`; do NOT enable in any env before the ON-vs-OFF churn is hand-reviewed. Toggle: `cf set-env tutorials-srv KG_COMMUNITY_WEIGHT 1.5 && cf restart tutorials-srv` (with `searchKgRerankEnabled=true`). +- **`KG_COMMUNITY_COVERAGE_NUDGE_THRESHOLD` (issue #1172)** — the `after('READ','KgCommunities')` decorator in `srv/admin-service.js` computes, per community at read time, mission-coverage % + dominant published mission + orphan-tutorial count (helper: `srv/lib/kg-community-coverage.js`) and populates virtual fields on `AdminService.KgCommunities`. Coverage is **published-missions-only** and the % denominator is **tutorial members only** (concept/tag-only communities render N/A, not 0%). `coverageHigh` (`>= threshold`, default **70**) is the single server-computed flag driving both the LR criticality badge and the FE promote-time `MessageBox.warning` ("~X% already in `` — extend instead?"). **Fail-quiet** in its own try/catch: any throw → warn-log, fields unset, no badge, never a 500. No new job/table/migration — computed live. Packet-safe: the covered-slug `.in()` is chunked at 500. SuperAdmin gate on `promoteCommunityToMission` unchanged; the nudge is advisory. Override: `cf set-env tutorials-srv KG_COMMUNITY_COVERAGE_NUDGE_THRESHOLD 80 && cf restart tutorials-srv`. DEV-only until the #1126 PROD Louvain rollout lands. +- **Cluster-level Q&A in Joule (issue #1173)** — `describeCommunity` Joule tool (`srv/lib/kg/joule-tool-describe-community.js`) answers "what's the AI cluster?" / "everything around RAP" by resolving a free-text topic to a labeled Louvain community. **LLM-side matching:** `communityCatalogLayer` in `srv/lib/chat-context.js` injects the labeled-cluster catalog (from `KgCommunityLabel`, cached ~5min, cap 40) into the learner system prompt **only when `communityPeersEnabled` is true**; the model passes the chosen label as `matched_label`, and `matchLabel` (`srv/lib/kg/community-label-match.js`, pure) does case-insensitive exact match + token-overlap fallback + ambiguity detection. Reuses the existing `communityPeersEnabled` flag (NO new flag/schema), the `community-peers-cards` SSE frame + `renderCommunityPeersCards` render path, and the extracted `resolveCommunityMembers` helper (`srv/lib/kg/community-members.js`). Fail-open throughout (never 500). **Gotcha:** `buildSystemPromptLines` in `chat-orchestrator.js` is DEAD at runtime — `buildSystemPrompt` (chat-context.js) never calls it; the live guidance ships via `communityCatalogLayer`. DEV-only until PROD Louvain data verifies. + +## HCQL protocol adapter (issue #995) + +- **HCQL protocol adapter** — CAP 10 beta feature. `@hcql` annotation on 9 read-heavy services (AdminService, AuthorService, AnalyticsService, ExportsService, ConsolidationService, KnowledgeGraphService, HomepageService, SearchService, DeveloperService) in `srv/hcql-enablement.cds` exposes each service at its existing OData URL to also accept CQN `SELECT` bodies (HCQL and OData share URLs; dispatch is by request-body shape). Auth inherited from existing `@readonly`/`@requires`. Writes intentionally unsupported (beta not stable cross-runtime). **Runtime hazard:** CAP 10.0.3 exits the process on malformed CQN — do not expose to untrusted clients until CAP hardens the adapter. Kill switch: delete `srv/hcql-enablement.cds`, `cds build --production`, redeploy. Full reference: [hcql-support.md](hcql-support.md). + +## Freshness detector + +- **Freshness detector grounding needs the corpus-embedding backfill** — the `checkFreshness`/`freshness-scan` engine cosine-searches `ApiDocs`/`Samples` embeddings. Those columns are populated by `srv/jobs/freshness-corpus-embedding-job.js` (nightly `17 3` + on-demand `runJob`). Until it runs in an env, grounding returns nothing and every API-obsolescence claim degrades to `confidence: Low` (fail-open, by design). LLM calls use the SAP AI SDK directly (`@sap-ai-sdk/orchestration`, forced tool-call), NOT `@cap-js/ai`; unit tests inject `globalThis.__FRESHNESS_TEST_IMPL__`. Bulk scan gated by `FRESHNESS_SCAN_ENABLED` (default OFF). **Tutorial markdown is sourced from `ContentFiles.sourceContent` via `getTutorialSource(slug)` in `srv/lib/content-store.js` — NOT from `Steps.description`** (Steps are never populated with step markdown; reading Steps would yield nothing). Findings carry a **global `codeBlockIndex`** across the whole-tutorial markdown — per-step attribution is deferred because the persisted source is not split per step. diff --git a/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase1-empty-facets.md b/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase1-empty-facets.md new file mode 100644 index 000000000..b2ee34f24 --- /dev/null +++ b/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase1-empty-facets.md @@ -0,0 +1,1215 @@ +# Tutorials Admin OP — Phase 1 (Empty Facets) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Populate the three empty Tutorials Object Page facets — Categories, Contributors, Validation Questions — by closing the fetch/publish pipeline gaps that leave their backing entities unpopulated. + +**Architecture:** Reuse the existing "fetch-sidecar → publish aux step → server REPLACE-per-slug inside `cds.tx()`" pattern (proven by `validate-answer-spec-publish.js`). Contributors and full validation rules get new `..json` cache sidecars written in the same `fetch-tutorials.ts` per-tutorial pass, POSTed by new client collectors in `publish-content.ts`, and upserted by new `srv/lib/*-publish.js` handlers. Categories are fixed by a fire-and-forget `classifyAndPersist('tutorial', id)` call after publish upserts a tutorial (publish bypasses the CAP `after('CREATE')` hook), plus a one-time backfill. + +**Tech Stack:** SAP CAP (Node.js, CDS), HANA (prod) / SQLite (unit), Fiori Elements annotations (`app/admin-annotations.cds`), Vitest (unit + hybrid projects). + +**Spec:** `docs/superpowers/specs/2026-08-31-tutorials-admin-op-enhancements-design.md` + +## Global Constraints + +- **Slugs are lowercase-canonical** — lowercase in both sidecar filename and JSON body; server resolves tutorial by lowercased slug. +- **REPLACE-per-slug** — publishing slug A must never touch slug B's rows. DELETE-by-`tutorial_ID` then INSERT inside `cds.tx()`. +- **srv-qa cp-list** — every new `srv/lib/*.js` MUST be appended to the `cp` command in `.deploy/mta.yaml` module `tutorials-srv-qa` (~line 175), or QA boot fails with `MODULE_NOT_FOUND`. +- **QA namespace guard** — new publish routes apply the `entity_not_in_model` fail-fast guard and skip the aux step on `channel === 'qa'` if the QA CDS model lacks the entity. +- **Never throw into the publish/completion tx** — fire-and-forget classification must `.catch(warn)`. +- **Schema changes** — `cds build --production`; register new persisted entities in `db/persistence.cds` (`@cds.persistence.journal`); never hand-author `.hdbmigrationtable`; run `npx cds deploy --to sqlite::memory:` before committing db changes. +- **BLOB reads stay raw `db.run()`** — never mix LOB + metadata in one CDS QL query (not expected in Phase 1). +- **Aux publish steps are non-fatal** — a sidecar publish failure warns, never fails the deploy. +- **Tests:** unit via `npm test` (in-memory SQLite); hybrid via `npm run test:hybrid` (real HANA, `--project hybrid`). Bare `vitest ` skips hybrid setup. +- **PR targets DEV, never main.** + +--- + +## File Structure + +**WS1 — Categories self-heal + backfill:** +- Modify: `srv/lib/content-publish-session.js` — add fire-and-forget `classifyAndPersist` loop over touched `tutorialIds` after the metadata/authorship block. +- Test: `test/unit/publish-category-selfheal.test.js`, `test/hybrid/publish-categories.test.js`. +- Ops: run `scripts/backfill-categories.cjs` (existing). + +**WS2 — Contributors:** +- Modify: `db/schema.cds` — add `login`/`avatarUrl`/`profileUrl` to `TutorialContributors`. +- Modify: `db/persistence.cds` — ensure `TutorialContributors` journaled. +- Modify: `scripts/fetch-tutorials.ts` — write `.contributors.json` sidecar. +- Create: `scripts/publish/publish-contributors.ts` (client collector), wired into `scripts/publish-content.ts`. +- Create: `srv/lib/contributors-publish.js` (server REPLACE handler). +- Modify: `srv/server.js` — mount the publish route. +- Modify: `app/admin-annotations.cds` — add `login` (GitHub link) + `avatarUrl` to `TutorialContributors` `@UI.LineItem`. +- Modify: `.deploy/mta.yaml` — add `contributors-publish.js` to srv-qa cp-list. +- Test: `test/unit/contributors-publish.test.js`, `test/hybrid/publish-contributors.test.js`. + +**WS3 — All validation rules:** +- Modify: `db/schema.cds` — add `TutorialValidationRules` entity. +- Modify: `db/persistence.cds` — journal `TutorialValidationRules`. +- Modify: `srv/admin-service.cds` — projection + `validationRules` association on `Tutorials`. +- Modify: `scripts/parsers/rules.ts` — add `collectAllRules()` alongside `collectAiGradedSpecs()`. +- Modify: `scripts/fetch-tutorials.ts` — write `.validation-rules.json` sidecar. +- Create: `scripts/publish/publish-validation-rules.ts` + wire into `publish-content.ts`. +- Create: `srv/lib/validation-rules-publish.js`. +- Modify: `srv/server.js` — mount route. +- Modify: `app/admin-annotations.cds` — relabel AI facet "AI-Graded Validation"; add "All Validation Rules" facet + LineItem. +- Modify: `.deploy/mta.yaml` — add `validation-rules-publish.js` to srv-qa cp-list. +- Test: `test/unit/collect-all-rules.test.js`, `test/unit/validation-rules-publish.test.js`, `test/hybrid/publish-validation-rules.test.js`. + +--- + +## WS1 — Categories: self-heal at publish + backfill + +### Task 1: Fire-and-forget classification after publish upsert + +**Files:** +- Modify: `srv/lib/content-publish-session.js` (~line 198-210, after `upsertTutorialMetadata`/`linkTutorialAuthorship`; `tutorialIds` already collected/returned per research) +- Test: `test/unit/publish-category-selfheal.test.js` + +**Interfaces:** +- Consumes: `classifyAndPersist(kind, id, _opts?)` — named export from `srv/lib/category-classifier.js:127`. +- Produces: nothing new; side effect is `TutorialCategories` rows for published tutorials. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/publish-category-selfheal.test.js +import { describe, it, expect, vi, beforeEach } from 'vitest' + +// Mock the classifier so the test asserts invocation without real embeddings/LLM. +const classifySpy = vi.fn().mockResolvedValue(undefined) +vi.mock('../../srv/lib/category-classifier.js', () => ({ + classifyAndPersist: classifySpy, +})) + +import { classifyTouchedTutorials } from '../../srv/lib/content-publish-session.js' + +describe('publish category self-heal', () => { + beforeEach(() => classifySpy.mockClear()) + + it('classifies every touched tutorial id, fire-and-forget', async () => { + await classifyTouchedTutorials(['id-a', 'id-b']) + expect(classifySpy).toHaveBeenCalledTimes(2) + expect(classifySpy).toHaveBeenCalledWith('tutorial', 'id-a') + expect(classifySpy).toHaveBeenCalledWith('tutorial', 'id-b') + }) + + it('never rejects even if a classification throws', async () => { + classifySpy.mockRejectedValueOnce(new Error('boom')) + await expect(classifyTouchedTutorials(['id-a'])).resolves.toBeUndefined() + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run test/unit/publish-category-selfheal.test.js --project unit` +Expected: FAIL — `classifyTouchedTutorials` is not exported. + +- [ ] **Step 3: Implement the helper + call site** + +In `srv/lib/content-publish-session.js`, add near the top (after existing imports): + +```js +const { classifyAndPersist } = require('./category-classifier.js') + +// Exported for unit testing; classifies touched tutorials without ever throwing +// into the publish tx (publish bypasses the CAP after('CREATE') classifier hook). +async function classifyTouchedTutorials(tutorialIds) { + await Promise.all( + (tutorialIds || []).map((id) => + Promise.resolve() + .then(() => classifyAndPersist('tutorial', id)) + .catch((e) => console.warn('[publish] category classify skipped', id, e?.message)), + ), + ) +} +module.exports.classifyTouchedTutorials = classifyTouchedTutorials +``` + +> Match the file's existing module system. Research shows `category-classifier.js` uses ES named exports; if `content-publish-session.js` is CommonJS, use dynamic `import()` inside the helper instead of top-level `require`. Verify the first two lines of `content-publish-session.js` before choosing. + +Then at the post-metadata call site (~line 198-210, where `tutorialIds` is in scope): + +```js +// Fire-and-forget: keep categories populated for publish-created tutorials. +classifyTouchedTutorials(tutorialIds) +``` + +(No `await` — must not block or fail the publish.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run test/unit/publish-category-selfheal.test.js --project unit` +Expected: PASS (both cases). + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/content-publish-session.js test/unit/publish-category-selfheal.test.js +git commit -m "feat(publish): self-heal categories via classifyAndPersist after upsert (#WS1)" +``` + +### Task 2: Hybrid guard — published tutorial gets categories + +**Files:** +- Test: `test/hybrid/publish-categories.test.js` + +**Interfaces:** +- Consumes: real `AdminService` + publish session against HANA (via `cds bind --exec`). + +- [ ] **Step 1: Write the hybrid test** + +```js +// test/hybrid/publish-categories.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' + +describe('publish populates categories (hybrid)', () => { + let db + beforeAll(async () => { db = await cds.connect.to('db') }) + + it('a freshly published tutorial has >= 0 category rows and no orphan write errors', async () => { + // Precondition: category seed embeddings must exist in this env. + const { Categories } = cds.entities('com.sap.developers.ims') + const seeds = await db.run(SELECT.from(Categories)) + expect(seeds.length).toBeGreaterThan(0) // else run embedAllSeeds first + + // Assert the classifier is reachable and idempotent for a known slug. + // (Use a slug known to exist in the bound DB.) + const { Tutorials, TutorialCategories } = cds.entities('com.sap.developers.ims') + const t = await db.run(SELECT.one.from(Tutorials).columns('ID', 'slug')) + expect(t).toBeTruthy() + const rows = await db.run(SELECT.from(TutorialCategories).where({ tutorial_ID: t.ID })) + expect(Array.isArray(rows)).toBe(true) + }) +}) +``` + +- [ ] **Step 2: Run the hybrid test** + +Run: `npm run test:hybrid -- test/hybrid/publish-categories.test.js` +Expected: PASS if seed embeddings exist. If `seeds.length === 0`, run the `embedAllSeeds` admin action first, then re-run. + +- [ ] **Step 3: Commit** + +```bash +git add test/hybrid/publish-categories.test.js +git commit -m "test(publish): hybrid guard for category population (#WS1)" +``` + +### Task 3: One-time backfill (ops step — documented, not code) + +**Files:** none (uses existing `scripts/backfill-categories.cjs`). + +- [ ] **Step 1: Confirm seed embeddings exist** — via admin action `embedAllSeeds` on the target env, or query `Categories` seed rows. +- [ ] **Step 2: Dry-run then run backfill** + +Run (against bound env): `node scripts/backfill-categories.cjs --dry-run` then without the flag. +Expected: rows inserted into `TutorialCategories` for previously-empty tutorials. + +- [ ] **Step 3: Spot-check in admin UI** — open the reference tutorial's Categories facet; confirm rows render. + +--- + +## WS2 — Contributors: map git list + GitHub links + +### Task 4: Schema — add GitHub columns to TutorialContributors + +**Files:** +- Modify: `db/schema.cds:451-457` (`TutorialContributors`) +- Modify: `db/persistence.cds` (ensure journaled) +- Test: `test/unit/schema-contributors.test.js` + +**Interfaces:** +- Produces: `TutorialContributors` now has `login : String(255)`, `avatarUrl : String(1024)`, `profileUrl : String(1024)`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/schema-contributors.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' + +describe('TutorialContributors schema', () => { + let m + beforeAll(async () => { m = await cds.load('*') }) + it('has GitHub link columns', () => { + const e = m.definitions['com.sap.developers.ims.TutorialContributors'] + expect(e.elements.login).toBeTruthy() + expect(e.elements.avatarUrl).toBeTruthy() + expect(e.elements.profileUrl).toBeTruthy() + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npx vitest run test/unit/schema-contributors.test.js --project unit` +Expected: FAIL — elements undefined. + +- [ ] **Step 3: Add the columns** + +In `db/schema.cds`, extend the `TutorialContributors` entity body: + +```cds +entity TutorialContributors : cuid, LegacyKeyed { + tutorial : Association to Tutorials; + name : String(255); + email : String(255); + role : String(50); + login : String(255); // GitHub handle + avatarUrl : String(1024); // https://github.com/.png + profileUrl : String(1024); // https://github.com/ + user : Association to Users; +} +``` + +Confirm `db/persistence.cds` journals `TutorialContributors` (add `@cds.persistence.journal` registration entry if a new column set requires a migration table — follow the existing entries' shape). + +- [ ] **Step 4: Verify schema + deploy dry-run** + +Run: `npx vitest run test/unit/schema-contributors.test.js --project unit` +Then: `npx cds deploy --to sqlite::memory:` +Expected: test PASS; deploy succeeds with no errors. + +- [ ] **Step 5: Build migration + commit** + +```bash +npx cds build --production +git add db/schema.cds db/persistence.cds db/src/gen test/unit/schema-contributors.test.js +git commit -m "feat(db): add GitHub link columns to TutorialContributors (#WS2)" +``` + +### Task 5: Fetch sidecar — write `.contributors.json` + +**Files:** +- Modify: `scripts/fetch-tutorials.ts` (~line 1043, beside the validate-answer sidecar write; `contributors` array in scope from ~`:936-948`) +- Test: `test/unit/contributors-sidecar.test.js` + +**Interfaces:** +- Produces: cache file `.contributors.json` = `{ slug, contributors: Array<{login,name,email,avatarUrl}> }` (max 10). + +- [ ] **Step 1: Write the failing test** (extract a pure helper to keep it testable) + +```js +// test/unit/contributors-sidecar.test.js +import { describe, it, expect } from 'vitest' +import { buildContributorsSidecar } from '../../scripts/parsers/contributors-sidecar' + +describe('buildContributorsSidecar', () => { + it('lowercases slug and caps at 10', () => { + const contribs = Array.from({ length: 12 }, (_, i) => ({ + login: `u${i}`, name: `N${i}`, email: `${i}@x.com`, avatarUrl: `a${i}`, + })) + const out = buildContributorsSidecar('My-Slug', contribs) + expect(out.slug).toBe('my-slug') + expect(out.contributors).toHaveLength(10) + expect(out.contributors[0]).toEqual({ login: 'u0', name: 'N0', email: '0@x.com', avatarUrl: 'a0' }) + }) + it('returns null when no contributors', () => { + expect(buildContributorsSidecar('s', [])).toBeNull() + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npx vitest run test/unit/contributors-sidecar.test.js --project unit` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement the helper + wire the write** + +Create `scripts/parsers/contributors-sidecar.ts`: + +```ts +export interface SidecarContributor { login: string; name: string; email: string; avatarUrl: string } +export interface ContributorsSidecar { slug: string; contributors: SidecarContributor[] } + +export function buildContributorsSidecar( + slug: string, + contributors: Array>, +): ContributorsSidecar | null { + if (!contributors || contributors.length === 0) return null + return { + slug: slug.toLowerCase(), + contributors: contributors.slice(0, 10).map((c) => ({ + login: c.login ?? '', name: c.name ?? '', email: c.email ?? '', avatarUrl: c.avatarUrl ?? '', + })), + } +} +``` + +In `scripts/fetch-tutorials.ts`, beside the validate-answer write (~`:1043`): + +```ts +import { buildContributorsSidecar } from './parsers/contributors-sidecar' +// ... +const contribSidecar = buildContributorsSidecar(t.slug, contributors) +if (contribSidecar) { + writeFileSync( + join(CACHE_DIR, `${t.slug.toLowerCase()}.contributors.json`), + JSON.stringify(contribSidecar, null, 2), + ) +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `npx vitest run test/unit/contributors-sidecar.test.js --project unit` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/parsers/contributors-sidecar.ts scripts/fetch-tutorials.ts test/unit/contributors-sidecar.test.js +git commit -m "feat(fetch): write contributors sidecar from git contributor list (#WS2)" +``` + +### Task 6: Server handler — REPLACE contributors per slug + +**Files:** +- Create: `srv/lib/contributors-publish.js` +- Modify: `srv/server.js` (mount route, mirror validate-answer mount) +- Modify: `.deploy/mta.yaml` (~line 175, srv-qa cp-list) +- Test: `test/unit/contributors-publish.test.js` + +**Interfaces:** +- Consumes: POST body `{ slug, contributors: [{login,name,email,avatarUrl}] }`. +- Produces: `publishContributors(req, res)` Express handler; REPLACE-by-`tutorial_ID` in `TutorialContributors`. + +- [ ] **Step 1: Write the failing test** (mirror `validate-answer-spec-publish` test shape, in-memory SQLite) + +```js +// test/unit/contributors-publish.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +import { replaceContributorsForSlug } from '../../srv/lib/contributors-publish.js' + +describe('replaceContributorsForSlug', () => { + let db + beforeAll(async () => { + await cds.test('serve', '--in-memory').in(process.cwd()) + db = await cds.connect.to('db') + }) + + it('replaces rows for the slug and derives profileUrl', async () => { + const { Tutorials, TutorialContributors } = cds.entities('com.sap.developers.ims') + const ID = cds.utils.uuid() + await db.run(INSERT.into(Tutorials).entries({ ID, slug: 'demo', title: 'Demo' })) + + await replaceContributorsForSlug(db, 'DEMO', [ + { login: 'octocat', name: 'Octo Cat', email: 'o@x.com', avatarUrl: 'https://github.com/octocat.png' }, + ]) + let rows = await db.run(SELECT.from(TutorialContributors).where({ tutorial_ID: ID })) + expect(rows).toHaveLength(1) + expect(rows[0].login).toBe('octocat') + expect(rows[0].profileUrl).toBe('https://github.com/octocat') + + // Second publish REPLACES, does not append. + await replaceContributorsForSlug(db, 'demo', [ + { login: 'hubot', name: 'Hubot', email: 'h@x.com', avatarUrl: 'https://github.com/hubot.png' }, + ]) + rows = await db.run(SELECT.from(TutorialContributors).where({ tutorial_ID: ID })) + expect(rows).toHaveLength(1) + expect(rows[0].login).toBe('hubot') + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npx vitest run test/unit/contributors-publish.test.js --project unit` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement the handler** + +Create `srv/lib/contributors-publish.js` (mirror `validate-answer-spec-publish.js` structure — namespace guard, per-slug REPLACE in `cds.tx()`): + +```js +const cds = require('@sap/cds') +const NS = 'com.sap.developers.ims' +const MAX_FIELD_BYTES = 2000 + +function githubProfileUrl(login) { + return login ? `https://github.com/${login}` : null +} + +// Core, unit-testable: REPLACE all contributor rows for one slug. +async function replaceContributorsForSlug(db, slug, contributors) { + const { Tutorials, TutorialContributors } = cds.entities(NS) + const lcSlug = String(slug || '').toLowerCase() + const tut = await db.run(SELECT.one.from(Tutorials).columns('ID').where({ slug: lcSlug })) + if (!tut) return { ok: false, reason: 'tutorial_not_found', slug: lcSlug } + + const entries = (contributors || []) + .filter((c) => c && (c.login || c.name || c.email)) + .slice(0, 10) + .map((c) => ({ + ID: cds.utils.uuid(), + tutorial_ID: tut.ID, + login: (c.login || '').slice(0, 255), + name: (c.name || '').slice(0, 255), + email: (c.email || '').slice(0, 255), + avatarUrl: (c.avatarUrl || '').slice(0, 1024), + profileUrl: githubProfileUrl(c.login), + })) + + await cds.tx(async (tx) => { + await tx.run(DELETE.from(TutorialContributors).where({ tutorial_ID: tut.ID })) + if (entries.length) await tx.run(INSERT.into(TutorialContributors).entries(entries)) + }) + return { ok: true, slug: lcSlug, count: entries.length } +} + +// Express handler mirroring validate-answer-spec-publish route. +async function publishContributors(req, res) { + try { + const { slug, contributors } = req.body || {} + if (!slug || !Array.isArray(contributors)) { + return res.status(400).json({ error: 'bad_request', detail: 'expected { slug, contributors[] }' }) + } + let entities + try { entities = cds.entities(NS) } catch { entities = null } + if (!entities || !entities.TutorialContributors) { + return res.status(409).json({ error: 'entity_not_in_model' }) + } + const db = await cds.connect.to('db') + const result = await replaceContributorsForSlug(db, slug, contributors) + if (!result.ok) return res.status(404).json(result) + return res.json(result) + } catch (e) { + return res.status(500).json({ error: 'internal', detail: e?.message }) + } +} + +module.exports = { replaceContributorsForSlug, publishContributors, githubProfileUrl } +``` + +Mount in `srv/server.js` beside the validate-answer route (guard with the same `CONTENT_API_KEY` middleware the other publish routes use): + +```js +const { publishContributors } = require('./lib/contributors-publish.js') +app.post('/content/publish-contributors', requireContentApiKey, express.json({ limit: '1mb' }), publishContributors) +``` + +> Verify the exact auth-middleware name and JSON body parser used by the existing `/content/publish` + validate-answer routes and match it. + +- [ ] **Step 4: Run to verify it passes** + +Run: `npx vitest run test/unit/contributors-publish.test.js --project unit` +Expected: PASS (both replace + derive assertions). + +- [ ] **Step 5: Add to srv-qa cp-list** + +In `.deploy/mta.yaml`, module `tutorials-srv-qa` `cp` command (~line 175), append `../../srv/lib/contributors-publish.js` to the `srv/lib/` copy segment. + +- [ ] **Step 6: Commit** + +```bash +git add srv/lib/contributors-publish.js srv/server.js .deploy/mta.yaml test/unit/contributors-publish.test.js +git commit -m "feat(publish): server REPLACE handler for TutorialContributors (#WS2)" +``` + +### Task 7: Client publish step + wire into publish-content + +**Files:** +- Create: `scripts/publish/publish-contributors.ts` +- Modify: `scripts/publish-content.ts` (~line 1327-1352, beside `publishValidateAnswerSpecs`; non-fatal aux step) +- Test: `test/unit/publish-contributors-client.test.js` + +**Interfaces:** +- Consumes: cache dir globbed for `*.contributors.json`; POSTs each to `/content/publish-contributors`. +- Produces: `publishContributors({ cacheDir, baseUrl, apiKey })`. + +- [ ] **Step 1: Write the failing test** (mock `fetch`) + +```js +// test/unit/publish-contributors-client.test.js +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { publishContributors } from '../../scripts/publish/publish-contributors' + +describe('publishContributors client', () => { + let dir + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'contrib-')) + writeFileSync(join(dir, 'demo.contributors.json'), + JSON.stringify({ slug: 'demo', contributors: [{ login: 'octocat', name: 'O', email: 'o@x', avatarUrl: 'a' }] })) + global.fetch = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ ok: true, count: 1 }) }) + }) + afterEach(() => { vi.restoreAllMocks() }) + + it('POSTs each sidecar to the endpoint', async () => { + const res = await publishContributors({ cacheDir: dir, baseUrl: 'http://x', apiKey: 'k' }) + expect(global.fetch).toHaveBeenCalledTimes(1) + const [url, opts] = global.fetch.mock.calls[0] + expect(url).toBe('http://x/content/publish-contributors') + expect(JSON.parse(opts.body).slug).toBe('demo') + expect(res.published).toBe(1) + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npx vitest run test/unit/publish-contributors-client.test.js --project unit` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement + wire** + +Create `scripts/publish/publish-contributors.ts`: + +```ts +import { readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' + +export async function publishContributors(opts: { cacheDir: string; baseUrl: string; apiKey: string }) { + const { cacheDir, baseUrl, apiKey } = opts + const files = readdirSync(cacheDir).filter((f) => f.endsWith('.contributors.json')) + let published = 0 + for (const f of files) { + const body = readFileSync(join(cacheDir, f), 'utf8') + const res = await fetch(`${baseUrl}/content/publish-contributors`, { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body, + }) + if (res.ok) published += 1 + else console.warn(`[publish-contributors] ${f} -> ${res.status}`) + } + return { published, total: files.length } +} +``` + +> Match the exact auth header name the existing publish client uses (research: validate-answer client — confirm `x-api-key` vs `authorization`). + +In `scripts/publish-content.ts`, beside `publishValidateAnswerSpecs` (~`:1327`), add a non-fatal aux step, skipping QA channel: + +```ts +if (channel !== 'qa') { + try { + const r = await publishContributors({ cacheDir: CACHE_DIR, baseUrl, apiKey }) + console.log(`[publish] contributors: ${r.published}/${r.total}`) + } catch (e) { + console.warn('[publish] contributors step failed (non-fatal)', (e as Error).message) + } +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `npx vitest run test/unit/publish-contributors-client.test.js --project unit` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/publish/publish-contributors.ts scripts/publish-content.ts test/unit/publish-contributors-client.test.js +git commit -m "feat(publish): non-fatal client step to publish contributors sidecars (#WS2)" +``` + +### Task 8: UI — GitHub-linked Contributors LineItem + +**Files:** +- Modify: `app/admin-annotations.cds:715-725` (`TutorialContributors` `@UI.LineItem`) +- Test: `test/unit/annotations-contributors.test.js` + +**Interfaces:** +- Consumes: `TutorialContributors.login`/`avatarUrl`/`profileUrl` (Task 4). + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/annotations-contributors.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' + +describe('Contributors LineItem', () => { + let m + beforeAll(async () => { m = await cds.load('*') }) + it('LineItem includes login column', () => { + const e = m.definitions['AdminService.TutorialContributors'] + const li = e['@UI.LineItem'] + const values = li.map((x) => x.Value?.['='] || x.Value) + expect(values).toContain('login') + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `npx vitest run test/unit/annotations-contributors.test.js --project unit` +Expected: FAIL — `login` not in LineItem. + +- [ ] **Step 3: Add columns + GitHub link** + +In `app/admin-annotations.cds`, extend the `TutorialContributors` `@UI.LineItem` (add `login` and render it as an external link to `profileUrl`): + +```cds +annotate AdminService.TutorialContributors with @( + UI.LineItem: [ + { Value: name, Label: 'Name' }, + { Value: login, Label: 'GitHub', @HTML5.LinkTarget: '_blank' }, + { Value: email, Label: 'Email' }, + { Value: role, Label: 'Role' } + ] +); +annotate AdminService.TutorialContributors with { + login @Common.Text: profileUrl @Common.TextArrangement: #TextOnly; +}; +``` + +> Preferred: make `login` a link via a `DataFieldWithUrl` pointing at `profileUrl` so the cell navigates to `github.com/`: +> ```cds +> { $Type: 'UI.DataFieldWithUrl', Value: login, Url: profileUrl, Label: 'GitHub' } +> ``` +> Use whichever renders as a clickable GitHub link in the current FE version; verify against the running admin UI. + +- [ ] **Step 4: Run to verify it passes** + +Run: `npx vitest run test/unit/annotations-contributors.test.js --project unit` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add app/admin-annotations.cds test/unit/annotations-contributors.test.js +git commit -m "feat(admin-ui): GitHub-linked login column on Contributors table (#WS2)" +``` + +### Task 9: Hybrid guard — publish links contributors to tutorial + +**Files:** +- Test: `test/hybrid/publish-contributors.test.js` + +- [ ] **Step 1: Write the hybrid test** + +```js +// test/hybrid/publish-contributors.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +import { replaceContributorsForSlug } from '../../srv/lib/contributors-publish.js' + +describe('contributors publish (hybrid)', () => { + let db + beforeAll(async () => { db = await cds.connect.to('db') }) + it('links contributor rows to an existing tutorial by slug', async () => { + const { Tutorials, TutorialContributors } = cds.entities('com.sap.developers.ims') + const t = await db.run(SELECT.one.from(Tutorials).columns('ID', 'slug')) + expect(t).toBeTruthy() + await replaceContributorsForSlug(db, t.slug, [ + { login: 'octocat', name: 'Octo', email: 'o@x.com', avatarUrl: 'https://github.com/octocat.png' }, + ]) + const rows = await db.run(SELECT.from(TutorialContributors).where({ tutorial_ID: t.ID, login: 'octocat' })) + expect(rows.length).toBe(1) + expect(rows[0].profileUrl).toBe('https://github.com/octocat') + // cleanup + await db.run(DELETE.from(TutorialContributors).where({ tutorial_ID: t.ID, login: 'octocat' })) + }) +}) +``` + +- [ ] **Step 2: Run** `npm run test:hybrid -- test/hybrid/publish-contributors.test.js` — Expected: PASS. +- [ ] **Step 3: Commit** + +```bash +git add test/hybrid/publish-contributors.test.js +git commit -m "test(publish): hybrid guard for contributor linking (#WS2)" +``` + +--- + +## WS3 — All validation rules + +### Task 10: Schema — `TutorialValidationRules` entity + +**Files:** +- Modify: `db/schema.cds` (add entity near `ValidateAnswerSpecs` ~`:865`) +- Modify: `db/persistence.cds` (journal it) +- Test: `test/unit/schema-validation-rules.test.js` + +**Interfaces:** +- Produces: `com.sap.developers.ims.TutorialValidationRules` with key `(tutorial, stepNumber, questionId)`, fields `questionText`, `ruleType`, `questionType`, `choiceMode`, `options` (LargeString JSON), `correctAnswer`, `aiGrading : Boolean`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/schema-validation-rules.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('TutorialValidationRules schema', () => { + let m + beforeAll(async () => { m = await cds.load('*') }) + it('exists with expected elements', () => { + const e = m.definitions['com.sap.developers.ims.TutorialValidationRules'] + expect(e).toBeTruthy() + for (const k of ['stepNumber','questionId','questionText','ruleType','questionType','choiceMode','options','correctAnswer','aiGrading']) + expect(e.elements[k]).toBeTruthy() + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — `npx vitest run test/unit/schema-validation-rules.test.js --project unit` → FAIL. + +- [ ] **Step 3: Add the entity** + +In `db/schema.cds`: + +```cds +entity TutorialValidationRules { + key tutorial : Association to Tutorials; + key stepNumber : Integer; + key questionId : String(100); + questionText : String(2000); + ruleType : String(50); // single-choice | multiple-choice | regex | exact-match | ... + questionType : String(20); // MCQ | TEXT + choiceMode : String(20); // single | multiple | null + options : LargeString; // JSON array of option strings (MCQ) or null + correctAnswer: LargeString; // reference answer (client-graded) or null when aiGrading + aiGrading : Boolean default false; +} +``` + +Register in `db/persistence.cds` mirroring the existing entries' `@cds.persistence.journal` shape. + +- [ ] **Step 4: Verify + deploy dry-run** — `npx vitest run test/unit/schema-validation-rules.test.js --project unit` then `npx cds deploy --to sqlite::memory:` → PASS + clean deploy. + +- [ ] **Step 5: Build migration + commit** + +```bash +npx cds build --production +git add db/schema.cds db/persistence.cds db/src/gen test/unit/schema-validation-rules.test.js +git commit -m "feat(db): add TutorialValidationRules entity for all rules.vr rules (#WS3)" +``` + +### Task 11: Parser — `collectAllRules()` + +**Files:** +- Modify: `scripts/parsers/rules.ts` (add beside `collectAiGradedSpecs` ~`:312`) +- Test: `test/unit/collect-all-rules.test.js` + +**Interfaces:** +- Consumes: `validationMap`, `ruleTypeByStepAndId`, `correctAnswerByStepAndId` (from `parseRulesVrEnriched`). +- Produces: `collectAllRules(map, ruleTypeMap, answerMap) => Array<{ stepNumber, questionId, questionText, ruleType, questionType, choiceMode, options, correctAnswer, aiGrading }>`. + +- [ ] **Step 1: Write the failing test** + +```ts +// test/unit/collect-all-rules.test.ts +import { describe, it, expect } from 'vitest' +import { collectAllRules } from '../../scripts/parsers/rules' + +describe('collectAllRules', () => { + it('includes non-AI MCQ rules with options + correctAnswer', () => { + const map = new Map([[1, [ + { id: 'validate-1', question: 'Pick one', type: 'QUESTION_TYPE_MCQ', options: ['A','B'], choiceMode: 'single', correctAnswer: 'A' }, + { id: 'validate-1b', question: 'AI graded', type: 'QUESTION_TYPE_TEXT', aiGrading: true }, + ]]]) + const ruleTypeMap = new Map([['1::validate-1', 'single-choice'], ['1::validate-1b', 'regex']]) + const answerMap = new Map([['1::validate-1', 'A']]) + const rows = collectAllRules(map, ruleTypeMap, answerMap) + expect(rows).toHaveLength(2) + const mcq = rows.find((r) => r.questionId === 'validate-1') + expect(mcq.aiGrading).toBe(false) + expect(mcq.questionType).toBe('MCQ') + expect(JSON.parse(mcq.options)).toEqual(['A','B']) + expect(mcq.correctAnswer).toBe('A') + const ai = rows.find((r) => r.questionId === 'validate-1b') + expect(ai.aiGrading).toBe(true) + expect(ai.correctAnswer).toBeNull() + }) +}) +``` + +> Confirm the exact key format of `ruleTypeByStepAndId`/`correctAnswerByStepAndId` in `rules.ts` (research indicated a `step::id` style). Adjust the test's key strings to match the real format before implementing. + +- [ ] **Step 2: Run to verify it fails** — `npx vitest run test/unit/collect-all-rules.test.ts --project unit` → FAIL. + +- [ ] **Step 3: Implement `collectAllRules`** + +In `scripts/parsers/rules.ts` (adapt key access to the confirmed map format): + +```ts +export interface AllRuleRow { + stepNumber: number; questionId: string; questionText: string; + ruleType: string; questionType: 'MCQ' | 'TEXT'; choiceMode: string | null; + options: string | null; correctAnswer: string | null; aiGrading: boolean; +} + +export function collectAllRules( + map: Map, + ruleTypeByStepAndId: Map, + correctAnswerByStepAndId: Map, +): AllRuleRow[] { + const rows: AllRuleRow[] = [] + for (const [stepNumber, questions] of map.entries()) { + for (const q of questions) { + const key = `${stepNumber}::${q.id}` + const isMcq = q.type === 'QUESTION_TYPE_MCQ' + const ai = Boolean((q as any).aiGrading) + rows.push({ + stepNumber, + questionId: q.id, + questionText: q.question, + ruleType: ruleTypeByStepAndId.get(key) ?? '', + questionType: isMcq ? 'MCQ' : 'TEXT', + choiceMode: (q as any).choiceMode ?? null, + options: isMcq && (q as any).options ? JSON.stringify((q as any).options) : null, + correctAnswer: ai ? null : (correctAnswerByStepAndId.get(key) ?? (q as any).correctAnswer ?? null), + aiGrading: ai, + }) + } + } + return rows +} +``` + +- [ ] **Step 4: Run to verify it passes** — `npx vitest run test/unit/collect-all-rules.test.ts --project unit` → PASS. +- [ ] **Step 5: Commit** + +```bash +git add scripts/parsers/rules.ts test/unit/collect-all-rules.test.ts +git commit -m "feat(parser): collectAllRules for full rules.vr rule set (#WS3)" +``` + +### Task 12: Fetch sidecar — `.validation-rules.json` + +**Files:** +- Modify: `scripts/fetch-tutorials.ts` (~`:975` where `validationMap` etc. are destructured; write beside other sidecars ~`:1043`) +- Test: covered by Task 11 helper + Task 13 server test; add a small write-path assertion. + +- [ ] **Step 1: Wire the sidecar write** + +```ts +import { collectAllRules } from './parsers/rules' +// ... +const allRules = collectAllRules(validationMap, ruleTypeByStepAndId, correctAnswerByStepAndId) +if (allRules.length > 0) { + writeFileSync( + join(CACHE_DIR, `${t.slug.toLowerCase()}.validation-rules.json`), + JSON.stringify({ slug: t.slug.toLowerCase(), rules: allRules }, null, 2), + ) +} +``` + +- [ ] **Step 2: Sanity build** — run `npm run fetch-tutorials` for a small subset if a `--slug`/limit flag exists, or type-check: `npx tsc --noEmit -p tsconfig.json` (confirm project has this). Expected: no type errors. +- [ ] **Step 3: Commit** + +```bash +git add scripts/fetch-tutorials.ts +git commit -m "feat(fetch): write validation-rules sidecar (all rule types) (#WS3)" +``` + +### Task 13: Server handler — REPLACE validation rules per slug + +**Files:** +- Create: `srv/lib/validation-rules-publish.js` +- Modify: `srv/server.js` (mount route) +- Modify: `.deploy/mta.yaml` (srv-qa cp-list ~line 175) +- Test: `test/unit/validation-rules-publish.test.js` + +**Interfaces:** +- Consumes: `{ slug, rules: AllRuleRow[] }`. +- Produces: `replaceValidationRulesForSlug(db, slug, rules)` + `publishValidationRules(req, res)`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/validation-rules-publish.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +import { replaceValidationRulesForSlug } from '../../srv/lib/validation-rules-publish.js' + +describe('replaceValidationRulesForSlug', () => { + let db + beforeAll(async () => { await cds.test('serve', '--in-memory').in(process.cwd()); db = await cds.connect.to('db') }) + it('replaces all-rule rows for a slug', async () => { + const { Tutorials, TutorialValidationRules } = cds.entities('com.sap.developers.ims') + const ID = cds.utils.uuid() + await db.run(INSERT.into(Tutorials).entries({ ID, slug: 'vr-demo', title: 'VR' })) + await replaceValidationRulesForSlug(db, 'VR-DEMO', [ + { stepNumber: 1, questionId: 'validate-1', questionText: 'Q', ruleType: 'single-choice', questionType: 'MCQ', choiceMode: 'single', options: '["A","B"]', correctAnswer: 'A', aiGrading: false }, + ]) + let rows = await db.run(SELECT.from(TutorialValidationRules).where({ tutorial_ID: ID })) + expect(rows).toHaveLength(1) + expect(rows[0].aiGrading).toBe(false) + await replaceValidationRulesForSlug(db, 'vr-demo', []) + rows = await db.run(SELECT.from(TutorialValidationRules).where({ tutorial_ID: ID })) + expect(rows).toHaveLength(0) + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — `npx vitest run test/unit/validation-rules-publish.test.js --project unit` → FAIL. + +- [ ] **Step 3: Implement the handler** (mirror `contributors-publish.js`) + +```js +const cds = require('@sap/cds') +const NS = 'com.sap.developers.ims' + +async function replaceValidationRulesForSlug(db, slug, rules) { + const { Tutorials, TutorialValidationRules } = cds.entities(NS) + const lcSlug = String(slug || '').toLowerCase() + const tut = await db.run(SELECT.one.from(Tutorials).columns('ID').where({ slug: lcSlug })) + if (!tut) return { ok: false, reason: 'tutorial_not_found', slug: lcSlug } + const entries = (rules || []).map((r) => ({ + tutorial_ID: tut.ID, + stepNumber: r.stepNumber, + questionId: String(r.questionId).slice(0, 100), + questionText: (r.questionText || '').slice(0, 2000), + ruleType: (r.ruleType || '').slice(0, 50), + questionType: (r.questionType || '').slice(0, 20), + choiceMode: r.choiceMode || null, + options: r.options || null, + correctAnswer: r.correctAnswer ?? null, + aiGrading: Boolean(r.aiGrading), + })) + await cds.tx(async (tx) => { + await tx.run(DELETE.from(TutorialValidationRules).where({ tutorial_ID: tut.ID })) + if (entries.length) await tx.run(INSERT.into(TutorialValidationRules).entries(entries)) + }) + return { ok: true, slug: lcSlug, count: entries.length } +} + +async function publishValidationRules(req, res) { + try { + const { slug, rules } = req.body || {} + if (!slug || !Array.isArray(rules)) return res.status(400).json({ error: 'bad_request' }) + let entities; try { entities = cds.entities(NS) } catch { entities = null } + if (!entities || !entities.TutorialValidationRules) return res.status(409).json({ error: 'entity_not_in_model' }) + const db = await cds.connect.to('db') + const result = await replaceValidationRulesForSlug(db, slug, rules) + return res.status(result.ok ? 200 : 404).json(result) + } catch (e) { return res.status(500).json({ error: 'internal', detail: e?.message }) } +} + +module.exports = { replaceValidationRulesForSlug, publishValidationRules } +``` + +Mount in `srv/server.js`: + +```js +const { publishValidationRules } = require('./lib/validation-rules-publish.js') +app.post('/content/publish-validation-rules', requireContentApiKey, express.json({ limit: '4mb' }), publishValidationRules) +``` + +- [ ] **Step 4: Run to verify it passes** — `npx vitest run test/unit/validation-rules-publish.test.js --project unit` → PASS. +- [ ] **Step 5: Add to srv-qa cp-list** — append `../../srv/lib/validation-rules-publish.js` to the `.deploy/mta.yaml` srv-qa `cp` command (~line 175). +- [ ] **Step 6: Commit** + +```bash +git add srv/lib/validation-rules-publish.js srv/server.js .deploy/mta.yaml test/unit/validation-rules-publish.test.js +git commit -m "feat(publish): server REPLACE handler for TutorialValidationRules (#WS3)" +``` + +### Task 14: Client publish step for validation rules + +**Files:** +- Create: `scripts/publish/publish-validation-rules.ts` +- Modify: `scripts/publish-content.ts` (beside contributors aux step) +- Test: `test/unit/publish-validation-rules-client.test.js` + +**Interfaces:** +- Produces: `publishValidationRules({ cacheDir, baseUrl, apiKey })` — globs `*.validation-rules.json`, POSTs to `/content/publish-validation-rules`. + +- [ ] **Step 1: Write the failing test** (mirror Task 7 client test, glob `*.validation-rules.json`, endpoint `/content/publish-validation-rules`). + +```js +// test/unit/publish-validation-rules-client.test.js +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { join } from 'node:path'; import { tmpdir } from 'node:os' +import { publishValidationRules } from '../../scripts/publish/publish-validation-rules' + +describe('publishValidationRules client', () => { + let dir + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'vr-')) + writeFileSync(join(dir, 'demo.validation-rules.json'), + JSON.stringify({ slug: 'demo', rules: [{ stepNumber: 1, questionId: 'validate-1' }] })) + global.fetch = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ ok: true }) }) + }) + afterEach(() => vi.restoreAllMocks()) + it('POSTs each sidecar', async () => { + const res = await publishValidationRules({ cacheDir: dir, baseUrl: 'http://x', apiKey: 'k' }) + expect(global.fetch).toHaveBeenCalledTimes(1) + expect(global.fetch.mock.calls[0][0]).toBe('http://x/content/publish-validation-rules') + expect(res.published).toBe(1) + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — → FAIL (module not found). + +- [ ] **Step 3: Implement + wire** (copy `publish-contributors.ts`, swap glob suffix + endpoint): + +```ts +import { readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +export async function publishValidationRules(opts: { cacheDir: string; baseUrl: string; apiKey: string }) { + const { cacheDir, baseUrl, apiKey } = opts + const files = readdirSync(cacheDir).filter((f) => f.endsWith('.validation-rules.json')) + let published = 0 + for (const f of files) { + const res = await fetch(`${baseUrl}/content/publish-validation-rules`, { + method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: readFileSync(join(cacheDir, f), 'utf8'), + }) + if (res.ok) published += 1; else console.warn(`[publish-validation-rules] ${f} -> ${res.status}`) + } + return { published, total: files.length } +} +``` + +In `scripts/publish-content.ts`, add a non-fatal aux step (skip `channel === 'qa'`), mirroring Task 7. + +- [ ] **Step 4: Run to verify it passes** — → PASS. +- [ ] **Step 5: Commit** + +```bash +git add scripts/publish/publish-validation-rules.ts scripts/publish-content.ts test/unit/publish-validation-rules-client.test.js +git commit -m "feat(publish): non-fatal client step to publish validation-rules sidecars (#WS3)" +``` + +### Task 15: Service projection + association + UI facets + +**Files:** +- Modify: `srv/admin-service.cds` (projection + association on `Tutorials`) +- Modify: `app/admin-annotations.cds` (relabel AI facet; add "All Validation Rules" facet + LineItem) +- Test: `test/unit/annotations-validation-rules.test.js` + +**Interfaces:** +- Consumes: `TutorialValidationRules` (Task 10). +- Produces: `AdminService.TutorialValidationRules` (read-only) + `Tutorials.validationRules` association. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/annotations-validation-rules.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('validation rules exposure + facet', () => { + let m + beforeAll(async () => { m = await cds.load('*') }) + it('AdminService exposes TutorialValidationRules read-only', () => { + expect(m.definitions['AdminService.TutorialValidationRules']).toBeTruthy() + }) + it('Tutorials has validationRules association', () => { + expect(m.definitions['AdminService.Tutorials'].elements.validationRules).toBeTruthy() + }) + it('OP facets include an All Validation Rules facet', () => { + const facets = m.definitions['AdminService.Tutorials']['@UI.Facets'] + const ids = facets.map((f) => f.ID) + expect(ids).toContain('AllValidationRulesFacet') + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — → FAIL. + +- [ ] **Step 3: Implement projection + association + facets** + +In `srv/admin-service.cds`: + +```cds +@readonly entity TutorialValidationRules as projection on ims.TutorialValidationRules; +``` + +Add to the `Tutorials` projection body (beside `validationSpecs` ~`:66`): + +```cds +validationRules : Association to many TutorialValidationRules on validationRules.tutorial = $self; +``` + +In `app/admin-annotations.cds`: +1. Relabel the existing AI facet (`ValidationSpecsFacet`, ~`:957`) `Label: 'AI-Graded Validation'`. +2. Add a LineItem + facet: + +```cds +annotate AdminService.TutorialValidationRules with @( + UI.LineItem: [ + { Value: stepNumber, Label: 'Step' }, + { Value: questionText, Label: 'Question' }, + { Value: questionType, Label: 'Type' }, + { Value: ruleType, Label: 'Rule' }, + { Value: aiGrading, Label: 'AI-Graded' }, + { Value: correctAnswer, Label: 'Correct Answer' } + ] +); +``` + +Add to the winning `@UI.Facets` block (~`:948-974`): + +```cds +{ $Type: 'UI.ReferenceFacet', Label: 'All Validation Rules', ID: 'AllValidationRulesFacet', Target: 'validationRules/@UI.LineItem' }, +``` + +- [ ] **Step 4: Run to verify it passes** — `npx vitest run test/unit/annotations-validation-rules.test.js --project unit` → PASS. Then `npx cds deploy --to sqlite::memory:` → clean. +- [ ] **Step 5: Commit** + +```bash +git add srv/admin-service.cds app/admin-annotations.cds test/unit/annotations-validation-rules.test.js +git commit -m "feat(admin-ui): All Validation Rules facet + relabel AI facet (#WS3)" +``` + +### Task 16: Hybrid guard — publish populates all rules + +**Files:** +- Test: `test/hybrid/publish-validation-rules.test.js` + +- [ ] **Step 1: Write the hybrid test** — publish a slug via `replaceValidationRulesForSlug` against HANA, assert both AI and non-AI rows land in `TutorialValidationRules`, and that AI rows still exist in `ValidateAnswerSpecs` (unchanged). Clean up after. + +```js +// test/hybrid/publish-validation-rules.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +import { replaceValidationRulesForSlug } from '../../srv/lib/validation-rules-publish.js' +describe('validation rules publish (hybrid)', () => { + let db; beforeAll(async () => { db = await cds.connect.to('db') }) + it('lands mixed AI + client rules for an existing slug', async () => { + const { Tutorials, TutorialValidationRules } = cds.entities('com.sap.developers.ims') + const t = await db.run(SELECT.one.from(Tutorials).columns('ID','slug')) + await replaceValidationRulesForSlug(db, t.slug, [ + { stepNumber: 99, questionId: 'vr-test-a', questionText: 'client', ruleType: 'single-choice', questionType: 'MCQ', choiceMode: 'single', options: '["A"]', correctAnswer: 'A', aiGrading: false }, + { stepNumber: 99, questionId: 'vr-test-b', questionText: 'ai', ruleType: 'regex', questionType: 'TEXT', choiceMode: null, options: null, correctAnswer: null, aiGrading: true }, + ]) + const rows = await db.run(SELECT.from(TutorialValidationRules).where({ tutorial_ID: t.ID, stepNumber: 99 })) + expect(rows.length).toBe(2) + await db.run(DELETE.from(TutorialValidationRules).where({ tutorial_ID: t.ID, stepNumber: 99 })) + }) +}) +``` + +- [ ] **Step 2: Run** `npm run test:hybrid -- test/hybrid/publish-validation-rules.test.js` → PASS. +- [ ] **Step 3: Commit** + +```bash +git add test/hybrid/publish-validation-rules.test.js +git commit -m "test(publish): hybrid guard for all-rules population (#WS3)" +``` + +--- + +## Final verification (whole phase) + +- [ ] Run full unit suite: `npm test` — Expected: all green. +- [ ] Run affected hybrid tests: `npm run test:hybrid -- test/hybrid/publish-categories.test.js test/hybrid/publish-contributors.test.js test/hybrid/publish-validation-rules.test.js` (requires `cf login` + `cds bind`). +- [ ] `npx cds deploy --to sqlite::memory:` clean (schema sanity). +- [ ] Confirm both new `srv/lib/*-publish.js` files are in the `.deploy/mta.yaml` srv-qa `cp` list. +- [ ] Open a PR targeting **DEV** summarizing WS1-WS3 + the one-time category backfill ops step. + +## Post-deploy validation (DEV) + +- [ ] After DEV deploy + a content publish, open the reference tutorial's OP: + - Categories facet shows rows (post-backfill + self-heal). + - Contributors table shows the git contributor list, each `login` linking to `github.com/`. + - "All Validation Rules" facet shows every rule; "AI-Graded Validation" still shows the AI subset. + +## Self-review notes (author) + +- **Spec coverage:** WS1 (Tasks 1-3), WS2 (Tasks 4-9), WS3 (Tasks 10-16) map to spec §WS1-WS3. WS4 (KG) and WS5 (media+freshness) are deferred to separate plans per the spec's phasing. +- **Assumptions flagged for the implementer to verify against live code before writing:** (a) module system of `content-publish-session.js` (CommonJS vs ESM) for Task 1; (b) exact auth-middleware + header name used by existing publish routes (Tasks 6/7/13/14); (c) exact key format of `ruleTypeByStepAndId`/`correctAnswerByStepAndId` (Task 11); (d) the FE link idiom that renders a clickable GitHub link (Task 8). These are grounded by research but must be confirmed at the touched lines. diff --git a/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase2-media-freshness.md b/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase2-media-freshness.md new file mode 100644 index 000000000..452cfd33d --- /dev/null +++ b/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase2-media-freshness.md @@ -0,0 +1,371 @@ +# Tutorials Admin OP — Phase 2 (Media + Freshness Header) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Surface per-tutorial object-store items (images + assets) with rich detail and a download/preview link, and add a Freshness report facet (last-run/model/cost/status) to the Tutorials Object Page. + +**Architecture:** `TutorialImages`/`TutorialAssets` are persisted but exposed on no service. Add `@readonly` projections on `AdminService` reachable from `Tutorials` via slug/channel associations; the `content : Composition of many Attachments` child auto-exposes with `@cap-js/attachments` media annotations (`@Core.MediaType`/`@Core.ContentDisposition`/`@UI.MediaResource`) so Fiori renders a native download link. Persist `byteSize` at image/asset ingest (already in memory as `buffer.length`). `FreshnessReport` is already exposed — add an association from `Tutorials` and a sorted LineItem facet. + +**Tech Stack:** SAP CAP (Node.js, CDS), `@cap-js/attachments` 4.0.0, Fiori Elements annotations, Vitest. + +**Spec:** `docs/superpowers/specs/2026-08-31-tutorials-admin-op-enhancements-design.md` (§WS5, decisions D1/D2) + +## Global Constraints + +- **D1 (prod S3 binding):** the Media facet works regardless of backing store; the missing `objectstore` resource in `mta.yaml` is a **separate ops task**, out of scope here. Do not add the S3 binding in this plan. +- **D2:** add `byteSize : Integer64`; **defer** width/height. +- **Read-only exposure** — projections are `@readonly`; use `@cds.redirection.target: false` (pattern at `srv/admin-service.cds:114,120`) to avoid stealing association redirects. +- **BLOB reads stay raw `db.run()`** — never mix LOB + metadata in one CDS QL query. +- **Schema changes** — `cds build --production`; register in `db/persistence.cds`; `npx cds deploy --to sqlite::memory:` before committing. +- **Associations on `slug`** — `TutorialImages`/`TutorialAssets` join `Tutorials` on `slug` (unmanaged `on`), filtered `channel = 'prod'`. +- **Tests:** `npm test` (unit), `npm run test:hybrid` (real HANA). PR targets DEV. + +## File Structure + +- Modify: `db/tutorial-images.cds`, `db/tutorial-assets.cds` — add `byteSize`. +- Modify: `db/persistence.cds` — journal (if needed for new column). +- Modify: `srv/lib/image-ingest-handler.js` (~`:77`) + the asset ingest handler — persist `byteSize`. +- Modify: `srv/admin-service.cds` — `@readonly` projections + `images`/`assets`/`freshnessReports` associations on `Tutorials`. +- Modify: `app/admin-annotations.cds` — Media facet (2 LineItems w/ download link + external sourceUrl) + Freshness Reports facet. +- Test: `test/unit/schema-media.test.js`, `test/unit/annotations-media.test.js`, `test/unit/annotations-freshness-facet.test.js`, `test/hybrid/media-exposure.test.js`. + +--- + +### Task 1: Schema — add `byteSize` to images + assets + +**Files:** +- Modify: `db/tutorial-images.cds:7-16`, `db/tutorial-assets.cds:6-16` +- Modify: `db/persistence.cds` +- Test: `test/unit/schema-media.test.js` + +**Interfaces:** +- Produces: `TutorialImages.byteSize : Integer64`, `TutorialAssets.byteSize : Integer64`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/schema-media.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('media byteSize', () => { + let m; beforeAll(async () => { m = await cds.load('*') }) + it('images + assets have byteSize', () => { + expect(m.definitions['com.sap.developers.ims.TutorialImages'].elements.byteSize).toBeTruthy() + expect(m.definitions['com.sap.developers.ims.TutorialAssets'].elements.byteSize).toBeTruthy() + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — `npx vitest run test/unit/schema-media.test.js --project unit` → FAIL. + +- [ ] **Step 3: Add the column** (both files): + +```cds +byteSize : Integer64; // original byte length captured at ingest +``` + +Register in `db/persistence.cds` if the column needs a migration-table entry (follow existing shape). + +- [ ] **Step 4: Verify + deploy dry-run** — test PASS; `npx cds deploy --to sqlite::memory:` clean. +- [ ] **Step 5: Build + commit** + +```bash +npx cds build --production +git add db/tutorial-images.cds db/tutorial-assets.cds db/persistence.cds db/src/gen test/unit/schema-media.test.js +git commit -m "feat(db): add byteSize to TutorialImages/TutorialAssets (#WS5)" +``` + +### Task 2: Persist `byteSize` at ingest + +**Files:** +- Modify: `srv/lib/image-ingest-handler.js` (~`:60-77`, where `contentHash`/`mimeType` are computed and `imageStore.put` is called) +- Modify: the asset ingest handler (per research: `srv/lib/attachment-source-handler.js:118` region / the asset ingest analog — confirm exact put call) +- Modify: `srv/lib/image-store.cjs` + `srv/lib/attachment-store.cjs` — accept/persist `byteSize` on the parent row +- Test: `test/unit/ingest-bytesize.test.js` + +**Interfaces:** +- Consumes: `buffer` at ingest. +- Produces: parent `TutorialImages`/`TutorialAssets` row carries `byteSize = buffer.length`. + +- [ ] **Step 1: Write the failing test** (unit-level against the store put, in-memory SQLite) + +```js +// test/unit/ingest-bytesize.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +const store = require('../../srv/lib/image-store.cjs') + +describe('image store persists byteSize', () => { + beforeAll(async () => { await cds.test('serve', '--in-memory').in(process.cwd()) }) + it('stores buffer length as byteSize', async () => { + const db = await cds.connect.to('db') + const buf = Buffer.from('hello world') + await store.put('https://raw.example/img.png', { buffer: buf, mimeType: 'image/png', contentHash: 'abc', slug: 'demo', channel: 'prod', byteSize: buf.length }) + const { TutorialImages } = cds.entities('com.sap.developers.ims') + const row = await db.run(SELECT.one.from(TutorialImages).where({ sourceUrl: 'https://raw.example/img.png' })) + expect(row.byteSize).toBe(11) + }) +}) +``` + +> Confirm `image-store.cjs` `put()` signature + how it writes the parent row before finalizing the test; adapt arg shape to the real API. + +- [ ] **Step 2: Run to verify it fails** — → FAIL (byteSize null/undefined). + +- [ ] **Step 3: Implement** + +In `srv/lib/image-ingest-handler.js`, pass `byteSize: buffer.length` into the `imageStore.put(...)` options (~`:77`). In `srv/lib/image-store.cjs`, include `byteSize` in the parent-row INSERT/UPSERT. Mirror both for assets (`attachment-store.cjs` + the asset ingest handler). + +- [ ] **Step 4: Run to verify it passes** — → PASS. +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/image-ingest-handler.js srv/lib/image-store.cjs srv/lib/attachment-store.cjs test/unit/ingest-bytesize.test.js +git commit -m "feat(media): persist byteSize at image/asset ingest (#WS5)" +``` + +> If the asset ingest handler lives in a distinct file, add it to this commit and to the srv-qa cp-list if not already present. + +### Task 3: Expose read-only Media projections + associations + +**Files:** +- Modify: `srv/admin-service.cds` +- Test: `test/unit/media-exposure.test.js` + +**Interfaces:** +- Produces: `AdminService.TutorialImages`, `AdminService.TutorialAssets` (`@readonly`); `Tutorials.images`, `Tutorials.assets` associations (join on `slug`, `channel='prod'`). + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/media-exposure.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('media exposure', () => { + let m; beforeAll(async () => { m = await cds.load('*') }) + it('exposes images + assets read-only', () => { + expect(m.definitions['AdminService.TutorialImages']).toBeTruthy() + expect(m.definitions['AdminService.TutorialAssets']).toBeTruthy() + }) + it('Tutorials has images + assets associations', () => { + const t = m.definitions['AdminService.Tutorials'].elements + expect(t.images).toBeTruthy() + expect(t.assets).toBeTruthy() + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — → FAIL. + +- [ ] **Step 3: Implement** + +In `srv/admin-service.cds`: + +```cds +@readonly @cds.redirection.target: false entity TutorialImages as projection on ims.TutorialImages; +@readonly @cds.redirection.target: false entity TutorialAssets as projection on ims.TutorialAssets; +``` + +Add to the `Tutorials` projection body: + +```cds +images : Association to many TutorialImages on images.slug = $self.slug and images.channel = 'prod'; +assets : Association to many TutorialAssets on assets.slug = $self.slug and assets.channel = 'prod'; +``` + +> The `content` Attachments composition auto-exposes when reachable from an exposed entity (@cap-js/attachments relies on this). + +- [ ] **Step 4: Run to verify it passes** — test PASS; `npx cds deploy --to sqlite::memory:` clean. +- [ ] **Step 5: Commit** + +```bash +git add srv/admin-service.cds test/unit/media-exposure.test.js +git commit -m "feat(admin): read-only Media projections + Tutorials associations (#WS5)" +``` + +### Task 4: UI — Media facet with download link + detail + +**Files:** +- Modify: `app/admin-annotations.cds` +- Test: `test/unit/annotations-media.test.js` + +**Interfaces:** +- Consumes: `AdminService.TutorialImages/TutorialAssets` + their `content` media child. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/annotations-media.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('Media facet', () => { + let m; beforeAll(async () => { m = await cds.load('*') }) + it('OP facets include Media', () => { + const ids = m.definitions['AdminService.Tutorials']['@UI.Facets'].map((f) => f.ID) + expect(ids).toContain('MediaImagesFacet') + expect(ids).toContain('MediaAssetsFacet') + }) + it('image LineItem shows sourceUrl + byteSize + mimeType', () => { + const li = m.definitions['AdminService.TutorialImages']['@UI.LineItem'] + const vals = li.map((x) => x.Value?.['='] || x.Value) + for (const c of ['sourceUrl','byteSize','mimeType','contentHash']) expect(vals).toContain(c) + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — → FAIL. + +- [ ] **Step 3: Implement** + +In `app/admin-annotations.cds`, add LineItems (render `sourceUrl` as external link via `DataFieldWithUrl`; the media download comes from the auto-exposed `content` child's ready-made annotations — a nested facet on `content` gives the download link): + +```cds +annotate AdminService.TutorialImages with @( + UI.LineItem: [ + { $Type: 'UI.DataFieldWithUrl', Value: sourceUrl, Url: sourceUrl, Label: 'Source (GitHub)' }, + { Value: mimeType, Label: 'Type' }, + { Value: byteSize, Label: 'Bytes' }, + { Value: contentHash, Label: 'Hash' }, + { Value: channel, Label: 'Channel' } + ] +); +annotate AdminService.TutorialAssets with @( + UI.LineItem: [ + { Value: filename, Label: 'File' }, + { $Type: 'UI.DataFieldWithUrl', Value: sourceUrl, Url: sourceUrl, Label: 'Source (GitHub)' }, + { Value: mimeType, Label: 'Type' }, + { Value: byteSize, Label: 'Bytes' }, + { Value: contentHash, Label: 'Hash' } + ] +); +``` + +Add facets to the winning `@UI.Facets` block: + +```cds +{ $Type: 'UI.ReferenceFacet', Label: 'Images', ID: 'MediaImagesFacet', Target: 'images/@UI.LineItem' }, +{ $Type: 'UI.ReferenceFacet', Label: 'Assets', ID: 'MediaAssetsFacet', Target: 'assets/@UI.LineItem' }, +``` + +> The @cap-js/attachments `content` child ships its own `@UI.LineItem` with a media download link. To surface a clickable download/preview, optionally add a nested facet targeting `images/content/@UI.LineItem` once verified against the running FE version. + +- [ ] **Step 4: Run to verify it passes** — → PASS; `npx cds deploy --to sqlite::memory:` clean. +- [ ] **Step 5: Commit** + +```bash +git add app/admin-annotations.cds test/unit/annotations-media.test.js +git commit -m "feat(admin-ui): Media facets (images/assets) with source link + byte size (#WS5)" +``` + +### Task 5: Freshness Reports facet (header) + +**Files:** +- Modify: `srv/admin-service.cds` — add `freshnessReports` association on `Tutorials` +- Modify: `app/admin-annotations.cds` — LineItem + facet, sorted by `runAt` desc via `@UI.PresentationVariant` +- Test: `test/unit/annotations-freshness-facet.test.js` + +**Interfaces:** +- Consumes: `AdminService.FreshnessReport` (already exposed `srv/admin-service.cds:129`). +- Produces: `Tutorials.freshnessReports` association + `FreshnessReportsFacet`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/annotations-freshness-facet.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('Freshness reports facet', () => { + let m; beforeAll(async () => { m = await cds.load('*') }) + it('Tutorials has freshnessReports association', () => { + expect(m.definitions['AdminService.Tutorials'].elements.freshnessReports).toBeTruthy() + }) + it('OP facets include FreshnessReportsFacet', () => { + const ids = m.definitions['AdminService.Tutorials']['@UI.Facets'].map((f) => f.ID) + expect(ids).toContain('FreshnessReportsFacet') + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — → FAIL. + +- [ ] **Step 3: Implement** + +In `srv/admin-service.cds` `Tutorials` projection: + +```cds +freshnessReports : Association to many FreshnessReport on freshnessReports.tutorial = $self; +``` + +In `app/admin-annotations.cds`: + +```cds +annotate AdminService.FreshnessReport with @( + UI.LineItem: [ + { Value: runAt, Label: 'Run At' }, + { Value: status, Label: 'Status' }, + { Value: model, Label: 'Model' }, + { Value: cost, Label: 'Cost' }, + { Value: openHighCount, Label: 'Open High' }, + { Value: error, Label: 'Error' } + ], + UI.PresentationVariant: { SortOrder: [{ Property: runAt, Descending: true }], Visualizations: ['@UI.LineItem'] } +); +``` + +Add to `@UI.Facets` (place above the existing Freshness findings facet): + +```cds +{ $Type: 'UI.ReferenceFacet', Label: 'Freshness Reports', ID: 'FreshnessReportsFacet', Target: 'freshnessReports/@UI.PresentationVariant' }, +``` + +- [ ] **Step 4: Run to verify it passes** — → PASS; `npx cds deploy --to sqlite::memory:` clean. +- [ ] **Step 5: Commit** + +```bash +git add srv/admin-service.cds app/admin-annotations.cds test/unit/annotations-freshness-facet.test.js +git commit -m "feat(admin-ui): Freshness Reports facet (runAt/model/cost/status) (#WS5)" +``` + +### Task 6: Hybrid guard — media exposure resolves + +**Files:** +- Test: `test/hybrid/media-exposure.test.js` + +- [ ] **Step 1: Write the hybrid test** + +```js +// test/hybrid/media-exposure.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('media exposure (hybrid)', () => { + let admin; beforeAll(async () => { admin = await cds.connect.to('AdminService') }) + it('reads images for a tutorial without LOB errors', async () => { + const t = await admin.run(SELECT.one.from('AdminService.Tutorials').columns('ID','slug')) + expect(t).toBeTruthy() + // metadata-only read (no BLOB mix) + const imgs = await admin.run(SELECT.from('AdminService.TutorialImages').columns('ID','sourceUrl','mimeType','byteSize').where({ slug: t.slug })) + expect(Array.isArray(imgs)).toBe(true) + }) +}) +``` + +- [ ] **Step 2: Run** `npm run test:hybrid -- test/hybrid/media-exposure.test.js` → PASS. +- [ ] **Step 3: Commit** + +```bash +git add test/hybrid/media-exposure.test.js +git commit -m "test(media): hybrid guard for media exposure (#WS5)" +``` + +--- + +## Final verification + +- [ ] `npm test` all green; `npx cds deploy --to sqlite::memory:` clean. +- [ ] Backfill media if needed: `npm run backfill-images` (+ assets) against the env so rows exist. +- [ ] DEV post-deploy: reference tutorial OP shows Images + Assets tables (source link, type, bytes, hash) and a Freshness Reports table sorted newest-first. + +## Self-review notes + +- **Spec coverage:** WS5 §Media (Tasks 1-4, 6) + §Freshness header (Task 5). D1 respected (no S3 binding change). D2 respected (byteSize yes, dimensions no). +- **Verify against live code:** `image-store.cjs`/`attachment-store.cjs` `put()` signatures (Task 2); the asset ingest handler path; FE `DataFieldWithUrl` + media-child download idiom for the current FE version (Task 4). diff --git a/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase3-knowledge-graph.md b/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase3-knowledge-graph.md new file mode 100644 index 000000000..2fec35da6 --- /dev/null +++ b/docs/superpowers/plans/2026-08-31-tutorials-admin-op-phase3-knowledge-graph.md @@ -0,0 +1,268 @@ +# Tutorials Admin OP — Phase 3 (Knowledge-Graph Facet) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Surface per-tutorial knowledge-graph data on the Tutorials Object Page — concepts taught (with confidence), PageRank importance, community label, and co-completed neighbor tutorials — all read-only from data already populated by nightly jobs. + +**Architecture:** These entities (`TutorialConceptLinks`, `TutorialRank`, `CoCompletions`, `KgCommunity`/`KgCommunityLabel`) live on `KnowledgeGraphService`, not `AdminService`. Add `@readonly` projections on `AdminService` + associations from `Tutorials`, then a "Knowledge Graph" facet group. Purely additive and read-only; no pipeline changes. Reads fail-open (any SELECT throw leaves fields unset — FE renders nothing rather than 500), mirroring the existing KG `after('READ')` decorators. + +**Tech Stack:** SAP CAP (Node.js, CDS), Fiori Elements annotations, Vitest. KG data populated by existing nightly jobs (PageRank #916, WCC #918, Louvain #917, community labels #1126). + +**Spec:** `docs/superpowers/specs/2026-08-31-tutorials-admin-op-enhancements-design.md` (§WS4) + +## Global Constraints + +- **Read-only, additive** — `@readonly` projections; `@cds.redirection.target: false` (pattern `srv/admin-service.cds:114,120`) to avoid stealing redirects. +- **Fail-open reads** — if a KG projection can throw at read time, guard with an `after('READ')` that leaves fields unset (mirror `KnowledgeGraphService.Concepts`/`AdminService.Tutorials` isolation decorators). Pure associations/projections need no decorator. +- **No new env flags, no schema, no jobs** — data already materialized. +- **DEV-only until PROD KG data verifies** — same posture as #1126 (PROD Louvain/community data may be sparse; empty facets render as FE "No data", never a 500). +- **Namespace** — KG entities are in the `com.sap.developers.ims` model (via `db/knowledge-graph*.cds`); some carry `@cds.autoexpose:false` (e.g. `TutorialRank`) requiring explicit projection. +- **Tests:** `npm test` (unit), `npm run test:hybrid`. PR targets DEV. + +## File Structure + +- Modify: `srv/admin-service.cds` — `@readonly` projections (`TutorialConceptLinks`, `TutorialRank`, `CoCompletions`) + associations (`conceptLinks`, `rank`, `coCompletions`) on `Tutorials`; community label reachable via existing `KgCommunityMembers` (`:1249`) join. +- Modify: `srv/admin-service.js` — optional fail-open `after('READ','Tutorials')` if a computed field is added (community label flatten). +- Modify: `app/admin-annotations.cds` — Knowledge Graph facet group (concepts taught LineItem, prerequisites, PageRank + community FieldGroup, co-completed neighbors LineItem). +- Test: `test/unit/kg-exposure.test.js`, `test/unit/annotations-kg.test.js`, `test/hybrid/kg-facet.test.js`. + +--- + +### Task 1: Expose read-only KG projections + associations + +**Files:** +- Modify: `srv/admin-service.cds` +- Test: `test/unit/kg-exposure.test.js` + +**Interfaces:** +- Consumes: `ims.TutorialConceptLinks` (`db/knowledge-graph.cds:61-70`, predicate teaches/extends + confidence), `ims.TutorialRank` (`:196-200`, PageRank score, `@cds.autoexpose:false`), `ims.CoCompletions` (`:158-162`, weighted A→B). +- Produces: `AdminService.TutorialConceptLinks`, `AdminService.TutorialRank`, `AdminService.CoCompletions` (`@readonly`); `Tutorials.conceptLinks` (many), `Tutorials.rank` (one), `Tutorials.coCompletions` (many). + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/kg-exposure.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('KG exposure on AdminService', () => { + let m; beforeAll(async () => { m = await cds.load('*') }) + it('exposes concept links, rank, co-completions read-only', () => { + expect(m.definitions['AdminService.TutorialConceptLinks']).toBeTruthy() + expect(m.definitions['AdminService.TutorialRank']).toBeTruthy() + expect(m.definitions['AdminService.CoCompletions']).toBeTruthy() + }) + it('Tutorials carries conceptLinks / rank / coCompletions', () => { + const t = m.definitions['AdminService.Tutorials'].elements + expect(t.conceptLinks).toBeTruthy() + expect(t.rank).toBeTruthy() + expect(t.coCompletions).toBeTruthy() + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — `npx vitest run test/unit/kg-exposure.test.js --project unit` → FAIL. + +> **Before implementing:** confirm exact entity + element names in `db/knowledge-graph.cds` — the join columns on `TutorialConceptLinks` (which side is the tutorial: `tutorial`/`source`), `TutorialRank` key/score field name (`score`/`pagerank`), and `CoCompletions` columns (`tutorialA`/`tutorialB`/`weight`). Adapt the `on` conditions below to the real names. + +- [ ] **Step 3: Implement** + +In `srv/admin-service.cds`: + +```cds +@readonly @cds.redirection.target: false entity TutorialConceptLinks as projection on ims.TutorialConceptLinks; +@readonly @cds.redirection.target: false entity TutorialRank as projection on ims.TutorialRank; +@readonly @cds.redirection.target: false entity CoCompletions as projection on ims.CoCompletions; +``` + +Add to the `Tutorials` projection body (adapt `on` to confirmed column names): + +```cds +conceptLinks : Association to many TutorialConceptLinks on conceptLinks.tutorial = $self; +rank : Association to one TutorialRank on rank.tutorial = $self; +coCompletions : Association to many CoCompletions on coCompletions.tutorialA = $self; +``` + +> If `Tutorials.conceptLinks` is already injected on the db entity (`db/knowledge-graph.cds:94-97`), the `*` in the projection may already carry it — in that case only expose the target projection and drop the redundant association line. Verify with the failing test after each change. + +- [ ] **Step 4: Run to verify it passes** — test PASS; `npx cds deploy --to sqlite::memory:` clean. +- [ ] **Step 5: Commit** + +```bash +git add srv/admin-service.cds test/unit/kg-exposure.test.js +git commit -m "feat(admin): read-only KG projections + Tutorials associations (#WS4)" +``` + +### Task 2: Community label reachable on the OP + +**Files:** +- Modify: `srv/admin-service.cds` (association to community membership/label) + optional `srv/admin-service.js` fail-open decorator +- Test: `test/unit/kg-community-link.test.js` + +**Interfaces:** +- Consumes: `AdminService.KgCommunityMembers` (already exposed `:1249`), `KgCommunityLabel` (`db/knowledge-graph-communities.cds:76-83`, keyed by `communityFingerprint`). +- Produces: a way to show the tutorial's community label on the OP — either an association `communityMembership` on `Tutorials` (many, filtered to this tutorial's slug) rendered as a small LineItem, or a flattened `virtual communityLabel` populated fail-open in `after('READ','Tutorials')`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/kg-community-link.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('community label reachable', () => { + let m; beforeAll(async () => { m = await cds.load('*') }) + it('Tutorials exposes community membership or virtual label', () => { + const t = m.definitions['AdminService.Tutorials'].elements + expect(t.communityMembership || t.communityLabel).toBeTruthy() + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — → FAIL. + +- [ ] **Step 3: Implement (prefer the association — no compute, lowest risk)** + +In `srv/admin-service.cds`, add an association from `Tutorials` to the exposed community-member rows for this tutorial's slug (adapt column names to `KgCommunityMembers`): + +```cds +communityMembership : Association to many KgCommunityMembers on communityMembership.memberSlug = $self.slug; +``` + +> If `KgCommunityMembers` does not carry a `memberSlug`/label directly, add a `virtual communityLabel : String` and populate it fail-open in `srv/admin-service.js` `after('READ','Tutorials')`: +> ```js +> srv.after('READ', 'Tutorials', async (rows) => { +> try { /* look up KgCommunity by slug → KgCommunityLabel; set r.communityLabel */ } +> catch (e) { /* fail-open: leave unset */ } +> }) +> ``` +> Choose the association path if the columns allow; only fall back to the virtual+decorator if a join isn't expressible. + +- [ ] **Step 4: Run to verify it passes** — → PASS; `npx cds deploy --to sqlite::memory:` clean. +- [ ] **Step 5: Commit** + +```bash +git add srv/admin-service.cds srv/admin-service.js test/unit/kg-community-link.test.js +git commit -m "feat(admin): expose community membership/label on Tutorials (#WS4)" +``` + +### Task 3: UI — Knowledge Graph facet group + +**Files:** +- Modify: `app/admin-annotations.cds` +- Test: `test/unit/annotations-kg.test.js` + +**Interfaces:** +- Consumes: Task 1 + Task 2 associations/projections. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/annotations-kg.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('Knowledge Graph facets', () => { + let m; beforeAll(async () => { m = await cds.load('*') }) + it('OP facets include KG facets', () => { + const ids = m.definitions['AdminService.Tutorials']['@UI.Facets'].map((f) => f.ID) + expect(ids).toContain('ConceptsTaughtFacet') + expect(ids).toContain('CoCompletionsFacet') + }) + it('concept links LineItem shows predicate + confidence', () => { + const li = m.definitions['AdminService.TutorialConceptLinks']['@UI.LineItem'] + const vals = li.map((x) => x.Value?.['='] || x.Value) + expect(vals).toContain('predicate') + expect(vals).toContain('confidence') + }) +}) +``` + +- [ ] **Step 2: Run to verify it fails** — → FAIL. + +- [ ] **Step 3: Implement** + +In `app/admin-annotations.cds` (adapt element names to confirmed schema): + +```cds +annotate AdminService.TutorialConceptLinks with @( + UI.LineItem: [ + { Value: concept_ID, Label: 'Concept' }, + { Value: predicate, Label: 'Relation' }, // teaches | extends + { Value: confidence, Label: 'Confidence' } + ] +); +annotate AdminService.CoCompletions with @( + UI.LineItem: [ + { Value: tutorialB_ID, Label: 'Also Completed' }, + { Value: weight, Label: 'Weight' } + ] +); +``` + +Add a "Knowledge Graph" FieldGroup for PageRank + community, and facets to the winning `@UI.Facets` block: + +```cds +annotate AdminService.Tutorials with @( + UI.FieldGroup #KnowledgeGraph: { Data: [ + { Value: rank.score, Label: 'PageRank' }, + { Value: communityLabel, Label: 'Community' } // or a nested membership ref + ]} +); +// facets: +{ $Type: 'UI.ReferenceFacet', Label: 'Knowledge Graph', ID: 'KgFieldsFacet', Target: '@UI.FieldGroup#KnowledgeGraph' }, +{ $Type: 'UI.ReferenceFacet', Label: 'Concepts Taught', ID: 'ConceptsTaughtFacet', Target: 'conceptLinks/@UI.LineItem' }, +{ $Type: 'UI.ReferenceFacet', Label: 'Co-Completed', ID: 'CoCompletionsFacet', Target: 'coCompletions/@UI.LineItem' }, +``` + +> `rank.score` path-navigation in a FieldGroup requires the to-one `rank` association from Task 1; if FE rejects the deep path, expose a flattened `virtual pageRank : Decimal` on `Tutorials` populated fail-open in `after('READ')` instead. + +- [ ] **Step 4: Run to verify it passes** — → PASS; `npx cds deploy --to sqlite::memory:` clean. +- [ ] **Step 5: Commit** + +```bash +git add app/admin-annotations.cds test/unit/annotations-kg.test.js +git commit -m "feat(admin-ui): Knowledge Graph facet (concepts/PageRank/community/co-completions) (#WS4)" +``` + +### Task 4: Hybrid guard — KG facet reads fail-open + +**Files:** +- Test: `test/hybrid/kg-facet.test.js` + +- [ ] **Step 1: Write the hybrid test** + +```js +// test/hybrid/kg-facet.test.js +import { describe, it, expect, beforeAll } from 'vitest' +import cds from '@sap/cds' +describe('KG facet reads (hybrid)', () => { + let admin; beforeAll(async () => { admin = await cds.connect.to('AdminService') }) + it('reads a tutorial with KG associations expanded without error', async () => { + const t = await admin.run(SELECT.one.from('AdminService.Tutorials').columns('ID','slug')) + expect(t).toBeTruthy() + const links = await admin.run(SELECT.from('AdminService.TutorialConceptLinks').where({ tutorial_ID: t.ID })) + expect(Array.isArray(links)).toBe(true) // may be empty in DEV — that's fine + const co = await admin.run(SELECT.from('AdminService.CoCompletions').limit(1)) + expect(Array.isArray(co)).toBe(true) + }) +}) +``` + +- [ ] **Step 2: Run** `npm run test:hybrid -- test/hybrid/kg-facet.test.js` → PASS (empty arrays acceptable). +- [ ] **Step 3: Commit** + +```bash +git add test/hybrid/kg-facet.test.js +git commit -m "test(kg): hybrid guard for KG facet reads (#WS4)" +``` + +--- + +## Final verification + +- [ ] `npm test` all green; `npx cds deploy --to sqlite::memory:` clean. +- [ ] DEV post-deploy: reference tutorial OP shows a Knowledge Graph facet — concepts taught with confidence, PageRank + community label, co-completed neighbors. Empty sections render as FE "No data" (never a 500). +- [ ] PR targets DEV. Note: DEV-only until PROD Louvain/community data verifies (#1126 posture). + +## Self-review notes + +- **Spec coverage:** WS4 concepts/prerequisites (Task 1/3), PageRank (Task 1/3), community label (Task 2/3), co-completed neighbors (Task 1/3). +- **Verify against live code before writing each task:** exact `TutorialConceptLinks` join column (`tutorial` vs `source`) + `predicate`/`confidence` names; `TutorialRank` score field name; `CoCompletions` column names (`tutorialA`/`tutorialB`/`weight`); whether `Tutorials.conceptLinks` is already carried via the db-entity injection (`db/knowledge-graph.cds:94-97`); whether `KgCommunityMembers` exposes a slug/label to associate on. Each is grounded by the inventory research but must be confirmed at the touched lines. Prefer associations over computed virtuals; only add a fail-open `after('READ')` decorator where a join isn't expressible. diff --git a/docs/superpowers/plans/2026-09-01-tag-tree-topics.md b/docs/superpowers/plans/2026-09-01-tag-tree-topics.md new file mode 100644 index 000000000..6856c3ec4 --- /dev/null +++ b/docs/superpowers/plans/2026-09-01-tag-tree-topics.md @@ -0,0 +1,1371 @@ +# Tag-Tree Topics Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Rebuild `/topics/` as a first-class CAP-served subsystem with the same technical quality as `/concepts/` — stable SAP-tag-hierarchy slugs, a server-rendered tree index + progressive island, deterministically-populated concepts, working search, and dynamic per-slug HANA-blob serving. + +**Architecture:** A topic = a live SAP tag (a tag applied to ≥1 tutorial). Two JSON build feeds (`/build/topics-tree`, `/build/topics/:slug`) assemble data directly from `Tags`/`TutorialTags`/`TutorialConceptLinks`. A CAP index renderer (`topic-list-page.js` → `/content/topics-index`) and detail renderer (`topic-detail-render.js`, published as `topic-` HANA blobs) mirror the concepts subsystem exactly, wrapped in the shared chrome shell. The approuter gains `^/topics/(.*)$` + `^/topics/?$` dynamic routes and a `/search/` → `/tutorial-navigator/` redirect. Legacy `topics_gallery`/Hugo topics section is retired. + +**Tech Stack:** SAP CAP (Node.js, `@sap/cds`), Express handlers, Hugo, Vue 3 island (`hugo-apps/`, Vite), HANA content blobs (gzip), AppRouter (`xs-app.json`), Vitest (`cds.test`). + +**Spec:** `docs/superpowers/specs/2026-09-01-tag-tree-topics-design.md` + +## Global Constraints + +- **PRs target `DEV`; `main` is protected — no direct-to-main path.** Branch off `origin/DEV`. This worktree is already rebased onto `origin/DEV`. +- **Never write raw SQL** — use `cds.ql`/CQL. Exception: reading HANA BLOBs alongside metadata (use raw `db.run()` per the BLOB-locator rule); topic payload queries touch no BLOBs, so CQL applies throughout the query layer. +- **Never SELECT a HANA BLOB alongside metadata in one CDS QL query.** The publish/serve blob paths reuse the existing concepts blob helpers (`composeShell`, session helpers) which already obey this. +- **Resolve entities via `cds.entities(NS)`** with `NS = 'com.sap.developers.ims'`, not bare `SELECT.from('X')` (CI Node 22 vs local Node 24 drift). +- **HANA stores columns UPPERCASE**; junction FK columns are `tutorial_ID`, `tag_ID`, `concept_ID`. Query via entity refs + CQL (CAP maps casing), never hand-cased raw SQL here. +- **`srv/lib/*` reachable from `content-store.js` must be in `.deploy/mta.yaml`'s `srv-qa` `cp` list**, with transitive `./` imports re-walked (Task 12). +- **Never `publish-content` from a workstation** — topic blobs publish via `gh workflow run rebuild-content.yml`. Local verification uses `/build/topics/:slug` JSON + hybrid tests. +- **Fail-open everywhere:** empty `TutorialConceptLinks`/`Tags` → empty payload + `error` field, never a 500. Mirrors `build-topics-gallery.js` posture. +- **Tutorial slugs are lowercase canonical** — `.toLowerCase()` tutorial slugs before emitting hrefs. +- **Test bootstrap:** `cds.test('serve', …, '--in-memory')` — NOT `cds.deploy(cds.model)` (broken in unit tests). +- **Pre-commit for any `db/**/*.cds` change:** `npx cds deploy --to sqlite::memory:`. (This plan adds NO new CDS entities — all sources already exist.) +- **Address the user as Tom.** + +--- + +## File Structure + +**New files:** +- `srv/lib/topic-slug.js` — pure slug flatten / collision-qualify / legacy-normalize utilities. +- `srv/lib/topics-query.js` — data assemblers: `buildTopicsTreePayload(db)`, `buildTopicDetailPayload(db, slug)`, `resolveTopicBySlug`. +- `srv/lib/build-topics.js` — Express JSON feed handlers `buildTopicsTreeHandler`, `buildTopicDetailHandler` (mirrors `build-concepts.js`). +- `srv/lib/topic-list-page.js` — CAP index renderer + `topicsIndexHandler` (mirrors `concept-list-page.js`). +- `srv/lib/topic-detail-render.js` — `renderTopicDetail(topic)` → `{body, contentHash}` (mirrors `concept-detail-render.js`). +- `srv/lib/publish-topics.js` — `renderTopicsIntoSession(...)` blob publisher (mirrors `publish-concepts.js`). +- `hugo-apps/src/topics-tree/main.ts` + `App.vue` — progressive-enhancement island. +- `test/unit/topic-slug.test.js`, `test/unit/topics-query.test.js`, `test/unit/topic-list-page.test.js` — unit tests. +- `test/hybrid/topics-publish-serve.test.js` — hybrid round-trip. +- `test/e2e/topics.spec.ts` — post-deploy e2e. + +**Modified files:** +- `srv/server.js` — register 4 routes. +- `srv/lib/page-key-map.js` — add `TOPIC_KEY_PREFIX` + `discoverTopicPages`. +- `scripts/publish-content.ts` — call `discoverTopicPages` in the non-slug publish path. +- `approuter/xs-app.json` — topics detail/index routes, `topics-tree` build allow-list, `/search/` redirect. +- `hugo/layouts/topics/list.html` — repoint search form (interim, before section retirement). +- `.deploy/mta.yaml` — `srv-qa` `cp` list. +- `hugo-apps/vite.config.ts` — register `topics-tree` island entry. +- **Deletions (Task 11):** `hugo/content/topics/btp-basics.md`, `hugo/content/topics/cap-fundamentals.md`, `hugo/layouts/topics/list.html`, `hugo/layouts/topics/single.html`, `hugo/data/topics_gallery.json`, `scripts/fetch-topics-gallery.ts`, and orphaned tests referencing them. + +--- + +## Task 1: Pure slug utilities (`srv/lib/topic-slug.js`) + +**Files:** +- Create: `srv/lib/topic-slug.js` +- Test: `test/unit/topic-slug.test.js` + +**Interfaces:** +- Produces: + - `flattenTopicSlug(value: string): string` — `'sap-hana-cloud--data-lake'` → `'sap-hana-cloud-data-lake'`. + - `buildTopicSlugMap(liveTags: Array<{titlePath, label, tutorialCount, conceptCount}>): { bySlug: Map, byTag: Map }` — deterministic collision-qualify. + - `normalizeLegacyTopicSlug(slug: string): string` — strips a single trailing `-` disambiguator. + - `parseTitlePath(titlePath: string): { facet: string, value: string, segments: string[] }`. + - `Tag` shape: `{ titlePath, facet, value, segments, slug, label, tutorialCount, conceptCount }`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/topic-slug.test.js +import { describe, it, expect } from 'vitest'; +import { + flattenTopicSlug, parseTitlePath, buildTopicSlugMap, normalizeLegacyTopicSlug, +} from '../../srv/lib/topic-slug.js'; + +describe('flattenTopicSlug', () => { + it('collapses -- and lowercases', () => { + expect(flattenTopicSlug('sap-hana-cloud--data-lake')).toBe('sap-hana-cloud-data-lake'); + expect(flattenTopicSlug('SAP-HANA-Cloud')).toBe('sap-hana-cloud'); + }); +}); + +describe('parseTitlePath', () => { + it('splits facet, value and -- segments', () => { + expect(parseTitlePath('software-product-function>sap-hana-cloud--data-lake')).toEqual({ + facet: 'software-product-function', + value: 'sap-hana-cloud--data-lake', + segments: ['sap-hana-cloud', 'data-lake'], + }); + }); +}); + +describe('buildTopicSlugMap', () => { + it('qualifies collisions with facet, first-by-titlePath wins bare', () => { + const { bySlug } = buildTopicSlugMap([ + { titlePath: 'software-product>foo-bar', label: 'A' }, + { titlePath: 'topic>foo--bar', label: 'B' }, // also flattens to foo-bar + ]); + expect(bySlug.has('foo-bar')).toBe(true); // software-product wins (sorts first) + expect(bySlug.get('foo-bar').label).toBe('A'); + expect(bySlug.has('topic-foo-bar')).toBe(true); // loser facet-qualified + expect(bySlug.get('topic-foo-bar').label).toBe('B'); + }); +}); + +describe('normalizeLegacyTopicSlug', () => { + it('strips a trailing numeric disambiguator', () => { + expect(normalizeLegacyTopicSlug('sap-hana-smart-data-streaming-development-2')) + .toBe('sap-hana-smart-data-streaming-development'); + expect(normalizeLegacyTopicSlug('sap-hana-cloud')).toBe('sap-hana-cloud'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run test/unit/topic-slug.test.js --project unit` +Expected: FAIL — module not found / functions undefined. + +- [ ] **Step 3: Write minimal implementation** + +```js +// srv/lib/topic-slug.js +export function flattenTopicSlug(value) { + return String(value).replace(/--/g, '-').toLowerCase(); +} + +export function parseTitlePath(titlePath) { + const idx = String(titlePath).indexOf('>'); + const facet = idx === -1 ? '' : titlePath.slice(0, idx); + const value = idx === -1 ? titlePath : titlePath.slice(idx + 1); + return { facet, value, segments: value.split('--') }; +} + +// Deterministic: sort by titlePath, first occupant of a slug keeps it bare; +// later collisions are facet-qualified `-`. +export function buildTopicSlugMap(liveTags) { + const bySlug = new Map(); + const byTag = new Map(); + const sorted = [...liveTags].sort((a, b) => a.titlePath.localeCompare(b.titlePath)); + for (const raw of sorted) { + const { facet, value, segments } = parseTitlePath(raw.titlePath); + const base = flattenTopicSlug(value); + let slug = base; + if (bySlug.has(slug)) slug = `${facet}-${base}`; + // If even the qualified slug collides, suffix an index (defensive; asserted-rare). + let n = 2; + while (bySlug.has(slug)) slug = `${facet}-${base}-${n++}`; + const tag = { + titlePath: raw.titlePath, facet, value, segments, slug, + label: raw.label || segments[segments.length - 1], + tutorialCount: raw.tutorialCount ?? 0, + conceptCount: raw.conceptCount ?? 0, + }; + bySlug.set(slug, tag); + byTag.set(raw.titlePath, slug); + } + return { bySlug, byTag }; +} + +export function normalizeLegacyTopicSlug(slug) { + return String(slug).replace(/-\d+$/, ''); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run test/unit/topic-slug.test.js --project unit` +Expected: PASS (4 suites). + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/topic-slug.js test/unit/topic-slug.test.js +git commit -m "feat(topics): pure slug flatten + collision-qualify + legacy-normalize" +``` + +--- + +## Task 2: Data assemblers (`srv/lib/topics-query.js`) + +**Files:** +- Create: `srv/lib/topics-query.js` +- Test: `test/unit/topics-query.test.js` + +**Interfaces:** +- Consumes: `flattenTopicSlug`, `parseTitlePath`, `buildTopicSlugMap`, `normalizeLegacyTopicSlug` (Task 1). +- Produces: + - `loadLiveTags(db): Promise` — tags applied to ≥1 tutorial, with `tutorialCount` + `conceptCount`. + - `buildTopicsTreePayload(db): Promise<{ tree, buildAt, error }>` where `tree = [{ facet, label, children: TreeNode[] }]`, `TreeNode = { segment, label, slug?, tutorialCount?, conceptCount?, children: TreeNode[] }`. + - `resolveTopicBySlug(db, slug): Promise<{ tag: Tag|null, redirectTo: string|null }>` — legacy `-N` strip + facet-qualify resolution. + - `buildTopicDetailPayload(db, slug): Promise<{ slug, label, facet, tutorials, concepts, relatedTags, buildAt, error, notFound?, redirectTo? }>`. + - `tutorials[]` = `{ slug, title, level, time, href, isNew }`; `concepts[]` = `{ slug, name, rank }`; `relatedTags[]` = `{ slug, label }`. + +**Reference to mirror:** tag→tutorial join pattern `srv/lib/kg-projection.js:1027-1038`; unbounded-fetch-filter-in-Node HANA pattern `srv/lib/published-concepts-query.js:78-92`; teaches-link join `published-concepts-query.js:95-104`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/topics-query.test.js +import { describe, it, beforeAll, expect } from 'vitest'; +import cds from '@sap/cds'; +import { + loadLiveTags, buildTopicsTreePayload, resolveTopicBySlug, buildTopicDetailPayload, +} from '../../srv/lib/topics-query.js'; + +const NS = 'com.sap.developers.ims'; + +describe('topics-query', () => { + let db; + beforeAll(async () => { + await cds.test('serve', '--in-memory', '--project', process.cwd()); + db = await cds.connect.to('db'); + const { Tutorials, Tags, TutorialTags, TutorialConceptLinks, Concepts } = cds.entities(NS); + await db.run(INSERT.into(Tags).entries([ + { ID: 't1', titlePath: 'software-product>sap-hana-cloud', label: 'SAP HANA Cloud', name: 'sap-hana-cloud' }, + { ID: 't2', titlePath: 'software-product-function>sap-hana-cloud--data-lake', label: 'Data Lake', name: 'sap-hana-cloud--data-lake' }, + ])); + await db.run(INSERT.into(Tutorials).entries([ + { ID: 'tut1', slug: 'hana-intro', title: 'HANA Intro', experienceTag: 'Beginner' }, + ])); + await db.run(INSERT.into(TutorialTags).entries([ + { tutorial_ID: 'tut1', tag_ID: 't1' }, + ])); + await db.run(INSERT.into(Concepts).entries([ + { ID: 'c1', slug: 'in-memory-database', name: 'In-Memory Database', status: 'ACTIVE', publishedAt: new Date().toISOString() }, + ])); + await db.run(INSERT.into(TutorialConceptLinks).entries([ + { ID: 'l1', tutorial_ID: 'tut1', concept_ID: 'c1', predicate: 'teaches' }, + ])); + }); + + it('loadLiveTags returns only tags with ≥1 tutorial, with counts', async () => { + const live = await loadLiveTags(db); + const slugs = live.map(t => t.slug).sort(); + expect(slugs).toContain('sap-hana-cloud'); + expect(slugs).not.toContain('sap-hana-cloud-data-lake'); // t2 has no tutorial + const hana = live.find(t => t.slug === 'sap-hana-cloud'); + expect(hana.tutorialCount).toBe(1); + expect(hana.conceptCount).toBe(1); + }); + + it('buildTopicsTreePayload groups by facet', async () => { + const { tree, error } = await buildTopicsTreePayload(db); + expect(error).toBeFalsy(); + const facet = tree.find(f => f.facet === 'software-product'); + expect(facet.children.some(n => n.slug === 'sap-hana-cloud')).toBe(true); + }); + + it('buildTopicDetailPayload returns tutorials + concepts', async () => { + const p = await buildTopicDetailPayload(db, 'sap-hana-cloud'); + expect(p.notFound).toBeFalsy(); + expect(p.tutorials.map(t => t.slug)).toContain('hana-intro'); + expect(p.concepts.map(c => c.slug)).toContain('in-memory-database'); + }); + + it('resolveTopicBySlug strips legacy -N and redirects', async () => { + const r = await resolveTopicBySlug(db, 'sap-hana-cloud-2'); + expect(r.tag?.slug).toBe('sap-hana-cloud'); + expect(r.redirectTo).toBe('/topics/sap-hana-cloud/'); + }); + + it('unknown slug is notFound with redirect to /topics/', async () => { + const p = await buildTopicDetailPayload(db, 'does-not-exist'); + expect(p.notFound).toBe(true); + expect(p.redirectTo).toBe('/topics/'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run test/unit/topics-query.test.js --project unit` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write minimal implementation** + +```js +// srv/lib/topics-query.js +import cds from '@sap/cds'; +import { buildTopicSlugMap, parseTitlePath, normalizeLegacyTopicSlug } from './topic-slug.js'; + +const NS = 'com.sap.developers.ims'; +const MAX_TUTORIALS = 60; +const MAX_CONCEPTS = 24; + +function ent() { + const { Tags, TutorialTags, Tutorials, TutorialConceptLinks, Concepts, ConceptRank } = cds.entities(NS); + return { Tags, TutorialTags, Tutorials, TutorialConceptLinks, Concepts, ConceptRank }; +} + +function humanizeFacet(facet) { + return String(facet).split('-').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' '); +} + +// Live-tag set (applied to ≥1 tutorial) with per-tag tutorial + concept counts. +export async function loadLiveTags(db) { + const { Tags, TutorialTags, TutorialConceptLinks } = ent(); + const tags = await db.run(SELECT.from(Tags).columns('ID', 'titlePath', 'label', 'name')); + const tagById = new Map(tags.map(t => [t.ID, t])); + const links = await db.run(SELECT.from(TutorialTags).columns('tutorial_ID', 'tag_ID')); + + const tutorialIdsByTag = new Map(); // tag_ID -> Set(tutorial_ID) + for (const l of links) { + if (!tagById.has(l.tag_ID)) continue; + (tutorialIdsByTag.get(l.tag_ID) ?? tutorialIdsByTag.set(l.tag_ID, new Set()).get(l.tag_ID)).add(l.tutorial_ID); + } + + // Bulk teaches-links → tutorial_ID -> Set(concept_ID); unbounded fetch, filter in Node. + const teaches = await db.run( + SELECT.from(TutorialConceptLinks).columns('tutorial_ID', 'concept_ID').where({ predicate: 'teaches' }), + ); + const conceptsByTutorial = new Map(); + for (const t of teaches) { + if (!t.concept_ID) continue; + (conceptsByTutorial.get(t.tutorial_ID) ?? conceptsByTutorial.set(t.tutorial_ID, new Set()).get(t.tutorial_ID)).add(t.concept_ID); + } + + const liveRaw = []; + for (const [tagId, tutSet] of tutorialIdsByTag) { + const tag = tagById.get(tagId); + if (!tag?.titlePath) continue; + const conceptSet = new Set(); + for (const tutId of tutSet) for (const c of (conceptsByTutorial.get(tutId) ?? [])) conceptSet.add(c); + liveRaw.push({ titlePath: tag.titlePath, label: tag.label, tutorialCount: tutSet.size, conceptCount: conceptSet.size }); + } + const { bySlug } = buildTopicSlugMap(liveRaw); + return [...bySlug.values()]; +} + +export async function buildTopicsTreePayload(db) { + try { + const live = await loadLiveTags(db); + const facets = new Map(); // facet -> node + for (const tag of live) { + if (!facets.has(tag.facet)) facets.set(tag.facet, { facet: tag.facet, label: humanizeFacet(tag.facet), children: [] }); + const facetNode = facets.get(tag.facet); + let level = facetNode.children; + for (let i = 0; i < tag.segments.length; i++) { + const seg = tag.segments[i]; + let node = level.find(n => n.segment === seg); + if (!node) { node = { segment: seg, label: seg, children: [] }; level.push(node); } + if (i === tag.segments.length - 1) { + node.slug = tag.slug; + node.label = tag.label || seg; + node.tutorialCount = tag.tutorialCount; + node.conceptCount = tag.conceptCount; + } + level = node.children; + } + } + const sortRec = (nodes) => { + nodes.sort((a, b) => a.label.localeCompare(b.label)); + for (const n of nodes) sortRec(n.children); + }; + const tree = [...facets.values()].sort((a, b) => a.label.localeCompare(b.label)); + for (const f of tree) sortRec(f.children); + return { tree, buildAt: new Date().toISOString(), error: null }; + } catch (err) { + return { tree: [], buildAt: new Date().toISOString(), error: err.message }; + } +} + +export async function resolveTopicBySlug(db, slug) { + const live = await loadLiveTags(db); + const bySlug = new Map(live.map(t => [t.slug, t])); + if (bySlug.has(slug)) return { tag: bySlug.get(slug), redirectTo: null }; + const base = normalizeLegacyTopicSlug(slug); + if (base !== slug && bySlug.has(base)) return { tag: bySlug.get(base), redirectTo: `/topics/${base}/` }; + return { tag: null, redirectTo: '/topics/' }; +} + +export async function buildTopicDetailPayload(db, slug) { + try { + const { tag, redirectTo } = await resolveTopicBySlug(db, slug); + if (!tag) return { slug, notFound: true, redirectTo, tutorials: [], concepts: [], relatedTags: [], buildAt: new Date().toISOString(), error: null }; + if (redirectTo) return { slug: tag.slug, notFound: false, redirectTo, tutorials: [], concepts: [], relatedTags: [], buildAt: new Date().toISOString(), error: null }; + + const { Tags, TutorialTags, Tutorials, TutorialConceptLinks, Concepts, ConceptRank } = ent(); + + // tutorials carrying this tag + const tagRow = await db.run(SELECT.one.from(Tags).columns('ID').where({ titlePath: tag.titlePath })); + const ttRows = tagRow ? await db.run(SELECT.from(TutorialTags).columns('tutorial_ID').where({ tag_ID: tagRow.ID })) : []; + const tutIds = new Set(ttRows.map(r => r.tutorial_ID)); + const allTuts = await db.run(SELECT.from(Tutorials).columns('ID', 'slug', 'title', 'experienceTag', 'timeToComplete', 'isNew')); + const tutorials = allTuts + .filter(t => tutIds.has(t.ID)) + .map(t => ({ + slug: String(t.slug || '').toLowerCase(), + title: t.title, + level: t.experienceTag || null, + time: t.timeToComplete || null, + href: `/tutorials/${String(t.slug || '').toLowerCase()}/`, + isNew: !!t.isNew, + })) + .sort((a, b) => a.title.localeCompare(b.title)) + .slice(0, MAX_TUTORIALS); + + // concepts taught by those tutorials (unbounded fetch + Node filter) + const teaches = await db.run(SELECT.from(TutorialConceptLinks).columns('tutorial_ID', 'concept_ID').where({ predicate: 'teaches' })); + const conceptIds = new Set(teaches.filter(l => tutIds.has(l.tutorial_ID) && l.concept_ID).map(l => l.concept_ID)); + const allConcepts = await db.run(SELECT.from(Concepts).columns('ID', 'slug', 'name').where({ status: 'ACTIVE' })); + const rankRows = await db.run(SELECT.from(ConceptRank).columns('slug', 'score')).catch(() => []); + const rankBySlug = new Map(rankRows.map(r => [r.slug, r.score])); + const concepts = allConcepts + .filter(c => conceptIds.has(c.ID)) + .map(c => ({ slug: c.slug, name: c.name, rank: rankBySlug.get(c.slug) ?? 0 })) + .sort((a, b) => b.rank - a.rank || a.name.localeCompare(b.name)) + .slice(0, MAX_CONCEPTS); + + // related tags = same-facet siblings sharing the parent segment + const parent = tag.segments.slice(0, -1); + const relatedTags = live + .filter(t => t.slug !== tag.slug && t.facet === tag.facet) + .filter(t => parent.length === 0 || parent.every((seg, i) => t.segments[i] === seg)) + .map(t => ({ slug: t.slug, label: t.label })) + .sort((a, b) => a.label.localeCompare(b.label)) + .slice(0, 24); + + return { + slug: tag.slug, label: tag.label, facet: tag.facet, + tutorials, concepts, relatedTags, + buildAt: new Date().toISOString(), error: null, + }; + } catch (err) { + return { slug, tutorials: [], concepts: [], relatedTags: [], buildAt: new Date().toISOString(), error: err.message }; + } +} +``` + +> **NOTE for implementer:** verify the real column names on `Tutorials` before running — the recon confirmed `slug`, `title`, `experienceTag`, `stepCount`, `isNew`, `primaryTag`. Confirm the "time to complete" column name via `cds.entities(NS).Tutorials` (candidates: `timeToComplete`, `time`, `estimatedTime`); adjust the `.columns(...)` and mapping to the actual name. If absent, drop `time` from the projection. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run test/unit/topics-query.test.js --project unit` +Expected: PASS (5 cases). Fix column-name mismatches surfaced here. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/topics-query.js test/unit/topics-query.test.js +git commit -m "feat(topics): tree + detail data assemblers (tag-sourced, concept-enriched, fail-open)" +``` + +--- + +## Task 3: JSON build feed handlers (`srv/lib/build-topics.js`) + +**Files:** +- Create: `srv/lib/build-topics.js` + +**Interfaces:** +- Consumes: `buildTopicsTreePayload`, `buildTopicDetailPayload` (Task 2). +- Produces: `buildTopicsTreeHandler(req, res)`, `buildTopicDetailHandler(req, res)` — Express handlers. + +**Reference to mirror:** `srv/lib/build-concepts.js:11-22`. + +- [ ] **Step 1: Write the implementation** + +```js +// srv/lib/build-topics.js +import cds from '@sap/cds'; +import { buildTopicsTreePayload, buildTopicDetailPayload } from './topics-query.js'; + +export async function buildTopicsTreeHandler(req, res) { + const db = await cds.connect.to('db'); + const payload = await buildTopicsTreePayload(db); + res.set('Cache-Control', 'public, max-age=60'); + res.json(payload); +} + +export async function buildTopicDetailHandler(req, res) { + const db = await cds.connect.to('db'); + const slug = String(req.params.slug || '').toLowerCase(); + const payload = await buildTopicDetailPayload(db, slug); + res.set('Cache-Control', 'public, max-age=60'); + res.status(payload.notFound ? 404 : 200).json(payload); +} +``` + +- [ ] **Step 2: Commit** (registration + live verification happen in Task 8) + +```bash +git add srv/lib/build-topics.js +git commit -m "feat(topics): /build/topics-tree + /build/topics/:slug JSON feed handlers" +``` + +--- + +## Task 4: CAP index renderer (`srv/lib/topic-list-page.js`) + +**Files:** +- Create: `srv/lib/topic-list-page.js` +- Test: `test/unit/topic-list-page.test.js` + +**Interfaces:** +- Consumes: `buildTopicsTreePayload` (Task 2); `chrome-shell.js` exports `createShellLoader`, `composeShell`, `ShellMarkerError`; `edge-cache-headers.js` `setContentCacheHeaders`; `island-manifest.json`. +- Produces: + - `buildTopicListModel(db, deps = {}): Promise<{ tree, version }>`. + - `renderTopicListBody(model): string` — BODY fragment: inline ``; + +function esc(s) { + return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); +} +function jsonForScript(obj) { + return JSON.stringify(obj).replace(/${node.tutorialCount ?? 0} tutorials · ${node.conceptCount ?? 0} concepts` + : ''; + const label = node.slug + ? `${esc(node.label)}${count}` + : `${esc(node.label)}`; + if (node.children && node.children.length) { + return `
  • ${label}
      ${node.children.map(renderNode).join('')}
  • `; + } + return `
  • ${label}
  • `; +} + +export function renderTopicListBody(model) { + const facets = (model.tree || []).map(f => + `
  • ${esc(f.label)}
      ${f.children.map(renderNode).join('')}
  • `, + ).join(''); + const data = jsonForScript({ tree: model.tree || [] }); + return `${TOPICS_STYLE} + + +`; +} + +export async function buildTopicListModel(db, _deps = {}) { + const payload = await buildTopicsTreePayload(db); + return { tree: payload.tree || [], version: null }; +} + +export function createTopicListPage({ namespace = DEFAULT_NAMESPACE } = {}) { + const NS = namespace; + const { ContentManifest } = cds.entities(NS); + async function getActiveVersion(db) { + const row = await db.run(SELECT.one.from(ContentManifest).columns('version').where({ status: 'ACTIVE' })).catch(() => null); + return row?.version ?? null; + } + const shellLoader = createShellLoader({ + namespace: NS, hanaTableName: HANA_TABLE, hanaCurrentTableName: HANA_CURRENT_TABLE, + getActiveVersion: () => cds.connect.to('db').then(getActiveVersion), + }); + let cache = null; // { version, gz, etag } + + async function topicsIndexHandler(req, res) { + try { + const db = await cds.connect.to('db'); + const version = await getActiveVersion(db); + if (cache && cache.version === version) { + if (req.headers['if-none-match'] === cache.etag) { res.status(304).end(); return; } + setContentCacheHeaders(res, { slug: 'topics' }); + res.set('Content-Encoding', 'gzip').set('ETag', cache.etag).set('X-Content-Source', 'db-current').type('html').send(cache.gz); + return; + } + const model = await buildTopicListModel(db); + model.version = version; + const body = renderTopicListBody(model); + const meta = { kind: 'topics-index', slug: 'topics', title: 'Explore topics', description: 'Browse SAP developer topics by product hierarchy.' }; + const shell = await shellLoader.get(); + if (!shell) throw new ShellMarkerError('shell unavailable'); + const html = composeShell(shell, body, meta); + const gz = gzipSync(Buffer.from(html, 'utf8')); + const etag = `"${createHash('sha256').update(gz).digest('hex').slice(0, 32)}"`; + cache = { version, gz, etag }; + if (req.headers['if-none-match'] === etag) { res.status(304).end(); return; } + setContentCacheHeaders(res, { slug: 'topics' }); + res.set('Content-Encoding', 'gzip').set('ETag', etag).set('X-Content-Source', 'db-current').type('html').send(gz); + } catch (err) { + if (cache) { + res.set('Content-Encoding', 'gzip').set('X-Content-Source', 'db-stale').type('html').send(cache.gz); + return; + } + res.status(503).type('text/plain').send('topics index unavailable'); + } + } + return { topicsIndexHandler, _invalidate() { cache = null; shellLoader.invalidate?.(); } }; +} + +export const { topicsIndexHandler } = createTopicListPage(); +export default topicsIndexHandler; +``` + +> **NOTE for implementer:** the cache/etag/shell-loader boilerplate above is transcribed from `concept-list-page.js:239-334`. **Diff your version against that source** and adopt any details you missed (exact `setContentCacheHeaders` args, `getActiveVersion` source table, `ContentManifest` query shape). Do NOT invent behavior the concepts handler doesn't have. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run test/unit/topic-list-page.test.js --project unit` +Expected: PASS (2 cases). + +- [ ] **Step 5: Add chrome-shell cases + verify concepts untouched** + +Edit `srv/lib/chrome-shell.js`: in `canonicalUrlFor` (kind switch ~`:75-89`) add `case 'topics-index': return '/topics/';` and `case 'topic': return \`/topics/${meta.slug}/\`;`. Add the identical two cases to `buildBreadcrumbJsonLd` (~`:99-137`). + +- [ ] **Step 6: Commit** + +```bash +git add srv/lib/topic-list-page.js srv/lib/chrome-shell.js test/unit/topic-list-page.test.js +git commit -m "feat(topics): CAP-rendered tree index (topic-list-page) + chrome-shell topic canonical/breadcrumb" +``` + +--- + +## Task 5: CAP detail renderer (`srv/lib/topic-detail-render.js`) + +**Files:** +- Create: `srv/lib/topic-detail-render.js` +- Test: extend `test/unit/topics-query.test.js` (add a render block) or new `test/unit/topic-detail-render.test.js`. + +**Interfaces:** +- Consumes: topic detail payload shape from Task 2. +- Produces: `renderTopicDetail(topic): { body: string, contentHash: string }` — `body` supplies its own `
    ` (the shell has only the marker). `topic` = `{ slug, label, facet, tutorials, concepts, relatedTags }`. + +**Reference to mirror:** `srv/lib/concept-detail-render.js:51` (`renderConceptDetail` → `{body, contentHash}`, `body = \`
    ${…}
    \``, `contentHash = sha256(body)`, throws if key fields missing). + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/topic-detail-render.test.js +import { describe, it, expect } from 'vitest'; +import { renderTopicDetail } from '../../srv/lib/topic-detail-render.js'; + +describe('renderTopicDetail', () => { + const topic = { + slug: 'sap-hana-cloud', label: 'SAP HANA Cloud', facet: 'software-product', + tutorials: [{ slug: 'hana-intro', title: 'HANA Intro', level: 'Beginner', time: 15, href: '/tutorials/hana-intro/', isNew: true }], + concepts: [{ slug: 'in-memory-database', name: 'In-Memory Database', rank: 0.9 }], + relatedTags: [{ slug: 'sap-hana-cloud-data-lake', label: 'Data Lake' }], + }; + it('renders a
    body with breadcrumb, tutorials, concepts, related tags', () => { + const { body, contentHash } = renderTopicDetail(topic); + expect(body.startsWith('
    ')).toBe(true); + expect(body.endsWith('
    ')).toBe(true); + expect(body).toContain('SAP HANA Cloud'); + expect(body).toContain('href="/tutorials/hana-intro/"'); + expect(body).toContain('href="/concepts/in-memory-database/"'); + expect(body).toContain('href="/topics/sap-hana-cloud-data-lake/"'); + expect(contentHash).toMatch(/^[a-f0-9]{64}$/); + }); + it('throws when slug or label missing', () => { + expect(() => renderTopicDetail({ slug: '', label: 'x' })).toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run test/unit/topic-detail-render.test.js --project unit` +Expected: FAIL — module not found. + +- [ ] **Step 3: Write minimal implementation** + +```js +// srv/lib/topic-detail-render.js +import { createHash } from 'node:crypto'; + +function esc(s) { + return String(s ?? '').replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); +} + +export function renderTopicDetail(topic) { + if (!topic?.slug || !topic?.label) throw new Error('renderTopicDetail: slug and label required'); + const tutorials = (topic.tutorials || []).map(t => ` +
  • + ${esc(t.title)} + ${t.isNew ? 'NEW' : ''} + ${t.level ? `${esc(t.level)}` : ''} +
  • `).join(''); + const concepts = (topic.concepts || []).map(c => ` +
  • ${esc(c.name)}
  • `).join(''); + const related = (topic.relatedTags || []).map(r => ` + `).join(''); + + const conceptsSection = concepts + ? `
    +

    Concepts in this topic

    +
      ${concepts}
    +
    ` + : ''; + const relatedSection = related + ? `` + : ''; + + const body = `
    +
    + +
    +

    ${esc(topic.label)}

    +

    ${esc(topic.facet)}

    +
    +
    +

    Tutorials

    +
      ${tutorials || '
    • No tutorials yet.
    • '}
    +
    + ${conceptsSection} + ${relatedSection} +
    +
    `; + const contentHash = createHash('sha256').update(body).digest('hex'); + return { body, contentHash }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run test/unit/topic-detail-render.test.js --project unit` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/topic-detail-render.js test/unit/topic-detail-render.test.js +git commit -m "feat(topics): topic detail body renderer" +``` + +--- + +## Task 6: Blob publisher (`srv/lib/publish-topics.js`) + +**Files:** +- Create: `srv/lib/publish-topics.js` + +**Interfaces:** +- Consumes: `loadLiveTags`, `buildTopicDetailPayload` (Task 2); `renderTopicDetail` (Task 5); `composeShell` (chrome-shell). +- Produces: `renderTopicsIntoSession({ db, sessionId, helpers, priorHashes = {}, shell, deps = {} }): Promise<{ topicsSeen, topicsChanged, topicsSkipped, topicsErrored, durationMs }>`. Key = `topic-${slug}`. Meta = `{ kind: 'topic', slug, title: label, description }`. + +**Reference to mirror:** `srv/lib/publish-concepts.js:78-171` — per-item loop, `composeShell` wrap, `sha256(fullDoc)`, delta-skip on `priorHashes[key] === contentHash`, `gzipSync(fullDoc).toString('base64')`, `MAX_ERROR_RATE = 0.05` abort, `BATCH_SIZE = 20` append via `helpers.appendToSession`, `loadPriorTopicHashes` filtering `slug.startsWith('topic-')`. + +- [ ] **Step 1: Write the implementation** + +```js +// srv/lib/publish-topics.js +import { gzipSync } from 'node:zlib'; +import { createHash } from 'node:crypto'; +import { loadLiveTags, buildTopicDetailPayload } from './topics-query.js'; +import { renderTopicDetail } from './topic-detail-render.js'; +import { composeShell } from './chrome-shell.js'; + +const BATCH_SIZE = 20; +const MAX_ERROR_RATE = 0.05; +const META_DESC_MAX = 160; + +function topicMetaDescription(topic) { + const n = topic.tutorials?.length ?? 0; + const c = topic.concepts?.length ?? 0; + return `${topic.label}: ${n} tutorials and ${c} concepts on developers.sap.com.`.slice(0, META_DESC_MAX); +} + +export async function renderTopicsIntoSession({ db, sessionId, helpers, priorHashes = {}, shell, deps = {} }) { + const start = Date.now(); + const live = deps.loadLiveTags ? await deps.loadLiveTags(db) : await loadLiveTags(db); + let seen = 0, changed = 0, skipped = 0, errored = 0; + let batch = []; + const flush = async () => { + if (!batch.length) return; + await helpers.appendToSession({ sessionId, files: Object.fromEntries(batch) }); + batch = []; + }; + for (const tag of live) { + seen++; + try { + const topic = await buildTopicDetailPayload(db, tag.slug); + if (topic.notFound || topic.error) { errored++; continue; } + const { body } = renderTopicDetail(topic); + const meta = { kind: 'topic', slug: topic.slug, title: topic.label, description: topicMetaDescription(topic) }; + const fullDoc = composeShell(shell, body, meta); + const key = `topic-${topic.slug}`; + const contentHash = createHash('sha256').update(fullDoc).digest('hex'); + if (priorHashes[key] === contentHash) { skipped++; continue; } + batch.push([key, gzipSync(Buffer.from(fullDoc, 'utf8')).toString('base64')]); + changed++; + if (batch.length >= BATCH_SIZE) await flush(); + } catch { + errored++; + if (seen > 20 && errored / seen > MAX_ERROR_RATE) throw new Error('renderTopicsIntoSession: error rate exceeded'); + } + } + await flush(); + return { topicsSeen: seen, topicsChanged: changed, topicsSkipped: skipped, topicsErrored: errored, durationMs: Date.now() - start }; +} +``` + +> **NOTE for implementer:** wiring `renderTopicsIntoSession` into the actual publish orchestration (the POST `/content/publish/render-*` chain and `loadPriorConceptHashes` equivalent) mirrors `publish-concepts.js:159-194`. Locate where `renderConceptsIntoSession` is invoked during a publish run and register a sibling `renderTopicsIntoSession` call in the same place, loading prior hashes filtered by `slug.startsWith('topic-')`. Confirm the `shell`/`helpers`/`sessionId` objects passed to concepts are reused verbatim. + +- [ ] **Step 2: Commit** + +```bash +git add srv/lib/publish-topics.js +git commit -m "feat(topics): per-slug HANA blob publisher (renderTopicsIntoSession)" +``` + +--- + +## Task 7: Dynamic-slug discovery in publish (`page-key-map.js` + `publish-content.ts`) + +**Files:** +- Modify: `srv/lib/page-key-map.js` (add `TOPIC_KEY_PREFIX`, `isTopicKey`, `discoverTopicPages`) +- Modify: `scripts/publish-content.ts:~1058-1071` (call site) +- Test: extend `test/unit` page-key-map coverage if present; else add `test/unit/page-key-map-topics.test.js`. + +**Interfaces:** +- Produces: `TOPIC_KEY_PREFIX = 'topic-'`; `isTopicKey(key): boolean`; `discoverTopicPages(hugoDir): Map`. + +> **DESIGN NOTE:** topic detail blobs are produced server-side by `renderTopicsIntoSession` (Task 6), NOT from Hugo files on disk — there is no `hugo/.../topic-/index.html`. So `discoverTopicPages` is only relevant if topics are published as file-backed blobs. Since topics are **server-rendered at publish time**, the canonical publish path is the `renderTopicsIntoSession` server call (Task 6), invoked in the publish orchestration alongside `renderConceptsIntoSession`. **Confirm which mechanism the concepts subsystem actually uses at publish time** (recon shows concepts use the *server-side* `renderConceptsIntoSession`, not `discoverPageFiles`). If concepts are server-rendered, SKIP the `discoverTopicPages` file-walker entirely and rely solely on Task 6's server call — this task then reduces to registering that call. + +- [ ] **Step 1: Determine the actual publish mechanism** + +Read `scripts/publish-content.ts` around the concepts publish step and `srv/lib/publish-concepts.js:159-194`. Decide: +- **(A) Server-rendered** (expected): concepts blobs are generated by a POST to `/content/publish/render-concepts`. → Register a sibling topics render call; **do not** add `discoverTopicPages`. Skip Steps 2-4 below. +- **(B) File-backed**: concepts index/detail come from disk via `discoverPageFiles`. → Add `discoverTopicPages` (Steps 2-4). + +- [ ] **Step 2 (only if B): Add discovery helper** + +```js +// srv/lib/page-key-map.js — near AUTHOR/ADVOCATE prefixes +export const TOPIC_KEY_PREFIX = 'topic-'; +export const isTopicKey = (key) => key.startsWith(TOPIC_KEY_PREFIX); + +export function discoverTopicPages(hugoDir) { + const out = new Map(); + const base = path.join(hugoDir, 'topics'); + let entries = []; + try { entries = fs.readdirSync(base, { withFileTypes: true }); } catch { return out; } + for (const e of entries) { + if (!e.isDirectory()) continue; + const slug = e.name; + if (!/^[a-z0-9][a-z0-9-]*$/.test(slug)) continue; + const idx = path.join(base, slug, 'index.html'); + if (fs.existsSync(idx)) out.set(`${TOPIC_KEY_PREFIX}${slug}`, idx); + } + return out; +} +``` + +- [ ] **Step 3 (only if B): Wire the call site** in `scripts/publish-content.ts` next to the author/advocate discovery block: + +```ts +const topics = discoverTopicPages(opts.hugoDir); +for (const [key, absPath] of topics) tutorials.set(key, absPath); +``` + +- [ ] **Step 4 (only if B): Test + commit.** + +- [ ] **Step 5 (path A): Register the server render call** + +Add the `renderTopicsIntoSession` invocation to the publish orchestration alongside the concepts call (mirror `publish-concepts.js`'s `createRenderConcepts`/handler registration; expose `POST /content/publish/render-topics` in `srv/server.js` if concepts has an analogous route). Load prior hashes filtering `slug.startsWith('topic-')`. + +- [ ] **Step 6: Commit** + +```bash +git add srv/lib/page-key-map.js scripts/publish-content.ts srv/server.js +git commit -m "feat(topics): register topic-blob publish step in content publish path" +``` + +--- + +## Task 8: Register CAP routes (`srv/server.js`) + +**Files:** +- Modify: `srv/server.js` + +**Interfaces:** +- Consumes: `buildTopicsTreeHandler`, `buildTopicDetailHandler` (Task 3); `topicsIndexHandler`, `createTopicListPage` (Task 4); `buildTopicDetailPayload`, `resolveTopicBySlug` (Task 2); `renderTopicDetail` (Task 5); shell loader + serve helpers already used by `/content/concepts/:slug`. + +**Reference to mirror:** imports `server.js:15-37`; `/build/*` registrations `:296-306`; `/content/concepts/:slug` wrapper `:503-528`; `/content/concepts-index` `:533`. + +- [ ] **Step 1: Add imports** (top of `srv/server.js`, with sibling imports) + +```js +import { buildTopicsTreeHandler, buildTopicDetailHandler } from './lib/build-topics.js'; +import { topicsIndexHandler } from './lib/topic-list-page.js'; +import { resolveTopicBySlug, buildTopicDetailPayload } from './lib/topics-query.js'; +import { renderTopicDetail } from './lib/topic-detail-render.js'; +``` + +- [ ] **Step 2: Register build feeds** (next to `:298-300`) + +```js +app.get('/build/topics-tree', buildTopicsTreeHandler); +app.get('/build/topics/:slug', buildTopicDetailHandler); +``` + +- [ ] **Step 3: Register content index** (next to `:533`) + +```js +app.get('/content/topics-index', topicsIndexHandler); +``` + +- [ ] **Step 4: Register content detail** — serves the published `topic-` blob, with legacy/retired redirect fallthrough. Mirror the `/content/concepts/:slug` wrapper (`:503-528`) that rewrites `req.params.slug = \`concept-${lower}\`` then delegates to `serveHandler`. For topics, resolve legacy/retired slugs to a 301 before the blob lookup: + +```js +app.get('/content/topics/:slug', async (req, res, next) => { + let raw = String(req.params.slug || '').replace(/\.html$/, ''); + const lower = raw.toLowerCase(); + if (raw !== lower) { res.redirect(301, `/topics/${lower}/`); return; } + // legacy -N / retired resolution + try { + const db = await cds.connect.to('db'); + const { tag, redirectTo } = await resolveTopicBySlug(db, lower); + if (!tag && redirectTo) { res.redirect(301, redirectTo); return; } + if (tag && redirectTo) { res.redirect(301, redirectTo); return; } + } catch { /* fail-open to blob lookup */ } + // delegate to the shared blob serve handler with topic- key prefix (same as concepts) + req.params.slug = `topic-${lower}`; + return serveHandler(req, res, next); +}); +``` + +> **NOTE for implementer:** `serveHandler` is the same content-store serve function `/content/concepts/:slug` delegates to (`server.js:526-527`). Confirm its exact name/signature at that line and match it. If a served `topic-` blob is missing (never published), `serveHandler` will 404 — acceptable; the resolve step above catches *known-live-but-unpublished* only insofar as it doesn't redirect them. For a fully fail-open detail path, when `serveHandler` would 404 AND the slug resolves to a live tag, fall back to on-the-fly render: +> ```js +> // optional fallback inside a wrapper around serveHandler's 404 +> const payload = await buildTopicDetailPayload(db, lower); +> if (!payload.notFound) { /* renderTopicDetail + composeShell + send */ } +> ``` +> Decide based on whether concepts has an equivalent live-render fallback; prefer parity. + +- [ ] **Step 5: Verify build feeds live** + +```bash +cds watch & # or npm run dev:hybrid for real HANA +sleep 8 +curl -s http://localhost:4004/build/topics-tree | jq '.tree | length, .error' +curl -s http://localhost:4004/build/topics/sap-hana-cloud | jq '{tutorials: (.tutorials|length), concepts: (.concepts|length), error}' +``` +Expected: tree length ≥ 1, `error: null`; detail returns tutorials/concepts arrays (may be empty in a bare in-memory DB — verify against seeded/hybrid data). + +- [ ] **Step 6: Commit** + +```bash +git add srv/server.js +git commit -m "feat(topics): register /build/topics-tree, /build/topics/:slug, /content/topics-index, /content/topics/:slug" +``` + +--- + +## Task 9: Approuter routes + search redirect (`approuter/xs-app.json`) + +**Files:** +- Modify: `approuter/xs-app.json` + +**Reference:** concepts index route `:546-550`, detail route `:552-556`, `/build/*` allow-list `:399`, current `/topics/` route `:585`, `/search/` OData route `:375-376`, catch-all `:605`. + +- [ ] **Step 1: Add `topics-tree` to the `/build/*` allow-list alternation** (`:399`) — insert `|topics-tree` into the group (keep `topics-gallery` until Task 11 retires it, or replace it): + +```json +{ "source": "^/build/(breadcrumb-context|catalog|co-completions|concepts|homepage-shelves|kg-stats|mission|my-progress|navigator|repo-catalog|slug-mapping|tag-labels|topics-gallery|topics-tree|topics)(/.*)?(\\?.*)?$", "target": "/build/$1$2$3", "destination": "srv-api", "authenticationType": "none" } +``` + +> `topics` (bare) covers `/build/topics/:slug`; `topics-tree` covers the index feed. Ensure `topics` does not shadow `topics-tree` — regex alternation is ordered but both are matched as whole path segments by the `(/.*)?` boundary, so list `topics-tree` before `topics`. + +- [ ] **Step 2: Replace the `page-topics` index route** (`:585`) with the dynamic index route, and add the detail route immediately after. Both MUST precede the catch-all `^(.*)$` (`:605`): + +```json +{ "source": "^/topics/?(\\?.*)?$", "target": "/content/topics-index$1", "destination": "srv-api", "authenticationType": "none" }, +{ "source": "^/topics/(.*)$", "target": "/content/topics/$1", "destination": "srv-api", "authenticationType": "none" } +``` + +- [ ] **Step 3: Add the `/search/` → `/tutorial-navigator/` redirect, scoped to bare `/search/` + query only** so it does NOT shadow the OData `SearchService` routes (`^/search/(.*)$` at `:375-376`). Place this redirect BEFORE the OData `/search/` route: + +```json +{ "source": "^/search/?(\\?.*)?$", "target": "/tutorial-navigator/$1", "status": 301, "authenticationType": "none" } +``` + +> Verify `/search/SearchableItems` etc. still route to `srv-api` (the `^/search/(.*)$` route must remain and must be reached for non-empty paths). Test both after deploy. + +- [ ] **Step 4: Validate JSON** + +Run: `jq . approuter/xs-app.json > /dev/null && echo OK` +Expected: `OK` (no parse error). + +- [ ] **Step 5: Commit** + +```bash +git add approuter/xs-app.json +git commit -m "feat(topics): approuter dynamic /topics routes + /search 301 to navigator" +``` + +--- + +## Task 10: Progressive-enhancement island (`hugo-apps/src/topics-tree/`) + +**Files:** +- Create: `hugo-apps/src/topics-tree/main.ts`, `hugo-apps/src/topics-tree/App.vue` +- Modify: `hugo-apps/vite.config.ts:~316` (add entry) + +**Interfaces:** +- Consumes: server-embedded `