feat(tags): automate Semaphore taxonomy sync (#2184)

.cdsrc.json

@@ -72,6 +72,12 @@
       "credentials": {
         "url": "http://localhost:0/ngds-mock"
       }
+    },
+    "semaphore": {
+      "kind": "rest",
+      "credentials": {
+        "url": "http://localhost:0/semaphore-mock"
+      }
     }
   }
 }

docs/superpowers/specs/2026-09-08-2184-semaphore-auto-sync-design.md

@@ -0,0 +1,100 @@
+# Semaphore Taxonomy Auto-Sync (#2184)
+
+**Date:** 2026-09-08
+**Issue:** [sap-tutorials/tutorials-ims#2184](https://github.com/sap-tutorials/tutorials-ims/issues/2184) — "Automate Loading Data from Semaphore"
+**Status:** implemented behind a default-OFF DB feature flag (`SEMAPHORE_SYNC_ENABLED`); ships dark, awaits a Service Account + `semaphore-destination`.
+
+## Problem
+
+Tag taxonomy is currently loaded from Progress/Smartlogic **Semaphore** by a manual,
+one-off batch. Terms drift (renames, new industry clusters, retired products) and the
+`Tags` table goes stale until someone re-runs the load by hand. The issue asks: can this
+be automated?
+
+## Answer
+
+Yes. Semaphore's **SES (Semantic Enhancement Server)** exposes a read-only REST API. The
+`allterms` command returns every term of a model as JSON, filterable by class:
+
+```
+GET {base}/{model}/{lang}/allterms.json[?FILTER=CL=<class>]
+e.g. https://sap.data.progress.cloud/semantic/prodses/SAPCore/en/allterms.json
+```
+
+Auth needs a Semaphore **Service Account** + API User role + token (requested by email in
+parallel — see the issue comment). Productive/unattended use requires a Service Account
+rather than a personal user.
+
+## Design
+
+Pipeline (weekly cron, all pure/injectable for test):
+
+```
+client.fetchAllTerms(SAPCore)   → SES allterms JSON
+  → mapper.mapAllTerms(data)    → [{ semaphoreId, label, name, titlePath, isActualTag, isInterestItem }]
+    → applier.applyTerms(rows)  → upsert into Tags, keyed on semaphoreId
+```
+
+### Components
+
+| File | Role |
+|---|---|
+| `srv/lib/semaphore-sync/client.js` | SES REST client. Resolves `semaphore-destination` (auth token / basic), builds the allterms URL, GETs with a 20s timeout, **fails shut** (throws) on any HTTP/parse/shape error. |
+| `srv/lib/semaphore-sync/mapper.js` | Pure transform SES term → tag row. Derives `titlePath` (human "A : B") and `name` (normalized). Class→flag mapping is **config-driven** (`actualTagClasses` / `interestItemClasses`). |
+| `srv/lib/semaphore-sync/applier.js` | Upsert engine keyed on `semaphoreId` (distinct from the CSV importer which keys on `name`). Adopts a legacy row by name when it has no `semaphoreId` yet. Idempotent; supports `dryRun`. |
+| `srv/jobs/semaphore-tag-sync-job.js` | Orchestrator. Flag-gated, config-driven, fail-shut. |
+
+### No schema change
+
+`Tags` already carries `semaphoreId`, `isActualTag`, `isInterestItem` (PR-1 of #385). The
+manual load and the one-time migration are the only current sources of `semaphoreId`; this
+job keeps it fresh.
+
+### Configuration (DB-driven — no env vars)
+
+Feature flag (registry `kind:'db'`, ImsConfig `flag.semaphore.sync`, default **OFF**, `dev-only`):
+
+- `SEMAPHORE_SYNC_ENABLED` — master switch. Off → job no-ops (`reason:'flag-off'`).
+
+ImsConfig string keys tune a run without redeploy:
+
+| Key | Default | Purpose |
+|---|---|---|
+| `semaphore.sync.model` | `SAPCore` | SES model name |
+| `semaphore.sync.lang` | `en` | language |
+| `semaphore.sync.filter` | — | optional SES `FILTER` clause |
+| `semaphore.sync.actualTagClasses` | — | comma-sep classes → `isActualTag=true` |
+| `semaphore.sync.interestItemClasses` | — | comma-sep classes → `isInterestItem=true` |
+| `semaphore.sync.dryRun` | `true` | **report-only** until validated against real data |
+
+### Destination
+
+`cds.requires.semaphore` (kind `rest`) → BTP destination `semaphore-destination`
+(`hybrid` + `production`). Base/dev/unit profiles use a mock URL in `.cdsrc.json`. The
+Service Account token lives in the destination / BTP Credential Store — **never** in source.
+
+### Safety
+
+- **DEV-first, not prod-only** (unlike NGDS/feedback) — the mapping must be validated on DEV
+  before PROD; `dryRun` defaults ON so the *first* enabled runs only report the plan.
+- **Fail-shut fetch:** a transient SES outage or garbled payload throws before the applier
+  is reached, so the taxonomy is never wiped by an empty response. A FAILED run is recorded.
+- **Idempotent upsert:** re-running with the same payload reports everything `unchanged`.
+- Weekly cadence: **Sunday 04:47 UTC**, 20-min lock (off the existing minute grid).
+
+## Open items (need the Service Account before flipping ON)
+
+1. Confirm the exact `paths` element shape and the class URIs that denote "actual tag" vs
+   "interest item" against a live SAPCore payload — the mapper is deliberately config-driven
+   and conservative (every term an actual tag, none an interest item) until then.
+2. Confirm the destination auth flavour (OAuth2 client-credentials vs. long-lived bearer);
+   `client.deriveAuth()` already handles token / basic.
+3. Run the job in `dryRun` on DEV, tune the class lists, then flip `dryRun` off, then enable
+   the flag on PROD.
+
+## Tests
+
+- `test/unit/semaphore-mapper.test.js` — transform, dedupe, class→flag, malformed input.
+- `test/unit/semaphore-client.test.js` — URL builder, auth derivation, HTTP/shape errors.
+- `test/unit/semaphore-sync-applier.test.js` — insert/update/adopt/idempotent/dryRun (DB-backed).
+- `test/unit/semaphore-tag-sync-job.test.js` — flag gate, dryRun default, write path, fail-shut.

package.json

@@ -328,6 +328,23 @@
           }
         }
       },
+      "semaphore": {
+        "kind": "rest",
+        "[hybrid]": {
+          "kind": "rest",
+          "credentials": {
+            "destination": "semaphore-destination",
+            "path": "/semantic/prodses"
+          }
+        },
+        "[production]": {
+          "kind": "rest",
+          "credentials": {
+            "destination": "semaphore-destination",
+            "path": "/semantic/prodses"
+          }
+        }
+      },
       "telemetry": {
         "kind": "telemetry-to-console",
         "tracing": {

srv/jobs/scheduler.js

@@ -969,6 +969,22 @@ export function registerJobs() {
     },
   });
 
+  // #2184 — weekly Semaphore taxonomy auto-sync (replaces the manual SES batch
+  // load). Sunday 04:47 UTC — off-cluster from the :00/:07/:11/:13/:17/:19/:23/
+  // :31/:37/:43/:57 minute grid. Gated by the SEMAPHORE_SYNC_ENABLED DB flag
+  // (default OFF); flag-off runs no-op. Fail-shut — a bad fetch never writes.
+  // Lazy-import keeps boot fast + avoids pulling @sap-cloud-sdk at module load.
+  registerJob({
+    jobName: 'semaphore-tag-sync',
+    schedule: '47 4 * * 0',
+    ttlMs: 20 * 60 * 1000,
+    description: 'Sync Semaphore SES taxonomy (SAPCore) into Tags, keyed on semaphoreId (weekly)',
+    fn: async (logId, opts) => {
+      const { runSemaphoreTagSync } = await import('./semaphore-tag-sync-job.js');
+      return runSemaphoreTagSync(logId, opts);
+    },
+  });
+
   // #1030 — Every 6 h at minute 17 (off :00/:30 to avoid stampede). Keeps the
   // Row 3 homepage events band fresh without incurring LLM cost — this job
   // ONLY re-pulls Khoros + RSS and upserts CommunityEvents (title, url,

srv/jobs/semaphore-tag-sync-job.js

@@ -0,0 +1,128 @@
+// srv/jobs/semaphore-tag-sync-job.js
+//
+// Weekly cron that replaces the manual, one-off Semaphore batch load (#2184).
+// Pulls the SAPCore taxonomy from the Semaphore SES `allterms` API, maps terms
+// to Tag rows, and upserts them keyed on `semaphoreId` (a renamed term updates
+// in place instead of duplicating).
+//
+//   client.fetchAllTerms → mapper.mapAllTerms → applier.applyTerms
+//
+// Gates + config (all DB-driven — Tom prefers admin config over env vars):
+//   - Feature flag SEMAPHORE_SYNC_ENABLED (ImsConfig flag.semaphore.sync,
+//     default OFF, dev-only). Flag off → job no-ops with reason:'flag-off'.
+//   - ImsConfig string keys tune the run without a redeploy:
+//       semaphore.sync.model               (default 'SAPCore')
+//       semaphore.sync.lang                (default 'en')
+//       semaphore.sync.filter              (optional SES FILTER clause)
+//       semaphore.sync.actualTagClasses    (comma-separated class names/URIs)
+//       semaphore.sync.interestItemClasses (comma-separated class names/URIs)
+//       semaphore.sync.dryRun              ('true'/'false', DEFAULT 'true') —
+//         first live runs only report the plan so the class→flag mapping can be
+//         validated against real data before it writes. Flip to 'false' to persist.
+//
+// FAIL-OPEN / FAIL-SHUT: a fetch or mapping error is caught, logged, and
+// returned as { ok:false, error } — the cron chassis records a FAILED run and
+// NOTHING is written. The applier is only reached on a well-formed, non-empty
+// terms[] payload, so a transient outage can never wipe the taxonomy.
+
+import cds from '@sap/cds';
+import { fetchAllTerms } from '../lib/semaphore-sync/client.js';
+import { mapAllTerms } from '../lib/semaphore-sync/mapper.js';
+import { applyTerms } from '../lib/semaphore-sync/applier.js';
+import { isFlagEnabled } from '../lib/feature-flags/db-flags.js';
+
+const LOG = cds.log('semaphore-sync');
+const NS = 'com.sap.developers.ims';
+
+const CONFIG_KEYS = [
+  'semaphore.sync.model',
+  'semaphore.sync.lang',
+  'semaphore.sync.filter',
+  'semaphore.sync.actualTagClasses',
+  'semaphore.sync.interestItemClasses',
+  'semaphore.sync.dryRun',
+];
+
+function splitList(v) {
+  return String(v ?? '')
+    .split(',')
+    .map((s) => s.trim())
+    .filter(Boolean);
+}
+
+// Read the semaphore.sync.* string config from ImsConfig in one SELECT.
+async function readConfig(db) {
+  const { ImsConfig } = cds.entities(NS);
+  let rows = [];
+  try {
+    rows = await db.run(
+      SELECT.from(ImsConfig).columns('key', 'value').where({ key: { in: CONFIG_KEYS } }),
+    );
+  } catch (e) {
+    LOG.warn(`ImsConfig read failed, using defaults: ${e.message}`);
+  }
+  const map = new Map(rows.map((r) => [r.key, r.value]));
+  return {
+    model: map.get('semaphore.sync.model') || 'SAPCore',
+    lang: map.get('semaphore.sync.lang') || 'en',
+    filter: map.get('semaphore.sync.filter') || undefined,
+    actualTagClasses: splitList(map.get('semaphore.sync.actualTagClasses')),
+    interestItemClasses: splitList(map.get('semaphore.sync.interestItemClasses')),
+    // dryRun defaults to TRUE — first live runs report the plan without writing.
+    dryRun: (map.get('semaphore.sync.dryRun') ?? 'true').toLowerCase() !== 'false',
+  };
+}
+
+/**
+ * @param {*} _logId  reserved for the cron chassis (unused)
+ * @param {object} [opts]  test/manual seam: { _deps } forwarded to fetchAllTerms
+ * @returns {Promise<object>} run summary
+ */
+export async function runSemaphoreTagSync(_logId, opts = {}) {
+  if (!isFlagEnabled('SEMAPHORE_SYNC_ENABLED')) {
+    LOG.info('semaphore-sync skipped: flag off');
+    return { ok: true, skipped: true, reason: 'flag-off' };
+  }
+
+  const db = cds.db ?? (await cds.connect.to('db'));
+  const cfg = await readConfig(db);
+
+  let data;
+  try {
+    data = await fetchAllTerms({
+      model: cfg.model,
+      lang: cfg.lang,
+      filter: cfg.filter,
+      _deps: opts._deps,
+    });
+  } catch (e) {
+    // Fail-shut: never write on a bad/empty fetch.
+    LOG.error(`semaphore-sync fetch failed: ${e.message}`);
+    return { ok: false, error: e.message, model: cfg.model };
+  }
+
+  const { rows, skipped } = mapAllTerms(data, {
+    actualTagClasses: cfg.actualTagClasses,
+    interestItemClasses: cfg.interestItemClasses,
+  });
+
+  let applied = { inserted: 0, updated: 0, unchanged: 0, total: rows.length };
+  try {
+    applied = await applyTerms(rows, { db, dryRun: cfg.dryRun });
+  } catch (e) {
+    LOG.error(`semaphore-sync upsert failed: ${e.message}`);
+    return { ok: false, error: e.message, mapped: rows.length, skipped: skipped.length };
+  }
+
+  const summary = {
+    ok: true,
+    model: cfg.model,
+    dryRun: cfg.dryRun,
+    fetched: Array.isArray(data.terms) ? data.terms.length : 0,
+    mapped: rows.length,
+    skipped: skipped.length,
+    ...applied,
+  };
+  LOG.info(`semaphore-sync summary: ${JSON.stringify(summary)}`);
+  return summary;
+}

srv/lib/feature-flags/registry.js

@@ -308,4 +308,12 @@ export const FEATURE_FLAGS = [
     description: 'When true, the nightly freshness-scan job runs the detector across the tutorial catalog. DB-driven config (ImsConfig key flag.freshness.scan); no env var. Default OFF.',
     howToChange: featureFlagUpsert('FRESHNESS_SCAN_ENABLED', 'flag.freshness.scan'),
   },
+  // ---- Taxonomy ----
+  {
+    key: 'SEMAPHORE_SYNC_ENABLED', label: 'Semaphore taxonomy auto-sync', category: 'Taxonomy',
+    kind: 'db', imsConfigKey: 'flag.semaphore.sync',
+    valueType: 'boolean', default: false, status: 'dev-only',
+    description: 'When true, the weekly semaphore-tag-sync job pulls the SAPCore model from the Semaphore SES allterms API and upserts Tags (keyed on semaphoreId). Fail-open: a fetch/mapping error records a FAILED run and never mutates tags. Pairs with ImsConfig keys semaphore.sync.{model,lang,filter,actualTagClasses,interestItemClasses,dryRun} and the semaphore-destination. DB-driven config (ImsConfig key flag.semaphore.sync); no env var. Default OFF (#2184).',
+    howToChange: featureFlagUpsert('SEMAPHORE_SYNC_ENABLED', 'flag.semaphore.sync'),
+  },
 ];

srv/lib/semaphore-sync/applier.js

@@ -0,0 +1,85 @@
+// srv/lib/semaphore-sync/applier.js
+//
+// Persist mapped Semaphore terms (from srv/lib/semaphore-sync/mapper.js) into the
+// `Tags` entity. This is a distinct upsert engine from the CSV importer's
+// srv/lib/tag-import/applier.js: the CSV path keys on `name` (author-typed rows
+// carry no Semaphore id), whereas taxonomy sync keys on the stable natural key
+// `semaphoreId`, so a renamed term updates in place instead of duplicating.
+//
+// Match order per row:
+//   1. by semaphoreId  → UPDATE (authoritative once a tag has been synced)
+//   2. by normalized name (only when semaphoreId not yet present on the row)
+//      → adopt: attach semaphoreId + flags to a legacy/CSV row
+//   3. no match → INSERT (ID assigned here: cds.db INSERT does NOT auto-fill a
+//      UUID key on HANA, only on SQLite — see memory cds-db-insert-omitting-uuid-key)
+//
+// Idempotent: a second run with the same payload reports every row `unchanged`.
+// dryRun:true computes the same plan without writing — used to validate the
+// class→flag mapping against the live SAPCore model before the flag is flipped.
+
+import cds from '@sap/cds';
+
+// Fields we consider when deciding whether an existing row needs an UPDATE.
+const TRACKED = ['label', 'name', 'titlePath', 'isActualTag', 'isInterestItem', 'semaphoreId'];
+
+function differs(existing, row) {
+  return TRACKED.some((f) => (existing[f] ?? null) !== (row[f] ?? null));
+}
+
+/**
+ * Upsert mapper rows into Tags.
+ *
+ * @param {Array}  rows  Output of mapAllTerms().rows
+ * @param {object} [opts]
+ * @param {boolean} [opts.dryRun=false]  compute the plan without writing
+ * @param {object}  [opts.db]            cds db (defaults to cds.db / connect)
+ * @returns {Promise<{inserted:number, updated:number, unchanged:number, total:number}>}
+ */
+export async function applyTerms(rows, opts = {}) {
+  const { dryRun = false } = opts;
+  const db = opts.db ?? cds.db ?? (await cds.connect.to('db'));
+  const { Tags } = cds.entities('com.sap.developers.ims');
+
+  let inserted = 0;
+  let updated = 0;
+  let unchanged = 0;
+  const total = Array.isArray(rows) ? rows.length : 0;
+
+  for (const row of rows ?? []) {
+    const fields = {
+      semaphoreId: row.semaphoreId,
+      label: row.label,
+      name: row.name,
+      titlePath: row.titlePath,
+      isActualTag: !!row.isActualTag,
+      isInterestItem: !!row.isInterestItem,
+    };
+
+    // 1. Existing by semaphoreId.
+    let existing = await db.run(SELECT.one.from(Tags).where({ semaphoreId: row.semaphoreId }));
+    // 2. Adopt a legacy/CSV row that matches by name but has no semaphoreId yet.
+    if (!existing) {
+      existing = await db.run(
+        SELECT.one.from(Tags).where({ name: row.name, semaphoreId: null }),
+      );
+    }
+
+    if (existing) {
+      if (differs(existing, fields)) {
+        if (!dryRun) await db.run(UPDATE(Tags, existing.ID).set(fields));
+        updated++;
+      } else {
+        unchanged++;
+      }
+      continue;
+    }
+
+    // 3. Insert. Assign the UUID key explicitly for HANA parity.
+    if (!dryRun) {
+      await db.run(INSERT.into(Tags).entries({ ID: cds.utils.uuid(), ...fields }));
+    }
+    inserted++;
+  }
+
+  return { inserted, updated, unchanged, total };
+}