feat(ci): probe production federation — nothing was checking whether remotes load

.github/workflows/prod-federation-probe.yml

@@ -0,0 +1,68 @@
+name: Prod federation probe
+
+# Answers, against the LIVE host, the one question no other check in this repo
+# asks: for each registered remote, would the browser actually get a working
+# module? prod-smoke.yml polls a health endpoint, post-prod-e2e.yml drives
+# flows, and e2e.yml checks federated assets only against a locally built
+# preview — so a remote that 404s in production had no detector.
+#
+# It is deliberately UNAUTHENTICATED: it probes the static asset paths, not
+# GET /api/apps (which requires a token and is scoped by org membership and
+# visibility, so it could not enumerate every app anyway).
+#
+# runs-on: ubuntu-latest, NOT the fuzefront ARC pool. This is a diagnostic that
+# has to be runnable exactly when the self-hosted pool is saturated — which is
+# when a prod incident is most likely to be under investigation.
+
+on:
+  workflow_dispatch:
+    inputs:
+      base_url:
+        description: 'Host origin to probe'
+        required: false
+        default: 'https://app.fuzefront.com'
+      slugs:
+        description: 'Comma-separated app slugs to probe'
+        required: false
+        # Derived from each product repo's registration/manifest.json.
+        # module-federation apps only — fuzehub and fuzeplan register as iframe
+        # and have no remoteEntry to probe.
+        # `slug` probes the same-origin layouts; `slug=<url>` probes the absolute
+        # entry a manifest declares (fuzekeys hosts its remote off-origin, so
+        # probing /apps/keys/ on the host would fabricate a 404).
+        default: 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys=https://keys.prod.fuzefront.com/apps/fuzekeys/remoteEntry.js,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex'
+  pull_request:
+    # Path-scoped on purpose: this runs against LIVE production, so it must not
+    # fire on every PR. It fires when the probe ITSELF changes, so a change to
+    # the instrument is validated by using it — the alternative is shipping a
+    # detector whose first real execution is after it merges.
+    paths:
+      - 'scripts/probe-prod-federation.mjs'
+      - 'scripts/check-federated-assets.mjs'
+      - '.github/workflows/prod-federation-probe.yml'
+  schedule:
+    # Daily. A remote can stop loading without any commit here — a product repo
+    # redeploys, an ingress changes, a chart bumps an image. Catching that needs
+    # a clock, not a PR.
+    - cron: '17 7 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  probe:
+    name: probe federated remotes
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: '24.x'
+
+      - name: Probe every federated remote
+        env:
+          BASE_URL: ${{ inputs.base_url || 'https://app.fuzefront.com' }}
+          SLUGS: ${{ inputs.slugs || 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys=https://keys.prod.fuzefront.com/apps/fuzekeys/remoteEntry.js,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex' }}
+        run: |
+          node scripts/probe-prod-federation.mjs --base "$BASE_URL" --slugs "$SLUGS"

scripts/probe-prod-federation.mjs

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+/**
+ * Ask the LIVE host, for every registered remote, the only question that matters:
+ * would the browser actually get a working module here?
+ *
+ * WHY THIS EXISTS. Nothing in this repo checks federation against production.
+ * `prod-smoke.yml` polls a health endpoint; `post-prod-e2e.yml` drives flows;
+ * `e2e.yml` runs scripts/check-federated-assets.mjs but only against a locally
+ * built preview. So the one failure the shell is most prone to — a remote whose
+ * entry or chunks 404 in prod while every green check in the pipeline describes
+ * a different artifact — had no detector at all. The portal reported these apps
+ * as HEALTHY the whole time, because backend/src/routes/apps.ts:206 treats any
+ * status < 500 as healthy, 404 included.
+ *
+ * WHAT IT DOES NOT ASSUME. The serve path is a free variable (see CLAUDE.md,
+ * "slug, display name, and the federated serve path are THREE INDEPENDENT
+ * questions"). Repos legitimately differ: most publish
+ * `/apps/<slug>/remoteEntry.js` (Vite `assetsDir: ''`), while FuzeFront's own
+ * fuzequality publishes `/apps/<slug>/assets/remoteEntry.js`. So each app is
+ * probed at BOTH candidates and the report names which one answered — the probe
+ * DISCOVERS the layout instead of re-asserting the convention it is meant to
+ * check. An app answering at neither is the finding.
+ *
+ * Chunk verification is delegated to scripts/check-federated-assets.mjs, which
+ * already encodes the two subtleties that make this test real rather than a
+ * re-derivation of config: chunk specifiers resolve against remoteEntry.js's
+ * OWN url, and an SPA fallback answering a missing chunk with 200 + HTML is the
+ * 404 it really is.
+ *
+ *   node scripts/probe-prod-federation.mjs --base <url> --slugs a,b,c
+ *
+ * Each entry is either `slug` (probe the same-origin layouts) or
+ * `slug=<absolute url>` (probe the off-origin entry a manifest declares).
+ *
+ * Exits non-zero if ANY app fails, or if ZERO apps were probed — a probe that
+ * checks nothing must never report success.
+ */
+
+import { spawn } from 'node:child_process'
+import { fileURLToPath } from 'node:url'
+import { dirname, join } from 'node:path'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const CHECKER = join(__dirname, 'check-federated-assets.mjs')
+
+const args = process.argv.slice(2)
+const argOf = name => {
+  const i = args.indexOf(name)
+  return i >= 0 ? args[i + 1] : null
+}
+
+const base = (argOf('--base') || 'https://app.fuzefront.com').replace(/\/+$/, '')
+const slugs = (argOf('--slugs') || '')
+  .split(',')
+  .map(s => s.trim())
+  .filter(Boolean)
+
+if (slugs.length === 0) {
+  console.error('::error title=Prod federation probe::No slugs given — refusing to report success on an empty probe.')
+  process.exit(2)
+}
+
+const JS_CT = /(javascript|ecmascript|text\/jsx?)/i
+const HTML_CT = /text\/html/i
+
+async function head(url) {
+  try {
+    const res = await fetch(url, { redirect: 'follow' })
+    const ct = res.headers.get('content-type') || ''
+    // Read a small prefix: enough to tell a module from an SPA shell without
+    // pulling whole bundles for 18 apps.
+    const body = (await res.text()).slice(0, 400)
+    return { status: res.status, ct, body }
+  } catch (err) {
+    return { status: 0, ct: '', body: '', error: err.message }
+  }
+}
+
+function verdictFor({ status, ct, body, error }) {
+  if (error) return { ok: false, why: `network error: ${error}` }
+  if (status !== 200) return { ok: false, why: `HTTP ${status}` }
+  if (HTML_CT.test(ct) || /^\s*<(!doctype|html)/i.test(body)) {
+    return { ok: false, why: `HTTP 200 but HTML — SPA fallback, not a module` }
+  }
+  if (!JS_CT.test(ct)) return { ok: false, why: `HTTP 200 but content-type '${ct || 'none'}'` }
+  return { ok: true, why: 'entry served as JS' }
+}
+
+function runChecker(entryUrl) {
+  return new Promise(resolve => {
+    const p = spawn(process.execPath, [CHECKER, entryUrl, '--origin', base], {
+      stdio: ['ignore', 'pipe', 'pipe'],
+    })
+    let out = ''
+    p.stdout.on('data', d => (out += d))
+    p.stderr.on('data', d => (out += d))
+    p.on('close', code => resolve({ code, out: out.trim() }))
+  })
+}
+
+const rows = []
+let failures = 0
+
+for (const spec of slugs) {
+  // `slug` probes the two same-origin layouts. `slug=<absolute url>` probes the
+  // entry the app's manifest actually declares.
+  //
+  // This second form is not a convenience — omitting it made the probe LIE. The
+  // frozen contract allows an absolute http(s) remoteEntry for remotes hosted
+  // outside the cluster, and fuzekeys uses one
+  // (https://keys.prod.fuzefront.com/apps/fuzekeys/remoteEntry.js). The first
+  // version of this probe tried only /apps/keys/... on the host origin, got a
+  // 404 that was CORRECT — nothing is supposed to be there — and reported
+  // fuzekeys as broken. A probe that fabricates a failure is as harmful as one
+  // that hides a real one; both make the report untrustworthy.
+  const eq = spec.indexOf('=')
+  const slug = eq >= 0 ? spec.slice(0, eq) : spec
+  const declared = eq >= 0 ? spec.slice(eq + 1) : null
+
+  const candidates = declared
+    ? [declared]
+    : [
+        `${base}/apps/${slug}/remoteEntry.js`,
+        `${base}/apps/${slug}/assets/remoteEntry.js`,
+      ]
+
+  let served = null
+  const attempts = []
+  for (const url of candidates) {
+    const res = await head(url)
+    const v = verdictFor(res)
+    attempts.push(`${url.startsWith(base) ? url.replace(base, '') : url} -> ${v.why}`)
+    if (v.ok) {
+      served = url
+      break
+    }
+  }
+
+  if (!served) {
+    failures++
+    rows.push({ slug, entry: '—', chunks: '—', status: '❌ entry', detail: attempts.join(' ; ') })
+    console.error(`::error title=${slug}::no remoteEntry served. ${attempts.join(' ; ')}`)
+    continue
+  }
+
+  const { code, out } = await runChecker(served)
+  if (code === 0) {
+    rows.push({ slug, entry: served.startsWith(base) ? served.replace(base, '') : served, chunks: 'all 200 + JS', status: '✅', detail: '' })
+  } else {
+    failures++
+    rows.push({
+      slug,
+      entry: served.startsWith(base) ? served.replace(base, '') : served,
+      chunks: 'BROKEN',
+      status: '❌ chunks',
+      detail: out.split('\n').slice(0, 4).join(' ; '),
+    })
+    console.error(`::error title=${slug}::entry serves but chunks fail. ${out.split('\n')[0] || ''}`)
+  }
+}
+
+const table = [
+  `### Prod federation probe — ${base}`,
+  '',
+  `Probed **${slugs.length}** app(s). **${slugs.length - failures} ok / ${failures} broken.**`,
+  '',
+  '| app | entry path served | chunks | result | detail |',
+  '|---|---|---|---|---|',
+  ...rows.map(r => `| \`${r.slug}\` | \`${r.entry}\` | ${r.chunks} | ${r.status} | ${r.detail.slice(0, 180)} |`),
+  '',
+  '`❌ entry` = neither candidate path served a JS module — the remote is not being served at all.',
+  '`❌ chunks` = the entry serves but something it imports 404s or returns HTML, which is the failure',
+  'that renders a blank panel while every healthcheck stays green.',
+].join('\n')
+
+console.log(table)
+
+if (process.env.GITHUB_STEP_SUMMARY) {
+  const { appendFileSync } = await import('node:fs')
+  appendFileSync(process.env.GITHUB_STEP_SUMMARY, table + '\n')
+}
+
+if (failures > 0) {
+  console.error(`::error title=Prod federation probe::${failures} of ${slugs.length} app(s) would not load in the browser.`)
+  process.exit(1)
+}