From 7e73b709433e87ee6fa72aa932823b50014ce3ad Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 23:07:10 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat(ci):=20probe=20production=20federation?= =?UTF-8?q?=20=E2=80=94=20nothing=20was=20checking=20whether=20remotes=20l?= =?UTF-8?q?oad?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The portal currently shows 13 products and NONE of them renders its UI. Trying to diagnose that surfaced the reason it went unnoticed: no check in this repo ever asks production whether a federated remote actually loads. prod-smoke.yml polls a health endpoint post-prod-e2e.yml drives user flows e2e.yml runs check-federated-assets.mjs — against a LOCAL preview So the failure the shell is most prone to — an entry or chunk that 404s in prod while every green check in the pipeline describes a different artifact — had no detector at all. Worse, the portal actively reported these apps as healthy. apps.ts:206 is `return response.status < 500`, with the comment "Consider 2xx, 3xx, 4xx as healthy". A remote whose assets 404 is reported HEALTHY. That is not a diagnostic gap, it is a diagnostic that says the opposite of the truth, and it is why "13 products, none loading" looked like a working system. Fixing that health check is a separate change; this one builds the instrument that can prove what is actually being served. WHAT IT DOES NOT ASSUME. The serve path is a free variable (CLAUDE.md, "slug, display name, and the federated serve path are THREE INDEPENDENT questions"). Most repos publish /apps//remoteEntry.js (Vite assetsDir: ''); FuzeFront's own fuzequality publishes /apps//assets/remoteEntry.js. So each app is probed at BOTH candidates and the report names which answered — it DISCOVERS the layout rather than re-asserting the convention it exists to check. An app answering at neither is the finding, not a probe bug. Chunk verification delegates to the existing check-federated-assets.mjs, which already encodes the two things that make this real rather than a re-derivation of config: chunk specifiers resolve against remoteEntry.js's OWN url, and a 200-with-HTML body is the 404 it really is. Unauthenticated by design: it probes static asset paths, not GET /api/apps, which needs a token AND is scoped by org membership + visibility — so it could not enumerate every app even with one. runs-on: ubuntu-latest, deliberately NOT the fuzefront ARC pool. A prod diagnostic has to be runnable exactly when the self-hosted pool is saturated, which is when an incident is most likely under investigation. NOT VACUOUS, and mutation-tested in both directions rather than only asserted: empty slug list -> exit 2 (a probe that checks nothing never passes) unreachable host -> exit 1, "no remoteEntry served" entry + chunk both JS -> exit 0 entry JS, chunk 200 + HTML -> exit 1, "BROKEN", 1 referenced / 0 loadable The last case is the one that matters: it is precisely the blank-panel failure, and a probe that only fetched the entry would have called it green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GaPa3JgrVNtWrGvqQEAEqv --- .github/workflows/prod-federation-probe.yml | 56 +++++++ scripts/probe-prod-federation.mjs | 167 ++++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 .github/workflows/prod-federation-probe.yml create mode 100644 scripts/probe-prod-federation.mjs diff --git a/.github/workflows/prod-federation-probe.yml b/.github/workflows/prod-federation-probe.yml new file mode 100644 index 00000000..873913bd --- /dev/null +++ b/.github/workflows/prod-federation-probe.yml @@ -0,0 +1,56 @@ +name: Prod federation probe + +# Answers, against the LIVE host, the one question no other check in this repo +# asks: for each registered remote, would the browser actually get a working +# module? prod-smoke.yml polls a health endpoint, post-prod-e2e.yml drives +# flows, and e2e.yml checks federated assets only against a locally built +# preview — so a remote that 404s in production had no detector. +# +# It is deliberately UNAUTHENTICATED: it probes the static asset paths, not +# GET /api/apps (which requires a token and is scoped by org membership and +# visibility, so it could not enumerate every app anyway). +# +# runs-on: ubuntu-latest, NOT the fuzefront ARC pool. This is a diagnostic that +# has to be runnable exactly when the self-hosted pool is saturated — which is +# when a prod incident is most likely to be under investigation. + +on: + workflow_dispatch: + inputs: + base_url: + description: 'Host origin to probe' + required: false + default: 'https://app.fuzefront.com' + slugs: + description: 'Comma-separated app slugs to probe' + required: false + # Derived from each product repo's registration/manifest.json `slug`. + # module-federation apps only — fuzehub and fuzeplan register as iframe + # and have no remoteEntry to probe. + default: 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex' + schedule: + # Daily. A remote can stop loading without any commit here — a product repo + # redeploys, an ingress changes, a chart bumps an image. Catching that needs + # a clock, not a PR. + - cron: '17 7 * * *' + +permissions: + contents: read + +jobs: + probe: + name: probe federated remotes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '24.x' + + - name: Probe every federated remote + env: + BASE_URL: ${{ inputs.base_url || 'https://app.fuzefront.com' }} + SLUGS: ${{ inputs.slugs || 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex' }} + run: | + node scripts/probe-prod-federation.mjs --base "$BASE_URL" --slugs "$SLUGS" diff --git a/scripts/probe-prod-federation.mjs b/scripts/probe-prod-federation.mjs new file mode 100644 index 00000000..94b9b84f --- /dev/null +++ b/scripts/probe-prod-federation.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node +/** + * Ask the LIVE host, for every registered remote, the only question that matters: + * would the browser actually get a working module here? + * + * WHY THIS EXISTS. Nothing in this repo checks federation against production. + * `prod-smoke.yml` polls a health endpoint; `post-prod-e2e.yml` drives flows; + * `e2e.yml` runs scripts/check-federated-assets.mjs but only against a locally + * built preview. So the one failure the shell is most prone to — a remote whose + * entry or chunks 404 in prod while every green check in the pipeline describes + * a different artifact — had no detector at all. The portal reported these apps + * as HEALTHY the whole time, because backend/src/routes/apps.ts:206 treats any + * status < 500 as healthy, 404 included. + * + * WHAT IT DOES NOT ASSUME. The serve path is a free variable (see CLAUDE.md, + * "slug, display name, and the federated serve path are THREE INDEPENDENT + * questions"). Repos legitimately differ: most publish + * `/apps//remoteEntry.js` (Vite `assetsDir: ''`), while FuzeFront's own + * fuzequality publishes `/apps//assets/remoteEntry.js`. So each app is + * probed at BOTH candidates and the report names which one answered — the probe + * DISCOVERS the layout instead of re-asserting the convention it is meant to + * check. An app answering at neither is the finding. + * + * Chunk verification is delegated to scripts/check-federated-assets.mjs, which + * already encodes the two subtleties that make this test real rather than a + * re-derivation of config: chunk specifiers resolve against remoteEntry.js's + * OWN url, and an SPA fallback answering a missing chunk with 200 + HTML is the + * 404 it really is. + * + * node scripts/probe-prod-federation.mjs --base --slugs a,b,c + * + * Exits non-zero if ANY app fails, or if ZERO apps were probed — a probe that + * checks nothing must never report success. + */ + +import { spawn } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' + +const __dirname = dirname(fileURLToPath(import.meta.url)) +const CHECKER = join(__dirname, 'check-federated-assets.mjs') + +const args = process.argv.slice(2) +const argOf = name => { + const i = args.indexOf(name) + return i >= 0 ? args[i + 1] : null +} + +const base = (argOf('--base') || 'https://app.fuzefront.com').replace(/\/+$/, '') +const slugs = (argOf('--slugs') || '') + .split(',') + .map(s => s.trim()) + .filter(Boolean) + +if (slugs.length === 0) { + console.error('::error title=Prod federation probe::No slugs given — refusing to report success on an empty probe.') + process.exit(2) +} + +const JS_CT = /(javascript|ecmascript|text\/jsx?)/i +const HTML_CT = /text\/html/i + +async function head(url) { + try { + const res = await fetch(url, { redirect: 'follow' }) + const ct = res.headers.get('content-type') || '' + // Read a small prefix: enough to tell a module from an SPA shell without + // pulling whole bundles for 18 apps. + const body = (await res.text()).slice(0, 400) + return { status: res.status, ct, body } + } catch (err) { + return { status: 0, ct: '', body: '', error: err.message } + } +} + +function verdictFor({ status, ct, body, error }) { + if (error) return { ok: false, why: `network error: ${error}` } + if (status !== 200) return { ok: false, why: `HTTP ${status}` } + if (HTML_CT.test(ct) || /^\s*<(!doctype|html)/i.test(body)) { + return { ok: false, why: `HTTP 200 but HTML — SPA fallback, not a module` } + } + if (!JS_CT.test(ct)) return { ok: false, why: `HTTP 200 but content-type '${ct || 'none'}'` } + return { ok: true, why: 'entry served as JS' } +} + +function runChecker(entryUrl) { + return new Promise(resolve => { + const p = spawn(process.execPath, [CHECKER, entryUrl, '--origin', base], { + stdio: ['ignore', 'pipe', 'pipe'], + }) + let out = '' + p.stdout.on('data', d => (out += d)) + p.stderr.on('data', d => (out += d)) + p.on('close', code => resolve({ code, out: out.trim() })) + }) +} + +const rows = [] +let failures = 0 + +for (const slug of slugs) { + // Both layouts are legitimate; discover which one this app uses. + const candidates = [ + `${base}/apps/${slug}/remoteEntry.js`, + `${base}/apps/${slug}/assets/remoteEntry.js`, + ] + + let served = null + const attempts = [] + for (const url of candidates) { + const res = await head(url) + const v = verdictFor(res) + attempts.push(`${url.replace(base, '')} -> ${v.why}`) + if (v.ok) { + served = url + break + } + } + + if (!served) { + failures++ + rows.push({ slug, entry: '—', chunks: '—', status: '❌ entry', detail: attempts.join(' ; ') }) + console.error(`::error title=${slug}::no remoteEntry served. ${attempts.join(' ; ')}`) + continue + } + + const { code, out } = await runChecker(served) + if (code === 0) { + rows.push({ slug, entry: served.replace(base, ''), chunks: 'all 200 + JS', status: '✅', detail: '' }) + } else { + failures++ + rows.push({ + slug, + entry: served.replace(base, ''), + chunks: 'BROKEN', + status: '❌ chunks', + detail: out.split('\n').slice(0, 4).join(' ; '), + }) + console.error(`::error title=${slug}::entry serves but chunks fail. ${out.split('\n')[0] || ''}`) + } +} + +const table = [ + `### Prod federation probe — ${base}`, + '', + `Probed **${slugs.length}** app(s). **${slugs.length - failures} ok / ${failures} broken.**`, + '', + '| app | entry path served | chunks | result | detail |', + '|---|---|---|---|---|', + ...rows.map(r => `| \`${r.slug}\` | \`${r.entry}\` | ${r.chunks} | ${r.status} | ${r.detail.slice(0, 180)} |`), + '', + '`❌ entry` = neither candidate path served a JS module — the remote is not being served at all.', + '`❌ chunks` = the entry serves but something it imports 404s or returns HTML, which is the failure', + 'that renders a blank panel while every healthcheck stays green.', +].join('\n') + +console.log(table) + +if (process.env.GITHUB_STEP_SUMMARY) { + const { appendFileSync } = await import('node:fs') + appendFileSync(process.env.GITHUB_STEP_SUMMARY, table + '\n') +} + +if (failures > 0) { + console.error(`::error title=Prod federation probe::${failures} of ${slugs.length} app(s) would not load in the browser.`) + process.exit(1) +} From afd1072d06d59c3bcc997535b191d2dfb470ce3f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 23:07:40 +0000 Subject: [PATCH 2/3] ci: run the prod federation probe on PRs that change the probe itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Path-scoped, not on every PR — it hits LIVE production. It fires when the instrument changes, so a change to the detector is validated by using it rather than first executing for real only after it merges. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GaPa3JgrVNtWrGvqQEAEqv --- .github/workflows/prod-federation-probe.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/prod-federation-probe.yml b/.github/workflows/prod-federation-probe.yml index 873913bd..e58fa24c 100644 --- a/.github/workflows/prod-federation-probe.yml +++ b/.github/workflows/prod-federation-probe.yml @@ -28,6 +28,15 @@ on: # module-federation apps only — fuzehub and fuzeplan register as iframe # and have no remoteEntry to probe. default: 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex' + pull_request: + # Path-scoped on purpose: this runs against LIVE production, so it must not + # fire on every PR. It fires when the probe ITSELF changes, so a change to + # the instrument is validated by using it — the alternative is shipping a + # detector whose first real execution is after it merges. + paths: + - 'scripts/probe-prod-federation.mjs' + - 'scripts/check-federated-assets.mjs' + - '.github/workflows/prod-federation-probe.yml' schedule: # Daily. A remote can stop loading without any commit here — a product repo # redeploys, an ingress changes, a chart bumps an image. Catching that needs From 6247391d89e9347c267a35a3283922640c75975d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 23:10:20 +0000 Subject: [PATCH 3/3] =?UTF-8?q?fix(probe):=20honour=20an=20absolute=20decl?= =?UTF-8?q?ared=20remoteEntry=20=E2=80=94=20the=20probe=20fabricated=20a?= =?UTF-8?q?=20404?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first run reported 16 of 16 apps broken. Fifteen of those are real. The sixteenth, `keys`, was the probe's own bug. The frozen contract allows an absolute http(s) `remoteEntry` for remotes hosted outside the cluster, and fuzekeys uses one: https://keys.prod.fuzefront.com/apps/fuzekeys/remoteEntry.js The probe only ever tried /apps//... on the host origin. For fuzekeys that 404 was CORRECT — nothing is supposed to be served there — and the probe turned it into a failure report. A probe that manufactures a failure is as harmful as one that hides a real failure: both make the whole report untrustworthy, and this one would have sent someone to debug an ingress that is not meant to exist. Each entry is now either `slug` (probe the two same-origin layouts) or `slug=` (probe exactly what the manifest declares). The workflow default carries fuzekeys in the second form. Verified, not assumed — the absolute form is proven to actually bypass --base: --base https://app.example.invalid --slugs aliased=http://127.0.0.1:8791/... -> exit 0, entry reported as the absolute URL. If the code had still resolved against --base, that case would have failed. Same-origin form re-tested unchanged, and the four earlier non-vacuity cases still hold. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GaPa3JgrVNtWrGvqQEAEqv --- .github/workflows/prod-federation-probe.yml | 9 +++-- scripts/probe-prod-federation.mjs | 37 ++++++++++++++++----- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/workflows/prod-federation-probe.yml b/.github/workflows/prod-federation-probe.yml index e58fa24c..15e6219a 100644 --- a/.github/workflows/prod-federation-probe.yml +++ b/.github/workflows/prod-federation-probe.yml @@ -24,10 +24,13 @@ on: slugs: description: 'Comma-separated app slugs to probe' required: false - # Derived from each product repo's registration/manifest.json `slug`. + # Derived from each product repo's registration/manifest.json. # module-federation apps only — fuzehub and fuzeplan register as iframe # and have no remoteEntry to probe. - default: 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex' + # `slug` probes the same-origin layouts; `slug=` probes the absolute + # entry a manifest declares (fuzekeys hosts its remote off-origin, so + # probing /apps/keys/ on the host would fabricate a 404). + default: 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys=https://keys.prod.fuzefront.com/apps/fuzekeys/remoteEntry.js,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex' pull_request: # Path-scoped on purpose: this runs against LIVE production, so it must not # fire on every PR. It fires when the probe ITSELF changes, so a change to @@ -60,6 +63,6 @@ jobs: - name: Probe every federated remote env: BASE_URL: ${{ inputs.base_url || 'https://app.fuzefront.com' }} - SLUGS: ${{ inputs.slugs || 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex' }} + SLUGS: ${{ inputs.slugs || 'finance,fuzequality,fuzeagent,fuzebi,call,contact,deploy,executive,keys=https://keys.prod.fuzefront.com/apps/fuzekeys/remoteEntry.js,market,fuzemerchandize,picker,sales,service,fuzesocial,fuzex' }} run: | node scripts/probe-prod-federation.mjs --base "$BASE_URL" --slugs "$SLUGS" diff --git a/scripts/probe-prod-federation.mjs b/scripts/probe-prod-federation.mjs index 94b9b84f..554c5b02 100644 --- a/scripts/probe-prod-federation.mjs +++ b/scripts/probe-prod-federation.mjs @@ -29,6 +29,9 @@ * * node scripts/probe-prod-federation.mjs --base --slugs a,b,c * + * Each entry is either `slug` (probe the same-origin layouts) or + * `slug=` (probe the off-origin entry a manifest declares). + * * Exits non-zero if ANY app fails, or if ZERO apps were probed — a probe that * checks nothing must never report success. */ @@ -98,19 +101,35 @@ function runChecker(entryUrl) { const rows = [] let failures = 0 -for (const slug of slugs) { - // Both layouts are legitimate; discover which one this app uses. - const candidates = [ - `${base}/apps/${slug}/remoteEntry.js`, - `${base}/apps/${slug}/assets/remoteEntry.js`, - ] +for (const spec of slugs) { + // `slug` probes the two same-origin layouts. `slug=` probes the + // entry the app's manifest actually declares. + // + // This second form is not a convenience — omitting it made the probe LIE. The + // frozen contract allows an absolute http(s) remoteEntry for remotes hosted + // outside the cluster, and fuzekeys uses one + // (https://keys.prod.fuzefront.com/apps/fuzekeys/remoteEntry.js). The first + // version of this probe tried only /apps/keys/... on the host origin, got a + // 404 that was CORRECT — nothing is supposed to be there — and reported + // fuzekeys as broken. A probe that fabricates a failure is as harmful as one + // that hides a real one; both make the report untrustworthy. + const eq = spec.indexOf('=') + const slug = eq >= 0 ? spec.slice(0, eq) : spec + const declared = eq >= 0 ? spec.slice(eq + 1) : null + + const candidates = declared + ? [declared] + : [ + `${base}/apps/${slug}/remoteEntry.js`, + `${base}/apps/${slug}/assets/remoteEntry.js`, + ] let served = null const attempts = [] for (const url of candidates) { const res = await head(url) const v = verdictFor(res) - attempts.push(`${url.replace(base, '')} -> ${v.why}`) + attempts.push(`${url.startsWith(base) ? url.replace(base, '') : url} -> ${v.why}`) if (v.ok) { served = url break @@ -126,12 +145,12 @@ for (const slug of slugs) { const { code, out } = await runChecker(served) if (code === 0) { - rows.push({ slug, entry: served.replace(base, ''), chunks: 'all 200 + JS', status: '✅', detail: '' }) + rows.push({ slug, entry: served.startsWith(base) ? served.replace(base, '') : served, chunks: 'all 200 + JS', status: '✅', detail: '' }) } else { failures++ rows.push({ slug, - entry: served.replace(base, ''), + entry: served.startsWith(base) ? served.replace(base, '') : served, chunks: 'BROKEN', status: '❌ chunks', detail: out.split('\n').slice(0, 4).join(' ; '),