diff --git a/backend/src/routes/appHealth.ts b/backend/src/routes/appHealth.ts new file mode 100644 index 00000000..265e2bf3 --- /dev/null +++ b/backend/src/routes/appHealth.ts @@ -0,0 +1,244 @@ +/** + * Health probing for registered apps. Deliberately its own module with NO + * express and NO database import, so it can be unit-tested as the pure function + * it is — the rest of routes/apps.ts needs a live Postgres to import. + */ + +/** The subset of an apps row this probe reads. */ +export interface ProbeableApp { + name: string + url: string + integration_type: 'iframe' | 'module-federation' | 'web-component' | 'spa' + remote_url: string +} + +/** + * HEALTH FOR A FEDERATED APP MEANS "THE MODULE LOADS", NOT "SOMETHING ANSWERED". + * + * This function used to be: + * + * const healthUrl = `${app.url}` + * ... + * return response.status < 500 // "Consider 2xx, 3xx, 4xx as healthy" + * + * which reported an app healthy when its URL returned **404**. A production probe + * of all 16 registered remotes found 0 serving a module and the portal calling + * every one of them green: 12 returned 404 (nothing mounted at the path at all), + * 3 returned 503 (route exists, no healthy backend), and 1 returned 200 with HTML + * — the host shell's own SPA fallback answering for a file that does not exist. + * A check that cannot tell "nothing is served here" from "this works" is not a + * health check. + * + * So the probe now depends on what the app actually is: + * + * - `module-federation`: fetch the registered remote entry and require HTTP 200 + * **and** a JavaScript body. The content-type/HTML test is not belt-and-braces + * — it is the only thing that catches the SPA-fallback case, where the status + * line says 200 and the bytes are the host's index.html. Same rule as + * scripts/check-federated-assets.mjs, deliberately, so the CI probe and the + * runtime check cannot disagree about what "serving" means. + * + * - everything else (`iframe`, `spa`, `web-component`): fetch the app URL and + * require < 400. A 404 on an app's own root means nothing is there; that is + * the whole point. + * + * `remote_url` may be same-origin-relative (`/apps//remoteEntry.js`, the + * stored form since migration 011) or an absolute URL (legacy cross-origin + * registrations). Relative values resolve against `origin`, matching how + * frontend/src/utils/loadFederatedApp.ts resolves them in the browser; absolute + * values are probed verbatim, so a cross-origin registration is judged on the + * host it actually names rather than on a path it never claimed. + */ +const LOOKS_LIKE_HTML = /^\s*(?: controller.abort(), 5000) + try { + const response = await fetch(url, { + method: 'GET', + signal: controller.signal, + headers: { Accept: accept }, + }) + return { ok: true as const, response } + } catch (error: unknown) { + return { + ok: false as const, + message: error instanceof Error ? error.message : 'Unknown error', + } + } finally { + clearTimeout(timeoutId) + } +} + +/** + * The origin a same-origin `remote_url` resolves against. + * + * CONFIGURATION ONLY — deliberately NOT derived from the request. An earlier + * revision fell back to `x-forwarded-host` / `x-forwarded-proto` / `Host` when + * FEDERATION_PROBE_ORIGIN was unset. Every one of those is caller-controlled: + * anyone able to reach the API could set `Host:` and choose which origin this + * server probes, making the health of every registered app — and therefore what + * the portal displays — an attacker-selected value. A probe whose target the + * caller picks is not a health check. + * + * So an unset FEDERATION_PROBE_ORIGIN returns '' and the caller FAILS CLOSED: + * see checkAppHealth, which reports unhealthy naming the missing configuration + * rather than guessing an origin or, worse, reporting healthy. That is the same + * rule this whole module exists to enforce — not knowing must never render as + * "fine". + * + * `req` is retained for signature stability with the route, and intentionally + * unread. + */ +export function probeOrigin(_req?: any): string { + return (process.env.FEDERATION_PROBE_ORIGIN || '').trim().replace(/\/+$/, '') +} + +export async function checkAppHealth( + app: ProbeableApp, + origin: string +): Promise { + const federated = app.integration_type === 'module-federation' + let target: string + + if (federated) { + const raw = (app.remote_url || '').trim() + if (!raw) { + return { + isHealthy: false, + httpStatus: null, + reason: + 'registered as module-federation but has no remote_url — there is nothing for the host to import', + } + } + // Fail closed on a same-origin entry with no configured origin. Guessing one + // from request headers is what this deliberately does not do (see + // probeOrigin); guessing wrong would report a 404 for a path the app never + // claimed, and trusting the header would let the caller choose the target. + const entryIsAbsolute = /^https?:\/\//i.test(raw) + if (!entryIsAbsolute && !origin) { + return { + isHealthy: false, + httpStatus: null, + reason: + `remote_url '${raw}' is same-origin, but FEDERATION_PROBE_ORIGIN is not set, ` + + `so there is no trustworthy origin to resolve it against. Set it to the ` + + `portal's public origin (e.g. https://app.fuzefront.com). It is not derived ` + + `from the Host / x-forwarded-host header on purpose: those are ` + + `caller-controlled, and trusting them would make every app's reported health ` + + `selectable by whoever sends the request.`, + } + } + try { + target = new URL(raw, origin || undefined).toString() + } catch { + return { + isHealthy: false, + httpStatus: null, + reason: `remote_url '${raw}' is not resolvable against origin '${origin}'`, + } + } + } else { + target = app.url + if (!target) { + return { isHealthy: false, httpStatus: null, reason: 'no url registered' } + } + } + + const probe = await fetchWithTimeout( + target, + federated + ? 'application/javascript,text/javascript,*/*' + : 'text/html,application/json' + ) + if (!probe.ok) { + console.log(`Health check failed for ${app.name} (${target}):`, probe.message) + return { + isHealthy: false, + httpStatus: null, + reason: `could not reach ${target}: ${probe.message}`, + } + } + const response = probe.response + + if (!federated) { + return response.status < 400 + ? { ...HEALTHY, httpStatus: response.status } + : { + isHealthy: false, + httpStatus: response.status, + reason: `${target} returned ${response.status}`, + } + } + + if (response.status !== 200) { + return { + isHealthy: false, + httpStatus: response.status, + reason: + response.status === 404 + ? `remote entry ${target} returned 404 — nothing is mounted at that path` + : response.status === 503 + ? `remote entry ${target} returned 503 — the route exists but has no healthy backend` + : `remote entry ${target} returned ${response.status}, not 200`, + } + } + + const contentType = response.headers.get('content-type') || '' + + if (/text\/html/i.test(contentType)) { + return { + isHealthy: false, + httpStatus: 200, + reason: `remote entry ${target} returned 200 but is served as HTML — this is an SPA fallback answering for a file that does not exist`, + } + } + + // A JavaScript content-type settles it. Return WITHOUT reading the body: a + // remoteEntry bundle can be hundreds of KB and this runs once per app on every + // listing request, so downloading it to confirm what the header already said + // would be the expensive way to learn nothing. + if (JS_CONTENT_TYPE.test(contentType)) { + return { ...HEALTHY, httpStatus: 200 } + } + + // Inconclusive content-type (missing, or application/octet-stream from a + // misconfigured static server). Only here is the body worth sniffing, and only + // its first bytes — enough to tell an HTML document from anything else. + let head = '' + try { + head = (await response.text()).slice(0, 512) + } catch { + /* unreadable body: fall through to the content-type verdict below */ + } + + if (LOOKS_LIKE_HTML.test(head)) { + return { + isHealthy: false, + httpStatus: 200, + reason: `remote entry ${target} returned 200 with content-type '${contentType || 'none'}' and an HTML body — an SPA fallback answering for a file that does not exist`, + } + } + + return { + isHealthy: false, + httpStatus: 200, + reason: `remote entry ${target} returned 200 with content-type '${contentType || 'none'}', which is not JavaScript — the host imports this as a module, so a non-JS body cannot load`, + } +} diff --git a/backend/src/routes/apps.ts b/backend/src/routes/apps.ts index 257e6e6c..6f80fa79 100644 --- a/backend/src/routes/apps.ts +++ b/backend/src/routes/apps.ts @@ -6,6 +6,9 @@ import { requireAppPermission } from '../middleware/permissions' import { App } from '../types/shared' import { isPrefixedIdsEnabled } from '../identity/flags' import { prefixDtoIds } from '../identity/serializer' +// Health probing lives in its own module so it can be unit-tested without a +// database — see appHealth.ts for why a 404 is no longer 'healthy'. +import { checkAppHealth, probeOrigin } from './appHealth' import { ROOT_ORG_ID } from '../migrations/015_seed_root_platform_organization' const router = express.Router() @@ -221,95 +224,6 @@ function scopeAppsQuery(query: any, memberOrgIds: string[]) { }) } -// Health check function for individual apps. -// -// This previously ended in `return response.status < 500`, with the comment -// "Consider 2xx, 3xx, 4xx as healthy". That reported an app whose assets 404 as -// HEALTHY, so the only thing it actually proved was that SOME server answered -// on that host — which, for every app mounted behind the shell's own ingress, -// is true even when nothing is deployed behind the route. -// -// That is not a missing check, it is a check asserting the opposite of the -// truth, and it is why federated remotes could serve nothing in production -// while the portal showed them green. Measured 2026-08-24 by the prod -// federation probe: of 16 remotes, 12 answered 404, 3 answered 503, and 1 -// answered 200 with the shell's own HTML. Only the three 503s would have been -// reported unhealthy by the old rule. -// -// A 4xx now means unhealthy. For a module-federation app the meaningful signal -// is stronger still: the remote entry must serve a JAVASCRIPT MODULE. An SPA -// fallback answering 200 with HTML satisfies any status-code check while the -// browser gets markup where it expected a module and the panel dies — that is -// exactly the `fuzequality` case, and a status-only check cannot see it. -const HEALTH_JS_CONTENT_TYPE = /(javascript|ecmascript)/i -const HEALTH_HTML_CONTENT_TYPE = /text\/html/i - -async function checkAppHealth(app: AppRow): Promise { - try { - // A module-federation remote entry may be declared as a same-origin PATH - // (`/apps//remoteEntry.js`). The browser resolves that against the - // page origin; this process has no reliable public origin to resolve it - // against, so the entry is only probed when the manifest declares an - // absolute URL. The relative case is covered by - // scripts/probe-prod-federation.mjs, which runs where a real origin is - // known. Deliberately NOT reconstructed from a request header — a spoofable - // Host would make the health of every app caller-controlled. - const isFederated = app.integration_type === 'module-federation' - const entryIsAbsolute = - typeof app.remote_url === 'string' && /^https?:\/\//i.test(app.remote_url) - const checkEntry = isFederated && entryIsAbsolute - const healthUrl = checkEntry ? app.remote_url : app.url - - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), 5000) // 5 second timeout - - const response = await fetch(healthUrl, { - method: 'GET', - signal: controller.signal, - headers: { - Accept: checkEntry - ? 'application/javascript,text/javascript' - : 'text/html,application/json', - }, - }) - - clearTimeout(timeoutId) - - // 4xx is NOT healthy. A 404 means the route resolves to nothing. - if (!response.ok) { - console.log( - `Health check failed for ${app.name} (${healthUrl}): HTTP ${response.status}` - ) - return false - } - - if (checkEntry) { - const contentType = response.headers.get('content-type') || '' - if ( - HEALTH_HTML_CONTENT_TYPE.test(contentType) || - !HEALTH_JS_CONTENT_TYPE.test(contentType) - ) { - console.log( - `Health check failed for ${app.name} (${healthUrl}): remote entry ` + - `returned 200 with content-type '${contentType || 'none'}' — an SPA ` + - `fallback, not a JavaScript module` - ) - return false - } - } - - return true - } catch (error: unknown) { - const errorMessage = - error instanceof Error ? error.message : 'Unknown error' - console.log( - `Health check failed for ${app.name} (${app.url}):`, - errorMessage - ) - return false - } -} - // GET /api/apps/health - Check health of apps the caller is entitled to see router.get('/health', authenticateToken, async (req: any, res) => { try { @@ -321,14 +235,24 @@ router.get('/health', authenticateToken, async (req: any, res) => { memberOrgIds ).orderBy('name') + // Resolve same-origin remote entries against the origin this request + // arrived on, the same way the browser does. + const origin = probeOrigin(req) + const healthChecks = await Promise.all( apps.map(async (app: AppRow) => { - const isHealthy = await checkAppHealth(app) + const result = await checkAppHealth(app, origin) return { id: app.id, name: app.name, url: app.url, - isHealthy, + isHealthy: result.isHealthy, + // `reason` and `httpStatus` are additive. An unhealthy app used to be + // an unexplained false; naming WHY is what makes 404 (no route) vs 503 + // (route up, backend down) vs 200-with-HTML actionable, and those three + // have different owners. + httpStatus: result.httpStatus, + reason: result.reason, lastChecked: new Date().toISOString(), } }) @@ -390,17 +314,28 @@ router.get('/', authenticateToken, async (req: any, res) => { memberOrgIds ).orderBy('name') - // Get health status for all apps + // Get health status for all apps. + // + // NOTE for whoever touches `healthyOnly` next: nothing in frontend/ or sdk/ + // passes it today (verified), so the portal lists apps regardless of health + // and this correctness fix cannot shrink what users see. If a caller ever + // does start passing it, be aware that it now means "actually serving a + // module" rather than "the server answered with anything at all", and with + // production currently at 0 of 16 remotes serving, it would filter the list + // to nothing. That would be the check telling the truth, not a regression — + // but it should be a deliberate choice, not a surprise. + const origin = probeOrigin(req) const appsWithHealth = await Promise.all( apps.map(async (app: AppRow) => { - const isHealthy = await checkAppHealth(app) + const health = await checkAppHealth(app, origin) return { id: app.id, name: app.name, url: app.url, iconUrl: app.icon_url, isActive: Boolean(app.is_active), - isHealthy: isHealthy, + isHealthy: health.isHealthy, + healthReason: health.reason, integrationType: app.integration_type as | 'module-federation' | 'iframe' diff --git a/backend/tests/appHealth.test.ts b/backend/tests/appHealth.test.ts new file mode 100644 index 00000000..19fcca61 --- /dev/null +++ b/backend/tests/appHealth.test.ts @@ -0,0 +1,233 @@ +/** + * These tests exist to prove the health probe is NOT VACUOUS. + * + * The version this replaced returned `response.status < 500`, so a 404 counted + * as healthy. A production probe of all 16 registered remotes found 0 serving a + * module while the portal reported every one of them green. A test suite that + * only asserted "a working app is healthy" would have passed against that code + * too — so the weight here is on the cases that MUST report unhealthy. + */ +import { checkAppHealth, probeOrigin, ProbeableApp } from '../src/routes/appHealth' + +const ORIGIN = 'https://app.example.test' + +function mf(remote_url: string): ProbeableApp { + return { + name: 'demo', + url: 'https://demo.example.test', + integration_type: 'module-federation', + remote_url, + } +} + +function iframe(url: string): ProbeableApp { + return { name: 'demo', url, integration_type: 'iframe', remote_url: '' } +} + +/** Stub global fetch with a fixed reply. Returns the URLs it was asked for. */ +function stubFetch( + reply: { status: number; contentType?: string; body?: string } | Error +) { + const seen: string[] = [] + ;(global as any).fetch = jest.fn(async (url: string) => { + seen.push(String(url)) + if (reply instanceof Error) throw reply + return { + status: reply.status, + headers: { + get: (h: string) => + h.toLowerCase() === 'content-type' ? (reply.contentType ?? '') : null, + }, + text: async () => reply.body ?? '', + } + }) + return seen +} + +afterEach(() => { + jest.restoreAllMocks() +}) + +describe('checkAppHealth — the cases that must NOT be healthy', () => { + it('reports a 404 remote entry as UNHEALTHY (the whole bug)', async () => { + stubFetch({ status: 404, contentType: 'text/plain' }) + const r = await checkAppHealth(mf('/apps/demo/remoteEntry.js'), ORIGIN) + expect(r.isHealthy).toBe(false) + expect(r.httpStatus).toBe(404) + expect(r.reason).toMatch(/nothing is mounted/i) + }) + + it('reports a 503 remote entry as UNHEALTHY and names it as a backend problem', async () => { + stubFetch({ status: 503 }) + const r = await checkAppHealth(mf('/apps/demo/remoteEntry.js'), ORIGIN) + expect(r.isHealthy).toBe(false) + expect(r.httpStatus).toBe(503) + expect(r.reason).toMatch(/no healthy backend/i) + }) + + it('reports 200 + HTML as UNHEALTHY — the SPA fallback "200 that isn\'t"', async () => { + stubFetch({ + status: 200, + contentType: 'text/html; charset=utf-8', + body: 'portal shell', + }) + const r = await checkAppHealth(mf('/apps/demo/remoteEntry.js'), ORIGIN) + expect(r.isHealthy).toBe(false) + expect(r.httpStatus).toBe(200) + expect(r.reason).toMatch(/SPA fallback/i) + }) + + it('reports 200 + HTML body as UNHEALTHY even when the content-type does not say html', async () => { + stubFetch({ + status: 200, + contentType: 'application/octet-stream', + body: ' \n', + }) + const r = await checkAppHealth(mf('/apps/demo/remoteEntry.js'), ORIGIN) + expect(r.isHealthy).toBe(false) + expect(r.reason).toMatch(/SPA fallback/i) + }) + + it('reports a module-federation app with no remote_url as UNHEALTHY', async () => { + stubFetch({ status: 200, contentType: 'application/javascript' }) + const r = await checkAppHealth(mf(''), ORIGIN) + expect(r.isHealthy).toBe(false) + expect(r.reason).toMatch(/no remote_url/i) + }) + + it('reports a network failure as UNHEALTHY and names the error', async () => { + stubFetch(new Error('ECONNREFUSED')) + const r = await checkAppHealth(mf('/apps/demo/remoteEntry.js'), ORIGIN) + expect(r.isHealthy).toBe(false) + expect(r.httpStatus).toBeNull() + expect(r.reason).toMatch(/ECONNREFUSED/) + }) + + it('reports a 404 on a NON-federated app root as UNHEALTHY (was < 500 before)', async () => { + stubFetch({ status: 404 }) + const r = await checkAppHealth(iframe('https://demo.example.test'), ORIGIN) + expect(r.isHealthy).toBe(false) + expect(r.httpStatus).toBe(404) + }) +}) + +describe('checkAppHealth — the cases that must be healthy', () => { + it('accepts 200 + a JavaScript content-type', async () => { + stubFetch({ status: 200, contentType: 'application/javascript' }) + const r = await checkAppHealth(mf('/apps/demo/remoteEntry.js'), ORIGIN) + expect(r.isHealthy).toBe(true) + expect(r.reason).toBeNull() + }) + + it('does NOT download the bundle when the content-type already settles it', async () => { + const textSpy = jest.fn(async () => 'x'.repeat(1_000_000)) + ;(global as any).fetch = jest.fn(async () => ({ + status: 200, + headers: { get: () => 'text/javascript' }, + text: textSpy, + })) + const r = await checkAppHealth(mf('/apps/demo/remoteEntry.js'), ORIGIN) + expect(r.isHealthy).toBe(true) + expect(textSpy).not.toHaveBeenCalled() + }) + + it('accepts a 2xx/3xx root for a non-federated app', async () => { + stubFetch({ status: 302 }) + const r = await checkAppHealth(iframe('https://demo.example.test'), ORIGIN) + expect(r.isHealthy).toBe(true) + }) +}) + +describe('remote_url resolution', () => { + it('resolves a relative remote_url against the portal origin', async () => { + const seen = stubFetch({ status: 200, contentType: 'application/javascript' }) + await checkAppHealth(mf('/apps/demo/remoteEntry.js'), ORIGIN) + expect(seen[0]).toBe(`${ORIGIN}/apps/demo/remoteEntry.js`) + }) + + it('probes an ABSOLUTE remote_url verbatim, not against the portal origin', async () => { + // A cross-origin registration must be judged on the host it names. Probing + // it same-origin would report a 404 for a path the app never claimed — + // which is exactly how the first production probe mis-scored fuzekeys. + const seen = stubFetch({ status: 200, contentType: 'application/javascript' }) + await checkAppHealth( + mf('https://keys.example.test/apps/demo/remoteEntry.js'), + ORIGIN + ) + expect(seen[0]).toBe('https://keys.example.test/apps/demo/remoteEntry.js') + }) +}) + +describe('probeOrigin', () => { + it('prefers an explicit FEDERATION_PROBE_ORIGIN', () => { + const prev = process.env.FEDERATION_PROBE_ORIGIN + process.env.FEDERATION_PROBE_ORIGIN = 'https://configured.example.test/' + expect(probeOrigin({ headers: { host: 'ignored' } })).toBe( + 'https://configured.example.test' + ) + if (prev === undefined) delete process.env.FEDERATION_PROBE_ORIGIN + else process.env.FEDERATION_PROBE_ORIGIN = prev + }) + + it('IGNORES caller-controlled headers and returns empty when unconfigured', () => { + // The security property under test. probeOrigin used to fall back to + // x-forwarded-host / Host, which let whoever sent the request choose the + // origin this server probes — and therefore choose the health every app + // reports. Asserting the headers are ignored is the only way this stays + // true: the previous implementation passed a test that asserted the + // opposite, so a "does it work" test would have agreed with the bug. + const prev = process.env.FEDERATION_PROBE_ORIGIN + delete process.env.FEDERATION_PROBE_ORIGIN + expect( + probeOrigin({ + protocol: 'http', + headers: { + host: 'attacker.example.test', + 'x-forwarded-proto': 'https', + 'x-forwarded-host': 'attacker.example.test', + }, + }) + ).toBe('') + if (prev !== undefined) process.env.FEDERATION_PROBE_ORIGIN = prev + }) + + it('strips a trailing slash from the configured origin', () => { + const prev = process.env.FEDERATION_PROBE_ORIGIN + process.env.FEDERATION_PROBE_ORIGIN = 'https://app.fuzefront.com//' + expect(probeOrigin({})).toBe('https://app.fuzefront.com') + if (prev === undefined) delete process.env.FEDERATION_PROBE_ORIGIN + else process.env.FEDERATION_PROBE_ORIGIN = prev + }) +}) + +describe('fail-closed when the origin is unconfigured', () => { + it('reports a same-origin remote_url as UNHEALTHY, naming the missing config', async () => { + const seen = stubFetch({ status: 200, contentType: 'application/javascript' }) + const r = await checkAppHealth(mf('/apps/demo/remoteEntry.js'), '') + expect(r.isHealthy).toBe(false) + expect(r.httpStatus).toBeNull() + expect(r.reason).toMatch(/FEDERATION_PROBE_ORIGIN is not set/i) + // and it must not have probed anything at all + expect(seen).toHaveLength(0) + }) + + it('still probes an ABSOLUTE remote_url with no configured origin', async () => { + // An absolute entry names its own host, so no origin is needed and there is + // nothing for a header to influence. Failing closed here would be + // over-correction, turning a working check off. + const seen = stubFetch({ status: 200, contentType: 'application/javascript' }) + const r = await checkAppHealth( + mf('https://keys.example.test/apps/demo/remoteEntry.js'), + '' + ) + expect(r.isHealthy).toBe(true) + expect(seen[0]).toBe('https://keys.example.test/apps/demo/remoteEntry.js') + }) + + it('does not affect non-federated apps, which probe their own absolute url', async () => { + const seen = stubFetch({ status: 200 }) + const r = await checkAppHealth(iframe('https://demo.example.test'), '') + expect(r.isHealthy).toBe(true) + expect(seen[0]).toBe('https://demo.example.test') + }) +})