diff --git a/backend/security/src/migrations/015_root_membership_backfill_and_personal_org_reclassify.ts b/backend/security/src/migrations/015_root_membership_backfill_and_personal_org_reclassify.ts index 014f2537..0e13caa0 100644 --- a/backend/security/src/migrations/015_root_membership_backfill_and_personal_org_reclassify.ts +++ b/backend/security/src/migrations/015_root_membership_backfill_and_personal_org_reclassify.ts @@ -11,15 +11,36 @@ import { ROOT_ORG_ID } from './014_seed_root_platform_organization' * amendment below — they are NOT independent): * * (a) BACKFILL — every user without an existing `organization_memberships` - * row in the root org (ROOT_ORG_ID, any role) gets one with - * `role='member', status='active'`. A user who already has a row (most - * commonly the root org's own owner, seeded by migration 014) is left - * untouched — this is a "fill the gap", not a "reset everyone to - * member" migration. Runs even if `organizations` has no root row yet - * (defers to 0 rows matched, self-heals on a later boot once 014/015 or - * the monolith's chain seeds it) — a user whose org can't be resolved - * still isn't skipped outright, they simply pick up their root row the - * next time this migration (or the provisioning self-heal path) runs. + * row in the root org gets one with `role='member', status='active'`. + * A user who already has a row (most commonly the root org's own owner, + * seeded by migration 014) is left untouched — this is a "fill the gap", + * not a "reset everyone to member" migration. Runs even if + * `organizations` has no root row yet (skips outright, self-heals on a + * later boot once 014/015 or the monolith's chain seeds it) — a user + * whose org can't be resolved still isn't skipped outright, they simply + * pick up their root row the next time this migration (or the + * provisioning self-heal path) runs. + * + * ROOT ORG RESOLUTION (2026-08-16 P1 fix): this step does NOT hardcode + * `ROOT_ORG_ID` into the INSERT. It resolves the actual root org row the + * SAME WAY `portalRepository.ensureRootPortal()` does — prefer the row + * whose id is `ROOT_ORG_ID`, else fall back to the oldest + * `organizations` row of `type='platform'`. That fallback is required + * because migration 014's own "adopt a pre-existing platform org rather + * than creating a second one" branch can leave a prod DB with a real + * platform-root org under a DIFFERENT id and NO `ROOT_ORG_ID` row at + * all — exactly what happened on the 2026-07-29 rebuild (the adopted + * org is `92f2020b-2bdb-41f0-98ff-1ef759b41741`, slug `fuzefront`, + * which also means a second platform org can never be created under + * `ROOT_ORG_ID` — that slug is taken). Hardcoding `ROOT_ORG_ID` here + * made every INSERT violate `organization_memberships_organization_id_ + * foreign` (23503) on such a DB, which is NOT caught by + * `ON CONFLICT DO NOTHING` (that only dedupes committed conflicts, it + * does not catch a failed insert), and knex propagated the error out of + * `initializeDatabase()` on every boot — the fuzefront-backend / + * fuzefront-security 2026-08-16 P1 crashloop. If NO platform org exists + * at all yet (fresh/schema-only DB), this step is skipped outright and + * self-heals once one exists. * * (b) RECLASSIFY — every `organizations` row with `type='personal'` becomes * `type='organization'`. Non-destructive: nothing is deleted, no other @@ -28,14 +49,16 @@ import { ROOT_ORG_ID } from './014_seed_root_platform_organization' * enum value is kept for back-compat (Postgres cannot drop an enum * value) but is no longer written once the flag is ON. * - * IDEMPOTENT: (a) is an INSERT..SELECT guarded by NOT EXISTS on the unique - * (user_id, organization_id) pair, PLUS `ON CONFLICT DO NOTHING` as a second - * belt-and-braces guard against a concurrent insert of the same pair between - * the SELECT and the INSERT. (b) is a plain UPDATE whose WHERE clause matches - * zero rows once every personal org has already been reclassified. Both are - * safe to run any number of times with no diff after the first successful run - * — verified by `tests/migrations.rootMembershipBackfill.test.ts` (run + re-run - * on a scratch DB). + * IDEMPOTENT: (a) is a root-org lookup (safe to repeat; re-resolves the same + * row every run) followed by an INSERT..SELECT guarded by NOT EXISTS on the + * unique (user_id, organization_id) pair, PLUS `ON CONFLICT DO NOTHING` as a + * second belt-and-braces guard against a concurrent insert of the same pair + * between the SELECT and the INSERT. (b) is a plain UPDATE whose WHERE clause + * matches zero rows once every personal org has already been reclassified. + * Both are safe to run any number of times with no diff after the first + * successful run — verified by `tests/migrations.rootMembershipBackfill.test.ts` + * (run + re-run on a scratch DB, PLUS a dedicated "ROOT_ORG_ID row absent, + * only an adopted platform org exists" regression case for the P1 above). * * PRECONDITION on BOTH (a) and (b): the `ROOT_ORG_ID` row must exist. It is a * hard-coded constant, not a lookup, and migration 014 has two paths that @@ -68,33 +91,21 @@ import { ROOT_ORG_ID } from './014_seed_root_platform_organization' * land in a deploy window (`deploy-window` label, FF-EPIC-17-S2 DoD). */ export async function up(knex: Knex): Promise { - // (a) Backfill root membership for every user lacking one. - // - // PRECONDITION: the root org row must actually exist. `ROOT_ORG_ID` is a - // hard-coded constant, not a lookup, and migration 014 has two paths that - // legitimately leave the row ABSENT — it adopts a pre-existing platform org - // under a different id, and it defers entirely when there is no user yet. - // Neither is an error there, but inserting memberships that reference a - // missing organization is a foreign-key violation (23503) that aborts the - // migration chain and crash-loops the service on boot. Verify, don't assume: - // a step that cannot tell "nothing to do" from "the thing I need is missing" - // eventually reports success for the wrong reason. - const rootOrg = await knex('organizations').where({ id: ROOT_ORG_ID }).first() - if (!rootOrg) { - const platform = await knex('organizations') + // (a) Resolve the ACTUAL root org row (see the P1 note above), then + // backfill root membership for every user lacking one against ITS id — + // never the bare ROOT_ORG_ID constant, which may have no `organizations` + // row at all on a DB that adopted a pre-existing platform org. + const rootOrg = + (await knex('organizations').where({ id: ROOT_ORG_ID }).first()) ?? + (await knex('organizations') .where({ type: 'platform' }) .orderBy('created_at', 'asc') - .first() - console.error( - `[015] SKIPPING root-membership backfill: organization ${ROOT_ORG_ID} does ` + - 'not exist. ' + - (platform - ? `The platform organization in this database is ${platform.id}, which ` + - 'migration 014 adopted rather than repointed. Resolve that first — ' + - 'until then every ROOT_ORG_ID call site misses.' - : 'No platform organization exists at all; migration 014 deferred.') + - ' Not inserting memberships that would violate' + - ' organization_memberships_organization_id_foreign.' + .first()) + + if (!rootOrg) { + console.log( + '[015] no platform root organization exists yet — skipping root-membership ' + + 'backfill (self-heals on a later boot once 014/ensureRootPortal() seeds one)' ) } else { const backfillResult = await knex.raw( @@ -107,10 +118,12 @@ export async function up(knex: Knex): Promise { WHERE om.user_id = u.id AND om.organization_id = ? ) ON CONFLICT (user_id, organization_id) DO NOTHING`, - [ROOT_ORG_ID, ROOT_ORG_ID] + [rootOrg.id, rootOrg.id] ) console.log( - `[015] root-membership backfill: inserted ${backfillResult.rowCount ?? 0} row(s)` + `[015] root-membership backfill: inserted ${backfillResult.rowCount ?? 0} row(s) ` + + `against root org ${rootOrg.id}` + + (rootOrg.id === ROOT_ORG_ID ? '' : ' (adopted platform org, not the canonical ROOT_ORG_ID)') ) } diff --git a/backend/security/tests/migrations.rootMembershipBackfill.test.ts b/backend/security/tests/migrations.rootMembershipBackfill.test.ts index 02723ba5..e0b3420e 100644 --- a/backend/security/tests/migrations.rootMembershipBackfill.test.ts +++ b/backend/security/tests/migrations.rootMembershipBackfill.test.ts @@ -287,3 +287,156 @@ describe('migration 015 — root-membership backfill + personal-org reclassify ( expect(rootOrg).toBeDefined() }) }) + +/** + * 2026-08-16 P1 regression — reproduces the exact prod condition that put + * fuzefront-backend/-security into a crashloop: `organizations` has NO row + * whose id is the canonical `ROOT_ORG_ID` (`…0010`), because migration 014's + * "adopt a pre-existing platform org rather than creating a second one" + * branch had already run against a DB whose platform org was created under a + * random id (prod: `92f2020b-2bdb-41f0-98ff-1ef759b41741`, slug `fuzefront`). + * Pre-fix, migration 015 hardcoded `ROOT_ORG_ID` into the backfill INSERT, + * which violated `organization_memberships_organization_id_foreign` (23503) + * on exactly this DB shape — uncaught by `ON CONFLICT DO NOTHING` (that only + * dedupes committed conflicts, not a failed insert) — and knex propagated the + * error out of `initializeDatabase()` on every boot. + * + * Uses its own scratch DB (not the suite above) so this describe block can + * freely omit ROOT_ORG_ID from `organizations` without disturbing the + * happy-path tests, which assume it exists. + */ +describe('migration 015 — resilient when the canonical ROOT_ORG_ID row is absent (2026-08-16 P1 regression)', () => { + let reachable = false + let db: any + let ROOT_ORG_ID: string + let migration015: { up(knex: any): Promise; down(knex: any): Promise } + const TEST_DB2 = 'fuzefront_security_root_backfill_missing_root_test' + + beforeAll(async () => { + reachable = await pgReachable() + if (!reachable) return + + process.env.USE_POSTGRES = 'true' + process.env.NODE_ENV = 'production' + process.env.DB_HOST = HOST + process.env.DB_PORT = String(PORT) + process.env.DB_USER = USER + process.env.DB_PASSWORD = PASSWORD + process.env.DB_NAME = TEST_DB2 + + const admin = new Client({ host: HOST, port: PORT, user: USER, password: PASSWORD, database: 'postgres' }) + await admin.connect() + await admin.query(`DROP DATABASE IF EXISTS ${TEST_DB2}`) + await admin.query(`CREATE DATABASE ${TEST_DB2}`) + await admin.end() + + const core = require('@fuzefront/core') + const migDir = path.join(__dirname, '..', 'dist', 'migrations') + // Run only the schema chain, NOT 014/015 — this test seeds the + // "adopted platform org, ROOT_ORG_ID absent" condition by hand and then + // invokes 015's up() directly, mirroring how the real deploy runs it. + await core.runMigrations({ migrationsTableName: 'knex_migrations', migrationsDir: migDir }) + core.initializeDatabaseConnection({ migrationsTableName: 'knex_migrations', migrationsDir: migDir }) + db = core.db + + const rootOrgMigration = require('../src/migrations/014_seed_root_platform_organization') + ROOT_ORG_ID = rootOrgMigration.ROOT_ORG_ID + migration015 = require('../src/migrations/015_root_membership_backfill_and_personal_org_reclassify') + + // The full chain above already ran 014/015 on a zero-user DB, so they + // deferred (no organizations row at all yet) — confirmed below. Now + // simulate the prod condition: a platform org exists under a RANDOM id + // (adopted, e.g. by an earlier ensureRootPortal() run), and ROOT_ORG_ID + // itself has no row. + const preexisting = await db('organizations').where({ id: ROOT_ORG_ID }).first() + if (preexisting) { + throw new Error('test setup invariant violated: ROOT_ORG_ID row should not exist yet') + } + + const adoptedOwnerId = uuidv4() + await db('users').insert({ + id: adoptedOwnerId, + email: 'adopted-owner@test.local', + first_name: 'Adopted', + last_name: 'Owner', + roles: JSON.stringify(['admin', 'user']), + created_at: new Date(), + updated_at: new Date(), + }) + await db('organizations').insert({ + id: uuidv4(), + name: 'FuzeFront', + slug: 'fuzefront', // matches prod: this slug is already taken by the adopted org + owner_id: adoptedOwnerId, + type: 'platform', + settings: JSON.stringify({}), + metadata: JSON.stringify({ root: true }), + is_active: true, + provisioning_state: 'active', + }) + }, 60000) + + afterAll(async () => { + if (db) await db.destroy() + }) + + async function createUser(): Promise { + const id = uuidv4() + await db('users').insert({ + id, + email: `missing-root-${id.slice(0, 8)}@test.local`, + first_name: 'MissingRoot', + last_name: 'Test', + roles: JSON.stringify(['user']), + created_at: new Date(), + updated_at: new Date(), + }) + return id + } + + it('does NOT throw an FK violation, backfills against the ADOPTED org, and creates no second platform org', async () => { + if (!reachable) return console.warn('Postgres unreachable — skipping') + + // Precondition matching prod exactly. + const canonical = await db('organizations').where({ id: ROOT_ORG_ID }).first() + expect(canonical).toBeUndefined() + const adopted = await db('organizations').where({ type: 'platform' }).first() + expect(adopted).toBeDefined() + + const user = await createUser() + + // Pre-fix this threw a Postgres 23503 FK violation. Post-fix it must + // resolve to the adopted org and succeed. + await expect(migration015.up(db)).resolves.toBeUndefined() + + const membership = await db('organization_memberships') + .where({ user_id: user, organization_id: adopted.id }) + .first() + expect(membership).toMatchObject({ role: 'member', status: 'active' }) + + // No row was ever inserted under the (still-nonexistent) canonical id. + const canonicalMembership = await db('organization_memberships') + .where({ organization_id: ROOT_ORG_ID }) + .first() + expect(canonicalMembership).toBeUndefined() + + // Exactly ONE platform org — the fix must never create a second one. + const platformOrgs = await db('organizations').where({ type: 'platform' }) + expect(platformOrgs).toHaveLength(1) + expect(platformOrgs[0].id).toBe(adopted.id) + }) + + it('is idempotent when re-run against the same adopted-root DB', async () => { + if (!reachable) return console.warn('Postgres unreachable — skipping') + + const membershipsBefore = await db('organization_memberships').select('id').orderBy('id') + const orgsBefore = await db('organizations').select('id', 'type').orderBy('id') + + await expect(migration015.up(db)).resolves.toBeUndefined() + + const membershipsAfter = await db('organization_memberships').select('id').orderBy('id') + const orgsAfter = await db('organizations').select('id', 'type').orderBy('id') + expect(membershipsAfter).toEqual(membershipsBefore) + expect(orgsAfter).toEqual(orgsBefore) + }) +}) diff --git a/backend/src/migrations/022_root_membership_backfill_and_personal_org_reclassify.ts b/backend/src/migrations/022_root_membership_backfill_and_personal_org_reclassify.ts index a78728d9..7eb43812 100644 --- a/backend/src/migrations/022_root_membership_backfill_and_personal_org_reclassify.ts +++ b/backend/src/migrations/022_root_membership_backfill_and_personal_org_reclassify.ts @@ -16,13 +16,33 @@ import { ROOT_ORG_ID } from './015_seed_root_platform_organization' * amendment below — they are NOT independent): * * (a) BACKFILL — every user without an existing `organization_memberships` - * row in the root org (ROOT_ORG_ID, any role) gets one with - * `role='member', status='active'`. A user who already has a row (most - * commonly the root org's own owner, seeded by migration 015) is left - * untouched — this is a "fill the gap", not a "reset everyone to - * member" migration. A user whose org can't be resolved still isn't - * skipped outright — root membership is universal and never depends on - * org resolution. + * row in the root org gets one with `role='member', status='active'`. + * A user who already has a row (most commonly the root org's own owner, + * seeded by migration 015) is left untouched — this is a "fill the gap", + * not a "reset everyone to member" migration. A user whose org can't be + * resolved still isn't skipped outright — root membership is universal + * and never depends on org resolution. + * + * ROOT ORG RESOLUTION (2026-08-16 P1 fix): this step does NOT hardcode + * `ROOT_ORG_ID` into the INSERT. It resolves the actual root org row the + * SAME WAY `portalRepository.ensureRootPortal()` does — prefer the row + * whose id is `ROOT_ORG_ID`, else fall back to the oldest + * `organizations` row of `type='platform'`. That fallback is required + * because migration 015's own "adopt a pre-existing platform org rather + * than creating a second one" branch can leave a prod DB with a real + * platform-root org under a DIFFERENT id and NO `ROOT_ORG_ID` row at + * all — exactly what happened on the 2026-07-29 rebuild (the adopted + * org is `92f2020b-2bdb-41f0-98ff-1ef759b41741`, slug `fuzefront`, + * which also means a second platform org can never be created under + * `ROOT_ORG_ID` — that slug is taken). Hardcoding `ROOT_ORG_ID` here + * made every INSERT violate `organization_memberships_organization_id_ + * foreign` (23503) on such a DB, which is NOT caught by + * `ON CONFLICT DO NOTHING` (that only dedupes committed conflicts, it + * does not catch a failed insert), and knex propagated the error out of + * `initializeDatabase()` on every boot — the fuzefront-backend / + * fuzefront-security 2026-08-16 P1 crashloop. If NO platform org exists + * at all yet (fresh/schema-only DB), this step is skipped outright and + * self-heals once one exists. * * (b) RECLASSIFY — every `organizations` row with `type='personal'` becomes * `type='organization'`. Non-destructive: nothing is deleted, no other @@ -31,15 +51,18 @@ import { ROOT_ORG_ID } from './015_seed_root_platform_organization' * enum value is kept for back-compat (Postgres cannot drop an enum * value) but is no longer written once the flag is ON. * - * IDEMPOTENT: (a) is an INSERT..SELECT guarded by NOT EXISTS on the unique - * (user_id, organization_id) pair, PLUS `ON CONFLICT DO NOTHING` as a second - * belt-and-braces guard against a concurrent insert of the same pair between - * the SELECT and the INSERT. (b) is a plain UPDATE whose WHERE clause matches - * zero rows once every personal org has already been reclassified. Both are - * safe to run any number of times with no diff after the first successful - * run — verified by `backend/security/tests/ - * migrations.rootMembershipBackfill.test.ts` (run + re-run on a scratch DB); - * this monolith copy is byte-identical SQL, so the same proof applies. + * IDEMPOTENT: (a) is a root-org lookup (safe to repeat; re-resolves the same + * row every run) followed by an INSERT..SELECT guarded by NOT EXISTS on the + * unique (user_id, organization_id) pair, PLUS `ON CONFLICT DO NOTHING` as a + * second belt-and-braces guard against a concurrent insert of the same pair + * between the SELECT and the INSERT. (b) is a plain UPDATE whose WHERE clause + * matches zero rows once every personal org has already been reclassified. + * Both are safe to run any number of times with no diff after the first + * successful run — verified by `backend/security/tests/ + * migrations.rootMembershipBackfill.test.ts` (run + re-run on a scratch DB, + * PLUS a dedicated "ROOT_ORG_ID row absent, only an adopted platform org + * exists" regression case for the P1 above); this monolith copy is + * byte-identical SQL, so the same proof applies. * * PRECONDITION on BOTH (a) and (b): the `ROOT_ORG_ID` row must exist. It is a * hard-coded constant, not a lookup, and migration 015 has two paths that @@ -72,33 +95,21 @@ import { ROOT_ORG_ID } from './015_seed_root_platform_organization' * land in a deploy window (`deploy-window` label, FF-EPIC-17-S2 DoD). */ export async function up(knex: Knex): Promise { - // (a) Backfill root membership for every user lacking one. - // - // PRECONDITION: the root org row must actually exist. `ROOT_ORG_ID` is a - // hard-coded constant, not a lookup, and migration 015 has two paths that - // legitimately leave the row ABSENT — it adopts a pre-existing platform org - // under a different id, and it defers entirely when there is no user yet. - // Neither is an error there, but inserting memberships that reference a - // missing organization is a foreign-key violation (23503) that aborts the - // migration chain and crash-loops the service on boot. Verify, don't assume: - // a step that cannot tell "nothing to do" from "the thing I need is missing" - // eventually reports success for the wrong reason. - const rootOrg = await knex('organizations').where({ id: ROOT_ORG_ID }).first() - if (!rootOrg) { - const platform = await knex('organizations') + // (a) Resolve the ACTUAL root org row (see the P1 note above), then + // backfill root membership for every user lacking one against ITS id — + // never the bare ROOT_ORG_ID constant, which may have no `organizations` + // row at all on a DB that adopted a pre-existing platform org. + const rootOrg = + (await knex('organizations').where({ id: ROOT_ORG_ID }).first()) ?? + (await knex('organizations') .where({ type: 'platform' }) .orderBy('created_at', 'asc') - .first() - console.error( - `[022] SKIPPING root-membership backfill: organization ${ROOT_ORG_ID} does ` + - 'not exist. ' + - (platform - ? `The platform organization in this database is ${platform.id}, which ` + - 'migration 015 adopted rather than repointed. Resolve that first — ' + - 'until then every ROOT_ORG_ID call site misses.' - : 'No platform organization exists at all; migration 015 deferred.') + - ' Not inserting memberships that would violate' + - ' organization_memberships_organization_id_foreign.' + .first()) + + if (!rootOrg) { + console.log( + '[022] no platform root organization exists yet — skipping root-membership ' + + 'backfill (self-heals on a later boot once 015/ensureRootPortal() seeds one)' ) } else { const backfillResult = await knex.raw( @@ -111,10 +122,12 @@ export async function up(knex: Knex): Promise { WHERE om.user_id = u.id AND om.organization_id = ? ) ON CONFLICT (user_id, organization_id) DO NOTHING`, - [ROOT_ORG_ID, ROOT_ORG_ID] + [rootOrg.id, rootOrg.id] ) console.log( - `[022] root-membership backfill: inserted ${backfillResult.rowCount ?? 0} row(s)` + `[022] root-membership backfill: inserted ${backfillResult.rowCount ?? 0} row(s) ` + + `against root org ${rootOrg.id}` + + (rootOrg.id === ROOT_ORG_ID ? '' : ' (adopted platform org, not the canonical ROOT_ORG_ID)') ) }