From 329b71aa94edbbb1ec2a35bd229e7f23415fb527 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:48:57 +0700 Subject: [PATCH 1/2] fix(dos-id): consume unknown-entity webhook events as idempotent no-ops DOS ID broadcasts ecosystem-wide events, including organisations whose owners never used Crove Sign (no JIT provisioning), so entity lookups fail permanently. The job runner retried those failures endlessly: ~1.4k failed process-dos-webhook jobs in a few hours, drowning real errors in the logs and burning job budget. Entity-not-found on team/org-member/user/org events is now consumed as a success no-op (matching the existing user-removed and team-removed semantics), so the in-flight retry queue drains instead of looping. Malformed payloads (missing required fields) keep failing loudly so contract breaks stay visible. Proven on prod before this fix: none of the failing organisation ids exist in sign.Organisation (checked read-only against the production database). --- .../server-only/dos-id/handle-dos-webhook.ts | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/lib/server-only/dos-id/handle-dos-webhook.ts b/packages/lib/server-only/dos-id/handle-dos-webhook.ts index 3e5b64a3ab..9f758a1430 100644 --- a/packages/lib/server-only/dos-id/handle-dos-webhook.ts +++ b/packages/lib/server-only/dos-id/handle-dos-webhook.ts @@ -49,6 +49,13 @@ export const handleDosWebhookEvent = async ( // ========================================== // ORGANISATION EVENTS // ========================================== + // DOS ID broadcasts events for the whole ecosystem, including + // organisations whose owners never used Crove Sign (no JIT provisioning). + // An entity-not-found lookup is permanent, so those events are consumed + // as idempotent no-ops instead of failures: retrying never succeeds and + // the retry storm once produced ~1.4k failed jobs per hour. Malformed + // payloads (missing required fields) keep failing so contract breaks + // stay loud. case 'organization.created': case 'org.created': { const orgId = (data.org_id || data.id) as string | undefined; @@ -102,7 +109,7 @@ export const handleDosWebhookEvent = async ( }); if (!org) { - return { success: false, message: 'Organization not found' }; + return { success: true, message: 'Organization not found in sign schema, nothing to update' }; } await prisma.organisation.update({ @@ -132,7 +139,7 @@ export const handleDosWebhookEvent = async ( }); if (!org) { - return { success: false, message: 'Organization not found for deletion' }; + return { success: true, message: 'Organization not found in sign schema, nothing to delete' }; } await deleteOrganisation({ @@ -164,7 +171,7 @@ export const handleDosWebhookEvent = async ( }); if (!org) { - return { success: false, message: 'Organization not found' }; + return { success: true, message: 'Organization not found in sign schema, nothing to update' }; } let user = await prisma.user.findFirst({ @@ -269,7 +276,7 @@ export const handleDosWebhookEvent = async ( }); if (!org) { - return { success: false, message: `Organisation ${orgId} not found for team.created` }; + return { success: true, message: `Organisation ${orgId} not found in sign schema, nothing to create` }; } // Check if team already exists @@ -318,7 +325,7 @@ export const handleDosWebhookEvent = async ( }); if (!team) { - return { success: false, message: 'Team not found for update' }; + return { success: true, message: 'Team not found in sign schema, nothing to update' }; } await prisma.team.update({ @@ -354,7 +361,7 @@ export const handleDosWebhookEvent = async ( }); if (!team) { - return { success: false, message: 'Team not found for deletion' }; + return { success: true, message: 'Team not found in sign schema, nothing to delete' }; } await prisma.$transaction(async (tx) => { @@ -413,7 +420,7 @@ export const handleDosWebhookEvent = async ( }); if (!targetOrg) { - return { success: false, message: 'Target organisation not found' }; + return { success: true, message: 'Target organisation not found in sign schema, nothing to add' }; } const finalOrgId = targetOrg.id; From 7c5af71b6ed5847294708c20351e957c3556b2e0 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:59:01 +0700 Subject: [PATCH 2/2] fix(dos-id): surface no-op reasons in job logs and guard empty org lookups Address review findings on the no-op change: log consumed-as-no-op reasons as warnings so the ignored bucket stays auditable (the job runner otherwise never logs the result message), and add missing-field guards to org.updated/org.deleted so an empty OR clause can never reach Prisma on a destructive path. Also correct the policy comment: org absence stems from pre-integration/failed-provisioning/deleted orgs, not a missing JIT path (org.created provisions anything it receives). --- .../internal/process-dos-webhook.handler.ts | 6 ++++++ .../server-only/dos-id/handle-dos-webhook.ts | 21 +++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/lib/jobs/definitions/internal/process-dos-webhook.handler.ts b/packages/lib/jobs/definitions/internal/process-dos-webhook.handler.ts index f091a49262..769a388fbb 100644 --- a/packages/lib/jobs/definitions/internal/process-dos-webhook.handler.ts +++ b/packages/lib/jobs/definitions/internal/process-dos-webhook.handler.ts @@ -14,6 +14,12 @@ export const run = async ({ payload, io }: { payload: TProcessDosWebhookJobDefin io.logger.info(`[DOS Webhook Job] Successfully processed event: ${payload.event}`); + // No-op consumptions (unknown entities) succeed silently; surface their + // reason as a warning so the "ignored" bucket stays auditable in logs. + if (/not found/i.test(result.message)) { + io.logger.warn(`[DOS Webhook Job] Event consumed as no-op: ${result.message}`); + } + return { success: true, message: result.message, diff --git a/packages/lib/server-only/dos-id/handle-dos-webhook.ts b/packages/lib/server-only/dos-id/handle-dos-webhook.ts index 9f758a1430..ac202114bc 100644 --- a/packages/lib/server-only/dos-id/handle-dos-webhook.ts +++ b/packages/lib/server-only/dos-id/handle-dos-webhook.ts @@ -50,10 +50,11 @@ export const handleDosWebhookEvent = async ( // ORGANISATION EVENTS // ========================================== // DOS ID broadcasts events for the whole ecosystem, including - // organisations whose owners never used Crove Sign (no JIT provisioning). - // An entity-not-found lookup is permanent, so those events are consumed - // as idempotent no-ops instead of failures: retrying never succeeds and - // the retry storm once produced ~1.4k failed jobs per hour. Malformed + // organisations absent from the sign schema (created before webhook + // integration, failed provisioning, or deleted upstream). An + // entity-not-found lookup is permanent, so those events are consumed as + // idempotent no-ops instead of failures: retrying never succeeds and the + // retry storm once produced ~1.4k failed jobs per hour. Malformed // payloads (missing required fields) keep failing so contract breaks // stay loud. case 'organization.created': @@ -102,6 +103,12 @@ export const handleDosWebhookEvent = async ( const slug = data.slug as string | undefined; const name = data.name as string | undefined; + // An empty where clause must never reach Prisma: a missing id AND slug + // is a malformed payload, not an entity to resolve. + if (!orgId && !slug) { + return { success: false, message: 'Missing org_id or slug in org.updated' }; + } + const org = await prisma.organisation.findFirst({ where: { OR: [...(orgId ? [{ id: orgId }] : []), ...(slug ? [{ url: slug }] : [])], @@ -128,6 +135,12 @@ export const handleDosWebhookEvent = async ( const orgId = (data.org_id || data.id) as string | undefined; const slug = data.slug as string | undefined; + // Same malformed-payload guard as org.updated: an empty OR clause must + // never reach Prisma on a destructive path. + if (!orgId && !slug) { + return { success: false, message: 'Missing org_id or slug in org.deleted' }; + } + const org = await prisma.organisation.findFirst({ where: { OR: [...(orgId ? [{ id: orgId }] : []), ...(slug ? [{ url: slug }] : [])],