From 7a17493b2234f19d369c5fb946b98e0c274327c3 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:22:04 -0700 Subject: [PATCH 1/2] Add e2e repro for vault write failure during token refresh --- e2e/cloud/vault-write-outage.test.ts | 313 +++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 e2e/cloud/vault-write-outage.test.ts diff --git a/e2e/cloud/vault-write-outage.test.ts b/e2e/cloud/vault-write-outage.test.ts new file mode 100644 index 000000000..4c4c85809 --- /dev/null +++ b/e2e/cloud/vault-write-outage.test.ts @@ -0,0 +1,313 @@ +// Cloud: reproduce the production "StorageError: WorkOS Vault secret write +// failed" Sentry events (NODE-CLOUDFLARE-WORKERS-52/53/4T) at the real +// upstream — faults armed on the WorkOS emulator's Vault KV PUT, the same +// emulator the product's real WorkOS SDK talks to. No product code touched. +// +// The production chain, staged here end-to-end: +// 1. An OAuth connection's access token expires (per-client TTL of 1s via +// the emulator's DCR `access_token_ttl_seconds` extension, well inside +// the 60s refresh skew). +// 2. A health check resolves the connection → triggers the refresh-token +// grant. The grant SUCCEEDS and the authorization server ROTATES the +// refresh token (single use — the emulator mirrors AuthKit). +// 3. Persisting the new tokens does read → `PUT /vault/v1/kv/:id`. The PUT +// fails (in prod: 409 version conflict, an OAuth-shaped 400, or an HTML +// error page; here: the armed 400 with `error`/`error_description`, the +// exact shape the WorkOS SDK maps to OauthException — Sentry issue -53). +// 4. The write failure surfaces as `StorageError: WorkOS Vault secret write +// failed` → the health endpoint answers a typed InternalError (the 500 +// Sentry records). +// +// The second scenario pins the DAMAGE, not just the surface error: the vault +// write failed AFTER the refresh token was consumed and rotated, so the new +// refresh token is lost and the stored one is revoked. Once the vault +// recovers, the connection must recover too — today it cannot (the next +// refresh gets invalid_grant → the connection reads "expired" and demands a +// re-auth). Red until the refresh/persist ordering is made crash-safe. +import { randomBytes } from "node:crypto"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import type { HttpApiClient } from "effect/unstable/httpapi"; +import { composePluginApi } from "@executor-js/api/server"; +import { connectEmulator, type EmulatorClient } from "@executor-js/emulate"; +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; +import { + AuthTemplateSlug, + ConnectionName, + IntegrationSlug, + OAuthClientSlug, +} from "@executor-js/sdk/shared"; + +import { scenario } from "../src/scenario"; +import { Api, Target } from "../src/services"; +import type { Target as TargetShape } from "../src/target"; +import { WORKOS_EMULATOR_PORT } from "../targets/cloud"; + +const api = composePluginApi([openApiHttpPlugin()] as const); +type Client = HttpApiClient.ForApi; + +const TEMPLATE = AuthTemplateSlug.make("oauth2"); +const CONNECTION = ConnectionName.make("main"); +const WORKOS_EMULATOR_URL = `http://127.0.0.1:${WORKOS_EMULATOR_PORT}`; + +const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; + +/** Minimal OpenAPI spec with a single GET /ping — never contacted (the + * integration declares no health-check spec, so checkHealth takes the + * credential-resolution path, which is where the vault write lives). */ +const pingSpec = JSON.stringify({ + openapi: "3.0.3", + info: { title: "Ping API", version: "1.0.0" }, + paths: { + "/ping": { + get: { operationId: "ping", summary: "Ping", responses: { "200": { description: "pong" } } }, + }, + }, +}); + +// The vault-write outage: `PUT /vault/v1/kv/:id` (the WorkOS SDK's +// updateObject) starts answering with the OAuth-shaped 400 the production +// events carry ("OauthException: Error: Invalid request parameters" — the +// SDK maps any non-{401,404,409,422,429} status whose body has +// `error`/`error_description` to OauthException). Reads (GET) and creates +// (POST) stay healthy: only the update leg fails, as in production. +// `times` bounds the blast radius; the finalizer clears whatever remains. +const VAULT_UPDATE_FAULT = { + match: { method: "PUT", pathPattern: "/vault/v1/kv/*" }, + response: { + status: 400, + body: { + code: "invalid_request", + message: "Invalid request parameters", + error: "Invalid request parameters", + error_description: "Invalid request parameters", + }, + }, + times: 8, +} as const; + +/** DCR-register an OAuth client directly on the WorkOS emulator with a 1s + * access-token TTL (per-client, so the emulator's default — which the + * product's own AuthKit sessions depend on — is untouched). 1s is inside + * executor's 60s refresh skew: the first resolve after connect refreshes. */ +const registerShortLivedOAuthClient = (redirectUri: string) => + Effect.promise(async () => { + const response = await fetch(`${WORKOS_EMULATOR_URL}/oauth2/register`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + client_name: "vault-write-outage-e2e", + redirect_uris: [redirectUri], + access_token_ttl_seconds: 1, + }), + }); + if (response.status !== 201) { + throw new Error(`WorkOS emulator DCR failed: ${response.status}`); + } + const body = (await response.json()) as { readonly client_id: string }; + return body.client_id; + }); + +/** Complete the emulator's authorize hop headlessly: `login_hint` makes the + * authorize endpoint 302 straight back with a code (no consent page). */ +const completeConsent = (authorizationUrl: string, email: string) => + Effect.promise(async () => { + const url = new URL(authorizationUrl); + url.searchParams.set("login_hint", email); + const callback = await fetch(url, { redirect: "manual" }); + const location = callback.headers.get("location"); + if (callback.status !== 302 || !location) { + throw new Error(`WorkOS emulator authorize did not redirect: ${callback.status}`); + } + const code = new URL(location).searchParams.get("code"); + if (!code) throw new Error("WorkOS emulator callback did not include a code"); + return code; + }); + +/** Register a fresh integration + OAuth app against the emulator's generic + * authorize/token endpoints and connect it. `offline_access` mints a + * refresh token, so the expired-token path refreshes instead of re-authing. */ +const connectExpiringOAuthConnection = (input: { + readonly client: Client; + readonly target: TargetShape; + readonly integration: IntegrationSlug; + readonly oauthClient: OAuthClientSlug; +}) => + Effect.gen(function* () { + const redirectUri = new URL("/api/oauth/callback", input.target.baseUrl).toString(); + + yield* input.client.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: pingSpec }, + slug: input.integration, + baseUrl: "http://127.0.0.1:59999", // never contacted + authenticationTemplate: [ + { + slug: "oauth2", + kind: "oauth2", + authorizationUrl: `${WORKOS_EMULATOR_URL}/oauth2/authorize`, + tokenUrl: `${WORKOS_EMULATOR_URL}/oauth2/token`, + scopes: ["offline_access"], + }, + ], + }, + }); + + const clientId = yield* registerShortLivedOAuthClient(redirectUri); + yield* input.client.oauth.createClient({ + payload: { + owner: "org", + slug: input.oauthClient, + grant: "authorization_code", + authorizationUrl: `${WORKOS_EMULATOR_URL}/oauth2/authorize`, + tokenUrl: `${WORKOS_EMULATOR_URL}/oauth2/token`, + clientId, + // The emulator's DCR clients are public (auth method "none"); the + // secret is carried but never validated. + clientSecret: "unused", + originIntegration: input.integration, + }, + }); + + const started = yield* input.client.oauth.start({ + payload: { + client: input.oauthClient, + clientOwner: "org", + owner: "org", + name: CONNECTION, + integration: input.integration, + template: TEMPLATE, + redirectUri, + }, + }); + expect(started.status, "OAuth starts with an emulator redirect").toBe("redirect"); + if (started.status !== "redirect") return yield* Effect.die("OAuth unexpectedly connected"); + + const code = yield* completeConsent(started.authorizationUrl, "vault-outage@example.com"); + const completed = yield* input.client.oauth.complete({ + payload: { state: started.state, code }, + }); + expect(completed.integration, "OAuth completion creates the connection").toBe( + input.integration, + ); + }); + +const removeEverything = (input: { + readonly client: Client; + readonly integration: IntegrationSlug; + readonly oauthClient: OAuthClientSlug; +}) => + Effect.gen(function* () { + yield* input.client.connections + .remove({ params: { owner: "org", integration: input.integration, name: CONNECTION } }) + .pipe(Effect.ignore); + yield* input.client.oauth + .removeClient({ params: { slug: input.oauthClient }, payload: { owner: "org" } }) + .pipe(Effect.ignore); + yield* input.client.openapi + .removeSpec({ params: { slug: input.integration } }) + .pipe(Effect.ignore); + }); + +const checkHealth = (client: Client, integration: IntegrationSlug) => + client.connections.checkHealth({ + params: { owner: "org", integration, name: CONNECTION }, + query: { ifStaleMs: 0 }, + }); + +scenario( + "Vault · a vault write failure during token refresh surfaces as the internal storage error", + {}, + Effect.gen(function* () { + const target = yield* Target; + const { client: makeClient } = yield* Api; + const identity = yield* target.newIdentity(); + const client = yield* makeClient(api, identity); + const integration = IntegrationSlug.make(unique("vault_outage")); + const oauthClient = OAuthClientSlug.make(unique("vault_outage_app")); + const workos = yield* Effect.promise(() => connectEmulator({ baseUrl: WORKOS_EMULATOR_URL })); + + yield* Effect.ensuring( + Effect.gen(function* () { + yield* connectExpiringOAuthConnection({ client, target, integration, oauthClient }); + + // The 1s token is already inside the refresh skew. Break the vault's + // update leg, then run the health check that resolves the credential. + yield* Effect.promise(() => workos.faults.arm(VAULT_UPDATE_FAULT)); + + const error = yield* checkHealth(client, integration).pipe(Effect.flip); + expect( + (error as { _tag?: string })._tag, + "the failed vault write surfaces as the typed internal error (the prod 500)", + ).toBe("InternalError"); + + // The proof this is the production mechanism and not an incidental + // failure: the product performed the refresh grant and then hit the + // injected fault on the vault update. + const ledger = yield* Effect.promise(() => workos.ledger.list(200)); + const faultedPut = ledger.find((entry) => entry.faulted === true && entry.method === "PUT"); + expect(faultedPut?.path, "the failure came from the injected vault-update fault").toMatch( + /\/vault\/v1\/kv\//, + ); + const refreshGrant = ledger.find( + (entry) => + entry.path.endsWith("/oauth2/token") && + JSON.stringify(entry.request.body ?? "").includes("refresh_token"), + ); + expect( + refreshGrant, + "the vault write that failed was persisting a completed refresh grant", + ).toBeDefined(); + }), + Effect.gen(function* () { + yield* Effect.promise(() => workos.faults.clear()); + yield* removeEverything({ client, integration, oauthClient }); + }), + ); + }), +); + +scenario( + "Vault · a transient vault write outage does not invalidate the connection once the vault recovers", + {}, + Effect.gen(function* () { + const target = yield* Target; + const { client: makeClient } = yield* Api; + const identity = yield* target.newIdentity(); + const client = yield* makeClient(api, identity); + const integration = IntegrationSlug.make(unique("vault_recover")); + const oauthClient = OAuthClientSlug.make(unique("vault_recover_app")); + const workos = yield* Effect.promise(() => connectEmulator({ baseUrl: WORKOS_EMULATOR_URL })); + + yield* Effect.ensuring( + Effect.gen(function* () { + yield* connectExpiringOAuthConnection({ client, target, integration, oauthClient }); + + // One health check during the outage: the refresh grant consumes and + // ROTATES the single-use refresh token, then the vault write of the + // rotated pair fails. + yield* Effect.promise(() => workos.faults.arm(VAULT_UPDATE_FAULT)); + yield* checkHealth(client, integration).pipe(Effect.flip); + yield* Effect.promise(() => workos.faults.clear()); + + // The vault has recovered. A transient storage blip must not cost the + // user their grant: the connection has a live authorization at the AS + // and must come back healthy without a re-auth. Today it cannot — the + // rotated refresh token was never persisted, so the stored (revoked) + // one is replayed, the AS answers invalid_grant, and the connection + // reads "expired" until a human reconnects. This is the real damage + // behind the prod Sentry events. + const health = yield* checkHealth(client, integration); + expect( + health.status, + `a transient vault outage must not permanently invalidate the connection: ${JSON.stringify(health)}`, + ).toBe("healthy"); + }), + Effect.gen(function* () { + yield* Effect.promise(() => workos.faults.clear()); + yield* removeEverything({ client, integration, oauthClient }); + }), + ); + }), +); From 8a89dd561bcc18b0699aeedf84f9024ebeb4b9ac Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:54:43 -0700 Subject: [PATCH 2/2] Gate OAuth token refresh on credential-store writability A vault write failure while persisting a refreshed OAuth token lost the rotated refresh token: the grant had already consumed the single-use stored token at the authorization server, so once the vault recovered the next refresh replayed the revoked token, got invalid_grant, and the connection demanded a re-auth over what was only a storage blip. Before consuming the refresh token, rewrite the stored value in place as a writability probe. A store outage now fails the resolve before the grant, leaving the stored token valid so the connection recovers with the store. --- e2e/cloud/vault-write-outage.test.ts | 43 +++++++++++++++------------- packages/core/sdk/src/executor.ts | 11 +++++++ 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/e2e/cloud/vault-write-outage.test.ts b/e2e/cloud/vault-write-outage.test.ts index 4c4c85809..49e3d39e0 100644 --- a/e2e/cloud/vault-write-outage.test.ts +++ b/e2e/cloud/vault-write-outage.test.ts @@ -7,30 +7,31 @@ // 1. An OAuth connection's access token expires (per-client TTL of 1s via // the emulator's DCR `access_token_ttl_seconds` extension, well inside // the 60s refresh skew). -// 2. A health check resolves the connection → triggers the refresh-token -// grant. The grant SUCCEEDS and the authorization server ROTATES the -// refresh token (single use — the emulator mirrors AuthKit). -// 3. Persisting the new tokens does read → `PUT /vault/v1/kv/:id`. The PUT -// fails (in prod: 409 version conflict, an OAuth-shaped 400, or an HTML -// error page; here: the armed 400 with `error`/`error_description`, the -// exact shape the WorkOS SDK maps to OauthException — Sentry issue -53). -// 4. The write failure surfaces as `StorageError: WorkOS Vault secret write +// 2. A health check resolves the connection → the refresh path runs. Vault +// writes go through read → `PUT /vault/v1/kv/:id`; the PUT fails (in +// prod: 409 version conflict, an OAuth-shaped 400, or an HTML error +// page; here: the armed 400 with `error`/`error_description`, the exact +// shape the WorkOS SDK maps to OauthException — Sentry issue -53). +// 3. The write failure surfaces as `StorageError: WorkOS Vault secret write // failed` → the health endpoint answers a typed InternalError (the 500 -// Sentry records). +// Sentry records). Crucially it must fail BEFORE the refresh-token grant +// (the AS rotates the single-use token; consuming it with an unwritable +// store loses the rotated copy forever). // -// The second scenario pins the DAMAGE, not just the surface error: the vault -// write failed AFTER the refresh token was consumed and rotated, so the new -// refresh token is lost and the stored one is revoked. Once the vault -// recovers, the connection must recover too — today it cannot (the next -// refresh gets invalid_grant → the connection reads "expired" and demands a -// re-auth). Red until the refresh/persist ordering is made crash-safe. +// The second scenario pins the DAMAGE the surface error used to hide: when +// the vault write failed AFTER the refresh token was consumed and rotated, +// the rotated token was lost and the stored one already revoked — the next +// refresh got invalid_grant and the connection demanded a re-auth over a +// storage blip. The fix gates the grant on a proof-of-writability rewrite of +// the stored refresh token, so a vault outage fails BEFORE the single-use +// token is spent and the connection recovers with the vault. import { randomBytes } from "node:crypto"; import { expect } from "@effect/vitest"; import { Effect } from "effect"; import type { HttpApiClient } from "effect/unstable/httpapi"; import { composePluginApi } from "@executor-js/api/server"; -import { connectEmulator, type EmulatorClient } from "@executor-js/emulate"; +import { connectEmulator } from "@executor-js/emulate"; import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; import { AuthTemplateSlug, @@ -243,13 +244,15 @@ scenario( ).toBe("InternalError"); // The proof this is the production mechanism and not an incidental - // failure: the product performed the refresh grant and then hit the - // injected fault on the vault update. + // failure: the product hit the injected fault on the vault update leg. const ledger = yield* Effect.promise(() => workos.ledger.list(200)); const faultedPut = ledger.find((entry) => entry.faulted === true && entry.method === "PUT"); expect(faultedPut?.path, "the failure came from the injected vault-update fault").toMatch( /\/vault\/v1\/kv\//, ); + // The crash-safety contract: the write-gate fails BEFORE the refresh + // grant, so the single-use refresh token is never consumed while the + // store cannot persist its rotated successor. const refreshGrant = ledger.find( (entry) => entry.path.endsWith("/oauth2/token") && @@ -257,8 +260,8 @@ scenario( ); expect( refreshGrant, - "the vault write that failed was persisting a completed refresh grant", - ).toBeDefined(); + "no refresh grant is issued while the vault cannot persist the rotated token", + ).toBeUndefined(); }), Effect.gen(function* () { yield* Effect.promise(() => workos.faults.clear()); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 5f470a70e..15030d033 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -1570,6 +1570,17 @@ export const createExecutor =