From b848686f79c598f367e9c6848040002c32ed792a Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:15:57 -0700 Subject: [PATCH] Run connect and oauth handoff scenarios on per-run emulator instances The hosted emulator service hosts no longer serve a shared default instance (they are control plane only), so the three scenarios that pointed at resend.emulators.dev and microsoft.emulators.dev directly now create their own instance first, via a small shared helper. This also isolates their ledger assertions from other runs. --- .claude/skills/emulate/SKILL.md | 27 +++++---- AGENTS.md | 3 +- e2e/scenarios/connect-handoff-session.test.ts | 46 ++++++++------- e2e/scenarios/connect-handoff.test.ts | 58 +++++++++++-------- e2e/scenarios/oauth-client-handoff.test.ts | 11 ++-- e2e/src/emulator-instance.ts | 20 +++++++ 6 files changed, 101 insertions(+), 64 deletions(-) create mode 100644 e2e/src/emulator-instance.ts diff --git a/.claude/skills/emulate/SKILL.md b/.claude/skills/emulate/SKILL.md index f8e3515b5..db274e27a 100644 --- a/.claude/skills/emulate/SKILL.md +++ b/.claude/skills/emulate/SKILL.md @@ -32,18 +32,22 @@ const github = await createEmulator({ service: "github", port: 4501 }); `servers`) when a proxy fronts the emulator — the bind stays on `port`. **Attach to a running one** (another process, or hosted on Cloudflare with -Durable Object state at `https://.emulators.dev` / -`https://..emulators.dev` — catalog at -`GET https://emulators.dev/_emulate/services`): +Durable Object state at `https://..emulators.dev` — +catalog at `GET https://emulators.dev/_emulate/services`). Service hosts +(`https://.emulators.dev`) are control plane only, with no shared +default instance behind them: create a private instance first and connect to +the returned `providerBaseUrl`. ```ts import { connectEmulator } from "@executor-js/emulate"; -const resend = await connectEmulator({ baseUrl: "https://resend.emulators.dev" }); +const created = await fetch("https://resend.emulators.dev/_emulate/instances", { method: "POST" }); +const { providerBaseUrl } = await created.json(); +const resend = await connectEmulator({ baseUrl: providerBaseUrl }); // optional: { service: "resend" } verifies the manifest on connect ``` -Create a private hosted instance with `POST /_emulate/instances` on the -service host. +In e2e scenarios use `createEmulatorInstance` from +`e2e/src/emulator-instance.ts`, which wraps this. ## The typed control-plane client @@ -113,12 +117,11 @@ The emulate repo's own `AGENTS.md` / `README.md` carry the current build, publish, and deploy commands (npm + Cloudflare creds are in 1Password). Read them there rather than memorizing flags here — they move. -**A hot deploy can redden other people's e2e.** `*.emulators.dev` service -hosts are shared infrastructure; a control-plane regression there has failed -unrelated PRs' suites before. When a scenario needs isolation or a behavior -that isn't deployed yet, pin to a published package version or mint a private -per-run instance (`POST /_emulate/instances`) instead of mutating the shared -service host. +**A hot deploy can redden other people's e2e.** The `emulate-hosts` worker +behind `*.emulators.dev` is shared infrastructure; a control-plane regression +there has failed unrelated PRs' suites before. Scenarios always run on private +per-run instances (`POST /_emulate/instances`); when a scenario needs behavior +that isn't deployed yet, pin to a published package version. ## Gotchas diff --git a/AGENTS.md b/AGENTS.md index a154d5722..ce78abfab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,8 @@ For any test or demo needing an upstream API, OAuth/OIDC provider, or webhook source, use the `@executor-js/emulate` emulators instead of writing a stub: wire-level and stateful, real SDKs run unmodified, each serves a full OpenAPI spec, mints real-shaped credentials, and records every call in a request ledger -to assert against. Hosted at `https://.emulators.dev`. See the `emulate` +to assert against. Hosted on emulators.dev (create a per-run instance via +`POST https://.emulators.dev/_emulate/instances`). See the `emulate` skill (`.claude/skills/emulate/SKILL.md`). They are a standalone project (`github.com/UsefulSoftwareCo/emulate`) consumed here as the published package: full autonomy to change, publish, and deploy them on their `main`; don't re-vendor. diff --git a/e2e/scenarios/connect-handoff-session.test.ts b/e2e/scenarios/connect-handoff-session.test.ts index 6401c85a7..e91feeca3 100644 --- a/e2e/scenarios/connect-handoff-session.test.ts +++ b/e2e/scenarios/connect-handoff-session.test.ts @@ -7,32 +7,29 @@ // theater (src/clients/chat-theater.ts) presenting REAL mcporter MCP calls // — OAuth, execute, approval pause/resume all genuine, every tool spinner // on screen bracketing the actual call it narrates. The provider on the -// other side is real too (resend.emulators.dev); its request ledger is the -// final evidence. +// other side is real too (a per-run Resend instance on emulators.dev); its +// request ledger is the final evidence. import { randomBytes } from "node:crypto"; import { join } from "node:path"; import { expect } from "@effect/vitest"; import { Effect } from "effect"; +import { createEmulatorInstance } from "../src/emulator-instance"; import { scenario } from "../src/scenario"; import { Browser, Cli, Mcp, RunDir, Target } from "../src/services"; import { withChatTheater } from "../src/clients/chat-theater"; import type { McpSession } from "../src/surfaces/mcp"; -const EMULATOR_BASE = "https://resend.emulators.dev"; - const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; // The emulator serves its own OpenAPI document (bearer auth, base URL in // `servers`) — adding by URL with nothing else is exactly what an agent // does, and the platform derives the paste-a-token auth method from the // spec's security scheme. -const EMULATOR_SPEC_URL = `${EMULATOR_BASE}/openapi.json`; - -const addSpecCode = (slug: string) => ` +const addSpecCode = (slug: string, specUrl: string) => ` const added = await tools.executor.openapi.addSpec({ - spec: { kind: "url", url: ${JSON.stringify(EMULATOR_SPEC_URL)} }, + spec: { kind: "url", url: ${JSON.stringify(specUrl)} }, slug: ${JSON.stringify(slug)}, }); return added.ok ? { ok: true, slug: added.data.slug, toolCount: added.data.toolCount } : { ok: false, error: added.error }; @@ -76,17 +73,18 @@ const executeJson = (session: McpSession, code: string) => return JSON.parse(result.text) as Record; }); -const mintEmulatorApiKey = Effect.promise(async () => { - const response = await fetch(`${EMULATOR_BASE}/_emulate/credentials`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ type: "api-key" }), +const mintEmulatorApiKey = (emulatorBase: string) => + Effect.promise(async () => { + const response = await fetch(`${emulatorBase}/_emulate/credentials`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ type: "api-key" }), + }); + const body = (await response.json()) as { credential?: { token?: string } }; + const token = body.credential?.token; + if (!token) throw new Error(`emulator credential mint failed: ${JSON.stringify(body)}`); + return token; }); - const body = (await response.json()) as { credential?: { token?: string } }; - const token = body.credential?.token; - if (!token) throw new Error(`emulator credential mint failed: ${JSON.stringify(body)}`); - return token; -}); scenario( "Connect · developer session: agent chat → handoff link → paste key → verified send", @@ -101,7 +99,8 @@ scenario( const integration = unique("resendsesh"); const emailSubject = unique("dev-session"); - const apiKey = yield* mintEmulatorApiKey; + const emulatorBase = yield* createEmulatorInstance("resend", "dev-session"); + const apiKey = yield* mintEmulatorApiKey(emulatorBase); const identity = yield* target.newIdentity(); const session = mcp.session(identity); @@ -121,8 +120,11 @@ scenario( ); yield* chat.assistant("I'll register the Resend API in your Executor now."); const added = yield* chat.tool( - { name: "execute", input: addSpecCode(integration) }, - executeJson(session, addSpecCode(integration)), + { + name: "execute", + input: addSpecCode(integration, `${emulatorBase}/openapi.json`), + }, + executeJson(session, addSpecCode(integration, `${emulatorBase}/openapi.json`)), ); expect(added.ok, `addSpec succeeded: ${JSON.stringify(added)}`).toBe(true); @@ -191,7 +193,7 @@ scenario( // Final evidence: the emulator's ledger saw the send from Executor. const ledger = yield* Effect.promise(async () => - (await fetch(`${EMULATOR_BASE}/_emulate/ledger`)).text(), + (await fetch(`${emulatorBase}/_emulate/ledger`)).text(), ); expect( ledger.includes(emailSubject), diff --git a/e2e/scenarios/connect-handoff.test.ts b/e2e/scenarios/connect-handoff.test.ts index 424beb086..d4cc80692 100644 --- a/e2e/scenarios/connect-handoff.test.ts +++ b/e2e/scenarios/connect-handoff.test.ts @@ -2,8 +2,9 @@ // handoff URL (`coreTools.connections.createHandoff`), and the user opens that // URL in a browser to paste the credential. This scenario walks the WHOLE // path — the exact flow that failed in production with a "wrong / bad" URL — -// against a real emulated provider (resend.emulators.dev) so the failure -// point is captured with trace + screenshots instead of guessed at: +// against a real emulated Resend provider (a per-run emulators.dev instance) +// so the failure point is captured with trace + screenshots instead of +// guessed at: // // 1. MCP `execute` → `openapi.addSpec` registers the emulated Resend API // 2. MCP `execute` → `connections.createHandoff` returns the browser URL @@ -22,9 +23,10 @@ import { expect } from "@effect/vitest"; import { Effect } from "effect"; import { AccountHttpApi } from "@executor-js/api"; import { composePluginApi } from "@executor-js/api/server"; -import { connectEmulator } from "@executor-js/emulate"; +import { connectEmulator, type EmulatorClient } from "@executor-js/emulate"; import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; +import { createEmulatorInstance } from "../src/emulator-instance"; import { scenario } from "../src/scenario"; import { Api, Browser, Mcp, Target } from "../src/services"; import type { Identity, Target as TargetShape } from "../src/target"; @@ -35,18 +37,14 @@ const api = composePluginApi([openApiHttpPlugin()] as const); const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; -const EMULATOR_BASE = "https://resend.emulators.dev"; - // The emulator serves its own OpenAPI document (bearer auth, same shape as // real Resend — and as the Sentry spec that failed in prod). Adding it by URL // with no authenticationTemplate exercises exactly the agentic path: the // add-account modal must render a paste-a-token flow derived from the spec's // bare `http`/`bearer` security scheme. -const EMULATOR_SPEC_URL = `${EMULATOR_BASE}/openapi.json`; - -const addSpecCode = (slug: string) => ` +const addSpecCode = (slug: string, specUrl: string) => ` const added = await tools.executor.openapi.addSpec({ - spec: { kind: "url", url: ${JSON.stringify(EMULATOR_SPEC_URL)} }, + spec: { kind: "url", url: ${JSON.stringify(specUrl)} }, slug: ${JSON.stringify(slug)}, }); return added.ok ? { ok: true, slug: added.data.slug, toolCount: added.data.toolCount } : { ok: false, error: added.error }; @@ -104,17 +102,20 @@ const executeJson = (session: McpSession, code: string) => }); // The typed control-plane client — minting and ledger reads with real shapes -// instead of hand-rolled fetch + casts. -const emulator = Effect.promise(() => connectEmulator({ baseUrl: EMULATOR_BASE })); - -const mintEmulatorApiKey = Effect.gen(function* () { - const client = yield* emulator; - const credential = yield* Effect.promise(() => client.credentials.mint({ type: "api-key" })); - const token = credential.token; - if (!token) throw new Error(`emulator credential mint failed: ${JSON.stringify(credential)}`); - return token; +// instead of hand-rolled fetch + casts, against this scenario's own instance. +const emulator = Effect.gen(function* () { + const baseUrl = yield* createEmulatorInstance("resend", "connect-handoff"); + return yield* Effect.promise(() => connectEmulator({ baseUrl })); }); +const mintEmulatorApiKey = (client: EmulatorClient) => + Effect.gen(function* () { + const credential = yield* Effect.promise(() => client.credentials.mint({ type: "api-key" })); + const token = credential.token; + if (!token) throw new Error(`emulator credential mint failed: ${JSON.stringify(credential)}`); + return token; + }); + scenario( "Connect · the agentic handoff URL opens this deployment's add-account flow and the pasted key works", { timeout: 240_000 }, @@ -126,7 +127,8 @@ scenario( const integration = unique("resendhf"); const emailSubject = unique("connect-handoff"); - const apiKey = yield* mintEmulatorApiKey; + const emulatorClient = yield* emulator; + const apiKey = yield* mintEmulatorApiKey(emulatorClient); const identity = yield* target.newIdentity(); const session = mcp.session(identity); @@ -148,6 +150,7 @@ scenario( emailSubject, apiKey, orgSlug: orgSlug!, + emulatorClient, }).pipe( // Best-effort cleanup even on failure: drop the created connection(s) // over MCP, then the integration over the API. `connections.remove` is @@ -172,13 +175,23 @@ const runScenario = (input: { readonly emailSubject: string; readonly apiKey: string; readonly orgSlug: string; + readonly emulatorClient: EmulatorClient; }) => Effect.gen(function* () { - const { target, browser, session, identity, integration, emailSubject, apiKey, orgSlug } = - input; + const { + target, + browser, + session, + identity, + integration, + emailSubject, + apiKey, + orgSlug, + emulatorClient, + } = input; // 1. Agent registers the emulated provider over MCP. - const added = yield* executeJson(session, addSpecCode(integration)); + const added = yield* executeJson(session, addSpecCode(integration, emulatorClient.openapiUrl)); expect(added.ok, `addSpec succeeded: ${JSON.stringify(added)}`).toBe(true); // 2. Agent asks for the browser handoff URL. @@ -238,7 +251,6 @@ const runScenario = (input: { const sent = yield* executeJson(session, sendEmailCode(integration, emailSubject)); expect(sent.ok, `email sent through the pasted connection: ${JSON.stringify(sent)}`).toBe(true); - const emulatorClient = yield* emulator; const entries = yield* Effect.promise(() => emulatorClient.ledger.list()); const recorded = entries.find((entry) => JSON.stringify(entry.request.body ?? "").includes(emailSubject), diff --git a/e2e/scenarios/oauth-client-handoff.test.ts b/e2e/scenarios/oauth-client-handoff.test.ts index 3f797a4ce..258ce3276 100644 --- a/e2e/scenarios/oauth-client-handoff.test.ts +++ b/e2e/scenarios/oauth-client-handoff.test.ts @@ -33,6 +33,7 @@ import { import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; import { ConnectionName, IntegrationSlug, OAuthClientSlug } from "@executor-js/sdk/shared"; +import { createEmulatorInstance } from "../src/emulator-instance"; import { scenario } from "../src/scenario"; import { Api, Browser, Mcp, Target } from "../src/services"; import type { McpSession } from "../src/surfaces/mcp"; @@ -252,8 +253,6 @@ scenario( // browser; the agent connects. // --------------------------------------------------------------------------- -const EMULATOR_BASE = "https://microsoft.emulators.dev"; - const handoffForBrowserCode = (input: { readonly integration: string; readonly slug: string; @@ -336,11 +335,11 @@ scenario( const connection = "machine"; const template = MICROSOFT_CLIENT_CREDENTIALS_AUTH_TEMPLATE_SLUG; - // The hosted emulator mints a real-shaped client-credentials app and records - // every token exchange — the ledger is shared, so key everything off the - // unique minted clientId. + // A per-run hosted emulator instance mints a real-shaped client-credentials + // app and records every token exchange in its own isolated ledger. + const emulatorBase = yield* createEmulatorInstance("microsoft", "oauth-handoff"); const emulator: EmulatorClient = yield* Effect.promise(() => - connectEmulator({ baseUrl: EMULATOR_BASE, service: "microsoft" }), + connectEmulator({ baseUrl: emulatorBase, service: "microsoft" }), ); const minted = yield* Effect.promise(() => emulator.credentials.mint({ type: "oauth-client-credentials", name: "Executor E2E Graph" }), diff --git a/e2e/src/emulator-instance.ts b/e2e/src/emulator-instance.ts new file mode 100644 index 000000000..6d5174b20 --- /dev/null +++ b/e2e/src/emulator-instance.ts @@ -0,0 +1,20 @@ +import { Effect } from "effect"; + +// Hosted service hosts (e.g. resend.emulators.dev) are control plane only — +// there is no shared default instance behind them. Every scenario creates its +// own isolated instance and works against the returned providerBaseUrl, which +// also keeps ledger assertions free of cross-run pollution. The server +// generates an unguessable instance name; the label is a readable prefix. +export const createEmulatorInstance = (service: string, label = "e2e") => + Effect.promise(async () => { + const response = await fetch(`https://${service}.emulators.dev/_emulate/instances`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ instance: label }), + }); + if (!response.ok) { + throw new Error(`${service} emulator instance creation failed: ${response.status}`); + } + const instance = (await response.json()) as { readonly providerBaseUrl: string }; + return instance.providerBaseUrl; + });