diff --git a/e2e/scenarios/namespace-enumeration.test.ts b/e2e/scenarios/namespace-enumeration.test.ts new file mode 100644 index 000000000..b4c068472 --- /dev/null +++ b/e2e/scenarios/namespace-enumeration.test.ts @@ -0,0 +1,167 @@ +// Cross-target: an integration's tool catalog is enumerable from the sandbox. +// tools.search with a namespace and an EMPTY query is enumeration: the whole +// catalog, path-sorted and paged, whose `total` reconciles exactly against +// the toolCount that executor.integrations.list reports. Before this +// guarantee an agent could only lower-bound a catalog by unioning keyword +// searches (issue #1383): the count was available and the contents were not. +import { randomBytes } from "node:crypto"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { composePluginApi } from "@executor-js/api/server"; +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; +import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; + +import { scenario } from "../src/scenario"; +import { Api, Target } from "../src/services"; + +const api = composePluginApi([openApiHttpPlugin()] as const); + +const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; + +/** Three operations whose names share no verb, so no single keyword search + * could ever return all of them — only enumeration can. */ +const spec = (baseUrl: string): string => + JSON.stringify({ + openapi: "3.0.3", + info: { title: "Enumerable API", version: "1.0.0" }, + servers: [{ url: baseUrl }], + paths: { + "/alpha": { + get: { + operationId: "alphaOp", + summary: "First operation", + responses: { "200": { description: "ok" } }, + }, + }, + "/bravo": { + post: { + operationId: "bravoOp", + summary: "Second operation", + responses: { "200": { description: "ok" } }, + }, + }, + "/charlie": { + delete: { + operationId: "charlieOp", + summary: "Third operation", + responses: { "200": { description: "ok" } }, + }, + }, + }, + }); + +scenario( + "Discovery · an integration's full tool catalog is enumerable and reconciles with its reported toolCount", + {}, + Effect.gen(function* () { + const target = yield* Target; + const { client: makeClient } = yield* Api; + const identity = yield* target.newIdentity(); + const client = yield* makeClient(api, identity); + const slug = unique("enum"); + + yield* Effect.ensuring( + Effect.gen(function* () { + yield* client.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: spec("http://127.0.0.1:59999") }, + slug, + baseUrl: "http://127.0.0.1:59999", // never contacted: discovery only + authenticationTemplate: [ + { + slug: "apiKey", + type: "apiKey", + headers: { "x-api-key": [{ type: "variable", name: "token" }] }, + }, + ], + }, + }); + yield* client.connections.create({ + payload: { + owner: "org", + name: ConnectionName.make("main"), + integration: IntegrationSlug.make(slug), + template: AuthTemplateSlug.make("apiKey"), + value: "tok_enum", + }, + }); + + // Everything below runs inside the execute sandbox — the exact + // surface an agent has. + const executed = yield* client.executions.execute({ + payload: { + code: ` +const integrations = await tools.executor.integrations.list({ limit: 50 }); +const mine = integrations.items.find((item) => item.id === ${JSON.stringify(slug)}); + +const enumerated = await tools.search({ namespace: ${JSON.stringify(slug)}, query: "", limit: 50 }); + +// Paged enumeration walks the same census. +const paged = []; +let offset = 0; +for (let i = 0; i < 10; i++) { + const page = await tools.search({ namespace: ${JSON.stringify(slug)}, query: "", limit: 2, offset }); + paged.push(...page.items.map((item) => item.path)); + if (!page.hasMore) break; + offset = page.nextOffset; +} + +// No namespace + empty query stays empty (no arbitrary workspace dump). +const unscoped = await tools.search({ query: "" }); + +return JSON.stringify({ + toolCount: mine ? mine.toolCount : null, + enumeratedTotal: enumerated.total, + enumeratedPaths: enumerated.items.map((item) => item.path), + pagedPaths: paged, + unscopedTotal: unscoped.total, +}); +`, + autoApprove: true, + }, + }); + expect(executed.status, "the sandbox execution completed").toBe("completed"); + const outcome = JSON.parse(executed.text) as { + readonly toolCount: number | null; + readonly enumeratedTotal: number; + readonly enumeratedPaths: readonly string[]; + readonly pagedPaths: readonly string[]; + readonly unscopedTotal: number; + }; + + // THE guarantee: enumeration returns the whole catalog, and its + // total reconciles exactly against the integration's toolCount. + expect(outcome.toolCount, "the integration reports its toolCount").toBe(3); + expect(outcome.enumeratedTotal, "enumeration returns the full census").toBe(3); + expect( + outcome.enumeratedPaths.map((path) => path.split(".").at(-1)).sort(), + "all three operations are enumerated, despite sharing no searchable verb", + ).toEqual(["alphaOp", "bravoOp", "charlieOp"]); + expect( + outcome.enumeratedPaths, + "enumeration is path-sorted, so paging is deterministic", + ).toEqual([...outcome.enumeratedPaths].sort()); + expect(outcome.pagedPaths, "paged enumeration walks the same census in order").toEqual( + outcome.enumeratedPaths, + ); + expect( + outcome.unscopedTotal, + "an empty query without a namespace still returns nothing", + ).toBe(0); + }), + Effect.gen(function* () { + yield* client.connections + .remove({ + params: { + owner: "org", + integration: IntegrationSlug.make(slug), + name: ConnectionName.make("main"), + }, + }) + .pipe(Effect.ignore); + yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore); + }), + ); + }), +); diff --git a/packages/core/execution/src/tool-invoker.test.ts b/packages/core/execution/src/tool-invoker.test.ts index f054cbfcf..407667d43 100644 --- a/packages/core/execution/src/tool-invoker.test.ts +++ b/packages/core/execution/src/tool-invoker.test.ts @@ -31,6 +31,7 @@ import { createExecutionEngine } from "./engine"; import { ExecutionToolError } from "./errors"; import { describeTool, + listExecutorIntegrations, makeExecutorToolInvoker, searchTools, type ToolDiscoveryProvider, @@ -502,6 +503,105 @@ describe("tool discovery", () => { }), ); + it.effect("enumerates a namespace's full catalog for an empty query with a namespace", () => + Effect.gen(function* () { + const executor = yield* makeSearchExecutor(); + + // Enumeration, not search: every github tool, path-sorted, score 0. + // `total` is the census an agent can reconcile against the integration's + // reported toolCount — keyword search can only ever lower-bound it. + const enumerated = yield* searchTools(executor, "", 100, { namespace: "github" }); + expect(enumerated.items.length).toBeGreaterThan(0); + expect(enumerated.total).toBe(enumerated.items.length); + expect(enumerated.items.map((item) => item.path)).toEqual( + [...enumerated.items.map((item) => item.path)].sort((a, b) => a.localeCompare(b)), + ); + expect(enumerated.items.every((item) => item.integration === "github")).toBe(true); + expect(enumerated.items.every((item) => item.score === 0)).toBe(true); + + // The census matches what the integrations list reports for the same + // namespace — the reconciliation #1383 could not perform. + const integrations = yield* listExecutorIntegrations(executor, { limit: 50 }); + const github = integrations.items.find((item) => item.id === "github"); + expect(github?.toolCount).toBe(enumerated.total); + + // Enumeration pages like any other discovery result. + const firstPage = yield* searchTools(executor, "", 1, { namespace: "github" }); + expect(firstPage.items).toEqual([enumerated.items[0]]); + expect(firstPage.total).toBe(enumerated.total); + expect(firstPage.hasMore).toBe(enumerated.total > 1); + }), + ); + + it.effect("enumeration scopes by exact slug, not the prefix matching ranked search uses", () => + Effect.gen(function* () { + // Prefix-sibling integrations: "google" is a token prefix of both + // "google_gmail" and "google_sheets". Ranked search deliberately treats + // namespace as a fuzzy prefix filter; enumeration must NOT, or its + // `total` stops being a census of one integration. + const google = makeTestPlugin({ + pluginId: "google-test", + integration: "google", + tools: [ + { + name: "ping", + description: "Ping", + inputJsonSchema: EmptyInputJson, + validator: EmptyValidator, + handler: () => Effect.succeed([]), + }, + ], + }); + const gmail = makeTestPlugin({ + pluginId: "google-gmail-test", + integration: "google_gmail", + tools: [ + { + name: "listMessages", + description: "List messages", + inputJsonSchema: EmptyInputJson, + validator: EmptyValidator, + handler: () => Effect.succeed([]), + }, + { + name: "sendMessage", + description: "Send a message", + inputJsonSchema: EmptyInputJson, + validator: EmptyValidator, + handler: () => Effect.succeed([]), + }, + ], + }); + const executor = yield* makeExecutorWith([google, gmail] as const); + yield* provision(executor as never, [ + { pluginId: "google-test", integration: "google" }, + { pluginId: "google-gmail-test", integration: "google_gmail" }, + ]); + + const parent = yield* searchTools(executor, "", 100, { namespace: "google" }); + expect( + parent.items.map((item) => item.integration), + "enumerating 'google' returns only the google integration's tools", + ).toEqual(["google"]); + expect(parent.total).toBe(1); + + const sibling = yield* searchTools(executor, "", 100, { namespace: "google_gmail" }); + expect( + sibling.items.every((item) => item.integration === "google_gmail"), + "enumerating the sibling returns only its own tools", + ).toBe(true); + expect(sibling.total).toBe(2); + + // Ranked search keeps its fuzzy namespace semantics: a keyword search + // scoped to "google" may still match tools in google_gmail. + const fuzzy = yield* searchTools(executor, "message", 100, { namespace: "google" }); + expect( + fuzzy.items.some((item) => item.integration === "google_gmail"), + "ranked search still prefix-matches sibling namespaces", + ).toBe(true); + }), + ); + it.effect("paginates ranked matches via limit + offset with hasMore + nextOffset", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index 323fb0e85..5c427148d 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -94,7 +94,8 @@ const BUILTIN_TOOL_DESCRIPTIONS: ReadonlyMap = new Map< { path: "search", name: "search", - description: "Search available Executor tools.", + description: + "Search available Executor tools. An empty query with a namespace enumerates that integration's full catalog, sorted by path.", inputTypeScript: "{ query: string; namespace?: string; limit?: number; offset?: number; }", outputTypeScript: "{ items: ToolDiscoveryResult[]; total: number; hasMore: boolean; nextOffset: number | null; }", @@ -655,15 +656,20 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( ...(options?.namespace ? { "executor.search.namespace": options.namespace } : {}), }); - const empty: PagedResult = { - items: [], - total: 0, - hasMore: false, - nextOffset: null, - }; + const emptyQuery = normalizeSearchText(query).length === 0; + const hasNamespace = + options?.namespace !== undefined && normalizeSearchText(options.namespace).length > 0; - if (normalizeSearchText(query).length === 0) { - return empty; + // An empty query with no namespace stays empty: it carries neither a + // ranking signal nor a scope, and listing the whole workspace "by default" + // is exactly the arbitrary dump the ranked search refuses to be. + if (emptyQuery && !hasNamespace) { + return { + items: [], + total: 0, + hasMore: false, + nextOffset: null, + } satisfies PagedResult; } const all = yield* executor.tools.list({ includeAnnotations: false }).pipe( @@ -676,11 +682,31 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( ), ); const searchable = all.map(toSearchableTool); - const ranked = searchable - .filter((tool: SearchableTool) => matchesNamespace(tool, options?.namespace)) - .map((tool: SearchableTool) => scoreToolMatch(tool, query)) - .filter(Predicate.isNotNull) - .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path)); + + // An empty query WITH a namespace is enumeration, not search: there is no + // ranking signal, so the namespace's whole catalog comes back sorted by + // path (score 0) and paged. Enumeration scopes by EXACT integration slug — + // the token-prefix `matchesNamespace` used for ranked search would also + // sweep in prefix-sibling integrations (namespace "google" matching + // google_gmail and google_sheets), which would silently break the census + // guarantee: `total` here must reconcile against + // `executor.integrations.list`'s per-integration toolCount. + const ranked: readonly ToolDiscoveryResult[] = emptyQuery + ? searchable + .filter((tool) => tool.integration === options?.namespace?.trim()) + .sort((left, right) => left.path.localeCompare(right.path)) + .map((tool) => ({ + path: tool.path, + name: tool.name, + integration: tool.integration, + score: 0, + ...(tool.description !== undefined ? { description: tool.description } : {}), + })) + : searchable + .filter((tool: SearchableTool) => matchesNamespace(tool, options?.namespace)) + .map((tool: SearchableTool) => scoreToolMatch(tool, query)) + .filter(Predicate.isNotNull) + .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path)); const page = paginate(ranked, offset, limit); diff --git a/packages/kernel/runtime-deno-subprocess/src/deno-subprocess-worker.mjs b/packages/kernel/runtime-deno-subprocess/src/deno-subprocess-worker.mjs index 2920f6e29..00507f71b 100644 --- a/packages/kernel/runtime-deno-subprocess/src/deno-subprocess-worker.mjs +++ b/packages/kernel/runtime-deno-subprocess/src/deno-subprocess-worker.mjs @@ -43,7 +43,7 @@ const createToolCaller = (toolPath) => (args) => const toolsEnumerationMessage = (path) => (path.length === 0 ? "tools" : "tools." + path.join(".")) + - ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.'; + ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.'; const createToolsProxy = (path = []) => { const callable = () => undefined; diff --git a/packages/kernel/runtime-deno-subprocess/src/index.test.ts b/packages/kernel/runtime-deno-subprocess/src/index.test.ts index eb45af4de..92f4aba38 100644 --- a/packages/kernel/runtime-deno-subprocess/src/index.test.ts +++ b/packages/kernel/runtime-deno-subprocess/src/index.test.ts @@ -337,9 +337,9 @@ describe.skipIf(!isDenoAvailable())("runtime-deno-subprocess", () => { expect(output.error).toBeUndefined(); expect(output.result).toEqual({ - keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.', + keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.', spread: - 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.', + 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.', }); }), ); diff --git a/packages/kernel/runtime-dynamic-worker/src/invocation.test.ts b/packages/kernel/runtime-dynamic-worker/src/invocation.test.ts index 951c4b5a5..b54869a8f 100644 --- a/packages/kernel/runtime-dynamic-worker/src/invocation.test.ts +++ b/packages/kernel/runtime-dynamic-worker/src/invocation.test.ts @@ -945,9 +945,9 @@ describe("makeDynamicWorkerExecutor", () => { expect(result.error).toBeUndefined(); expect(result.result).toEqual({ - keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.', + keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.', spread: - 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.', + 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.', }); }), ); diff --git a/packages/kernel/runtime-dynamic-worker/src/module-template.ts b/packages/kernel/runtime-dynamic-worker/src/module-template.ts index 7cf87ba11..38c8c67ee 100644 --- a/packages/kernel/runtime-dynamic-worker/src/module-template.ts +++ b/packages/kernel/runtime-dynamic-worker/src/module-template.ts @@ -178,7 +178,7 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string => " };", " const __toolsEnumerationError = (path) => new Error(", " (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +", - " ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: \"...\" }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',", + ' \' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.\',', " );", " const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {", " get(_target, prop) {", diff --git a/packages/kernel/runtime-quickjs/src/index.test.ts b/packages/kernel/runtime-quickjs/src/index.test.ts index 83240df6f..ae1145bfa 100644 --- a/packages/kernel/runtime-quickjs/src/index.test.ts +++ b/packages/kernel/runtime-quickjs/src/index.test.ts @@ -363,9 +363,9 @@ describe("quickjs executor", () => { expect(result.error).toBeUndefined(); expect(result.result).toEqual({ - keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.', + keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.', spread: - 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.', + 'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.', }); }), ); diff --git a/packages/kernel/runtime-quickjs/src/index.ts b/packages/kernel/runtime-quickjs/src/index.ts index 8413b9957..2ccebbb25 100644 --- a/packages/kernel/runtime-quickjs/src/index.ts +++ b/packages/kernel/runtime-quickjs/src/index.ts @@ -170,7 +170,7 @@ const buildExecutionSource = (code: string): string => { "};", "const __toolsEnumerationError = (path) => new Error(", " (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +", - " ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: \"...\" }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',", + ' \' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.\',', ");", "const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {", " get(_target, prop) {",