Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 167 additions & 0 deletions e2e/scenarios/namespace-enumeration.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
// Cross-target: an integration's tool catalog is enumerable from the sandbox.
// tools.search with a namespace and an EMPTY query is enumeration: the whole
// catalog, path-sorted and paged, whose `total` reconciles exactly against
// the toolCount that executor.integrations.list reports. Before this
// guarantee an agent could only lower-bound a catalog by unioning keyword
// searches (issue #1383): the count was available and the contents were not.
import { randomBytes } from "node:crypto";

import { expect } from "@effect/vitest";
import { Effect } from "effect";
import { composePluginApi } from "@executor-js/api/server";
import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";

import { scenario } from "../src/scenario";
import { Api, Target } from "../src/services";

const api = composePluginApi([openApiHttpPlugin()] as const);

const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`;

/** Three operations whose names share no verb, so no single keyword search
* could ever return all of them — only enumeration can. */
const spec = (baseUrl: string): string =>
JSON.stringify({
openapi: "3.0.3",
info: { title: "Enumerable API", version: "1.0.0" },
servers: [{ url: baseUrl }],
paths: {
"/alpha": {
get: {
operationId: "alphaOp",
summary: "First operation",
responses: { "200": { description: "ok" } },
},
},
"/bravo": {
post: {
operationId: "bravoOp",
summary: "Second operation",
responses: { "200": { description: "ok" } },
},
},
"/charlie": {
delete: {
operationId: "charlieOp",
summary: "Third operation",
responses: { "200": { description: "ok" } },
},
},
},
});

scenario(
"Discovery · an integration's full tool catalog is enumerable and reconciles with its reported toolCount",
{},
Effect.gen(function* () {
const target = yield* Target;
const { client: makeClient } = yield* Api;
const identity = yield* target.newIdentity();
const client = yield* makeClient(api, identity);
const slug = unique("enum");

yield* Effect.ensuring(
Effect.gen(function* () {
yield* client.openapi.addSpec({
payload: {
spec: { kind: "blob", value: spec("http://127.0.0.1:59999") },
slug,
baseUrl: "http://127.0.0.1:59999", // never contacted: discovery only
authenticationTemplate: [
{
slug: "apiKey",
type: "apiKey",
headers: { "x-api-key": [{ type: "variable", name: "token" }] },
},
],
},
});
yield* client.connections.create({
payload: {
owner: "org",
name: ConnectionName.make("main"),
integration: IntegrationSlug.make(slug),
template: AuthTemplateSlug.make("apiKey"),
value: "tok_enum",
},
});

// Everything below runs inside the execute sandbox — the exact
// surface an agent has.
const executed = yield* client.executions.execute({
payload: {
code: `
const integrations = await tools.executor.integrations.list({ limit: 50 });
const mine = integrations.items.find((item) => item.id === ${JSON.stringify(slug)});

const enumerated = await tools.search({ namespace: ${JSON.stringify(slug)}, query: "", limit: 50 });

// Paged enumeration walks the same census.
const paged = [];
let offset = 0;
for (let i = 0; i < 10; i++) {
const page = await tools.search({ namespace: ${JSON.stringify(slug)}, query: "", limit: 2, offset });
paged.push(...page.items.map((item) => item.path));
if (!page.hasMore) break;
offset = page.nextOffset;
}

// No namespace + empty query stays empty (no arbitrary workspace dump).
const unscoped = await tools.search({ query: "" });

return JSON.stringify({
toolCount: mine ? mine.toolCount : null,
enumeratedTotal: enumerated.total,
enumeratedPaths: enumerated.items.map((item) => item.path),
pagedPaths: paged,
unscopedTotal: unscoped.total,
});
`,
autoApprove: true,
},
});
expect(executed.status, "the sandbox execution completed").toBe("completed");
const outcome = JSON.parse(executed.text) as {
readonly toolCount: number | null;
readonly enumeratedTotal: number;
readonly enumeratedPaths: readonly string[];
readonly pagedPaths: readonly string[];
readonly unscopedTotal: number;
};

// THE guarantee: enumeration returns the whole catalog, and its
// total reconciles exactly against the integration's toolCount.
expect(outcome.toolCount, "the integration reports its toolCount").toBe(3);
expect(outcome.enumeratedTotal, "enumeration returns the full census").toBe(3);
expect(
outcome.enumeratedPaths.map((path) => path.split(".").at(-1)).sort(),
"all three operations are enumerated, despite sharing no searchable verb",
).toEqual(["alphaOp", "bravoOp", "charlieOp"]);
expect(
outcome.enumeratedPaths,
"enumeration is path-sorted, so paging is deterministic",
).toEqual([...outcome.enumeratedPaths].sort());
expect(outcome.pagedPaths, "paged enumeration walks the same census in order").toEqual(
outcome.enumeratedPaths,
);
expect(
outcome.unscopedTotal,
"an empty query without a namespace still returns nothing",
).toBe(0);
}),
Effect.gen(function* () {
yield* client.connections
.remove({
params: {
owner: "org",
integration: IntegrationSlug.make(slug),
name: ConnectionName.make("main"),
},
})
.pipe(Effect.ignore);
yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore);
}),
);
}),
);
31 changes: 31 additions & 0 deletions packages/core/execution/src/tool-invoker.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import { createExecutionEngine } from "./engine";
import { ExecutionToolError } from "./errors";
import {
describeTool,
listExecutorIntegrations,
makeExecutorToolInvoker,
searchTools,
type ToolDiscoveryProvider,
Expand Down Expand Up @@ -502,6 +503,36 @@ describe("tool discovery", () => {
}),
);

it.effect("enumerates a namespace's full catalog for an empty query with a namespace", () =>
Effect.gen(function* () {
const executor = yield* makeSearchExecutor();

// Enumeration, not search: every github tool, path-sorted, score 0.
// `total` is the census an agent can reconcile against the integration's
// reported toolCount — keyword search can only ever lower-bound it.
const enumerated = yield* searchTools(executor, "", 100, { namespace: "github" });
expect(enumerated.items.length).toBeGreaterThan(0);
expect(enumerated.total).toBe(enumerated.items.length);
expect(enumerated.items.map((item) => item.path)).toEqual(
[...enumerated.items.map((item) => item.path)].sort((a, b) => a.localeCompare(b)),
);
expect(enumerated.items.every((item) => item.integration === "github")).toBe(true);
expect(enumerated.items.every((item) => item.score === 0)).toBe(true);

// The census matches what the integrations list reports for the same
// namespace — the reconciliation #1383 could not perform.
const integrations = yield* listExecutorIntegrations(executor, { limit: 50 });
const github = integrations.items.find((item) => item.id === "github");
expect(github?.toolCount).toBe(enumerated.total);

// Enumeration pages like any other discovery result.
const firstPage = yield* searchTools(executor, "", 1, { namespace: "github" });
expect(firstPage.items).toEqual([enumerated.items[0]]);
expect(firstPage.total).toBe(enumerated.total);
expect(firstPage.hasMore).toBe(enumerated.total > 1);
}),
);

it.effect("paginates ranked matches via limit + offset with hasMore + nextOffset", () =>
Effect.gen(function* () {
const executor = yield* makeSearchExecutor();
Expand Down
54 changes: 39 additions & 15 deletions packages/core/execution/src/tool-invoker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ const BUILTIN_TOOL_DESCRIPTIONS: ReadonlyMap<string, DescribedTool> = new Map<
{
path: "search",
name: "search",
description: "Search available Executor tools.",
description:
"Search available Executor tools. An empty query with a namespace enumerates that integration's full catalog, sorted by path.",
inputTypeScript: "{ query: string; namespace?: string; limit?: number; offset?: number; }",
outputTypeScript:
"{ items: ToolDiscoveryResult[]; total: number; hasMore: boolean; nextOffset: number | null; }",
Expand Down Expand Up @@ -655,15 +656,20 @@ export const searchTools = Effect.fn("executor.tools.search")(function* (
...(options?.namespace ? { "executor.search.namespace": options.namespace } : {}),
});

const empty: PagedResult<ToolDiscoveryResult> = {
items: [],
total: 0,
hasMore: false,
nextOffset: null,
};
const emptyQuery = normalizeSearchText(query).length === 0;
const hasNamespace =
options?.namespace !== undefined && normalizeSearchText(options.namespace).length > 0;

if (normalizeSearchText(query).length === 0) {
return empty;
// An empty query with no namespace stays empty: it carries neither a
// ranking signal nor a scope, and listing the whole workspace "by default"
// is exactly the arbitrary dump the ranked search refuses to be.
if (emptyQuery && !hasNamespace) {
return {
items: [],
total: 0,
hasMore: false,
nextOffset: null,
} satisfies PagedResult<ToolDiscoveryResult>;
}

const all = yield* executor.tools.list({ includeAnnotations: false }).pipe(
Expand All @@ -675,12 +681,30 @@ export const searchTools = Effect.fn("executor.tools.search")(function* (
}),
),
);
const searchable = all.map(toSearchableTool);
const ranked = searchable
.filter((tool: SearchableTool) => matchesNamespace(tool, options?.namespace))
.map((tool: SearchableTool) => scoreToolMatch(tool, query))
.filter(Predicate.isNotNull)
.sort((left, right) => right.score - left.score || left.path.localeCompare(right.path));
const inScope = all
.map(toSearchableTool)
.filter((tool: SearchableTool) => matchesNamespace(tool, options?.namespace));

// An empty query WITH a namespace is enumeration, not search: there is no
// ranking signal, so the namespace's whole catalog comes back sorted by
// path (score 0) and paged. This is the only way to list a catalog
// exhaustively — keyword search can only ever lower-bound it — and it makes
// `total` a census that reconciles against `executor.integrations.list`'s
// per-integration toolCount.
const ranked: readonly ToolDiscoveryResult[] = emptyQuery
? [...inScope]
.sort((left, right) => left.path.localeCompare(right.path))
.map((tool) => ({
path: tool.path,
name: tool.name,
integration: tool.integration,
score: 0,
...(tool.description !== undefined ? { description: tool.description } : {}),
}))
: inScope
.map((tool: SearchableTool) => scoreToolMatch(tool, query))
.filter(Predicate.isNotNull)
.sort((left, right) => right.score - left.score || left.path.localeCompare(right.path));

const page = paginate(ranked, offset, limit);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ const createToolCaller = (toolPath) => (args) =>

const toolsEnumerationMessage = (path) =>
(path.length === 0 ? "tools" : "tools." + path.join(".")) +
' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.';
' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.';

const createToolsProxy = (path = []) => {
const callable = () => undefined;
Expand Down
4 changes: 2 additions & 2 deletions packages/kernel/runtime-deno-subprocess/src/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -337,9 +337,9 @@ describe.skipIf(!isDenoAvailable())("runtime-deno-subprocess", () => {

expect(output.error).toBeUndefined();
expect(output.result).toEqual({
keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
spread:
'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
});
}),
);
Expand Down
4 changes: 2 additions & 2 deletions packages/kernel/runtime-dynamic-worker/src/invocation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -945,9 +945,9 @@ describe("makeDynamicWorkerExecutor", () => {

expect(result.error).toBeUndefined();
expect(result.result).toEqual({
keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
spread:
'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
});
}),
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string =>
" };",
" const __toolsEnumerationError = (path) => new Error(",
" (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +",
" ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: \"...\" }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',",
' \' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.\',',
" );",
" const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",
" get(_target, prop) {",
Expand Down
4 changes: 2 additions & 2 deletions packages/kernel/runtime-quickjs/src/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -363,9 +363,9 @@ describe("quickjs executor", () => {

expect(result.error).toBeUndefined();
expect(result.result).toEqual({
keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
keys: 'tools is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
spread:
'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',
'tools.github is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.',
});
}),
);
Expand Down
2 changes: 1 addition & 1 deletion packages/kernel/runtime-quickjs/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ const buildExecutionSource = (code: string): string => {
"};",
"const __toolsEnumerationError = (path) => new Error(",
" (path.length === 0 ? 'tools' : 'tools.' + path.join('.')) +",
" ' is a lazy proxy and cannot be enumerated. Use tools.search({ query: \"...\" }) to find tools, or tools.executor.coreTools.connections.list({}) to list saved connections.',",
' \' is a lazy proxy and cannot be enumerated. Use tools.search({ query: "..." }) to find tools, tools.search({ namespace: "<integration>", query: "" }) to list every tool in an integration, or tools.executor.coreTools.connections.list({}) to list saved connections.\',',
");",
"const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",
" get(_target, prop) {",
Expand Down
Loading