diff --git a/AGENTS.md b/AGENTS.md index acf9c12..61c2391 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ AXIS (Agent Experience Index Score) is a synthetic testing framework for AI agen Built-in adapters split into two factories. NDJSON-style adapters (`claude-code`, `codex`) are created via `createAgentAdapter(spec)` from `src/adapters/base/agent-adapter.ts`. ACP-based adapters (`claude-sdk`, `codex-sdk`, `gemini`, `goose`, `opencode`, `qwen-code`, `stakpak`, `blackbox`, `fast-agent`, `mistral-vibe`, `factory-droid`, `poolside`, `vtcode`, `cursor-agent`, `auggie`, `kimi`, `openhands`, `cline`, `kiro-cli`, `kilo`, `qoder`) are created via `createAcpBasedAdapter(spec)` from `src/adapters/base/acp-adapter.ts`. Each adapter is a plain factory function (e.g. `createGeminiAdapter()`) that returns an `AgentAdapter` -no classes, no inheritance. The factory owns the shared plumbing: -- Spawn + stdin.end + cleanup registration (SIGTERM on Ctrl-C) +- Spawn + cleanup registration (SIGTERM on Ctrl-C); stdin is closed immediately by default, or written with the prompt then closed when `promptVia: "stdin"` - 10-minute timeout → SIGTERM → SIGKILL after 5s grace (timer cleared on clean exit) - stderr capped at 100 KB - `close` event listener registered BEFORE stdout stream to avoid missing it @@ -46,20 +46,21 @@ The NDJSON-style adapters (`claude-code`, `codex`) use `lines` mode for NDJSON p Call `createAgentAdapter(spec)` with an `AgentAdapterSpec`. The spec is a single typed object -no class inheritance, no protected hooks: -| Spec field | Purpose | -| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `name` | Adapter name (registered in `src/adapters/registry.ts`) | -| `cliCommand?` | CLI binary for `resolveCommand`; omit if user-supplied | -| `timeoutMs?` | Execution timeout (default 10 min) | -| `requiredEnv?` | Env vars validated by the runner pre-flight (e.g. `ANTHROPIC_API_KEY`) | -| `hasLocalSession?` | Detect a usable local CLI login (e.g. `claude login`, `codex login`). Runner calls this only when `requiredEnv` is missing — explicit API keys always win | -| `isolationEnv?` | Isolation vars (e.g. `CLAUDE_CONFIG_DIR`, `CODEX_HOME`). Signature: `({ workspace, home }) => Record`. Point `*_HOME`-style paths under `home`, never `workspace` | -| `prepare?` | Side effects (mkdir, MCP / skills writers) before spawn | -| `resolveCommand?` | Override how the CLI command is resolved | -| `buildArgs` | Build CLI arguments (prefix args from command resolution prepended automatically) | -| `initialState` | Per-run mutable state used by `streamConfig` handlers and `getResult` | -| `streamConfig` | How to process agent stdout. Discriminated union: `{ mode: "lines", onLine, onEnd? }` or `{ mode: "aggregate", onChunk, onEnd? }` | -| `getResult` | Build final `{ result, metadata? }` from accumulated state after exit | +| Spec field | Purpose | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | Adapter name (registered in `src/adapters/registry.ts`) | +| `cliCommand?` | CLI binary for `resolveCommand`; omit if user-supplied | +| `timeoutMs?` | Execution timeout (default 10 min) | +| `requiredEnv?` | Env vars validated by the runner pre-flight (e.g. `ANTHROPIC_API_KEY`) | +| `hasLocalSession?` | Detect a usable local CLI login (e.g. `claude login`, `codex login`). Runner calls this only when `requiredEnv` is missing — explicit API keys always win | +| `isolationEnv?` | Isolation vars (e.g. `CLAUDE_CONFIG_DIR`, `CODEX_HOME`). Signature: `({ workspace, home }) => Record`. Point `*_HOME`-style paths under `home`, never `workspace` | +| `prepare?` | Side effects (mkdir, MCP / skills writers) before spawn | +| `resolveCommand?` | Override how the CLI command is resolved | +| `buildArgs` | Build CLI arguments (prefix args from command resolution prepended automatically) | +| `promptVia?` | How the prompt reaches the CLI: `"argv"` (default) -`buildArgs` places `input.prompt` on the command line; `"stdin"` -the base writes `input.prompt` to the child's stdin and closes it, and `buildArgs` must omit the prompt. `claude-code` and `codex` use `"stdin"`: argv rejects null bytes and caps argument length, and agent transcripts can contain either | +| `initialState` | Per-run mutable state used by `streamConfig` handlers and `getResult` | +| `streamConfig` | How to process agent stdout. Discriminated union: `{ mode: "lines", onLine, onEnd? }` or `{ mode: "aggregate", onChunk, onEnd? }` | +| `getResult` | Build final `{ result, metadata? }` from accumulated state after exit | The `streamConfig` field uses a discriminated union so the mode and its handler can never get out of sync -no runtime assertions needed. `getResult` returns `null` for "no result" (never `""`). Metadata overrides (e.g. upstream `durationMs`) are spread on top of base-computed fields. diff --git a/src/adapters/base/agent-adapter.ts b/src/adapters/base/agent-adapter.ts index 2b91e01..2b41ae5 100644 --- a/src/adapters/base/agent-adapter.ts +++ b/src/adapters/base/agent-adapter.ts @@ -133,6 +133,15 @@ export type AgentAdapterSpec = { /** Build the CLI arguments for the agent process. Prefix args from command resolution are prepended automatically. */ buildArgs: (input: AgentInput) => string[]; + /** + * How the prompt reaches the CLI. + * - "argv" (default): `buildArgs` places `input.prompt` on the command line. + * - "stdin": the base writes `input.prompt` to the child's stdin and closes it; + * `buildArgs` must NOT include the prompt. Use this for CLIs that read the + * prompt from a pipe — argv rejects null bytes and caps argument length. + */ + promptVia?: "argv" | "stdin"; + /** Per-run mutable state. Called once per run to create a fresh state bag for `streamConfig` handlers and `getResult`. */ initialState: () => State; @@ -250,7 +259,12 @@ export function createAgentAdapter(spec: AgentAdapterSpec): AgentA env: input.env ?? { ...process.env }, }); - child.stdin?.end(); + if (spec.promptVia === "stdin") { + child.stdin?.on("error", () => {}); // Prevent an unhandled stream error if the child closes stdin early. + child.stdin?.end(input.prompt); + } else { + child.stdin?.end(); + } // 6. Cleanup handler for Ctrl-C input.registerCleanup?.(() => { diff --git a/src/adapters/claude-code.ts b/src/adapters/claude-code.ts index f616edf..335d33a 100644 --- a/src/adapters/claude-code.ts +++ b/src/adapters/claude-code.ts @@ -15,6 +15,7 @@ export function createClaudeCodeAdapter(): AgentAdapter { return createAgentAdapter({ name: "claude-code", cliCommand: "claude", + promptVia: "stdin", requiredEnv: () => ["ANTHROPIC_API_KEY"], @@ -77,7 +78,7 @@ export function createClaudeCodeAdapter(): AgentAdapter { // Default dangerously-skip-permissions to true — AXIS runs agents headlessly const skipPermissions = flags["dangerously-skip-permissions"] ?? true; - const args = ["-p", input.prompt, "--output-format", "stream-json", "--verbose"]; + const args = ["-p", "--output-format", "stream-json", "--verbose"]; if (skipPermissions) args.push("--dangerously-skip-permissions"); if (input.config.model) args.push("--model", input.config.model); diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index fe64042..7c46eba 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -15,6 +15,7 @@ export function createCodexAdapter(): AgentAdapter { return createAgentAdapter({ name: "codex", cliCommand: "codex", + promptVia: "stdin", requiredEnv: () => ["CODEX_API_KEY"], @@ -77,8 +78,6 @@ export function createCodexAdapter(): AgentAdapter { } } - // Prompt is the final positional argument - args.push(input.prompt); return args; }, diff --git a/src/docs-site/src/pages/running.astro b/src/docs-site/src/pages/running.astro index 474772d..9bdb046 100644 --- a/src/docs-site/src/pages/running.astro +++ b/src/docs-site/src/pages/running.astro @@ -63,6 +63,16 @@ export default createAgentAdapter<{ stdout: string }>({ result: ctx.state.stdout.trim() || null, }), });`} +

+ The example above uses the default promptVia: "argv" behavior: buildArgs + places input.prompt on the command line. Set promptVia: "stdin" and AXIS + writes the prompt to the process's stdin and closes it instead -in that mode + buildArgs must omit the prompt. The built-in claude-code and + codex adapters use "stdin": argv rejects null bytes and caps argument + length, and agent transcripts can contain either. +

+ 
{`promptVia: "stdin",
+buildArgs: () => [],`}

Register it in axis.config.json:

diff --git a/src/scoring/prompt-templates.ts b/src/scoring/prompt-templates.ts index b482b54..d7025cd 100644 --- a/src/scoring/prompt-templates.ts +++ b/src/scoring/prompt-templates.ts @@ -41,6 +41,15 @@ export interface PromptTemplate { // Interpolation // --------------------------------------------------------------------------- +/** + * C0 control characters (except `\t`, `\n`, `\r`) and DEL. Some substituted + * values carry agent output, which can contain raw control bytes; a judge + * CLI can't render them and argv rejects null bytes outright, so they're + * stripped from every value at interpolation. + */ +// eslint-disable-next-line no-control-regex -- intentionally matching control characters +const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g; + /** * Replace `{{key}}` placeholders in `template` with values from `vars`. * @@ -52,7 +61,7 @@ export function interpolate(template: string, vars: Record ({ spawn: vi.fn(), @@ -24,7 +28,7 @@ function createMockProcess(opts: { const { stdout: stdoutLines = [], stderr: stderrLines = [], exitCode = 0, delayMs = 0, hang = false } = opts; const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { @@ -62,7 +66,9 @@ let setupCalls: SetupContext[] = []; let getResultCalls = 0; let resultOverride: Partial | null = null; -function createLinesTestAdapter(): AgentAdapter { +function createLinesTestAdapter( + overrides: Partial> = {}, +): AgentAdapter { setupCalls = []; getResultCalls = 0; resultOverride = null; @@ -96,6 +102,8 @@ function createLinesTestAdapter(): AgentAdapter { metadata: resultOverride ?? {}, }; }, + + ...overrides, }); } @@ -424,6 +432,34 @@ describe("createAgentAdapter", () => { expect(out.metadata.error?.length).toBeLessThan(200_000); }); + it("promptVia: stdin writes the prompt to child.stdin and resolves without throwing", async () => { + const fakeChild = createMockProcess({ stdout: ["ok\n"] }); + mockSpawn.mockImplementation((() => fakeChild) as any); + + const adapter = createLinesTestAdapter({ promptVia: "stdin" }); + + const prompt = "hello\0world"; + await expect(adapter.run(makeInput({ prompt }))).resolves.not.toThrow(); + + expect(fakeChild.stdin.end).toHaveBeenCalledWith(prompt); + }); + + it("promptVia omitted: stdin.end is called with no data and argv is untouched", async () => { + const fakeChild = createMockProcess({ stdout: ["ok\n"] }); + let captured: string[] = []; + mockSpawn.mockImplementation(((_cmd: string, args: string[]) => { + captured = args; + return fakeChild; + }) as any); + + const adapter = createLinesTestAdapter(); + + await adapter.run(makeInput({ prompt: "hello\0world" })); + + expect(fakeChild.stdin.end).toHaveBeenCalledWith(); + expect(captured).toEqual(["--flag"]); + }); + it("custom resolveCommand overrides default resolution", async () => { let usedCmd = ""; mockSpawn.mockImplementation(((cmd: string) => { diff --git a/test/unit/adapters/claude-code.test.ts b/test/unit/adapters/claude-code.test.ts index 12843b0..073115c 100644 --- a/test/unit/adapters/claude-code.test.ts +++ b/test/unit/adapters/claude-code.test.ts @@ -26,7 +26,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin }); // Push lines async so readline can consume them @@ -219,6 +219,25 @@ describe("ClaudeCodeAdapter", () => { expect(output.rawOutput).toBeUndefined(); }); + it("sends the prompt over stdin instead of argv", async () => { + let capturedArgs: string[] = []; + let proc: ReturnType; + + mockSpawn.mockImplementation(((_cmd: string, args: string[]) => { + capturedArgs = args as string[]; + proc = createMockProcess([JSON.stringify({ type: "result", result: "ok" })]); + return proc; + }) as any); + + const prompt = "do the thing\0with a null byte"; + await adapter.run(makeInput(prompt)); + + expect(capturedArgs[0]).toBe("-p"); + expect(capturedArgs[1]).toBe("--output-format"); + expect(capturedArgs).not.toContain(prompt); + expect(proc!.stdin.end).toHaveBeenCalledWith(prompt); + }); + it("passes --strict-mcp-config so only AXIS-declared MCP servers are used", async () => { let capturedArgs: string[] = []; diff --git a/test/unit/adapters/codex-e2e.test.ts b/test/unit/adapters/codex-e2e.test.ts index e825534..853598b 100644 --- a/test/unit/adapters/codex-e2e.test.ts +++ b/test/unit/adapters/codex-e2e.test.ts @@ -24,7 +24,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { diff --git a/test/unit/adapters/codex.test.ts b/test/unit/adapters/codex.test.ts index f71fd26..39c9bd0 100644 --- a/test/unit/adapters/codex.test.ts +++ b/test/unit/adapters/codex.test.ts @@ -23,7 +23,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { @@ -265,21 +265,25 @@ describe("CodexAdapter", () => { expect(capturedArgs).not.toContain("--color"); }); - it("puts prompt as last positional argument after exec --json", async () => { + it("sends the prompt over stdin instead of argv", async () => { let capturedArgs: string[] = []; + let proc: ReturnType; mockSpawn.mockImplementation(((_cmd: string, args: string[]) => { capturedArgs = args as string[]; - return createMockProcess([ + proc = createMockProcess([ JSON.stringify({ type: "item.completed", item: { type: "agent_message", text: "ok" } }), ]); + return proc; }) as any); - await adapter.run(makeInput("do the thing")); + const prompt = "do the thing\0with a null byte"; + await adapter.run(makeInput(prompt)); expect(capturedArgs[0]).toBe("exec"); expect(capturedArgs[1]).toBe("--json"); - expect(capturedArgs[capturedArgs.length - 1]).toBe("do the thing"); + expect(capturedArgs).not.toContain(prompt); + expect(proc!.stdin.end).toHaveBeenCalledWith(prompt); }); it("uses last agent_message when multiple are emitted", async () => { diff --git a/test/unit/adapters/mcp-e2e.test.ts b/test/unit/adapters/mcp-e2e.test.ts index f2eae18..59c866e 100644 --- a/test/unit/adapters/mcp-e2e.test.ts +++ b/test/unit/adapters/mcp-e2e.test.ts @@ -23,7 +23,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { diff --git a/test/unit/adapters/skills-e2e.test.ts b/test/unit/adapters/skills-e2e.test.ts index e7ee469..3647ea9 100644 --- a/test/unit/adapters/skills-e2e.test.ts +++ b/test/unit/adapters/skills-e2e.test.ts @@ -49,7 +49,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { diff --git a/test/unit/scoring/prompt-templates.test.ts b/test/unit/scoring/prompt-templates.test.ts index 3137f9e..a7b4612 100644 --- a/test/unit/scoring/prompt-templates.test.ts +++ b/test/unit/scoring/prompt-templates.test.ts @@ -37,6 +37,14 @@ describe("interpolate", () => { it("handles empty string variable value", () => { expect(interpolate("before{{gap}}after", { gap: "" })).toBe("beforeafter"); }); + + it("strips control characters from a substituted value", () => { + expect(interpolate("{{v}}", { v: "a\0b\x01c\x7F" })).toBe("abc"); + }); + + it("preserves tab, newline, and carriage return in a substituted value", () => { + expect(interpolate("{{v}}", { v: "x\ny\tz\r" })).toBe("x\ny\tz\r"); + }); }); describe("getPromptTemplates", () => {