From 527553def9141c82d850559866de0f135b29a0c7 Mon Sep 17 00:00:00 2001
From: bowlerjim <jcrews@gmail.com>
Date: Sun, 20 Sep 2026 23:33:05 -0400
Subject: [PATCH] fix(Evals): binary_tests no longer depends on GNU `timeout`
 and runs its command through a shell (#2188)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The grader shelled out to `timeout`, which is GNU coreutils and absent on a
stock macOS install, so every test file came back `bun: command not found:
timeout` and was graded failed — 0/N on a suite that passes. The same line
interpolated `${command}` into a tagged template, which escapes a multi-word
test command into a single argv token.

Spawn `/bin/sh -c` (`cmd /c` on win32) with Bun's own `timeout`/`killSignal`
instead. The pipe drain is capped, because a killed shell can leave an orphan
child holding stdout open: awaiting it outright made the timeout worthless
(measured 30s on a 1.5s budget).

Verified on macOS: passing file -> 1/1 (was 0/1), failing file -> 0/1,
`sleep 30` with timeout_ms 1500 -> 0/1 in 3.5s (was 30s).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PuS77R8a6Lhp7jK6dnwdTh
---
 .../Evals/Graders/CodeBased/BinaryTests.ts    | 42 +++++++++++++++----
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/LifeOS/install/skills/Evals/Graders/CodeBased/BinaryTests.ts b/LifeOS/install/skills/Evals/Graders/CodeBased/BinaryTests.ts
index f993a38494..bf3a334c0a 100755
--- a/LifeOS/install/skills/Evals/Graders/CodeBased/BinaryTests.ts
+++ b/LifeOS/install/skills/Evals/Graders/CodeBased/BinaryTests.ts
@@ -5,7 +5,22 @@
 
 import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
 import type { GraderConfig, GraderResult, BinaryTestsParams } from '../../Types/index.ts';
-import { $ } from 'bun';
+
+// Windows has no /bin/sh; cmd takes /c where POSIX shells take -c.
+const SHELL_PREFIX = process.platform === 'win32' ? ['cmd', '/c'] : ['/bin/sh', '-c'];
+
+async function drainStream(stream: ReadableStream<Uint8Array> | null, ms = 2000): Promise<string> {
+  if (!stream) return '';
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      new Response(stream).text(),
+      new Promise<string>((resolve) => { timer = setTimeout(() => resolve(''), ms); }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
 
 export class BinaryTestsGrader extends BaseGrader {
   type = 'binary_tests' as const;
@@ -30,16 +45,29 @@ export class BinaryTestsGrader extends BaseGrader {
         // Detect test command based on file extension
         const command = params.test_command ?? this.detectTestCommand(testFile);
 
-        const result = await $`cd ${workingDir} && timeout ${Math.ceil(timeout/1000)} ${command} ${testFile}`
-          .quiet()
-          .nothrow();
+        // Run through a shell rather than a tagged template: the interpolation
+        // would escape a multi-word test command into one argv token. The old
+        // form also shelled out to `timeout`, which is GNU coreutils and absent
+        // on a stock macOS box, so every file was graded failed there.
+        const proc = Bun.spawn([...SHELL_PREFIX, `${command} ${testFile}`], {
+          cwd: workingDir,
+          stdout: 'pipe',
+          stderr: 'pipe',
+          timeout,
+          killSignal: 'SIGKILL',
+        });
+        const exitCode = await proc.exited;
+        // A killed shell can leave an orphan child holding the pipe open, so the
+        // drain is capped rather than awaited outright — otherwise the timeout
+        // above buys nothing.
+        const [stdout, stderr] = await Promise.all([drainStream(proc.stdout), drainStream(proc.stderr)]);
 
-        const passed = result.exitCode === 0;
+        const passed = exitCode === 0;
         results.push({
           file: testFile,
           passed,
-          output: result.stdout.toString().slice(-500),  // Last 500 chars
-          error: passed ? undefined : result.stderr.toString().slice(-500),
+          output: stdout.slice(-500),  // Last 500 chars
+          error: passed ? undefined : stderr.slice(-500),
         });
       } catch (e) {
         results.push({