From b6849c1d355157853f13a420b6c67e6bf836e054 Mon Sep 17 00:00:00 2001 From: tiborsekera <34183854+tiborsekera@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:24:42 +0200 Subject: [PATCH 01/94] fix(codex): reject history sync for active sessions (#1895) * fix(codex): guard transcript sync for active sessions * test(codex): exercise divergent transcript prefix --- hub/src/web/routes/codexDesktop.test.ts | 39 ++++++++++++++++++++ hub/src/web/routes/codexDesktop.ts | 12 +++++++ web/src/components/SessionHeader.test.tsx | 44 +++++++++++++++++++++++ web/src/components/SessionHeader.tsx | 4 +-- 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/hub/src/web/routes/codexDesktop.test.ts b/hub/src/web/routes/codexDesktop.test.ts index ddfbd3f466..65c81ed440 100644 --- a/hub/src/web/routes/codexDesktop.test.ts +++ b/hub/src/web/routes/codexDesktop.test.ts @@ -437,6 +437,45 @@ describe('Codex Desktop import routes', () => { } }) + it('refuses a transcript import for an active Codex thread when stored messages do not match its prefix', async () => { + const codexHome = mkdtempSync(join(tmpdir(), 'hapi-codex-home-active-sync-test-')) + const store = new Store(':memory:') + const codexSessionId = '10101010-1010-4010-8010-101010101010' + process.env.CODEX_HOME = codexHome + + try { + createTranscript(codexHome, codexSessionId) + const liveSession = store.sessions.getOrCreateSession('live-session', { + path: 'C:\\work\\project', + flavor: 'codex', + codexSessionId + }, {}, 'default') + store.messages.addMessage(liveSession.id, { + role: 'user', + content: { type: 'text', text: 'different from the transcript' } + }, 'live-1') + const engine = { + getSessionsByNamespace: () => [{ ...liveSession, active: true }] + } as unknown as SyncEngine + + const result = await importSelectedCodexSessions({ + codexSessionIds: [codexSessionId], + store, + namespace: 'default', + getSyncEngine: () => engine + }) + + expect(result.success).toBe(false) + if (result.success) throw new Error('Expected active-session transcript import to fail') + expect(result.error).toContain('matching HAPI session is active') + expect(store.sessions.getSessionsByNamespace('default')).toHaveLength(1) + expect(store.messages.getAllMessages(liveSession.id)).toHaveLength(1) + } finally { + store.close() + rmSync(codexHome, { recursive: true, force: true }) + } + }) + it('updates an existing forked import when syncing the original Codex session id', async () => { const codexHome = mkdtempSync(join(tmpdir(), 'hapi-codex-home-source-test-')) const store = new Store(':memory:') diff --git a/hub/src/web/routes/codexDesktop.ts b/hub/src/web/routes/codexDesktop.ts index fb54b67ffe..05913d41dd 100644 --- a/hub/src/web/routes/codexDesktop.ts +++ b/hub/src/web/routes/codexDesktop.ts @@ -2013,6 +2013,18 @@ function importSingleCodexSession(options: { try { const candidates = collectImportCandidates(options.store, options.namespace, options.getSyncEngine) + const activeCandidate = candidates.find((candidate) => ( + candidate.active + && getCodexImportIds(candidate.metadata).includes(options.codexSessionId) + && ( + !options.machineId + || typeof candidate.metadata?.machineId !== 'string' + || candidate.metadata.machineId === options.machineId + ) + )) + if (activeCandidate) { + throw new Error('Cannot sync Codex transcript while the matching HAPI session is active') + } const target = selectImportTargetSession( options.store, candidates, diff --git a/web/src/components/SessionHeader.test.tsx b/web/src/components/SessionHeader.test.tsx index de968259af..a78780afa4 100644 --- a/web/src/components/SessionHeader.test.tsx +++ b/web/src/components/SessionHeader.test.tsx @@ -58,6 +58,18 @@ function renderHeader(session: Session, extra?: { serviceTier?: string | null; t ) } +function renderHeaderWithApi(session: Session, api: ApiClient) { + return render( + + + + + + + + ) +} + describe('resolveSessionHeaderMachineLabel', () => { it('prefers cached/display labels, then host, then short machine id', () => { expect(resolveSessionHeaderMachineLabel( @@ -83,6 +95,38 @@ describe('resolveSessionHeaderMachineLabel', () => { }) describe('SessionHeader', () => { + it('does not offer manual Codex sync while the HAPI session is active', () => { + const api = { + getMachines: vi.fn().mockResolvedValue({ machines: [] }), + getScratchlist: vi.fn().mockResolvedValue({ entries: [] }), + syncCodexSession: vi.fn() + } as unknown as ApiClient + + renderHeaderWithApi(baseSession({ + active: true, + metadata: { flavor: 'codex', path: '/repo', host: 'machine', codexSessionId: 'codex-thread-1' } + }), api) + + fireEvent.click(screen.getByRole('button', { name: /More/ })) + expect(screen.queryByRole('menuitem', { name: /Sync Codex/ })).toBeNull() + }) + + it('keeps manual Codex sync available for an inactive imported thread', () => { + const api = { + getMachines: vi.fn().mockResolvedValue({ machines: [] }), + getScratchlist: vi.fn().mockResolvedValue({ entries: [] }), + syncCodexSession: vi.fn() + } as unknown as ApiClient + + renderHeaderWithApi(baseSession({ + active: false, + metadata: { flavor: 'codex', path: '/repo', host: 'machine', codexSessionId: 'codex-thread-1' } + }), api) + + fireEvent.click(screen.getByRole('button', { name: /More/ })) + expect(screen.getByRole('menuitem', { name: /Sync Codex/ })).toBeInTheDocument() + }) + it('hides title generation when the Hub does not advertise the capability', () => { const api = { getMachines: vi.fn().mockResolvedValue({ machines: [] }), diff --git a/web/src/components/SessionHeader.tsx b/web/src/components/SessionHeader.tsx index f42ac39fe8..1939300f29 100644 --- a/web/src/components/SessionHeader.tsx +++ b/web/src/components/SessionHeader.tsx @@ -268,7 +268,7 @@ export function SessionHeader(props: { } const handleSyncCodex = async () => { - if (!api || !codexSessionId || isSyncingCodex) return + if (!api || !codexSessionId || isSyncingCodex || session.active) return setIsSyncingCodex(true) try { @@ -523,7 +523,7 @@ export function SessionHeader(props: { onMarkUnread={() => markSessionUnread(session.id, session.updatedAt)} onSetPinMode={api ? (mode) => void handleSetPinMode(mode) : undefined} onExport={() => setExportOpen(true)} - onSyncCodex={api && codexSessionId ? handleSyncCodex : undefined} + onSyncCodex={api && codexSessionId && !session.active ? handleSyncCodex : undefined} onSyncPi={api && piSessionId && !session.active ? handleSyncPi : undefined} onArchive={() => setArchiveOpen(true)} onReopen={props.canReopen === false ? undefined : handleReopen} From 97cf69bb5989e64569514a58de959e7c5f06e693 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Thu, 24 Sep 2026 00:25:17 +0100 Subject: [PATCH 02/94] feat(web): dictate keyboard shortcut (Ctrl/Cmd+Shift+D) (#1902) * feat(web): add Ctrl/Cmd+Shift+D dictate keyboard shortcut Toggle composer dictation or voice assistant via the same path as the mic buttons, with dialog/input guards and tooltip documentation. Fixes tiann/hapi#1900 Co-authored-by: Cursor * fix(web): address dictate hotkey bot findings on PR 1902 Ignore key repeat on the session listener and align assistant-mode invoke guards with UnifiedButton send mode (canSend). Co-authored-by: Cursor --------- Co-authored-by: Cursor --- .../AssistantChat/ComposerButtons.test.tsx | 2 +- .../AssistantChat/HappyComposer.tsx | 35 +++++++ web/src/components/SessionChat.tsx | 26 +++++ web/src/lib/composerDictateShortcut.test.ts | 99 +++++++++++++++++++ web/src/lib/composerDictateShortcut.ts | 62 ++++++++++++ web/src/lib/locales/en.ts | 2 +- web/src/lib/locales/zh-CN.ts | 2 +- 7 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 web/src/lib/composerDictateShortcut.test.ts create mode 100644 web/src/lib/composerDictateShortcut.ts diff --git a/web/src/components/AssistantChat/ComposerButtons.test.tsx b/web/src/components/AssistantChat/ComposerButtons.test.tsx index d6ac051be3..3b7df0152f 100644 --- a/web/src/components/AssistantChat/ComposerButtons.test.tsx +++ b/web/src/components/AssistantChat/ComposerButtons.test.tsx @@ -177,7 +177,7 @@ describe('DictationButton', () => { />, ) - fireEvent.click(getButton('Dictate')) + fireEvent.click(getButton(/^Dictate/)) expect(onVoiceToggle).toHaveBeenCalledOnce() }) }) diff --git a/web/src/components/AssistantChat/HappyComposer.tsx b/web/src/components/AssistantChat/HappyComposer.tsx index 78ddb24220..5be36c3942 100644 --- a/web/src/components/AssistantChat/HappyComposer.tsx +++ b/web/src/components/AssistantChat/HappyComposer.tsx @@ -21,6 +21,7 @@ import { useState } from 'react' import { useNarrowViewport } from '@/hooks/useNarrowViewport' +import { shouldInvokeComposerDictateShortcut } from '@/lib/composerDictateShortcut' import { isRichComposerMentionsEnabled, resolveComposerPlaceholderKey } from '@/lib/composerSegments' import type { SessionMentionResolveResult } from '@/components/AssistantChat/RichComposerInput' import { @@ -364,6 +365,8 @@ export function HappyComposer(props: { ) => Promise /** Parent disables DragDropZone / scratchlist promote while park is in flight. */ onScratchlistParkingChange?: (parking: boolean) => void + /** SessionChat binds Ctrl/Cmd+Shift+D; HappyComposer registers the effective voice toggle. */ + dictateHotkeyRef?: MutableRefObject<(() => void) | null> // Set when the most recent send failed (4xx/5xx/network). The composer // restores the original text once per `sendError.id` and renders an // inline error affordance until the user dismisses or starts editing. @@ -1623,6 +1626,38 @@ export function HappyComposer(props: { ) const showAbortButton = true const voiceEnabled = Boolean(effectiveVoiceToggle) + const routesToScratchlist = (props.scratchlistMode ?? false) && pendingSchedule == null + + const invokeDictateHotkey = useCallback(() => { + if (!shouldInvokeComposerDictateShortcut({ + controlsDisabled, + voiceEnabled, + dictationActive, + voiceStatus: effectiveVoiceStatus, + canSend, + routesToScratchlist, + })) { + return + } + effectiveVoiceToggle?.() + }, [ + controlsDisabled, + voiceEnabled, + dictationActive, + effectiveVoiceStatus, + canSend, + routesToScratchlist, + effectiveVoiceToggle, + ]) + + useEffect(() => { + const ref = props.dictateHotkeyRef + if (!ref) return + ref.current = invokeDictateHotkey + return () => { + ref.current = null + } + }, [props.dictateHotkeyRef, invokeDictateHotkey]) // Generic model/effort value buttons. The current value label doubles as // the button caption; clicking opens the settings sheet. Hidden on narrow diff --git a/web/src/components/SessionChat.tsx b/web/src/components/SessionChat.tsx index e8dd5e7745..6a2a562f13 100644 --- a/web/src/components/SessionChat.tsx +++ b/web/src/components/SessionChat.tsx @@ -35,6 +35,10 @@ import { } from '@/lib/codexModelCapabilities' import { createSerialAsyncQueue } from '@/lib/serialAsyncQueue' import { HappyComposer, type ComposerSendError } from '@/components/AssistantChat/HappyComposer' +import { + isDictateHotkeyBlockedTarget, + isDictateToggleHotkey, +} from '@/lib/composerDictateShortcut' import { codexModelAdvertisesFastTier, getEffectiveCodexServiceTier } from '@/components/AssistantChat/codexFastMode' import type { PendingSchedule } from '@/components/AssistantChat/ScheduleTimePicker' import { resolvePendingSchedule } from '@/components/AssistantChat/ScheduleTimePicker' @@ -767,6 +771,7 @@ function SessionChatInner(props: SessionChatProps) { if (isScratchlistParking) return setScratchlistMode((m) => !m) }, [isScratchlistParking]) + const dictateHotkeyRef = useRef<(() => void) | null>(null) /** * Global keyboard shortcut: Ctrl/Cmd + Shift + S toggles scratchlist * mode (open/close drawer + flip composer routing). @@ -800,6 +805,26 @@ function SessionChatInner(props: SessionChatProps) { window.addEventListener('keydown', onKeyDown) return () => window.removeEventListener('keydown', onKeyDown) }, [isScratchlistParking]) + /** + * Global keyboard shortcut: Ctrl/Cmd + Shift + D toggles composer + * dictation (Settings → Voice mode: dictation) or voice assistant, + * using the same effective toggle as the mic / dictate buttons in + * HappyComposer. Skipped for dialog / single-line input targets; + * rich composer input is allowed (see isDictateHotkeyBlockedTarget). + */ + useEffect(() => { + const onKeyDown = (e: globalThis.KeyboardEvent) => { + if (e.repeat) return + if (!isDictateToggleHotkey(e)) return + if (isDictateHotkeyBlockedTarget(e.target)) return + const invoke = dictateHotkeyRef.current + if (!invoke) return + e.preventDefault() + invoke() + } + window.addEventListener('keydown', onKeyDown) + return () => window.removeEventListener('keydown', onKeyDown) + }, []) /** * Global select-all takeover: see applyGlobalSelectAll. Bound at * window scope because the broken case is focus on the page body / @@ -2210,6 +2235,7 @@ function SessionChatInner(props: SessionChatProps) { onScratchlistToggle={handleScratchlistToggle} onParkScratchlist={onParkScratchlist} onScratchlistParkingChange={setIsScratchlistParking} + dictateHotkeyRef={dictateHotkeyRef} sendError={props.sendError ?? null} onClearSendError={handleClearSendError} onSuppressSendErrorRestore={props.onSuppressSendErrorRestore} diff --git a/web/src/lib/composerDictateShortcut.test.ts b/web/src/lib/composerDictateShortcut.test.ts new file mode 100644 index 0000000000..817bf1e1b4 --- /dev/null +++ b/web/src/lib/composerDictateShortcut.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest' +import { + isDictateHotkeyBlockedTarget, + isDictateToggleHotkey, + shouldInvokeComposerDictateShortcut, +} from './composerDictateShortcut' + +describe('isDictateToggleHotkey', () => { + function k(over: Partial<{ + metaKey: boolean; ctrlKey: boolean; shiftKey: boolean; altKey: boolean; key: string + }>): { metaKey: boolean; ctrlKey: boolean; shiftKey: boolean; altKey: boolean; key: string } { + return { metaKey: false, ctrlKey: false, shiftKey: false, altKey: false, key: '', ...over } + } + + it('matches Ctrl+Shift+D and Cmd+Shift+D', () => { + expect(isDictateToggleHotkey(k({ ctrlKey: true, shiftKey: true, key: 'D' }))).toBe(true) + expect(isDictateToggleHotkey(k({ metaKey: true, shiftKey: true, key: 'd' }))).toBe(true) + }) + + it('rejects bare D and Ctrl+D without shift', () => { + expect(isDictateToggleHotkey(k({ key: 'd' }))).toBe(false) + expect(isDictateToggleHotkey(k({ ctrlKey: true, key: 'd' }))).toBe(false) + }) + + it('rejects unrelated keys with the modifier chord', () => { + expect(isDictateToggleHotkey(k({ ctrlKey: true, shiftKey: true, key: 'S' }))).toBe(false) + }) +}) + +describe('isDictateHotkeyBlockedTarget', () => { + it('blocks single-line inputs and dialogs like scratchlist', () => { + const input = document.createElement('input') + expect(isDictateHotkeyBlockedTarget(input)).toBe(true) + }) + + it('allows the rich composer contentEditable host', () => { + const shell = document.createElement('div') + shell.setAttribute('data-testid', 'rich-composer-input') + const editor = document.createElement('div') + editor.setAttribute('contenteditable', 'plaintext-only') + shell.appendChild(editor) + document.body.appendChild(shell) + expect(isDictateHotkeyBlockedTarget(editor)).toBe(false) + document.body.removeChild(shell) + }) + + it('allows the fallback composer textarea', () => { + const textarea = document.createElement('textarea') + expect(isDictateHotkeyBlockedTarget(textarea)).toBe(false) + }) +}) + +describe('shouldInvokeComposerDictateShortcut', () => { + const base = { + controlsDisabled: false, + voiceEnabled: true, + dictationActive: false, + voiceStatus: 'disconnected' as const, + canSend: false, + routesToScratchlist: false, + } + + it('stops an active voice session', () => { + expect(shouldInvokeComposerDictateShortcut({ + ...base, + voiceStatus: 'connected', + })).toBe(true) + }) + + it('starts assistant voice on an empty composer', () => { + expect(shouldInvokeComposerDictateShortcut(base)).toBe(true) + }) + + it('does not start assistant voice when UnifiedButton is in send mode', () => { + expect(shouldInvokeComposerDictateShortcut({ ...base, canSend: true })).toBe(false) + }) + + it('starts dictation on an empty composer when scratchlist routing is off', () => { + expect(shouldInvokeComposerDictateShortcut({ + ...base, + dictationActive: true, + })).toBe(true) + }) + + it('allows dictation when a draft makes canSend true', () => { + expect(shouldInvokeComposerDictateShortcut({ + ...base, + dictationActive: true, + canSend: true, + })).toBe(true) + }) + + it('suppresses scratchlist-routed empty composer starts', () => { + expect(shouldInvokeComposerDictateShortcut({ + ...base, + routesToScratchlist: true, + })).toBe(false) + }) +}) diff --git a/web/src/lib/composerDictateShortcut.ts b/web/src/lib/composerDictateShortcut.ts new file mode 100644 index 0000000000..44f8df277c --- /dev/null +++ b/web/src/lib/composerDictateShortcut.ts @@ -0,0 +1,62 @@ +import type { ConversationStatus } from '@/realtime/types' + +/** + * True if the keystroke matches the composer dictation / voice toggle shortcut + * (Ctrl/Cmd + Shift + D, no Alt). Pure for unit tests. + * + * Modifier shape matches scratchlist (Ctrl/Cmd+Shift+S) and model cycle + * (Ctrl/Cmd+M). Shift+D avoids browser bookmark / devtools clashes on bare D. + */ +export function isDictateToggleHotkey(e: { + metaKey: boolean + ctrlKey: boolean + shiftKey: boolean + altKey: boolean + key: string +}): boolean { + if (!(e.metaKey || e.ctrlKey)) return false + if (!e.shiftKey) return false + if (e.altKey) return false + return e.key === 'D' || e.key === 'd' +} + +/** + * True when the global dictate hotkey should be SKIPPED for the event target. + * + * Same dialog / single-line input guards as scratchlist + * (`isScratchlistHotkeyBlockedTarget`), but the rich composer (contentEditable) + * is allowed so the shortcut works while typing in the main input. + */ +export function isDictateHotkeyBlockedTarget(target: EventTarget | null): boolean { + if (!(target instanceof HTMLElement)) return false + if (target.closest('[data-testid="rich-composer-input"]') !== null) return false + if (target.closest('[role="dialog"]') !== null) return true + if (target instanceof HTMLInputElement) return true + if (target instanceof HTMLSelectElement) return true + if (target.isContentEditable === true) return true + return target.getAttribute('contenteditable') === 'true' +} + +/** + * Mirrors DictationButton + UnifiedButton voice-start/stop rules so the hotkey + * does not send chat text or start voice when the mic UI would not. + */ +export function shouldInvokeComposerDictateShortcut(args: { + controlsDisabled: boolean + voiceEnabled: boolean + dictationActive: boolean + voiceStatus: ConversationStatus + canSend: boolean + routesToScratchlist: boolean +}): boolean { + if (args.controlsDisabled || !args.voiceEnabled) return false + const isVoiceActive = args.voiceStatus === 'connecting' || args.voiceStatus === 'connected' + if (isVoiceActive) return true + if (args.dictationActive) { + if (args.canSend) return true + return !args.routesToScratchlist + } + // UnifiedButton treats canSend (text or attachments) as Send mode for assistant voice. + if (args.canSend) return false + return !args.routesToScratchlist +} diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index 647b80054e..cfdc81fb6c 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -660,7 +660,7 @@ export default { 'composer.send': 'Send', 'composer.stop': 'Stop', 'composer.voice': 'Voice assistant', - 'composer.dictate': 'Dictate', + 'composer.dictate': 'Dictate (Ctrl/Cmd+Shift+D)', 'composer.scheduleSend': 'Schedule send', 'composer.scheduleRelativeTab': 'Relative', 'composer.scheduleSpecificTab': 'Specific', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index 6886e3ae5c..2c3bf40054 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -658,7 +658,7 @@ export default { 'composer.send': '发送', 'composer.stop': '停止', 'composer.voice': '语音助手', - 'composer.dictate': '语音输入', + 'composer.dictate': '语音输入 (Ctrl/Cmd+Shift+D)', 'composer.scheduleSend': '定时发送', 'composer.scheduleRelativeTab': '相对时间', 'composer.scheduleSpecificTab': '指定时间', From 86c88df93baf5d1f738dd4b202078bdf6dec376e Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Fri, 25 Sep 2026 09:15:08 +0100 Subject: [PATCH 03/94] fix(runner,hub): cross-platform orphan reap on archive / tracking loss (#1911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(runner,hub): reap detached driver CLI on archive and tracking loss Archive always verifies via stopRunnerSession; runner stopSession scans argv orphans and awaits webhook-timeout tree-kill so PPID=1 children cannot accumulate after KillSession miss or map drop. Fixes #1910 Co-authored-by: Cursor * fix(cli): keep --hapi-session-id reap-only; test orphan kill path Claude/cursor swallow the stamp without bootstrap reuse, and reapRunnerSpawnedOrphans is unit-tested as the stopSession map-miss path. Co-authored-by: Cursor * fix(cli,hub): address #1911 CI and bot Majors Use Vitest in orphanReap tests, add Pi session.on mock, refuse archive when machine StopSession is unreachable, trust full tree-kill results, and argv-sweep orphans after every mapped stop. Co-authored-by: Cursor * fix(runner): gate late-webhook kill on timeout orphans; fail closed on scan error Track webhook-timeout PIDs separately so recovered shared roots are not killed, and treat ps-list failures as still_alive rather than empty. Co-authored-by: Cursor * fix(runner): keep shared Codex wrapper alive when archiving one root StopSession must detach a single shared root without tree-killing the wrapper PID while sibling roots remain, including after KillSession has already marked the runtime inactive. Co-authored-by: Cursor * fix(runner): keep shared Codex siblings after restart tracking loss Before persisted-PID or argv orphan kills, consult the durable runtime registry for active sibling roots on the same wrapper PID so archiving one root after a runner restart cannot tree-kill live shared sessions. Co-authored-by: Cursor * fix(hub,cli): deliver hub-authored archive metadata to reconnecting CLIs When KillSession misses and archive is written via markSessionArchivedFromHub, emit Socket.IO update-session to the session room. On reconnect, ApiSessionClient reconciles GET /cli/sessions/:id metadata so hub-archived still fires. Co-authored-by: Cursor * fix(runner): argv-scan orphans even when shared siblings remain Remove session-wide early returns that skipped orphan reaping whenever any shared wrapper had active siblings. Keep only the PID-specific filter so older untracked CLIs with the same HAPI id are still reaped. Co-authored-by: Cursor * fix(runner): protect tracked shared wrappers when runtime registry is empty Argv orphan sweeps must exclude in-memory shared wrapper PIDs that still host sibling roots, not only durable registry siblings — otherwise a missing runtime record lets the sweep tree-kill the wrapper while archiving one root. Co-authored-by: Cursor * fix(runner): adopt untracked nonshared CLIs after runner restart When a runner-spawned Claude webhook arrives with no TrackedSession (typical after restart mid-bootstrap), durably adopt the PID instead of ignoring it so StopSession can reap. Keep killing only nonshared timeout orphans; never kill shared Codex wrappers on late webhooks. Co-authored-by: Cursor * fix(runner): keep recovered shared wrappers for registry siblings After restart adoption tracks only a newly reported Codex root, archiving it must consult the durable registry before tree-killing the wrapper PID so older active sibling roots survive. Co-authored-by: Cursor * fix(hub): do not archive on KillSession ack when StopSession is unknown KillSession returns before cleanupAndExit finishes. When the runner cannot find the PID (unknown), wait for the session KillSession handler to drop before archiving; refuse if the CLI socket remains reachable. Co-authored-by: Cursor * fix(hub,cli): confirm KillSession pid via StopSession before archive KillSession now returns the CLI pid. When session-id StopSession is unknown, archive confirms that OS pid (without treating socket loss as exit) and still refuses when confirmation stays unknown or still_alive. Co-authored-by: Cursor * fix(runner,hub): tree-kill adopted CLIs and require PID start markers Adopted post-restart runner sessions now use killProcessTreeByPid when no ChildProcess handle exists. KillSession carries a process-start marker so raw PID confirmation cannot tree-kill a reused OS pid. Co-authored-by: Cursor * fix(runner,hub): verify adopted PID markers and hub-archive after stop Adopted sessions compare the persisted start marker before tree-kill and drop stale tracking on mismatch. After confirmed StopSession, hub-authors archive metadata when the CLI did not finish its flush. Co-authored-by: Cursor * fix(runner): refuse unconfirmed shared/adopted stops Keep shared wrappers when siblings remain, but return unknown unless runtimeControl already ended the root. Adopted PIDs without a start marker no longer tree-kill by tracked PID alone. Co-authored-by: Cursor * fix(runner): require inactive binding before sibling-kept archive Siblings alone must not flip StopSession to stopped on retry, while an acknowledged inactive Codex registry binding remains stop proof. Handle PID- markers before any tracked kill so reuse cannot nuke strangers. Co-authored-by: Cursor * fix(runner): distinguish orphan reap from unknown keep-wrapper base Pass unknown into finishWithOrphanSweep so a successful argv reap stays stopped instead of being overwritten by an absent registry binding. Co-authored-by: Cursor * fix(runner): honor orphan reap when adopted PID lacks a start marker finishWithOrphanSweep with an unknown base lets a successful argv kill report stopped instead of forcing unknown after the marker probe fails. Co-authored-by: Cursor * fix(hub): confirm archive via metadata.hostPid tombstone When StopSession(sessionId) is unknown and KillSession supplies no confirmable pid+marker, probe PID-. already_gone allows archive; still_alive/unknown still refuse (#1911 Peer #1820). Co-authored-by: Cursor * fix(cli): escalate Windows taskkill to /F like SIGTERM→SIGKILL Soft taskkill /T is refused on win32 console trees; orphanReap left orphans still_alive. waitForProcessToDie now force-escalates on both platforms so cross-platform reaping matches. Co-authored-by: Cursor * fix(cli): widen process.kill mock signal to string | number Match @types/node process.kill(pid, signal?: string | number) so the Windows taskkill escalation mocks typecheck on tip 54f141368. Co-authored-by: Cursor * fix(cli): tasklist liveness + immediate /F after soft taskkill refuse Windows orphan reap still failed in 5ms on Teemo: soft taskkill refused but escalate waited on an unreliable process.kill(0) probe. Prefer tasklist for win32 liveness and escalate to /F as soon as the PID is still live after soft kill. Co-authored-by: Cursor * fix(cli): coerce win32 ps-list PIDs and escalate soft taskkill to /F String PIDs from ps-list made Number.isFinite fail so orphan kill returned still_alive without calling taskkill. Coerce PIDs, prefer tasklist liveness, and escalate soft taskkill to /F like SIGTERM→SIGKILL. Co-authored-by: Cursor * fix(cli): Windows orphan scan via CIM CommandLine + taskkill /F escalate ps-list/fastlist is missing CommandLine and often missing from single-exe bundles, so Teemo orphan reap returned scan_failed (logged as live PIDs). List via Win32_Process CIM, coerce PIDs, escalate soft taskkill to /F. Co-authored-by: Cursor * fix(cli): grace-wait after successful soft taskkill before /F Escalate immediately only when soft taskkill is refused. When soft succeeds but the PID is still draining, honor the existing grace period before force (Codex Major on #1911). Co-authored-by: Cursor * fix(runner): re-verify start marker before orphan argv tree-kill Capture getProcessStartMarker after the argv match list, re-check immediately before killTree, and skip on mismatch/null-while-alive so PID reuse cannot tree-kill an unrelated process on the orphan fallback. Co-authored-by: Cursor * fix(hub): preallocate HAPI id before fresh machine spawn Fresh POST /machines/:id/spawn omitted existingSessionId, so argv never got a reap stamp. Preallocate the hub row (fork/clear pattern) and pass that id so a runner restart before the first webhook remains reapeable. Co-authored-by: Cursor * fix(cli,hub): close three post-prealloc unreapable paths Pass --hapi-session-id into Claude/cursor/kimi/copilot bootstrap so the preallocated hub row is used. Treat null marker re-check while alive as still_alive (not stopped). Keep the preallocated stub when spawn-fail StopSession cannot confirm the child is gone. Co-authored-by: Cursor * fix(cli): adopt-stub via reservedSessionId, not existingSessionId `--hapi-session-id` binds a fresh create bootstrap to a hub-preallocated UUID (getOrCreateSession id) without taking the reopen path. Non-UUID stamps stay reap-only to match hub create schema. Restores integration spawn invariants broken by mapping hapi→existingSessionId. Co-authored-by: Cursor * fix(cli,hub): close Opus cold-read B1–B3 for #1911 B1: capture start markers from the same process snapshot as argv match (POSIX ps lstart / Win32 CreationDate); re-check before kill. Continue other orphans when one target is unconfirmed. B2: Windows killProcessTreeByPid verifies every collected descendant, not root-only — taskkill /T exit 0 is signalled, not gone. B3: runner sets childStarted:false on pre-exec rejection; hub deletes the preallocated stub without StopSession. Post-exec ambiguity still keeps the row until stop confirms. Terminal archives no longer refuse when no runner is connected. Co-authored-by: Cursor * fix(cli,hub): explicit adopt for hub-preallocated session stubs Fresh machine spawn preallocates with tag machine-spawn:; CLI create with a different tag and the same id 409'd on identity conflict. Add adoptPreallocatedSession (hub-internal stub check) and POST /cli/sessions { adopt: true, id } so create binds the reserved row and overwrites tag/metadata without guessing the hub tag. Tests fail-first on the real prealloc→CLI create path. Co-authored-by: Cursor * fix(cli): preserve snapshot markers + unify Windows CIM start markers Runner stopSession was still on findOrphans (PID-only), discarding same-snapshot markers and reopening the PID-reuse kill window. Switch both call sites to findTargets. Force UTC ISO 'o' CreationDate for list and single-PID probe so ConvertTo-Json cannot disagree with getProcessStartMarker on WinPS 5.1. Co-authored-by: Cursor * fix(cli): exercise production orphan sweep wiring in tests Extract findStopSessionOrphanTargets as the stopSession discovery path both run.ts sweeps call. Add tests that drive that helper end-to-end (not an injected findOrphans seam), assert run.ts source wiring, and cover CIM-list ISO vs probe ISO agreement plus the broken WinPS shape mismatch. Co-authored-by: Cursor * fix(hub,cli): never delete live spawn on RPC timeout; release stub tags Ambiguous machine-spawn failures (timeout / unset childStarted) no longer call stopRunnerSession or deleteSession — that path killed healthy late boots and CASCADE-wiped transcripts. Pre-exec delete still requires a real machine-spawn stub (store tag check). Metadata updates release the stub tag so reopen flavors are not permanently adoptable; adopt refuses archived stubs. POSIX pgrep ENOENT fails closed instead of root-only verify. Co-authored-by: Cursor * fix(cli): parseable Windows tree scan + fail closed on CIM failure Space-joined PowerShell for collectWindowsProcessTree was a WinPS parse error ($seen=@{} $bfs=@()), so every scan fell open to [root] and B2 false-stopped again. Join with newlines; return scan_failed on error/empty; killProcessTreeByPid refuses root-only verify. Assert on generated command text (Teemo-confirmed) so mocks cannot hide the next malformed script. Co-authored-by: Cursor * fix(cli,hub): reopen stamps existing-session-id; scan-fail signals root Opus Critical on #1911: runner always stamps --existing-session-id for known hub ids so Claude/Kimi/Copilot reopen does not hit adopt-stub 409. On pgrep/CIM tree-scan failure still signal the verified root but never claim stopped. Archive when both KillSession and machine RPC are missing. Empty Win process-list stdout and strict Codex registry reads fail closed. Co-authored-by: Cursor * fix(cli,hub): OK: sentinel for Windows tree scan; adopt CAS CIM failure under SilentlyContinue printed root-only with exit 0 — byte-identical to a healthy childless tree, so scan_failed never fired and killProcessTreeByPid false-stopped (#1911 Overseer B1). Require ErrorActionPreference=Stop, trap exit 1, and an OK: success sentinel; parse anything else as scan_failed. Same Stop/trap on the machine-wide list command. Execute the generated scripts under Docker pwsh in CI when the image is present. Treat pgrep status=null as scan_failed; coarsen Windows tasklist poll; wrap adoptPreallocatedSession in a transaction with stub-tag CAS. Co-authored-by: Cursor * fix(cli,hub): reservedSessionId for fresh prealloc; signal Win survivors Fresh machine-spawn stubs must not stamp --existing-session-id — that sends Codex down reopen without a thread binding (#1911 bot Major) and re-broke local HTTP non-UUID hints as reopen (round-4). Hub passes prealloc as reservedSessionId; buildCliArgs stamps --hapi-session-id for reserved/sessionId and --existing-session-id only for live reopen. Codex parses --hapi-session-id and adopts via bootstrapSession. Windows tree kill now taskkills surviving pre-scan PIDs after /T misses broken intermediate links. Co-authored-by: Cursor * fix(cli,hub): fail-loud Docker pwsh CI; scope Codex registry; stub leak CI skipped all four windowsProcessTree.pwsh tests on 48bb (image inspect without pull). Pull the image in test.yml and throw under CI when missing. Scope strict runtime-registry failure to Codex stop contexts so one bad JSON cannot block archive machine-wide. Stamp childStarted:false on machine RPC outside_workspace_roots. Add Codex reservedSessionId prealloc and runSharedCodex regression coverage for the #1911 Critical. Co-authored-by: Cursor * fix(cli,hub): single-use reservedSessionId; skip UUID HTTP hints Consume the machine-spawn reservation on first prepare() so a second Codex root/fork cannot re-adopt a consumed stub (409). Stop stamping UUID local-HTTP sessionId as --hapi-session-id (adopt hard-fail). Update stale hub spawn mocks for the reservedSessionId arity. Co-authored-by: Cursor * fix(hub): archive never-started machine-spawn stubs on unknown Ambiguous keep-stubs (startedBy=runner, no hostPid, still machine-spawn tagged) made StopSession return unknown forever while the runner was online, so archive threw permanently. Treat unknown + prealloc stub + no hostPid as already_gone. Also reject spawn success when the runner reports a different id than the preallocated stub. Co-authored-by: Cursor * fix(hub,cli): refuse dual-RPC-miss archive; generation-check Win survivors Both KillSession and StopSession missing is not proof a detached CLI exited — keep the row unconfirmed until the runner reconnects. Windows tree kill now re-checks process-start markers before individually signalling survivors so PID reuse is not killed. Co-authored-by: Cursor * fix(cli): refuse archived reopen in bootstrapExistingSession Hub-archived rows could be resurrected when a late-booting child hit --existing-session-id: buildSessionMetadata stamped lifecycleState=running and updateMetadata overwrote the archive. Adopt already rejected archived stubs; reopen now matches. Intentional revive still goes through hub reopenSession first. Correct stub-hatch comment (no post-archive argv sweep). Co-authored-by: Cursor * fix(hub,cli): reject un-archive in updateSessionMetadata CAS Hub-side transactional guard (Discovery #1911 M1): refuse metadata writes that clear archived lifecycle unless allowUnarchive (reopen clear). Client stops infinite updateMetadata backoff when mismatch applies archived. Clear archive immediately before spawn so Pi/PTY attempt snapshots stay valid. Co-authored-by: Cursor * fix(hub,cli): success+preserve hub archive; M3 reservedSessionId split Replace version-mismatch refuse (livelock in CAS backoff) with success + merge-preserve of hub-archived fields unless allowUnarchive. Ack path routes through noteHubArchived so the CLI exits (criterion 6). Clear- before-spawn remains the authorized revive. Thread --hapi-session-id to reservedSessionId for shared-parser flavors (adopt≠reopen). Co-authored-by: Cursor * fix(cli): fail-closed orphan ps signal; PID- registry; spawn timeouts Overseer B1: signalled ps (status null) is scan_failed, not empty table. B2: unreadable registry on PID-* stops returns null (no soft []). B3: 10s timeout on process/orphanReap spawn.sync. Narrow bootstrap archived belt to archivedBy=hub to match hub merge-preserve. Co-authored-by: Cursor * fix(cli,hub): hubArchived latch; PTY archiveSnapshot durability Criterion 6: noteHubArchived latches hubArchived so late registerKillSessionHandler registrations still EXIT (EventEmitter does not replay). Mirror Pi: persist archiveSnapshot on ptyResumeAttempt across clear-before-spawn and quarantine, restore on failed-resume cleanup (#1911 Discovery Majors). Co-authored-by: Cursor * fix(cli): Codex hub-archived exit; AC6 latch test production order C2: SharedCodexRoot registers hub-archived (and latch check) beside its private KillSession — Codex never called registerKillSessionHandler. C1: keep latch+subscribe; test write-then-register order Discovery proved. Co-authored-by: Cursor * fix(cli): type Codex hub-archived end mock for tsc vi.fn(async () => {}) inferred zero-arg calls; RootHost['end'] signature unblocks end.mock.calls[0][0] (TS2493). Co-authored-by: Cursor * fix(cli): catch hub-archived Codex host.end rejection void this.host.end() discarded archive-RPC failures as unhandled rejections; Codex has no unhandledRejection handler, so Node can exit the shared wrapper and kill sibling roots mid-turn. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- .github/workflows/test.yml | 3 + cli/src/agent/runners/runAgentSession.ts | 2 +- cli/src/agent/sessionFactory.test.ts | 115 +++ cli/src/agent/sessionFactory.ts | 49 ++ cli/src/agy/runAgy.ts | 7 +- cli/src/api/api.ts | 3 + cli/src/api/apiMachine.test.ts | 29 + cli/src/api/apiMachine.ts | 17 +- cli/src/api/apiSession.test.ts | 198 ++++- cli/src/api/apiSession.ts | 170 +++- .../claude/registerKillSessionHandler.test.ts | 69 +- cli/src/claude/registerKillSessionHandler.ts | 38 +- cli/src/claude/runClaude.ts | 10 +- cli/src/codex/shared/frontend.test.ts | 22 + cli/src/codex/shared/frontend.ts | 3 +- cli/src/codex/shared/launch.test.ts | 11 +- cli/src/codex/shared/launch.ts | 15 +- cli/src/codex/shared/registry.test.ts | 11 +- cli/src/codex/shared/registry.ts | 4 +- cli/src/codex/shared/root.test.ts | 37 +- cli/src/codex/shared/root.ts | 12 + cli/src/codex/shared/runtime.ts | 11 +- cli/src/commands/agentCommandOptions.test.ts | 11 +- cli/src/commands/agentCommandOptions.ts | 20 +- cli/src/commands/claude.test.ts | 26 +- cli/src/commands/claude.ts | 9 + cli/src/commands/codex.ts | 7 + cli/src/commands/copilot.ts | 15 + cli/src/commands/cursor.ts | 10 + cli/src/commands/kimi.ts | 15 + cli/src/copilot/runCopilot.ts | 7 +- cli/src/cursor/runCursor.ts | 7 +- cli/src/dsh/runDsh.ts | 7 +- cli/src/grok/runGrok.ts | 7 +- cli/src/kimi/runKimi.ts | 7 +- cli/src/modules/common/rpcTypes.ts | 16 +- cli/src/opencode/runOpencode.test.ts | 27 +- cli/src/opencode/runOpencode.ts | 7 +- cli/src/pi/runPi.test.ts | 1 + cli/src/pi/runPi.ts | 7 +- cli/src/runner/buildCliArgs.test.ts | 81 +- cli/src/runner/controlClient.ts | 4 +- cli/src/runner/controlServer.ts | 4 +- cli/src/runner/lateRunnerWebhook.test.ts | 31 + cli/src/runner/lateRunnerWebhook.ts | 26 + cli/src/runner/orphanReap.test.ts | 438 ++++++++++ cli/src/runner/orphanReap.ts | 332 ++++++++ cli/src/runner/run.ts | 592 +++++++++++--- cli/src/runner/runner.integration.test.ts | 5 +- cli/src/runner/sharedSessionStop.test.ts | 243 ++++++ cli/src/runner/sharedSessionStop.ts | 207 +++++ cli/src/utils/process.test.ts | 607 ++++++++++++-- cli/src/utils/process.ts | 286 ++++++- cli/src/utils/windowsProcessTree.pwsh.test.ts | 100 +++ hub/src/store/sessionStore.ts | 18 +- hub/src/store/sessions.test.ts | 348 ++++++++ hub/src/store/sessions.ts | 192 ++++- hub/src/sync/opencodeClear.test.ts | 3 + hub/src/sync/rpcGateway.ts | 51 +- hub/src/sync/sessionCache.ts | 26 +- hub/src/sync/sessionModel.test.ts | 56 +- hub/src/sync/syncEngine.ts | 422 ++++++++-- hub/src/sync/syncEngineArchiveSession.test.ts | 388 +++++++++ .../syncEngineReopenPreservesPtyId.test.ts | 112 ++- hub/src/sync/syncEngineSpawnPrealloc.test.ts | 745 ++++++++++++++++++ hub/src/web/routes/cli.ts | 37 +- hub/src/web/routes/machines.ts | 4 +- shared/src/apiTypes.ts | 13 + shared/src/schemas.ts | 8 + 69 files changed, 6019 insertions(+), 402 deletions(-) create mode 100644 cli/src/runner/lateRunnerWebhook.test.ts create mode 100644 cli/src/runner/lateRunnerWebhook.ts create mode 100644 cli/src/runner/orphanReap.test.ts create mode 100644 cli/src/runner/orphanReap.ts create mode 100644 cli/src/runner/sharedSessionStop.test.ts create mode 100644 cli/src/runner/sharedSessionStop.ts create mode 100644 cli/src/utils/windowsProcessTree.pwsh.test.ts create mode 100644 hub/src/sync/syncEngineArchiveSession.test.ts create mode 100644 hub/src/sync/syncEngineSpawnPrealloc.test.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c13d0de75..5a06a9f270 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,9 @@ jobs: - run: bun install --frozen-lockfile - run: bun typecheck - run: bunx playwright install --with-deps chromium + # Required by cli/src/utils/windowsProcessTree.pwsh.test.ts — image + # inspect alone does not pull; silent skip was fake B1 coverage (#1911). + - run: docker pull mcr.microsoft.com/powershell:latest - run: bun run test:e2e -- terminal-wrap-fidelity.spec.ts composer-copy.spec.ts session-list-scroll.spec.ts cold-initial-tail.spec.ts scrollbar-auto-hide.spec.ts - run: bun run test diff --git a/cli/src/agent/runners/runAgentSession.ts b/cli/src/agent/runners/runAgentSession.ts index efd33c28b3..d937ee54ce 100644 --- a/cli/src/agent/runners/runAgentSession.ts +++ b/cli/src/agent/runners/runAgentSession.ts @@ -161,7 +161,7 @@ export async function runAgentSession(opts: { } }; - registerKillSessionHandler(session.rpcHandlerManager, handleKillSession); + registerKillSessionHandler(session.rpcHandlerManager, handleKillSession, session); let sessionEndReason: SessionEndReason = 'completed'; try { diff --git a/cli/src/agent/sessionFactory.test.ts b/cli/src/agent/sessionFactory.test.ts index fb50342e75..7f0aed5505 100644 --- a/cli/src/agent/sessionFactory.test.ts +++ b/cli/src/agent/sessionFactory.test.ts @@ -135,6 +135,61 @@ describe('bootstrapExistingSession', () => { ) }) + it('refuses to reopen a hub-archived session (hub-archive resurrection guard)', async () => { + // #1911 M1 belt: hub-archived only (matches store merge-preserve scope). + const session = createSession() + const existing = session.metadata + if (!existing) throw new Error('expected metadata') + session.metadata = { + ...existing, + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'Archived from hub', + } + const sessionClient = { updateMetadata: vi.fn() } + getSessionMock.mockResolvedValue(session) + getOrCreateMachineMock.mockResolvedValue({ id: 'machine-1' }) + sessionSyncClientMock.mockReturnValue(sessionClient) + readSettingsMock.mockResolvedValue({ machineId: 'machine-1' }) + + await expect(bootstrapExistingSession({ + sessionId: 'hapi-session-1', + flavor: 'claude', + workingDirectory: '/tmp/project', + startedBy: 'runner', + })).rejects.toThrow(/hub-archived|archived/) + + expect(sessionClient.updateMetadata).not.toHaveBeenCalled() + expect(notifyRunnerSessionStartedMock).not.toHaveBeenCalled() + expect(sessionSyncClientMock).not.toHaveBeenCalled() + }) + + it('allows reopen of CLI self-archived sessions (archivedBy=cli)', async () => { + const session = createSession() + const existing = session.metadata + if (!existing) throw new Error('expected metadata') + session.metadata = { + ...existing, + lifecycleState: 'archived', + archivedBy: 'cli', + archiveReason: 'clean exit', + } + const sessionClient = { updateMetadata: vi.fn() } + getSessionMock.mockResolvedValue(session) + getOrCreateMachineMock.mockResolvedValue({ id: 'machine-1' }) + sessionSyncClientMock.mockReturnValue(sessionClient) + readSettingsMock.mockResolvedValue({ machineId: 'machine-1' }) + + await expect(bootstrapExistingSession({ + sessionId: 'hapi-session-1', + flavor: 'claude', + workingDirectory: '/tmp/project', + startedBy: 'runner', + })).resolves.toMatchObject({ session: sessionClient }) + + expect(sessionClient.updateMetadata).toHaveBeenCalledOnce() + }) + it('preserves existing native resume metadata when reactivating a session', async () => { const session = createSession() const existingMetadata = session.metadata @@ -364,6 +419,66 @@ describe('bootstrapSession HAPI_SESSION_ID export', () => { expect(result.sessionInfo.id).toBe('hub-session-42') expect(process.env[HAPI_SESSION_ID_ENV]).toBe('hub-session-42') }) + + it('passes reservedSessionId as getOrCreateSession id (adopt-stub, #1911)', async () => { + const session = createSession() + session.id = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' + getOrCreateSessionMock.mockResolvedValue(session) + getOrCreateMachineMock.mockResolvedValue({ id: 'machine-1' }) + sessionSyncClientMock.mockReturnValue({ isPending: () => false }) + readSettingsMock.mockResolvedValue({ machineId: 'machine-1' }) + + const result = await bootstrapSession({ + flavor: 'claude', + workingDirectory: '/tmp/project', + reservedSessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' + }) + + expect(getOrCreateSessionMock).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee', + adopt: true, + }) + ) + expect(result.sessionInfo.id).toBe('aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee') + }) + + it('does not bind non-UUID reservedSessionId (reap stamp only; hub rejects non-uuid id)', async () => { + const session = createSession() + session.id = 'minted-hub-id' + getOrCreateSessionMock.mockResolvedValue(session) + getOrCreateMachineMock.mockResolvedValue({ id: 'machine-1' }) + sessionSyncClientMock.mockReturnValue({ isPending: () => false }) + readSettingsMock.mockResolvedValue({ machineId: 'machine-1' }) + + await bootstrapSession({ + flavor: 'claude', + workingDirectory: '/tmp/project', + reservedSessionId: 'spawned-test-456' + }) + + expect(getOrCreateSessionMock).toHaveBeenCalledWith( + expect.not.objectContaining({ id: expect.anything() }) + ) + // Call args should omit `id` entirely: + const call = getOrCreateSessionMock.mock.calls[0][0] as { id?: string } + expect(call.id).toBeUndefined() + }) + + it('throws when hub returns a different id than reservedSessionId', async () => { + const session = createSession() + session.id = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + getOrCreateSessionMock.mockResolvedValue(session) + getOrCreateMachineMock.mockResolvedValue({ id: 'machine-1' }) + sessionSyncClientMock.mockReturnValue({ isPending: () => false }) + readSettingsMock.mockResolvedValue({ machineId: 'machine-1' }) + + await expect(bootstrapSession({ + flavor: 'claude', + workingDirectory: '/tmp/project', + reservedSessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' + })).rejects.toThrow(/unexpected session id/) + }) }) describe('buildMachineMetadata runner-only capabilities', () => { diff --git a/cli/src/agent/sessionFactory.ts b/cli/src/agent/sessionFactory.ts index 79a47ba159..82cac20b89 100644 --- a/cli/src/agent/sessionFactory.ts +++ b/cli/src/agent/sessionFactory.ts @@ -1,6 +1,7 @@ import os from 'node:os' import { randomUUID } from 'node:crypto' import { resolve } from 'node:path' +import { z } from 'zod' import { ApiClient } from '@/api/api' import type { ApiSessionClient } from '@/api/apiSession' @@ -20,6 +21,9 @@ export { HAPI_SESSION_ID_ENV, exportHapiSessionEnv, exportHapiHubAuthEnv } from export type SessionStartedBy = 'runner' | 'terminal' +/** Matches shared CreateOrLoadSessionRequestSchema `id`. */ +const HubReservedSessionIdSchema = z.string().uuid() + export type SessionBootstrapOptions = { reportStarted?: boolean /** Multi-session workers inject session identity into each child, never process.env. */ @@ -28,6 +32,16 @@ export type SessionBootstrapOptions = { startedBy?: SessionStartedBy workingDirectory?: string tag?: string + /** + * Hub-preallocated / runner-stamped id for a *fresh* create bootstrap. + * Passed as `getOrCreateSession({ id })` so create-time metadata still runs + * while binding the reserved row. Distinct from reopen via + * `bootstrapExistingSession` / `--existing-session-id`. + * + * Must be a UUID (hub create schema). Non-UUID stamps (legacy HTTP spawn + * hints) stay reap-only: argv retains the id, create mints a new hub row. + */ + reservedSessionId?: string agentState?: AgentState | null model?: string modelReasoningEffort?: string @@ -35,6 +49,19 @@ export type SessionBootstrapOptions = { metadataOverrides?: Partial } +/** Hub create/load accepts optional `id` only as UUID — match that gate. */ +export function resolveReservedHubSessionId(reservedSessionId: string | undefined): string | undefined { + if (!reservedSessionId) return undefined + const parsed = HubReservedSessionIdSchema.safeParse(reservedSessionId) + if (!parsed.success) { + logger.debug( + `[START] Ignoring non-UUID reservedSessionId for create bind (reap stamp only): ${reservedSessionId}` + ) + return undefined + } + return parsed.data +} + export type SessionBootstrapResult = { api: ApiClient session: ApiSessionClient @@ -204,6 +231,7 @@ export async function bootstrapSession(options: SessionBootstrapOptions): Promis const startedBy = options.startedBy ?? 'terminal' const sessionTag = options.tag ?? randomUUID() const agentState = options.agentState === undefined ? {} : options.agentState + const reservedHubId = resolveReservedHubSessionId(options.reservedSessionId) const api = await ApiClient.create() @@ -222,6 +250,9 @@ export async function bootstrapSession(options: SessionBootstrapOptions): Promis }) const sessionInfo = await api.getOrCreateSession({ + ...(reservedHubId + ? { id: reservedHubId, adopt: true as const } + : {}), tag: sessionTag, metadata, state: agentState, @@ -230,6 +261,12 @@ export async function bootstrapSession(options: SessionBootstrapOptions): Promis effort: options.effort }) + if (reservedHubId && sessionInfo.id !== reservedHubId) { + throw new Error( + `Hub returned unexpected session id ${sessionInfo.id} (reserved ${reservedHubId})` + ) + } + const session = api.sessionSyncClient(sessionInfo) if (options.exportSessionEnv !== false) exportHapiSessionEnv(sessionInfo.id) @@ -352,6 +389,18 @@ export async function bootstrapExistingSession(options: { }) const sessionInfo = await api.getSession(options.sessionId) + // #1911 M1 belt: match hub merge-preserve scope — only hub-authored archive. + // CLI self-archive (archivedBy=cli) must still reopen; hub clears before spawn + // for intentional revive. CAS closed hub-side (success+preserve + ack EXIT). + if ( + sessionInfo.metadata?.lifecycleState === 'archived' + && sessionInfo.metadata?.archivedBy === 'hub' + ) { + throw new Error( + `HAPI session ${options.sessionId} is hub-archived; refuse --existing-session-id reopen ` + + '(use hub reopen to clear archive metadata first)' + ) + } const baseMetadata = buildSessionMetadata({ flavor: options.flavor, startedBy, diff --git a/cli/src/agy/runAgy.ts b/cli/src/agy/runAgy.ts index b8170fd1c4..a0b14f03f1 100644 --- a/cli/src/agy/runAgy.ts +++ b/cli/src/agy/runAgy.ts @@ -22,6 +22,8 @@ export async function runAgy(opts: { effort?: string; resumeSessionId?: string; existingSessionId?: string; + /** Fresh machine-spawn stub (`--hapi-session-id`); adopt via bootstrapSession. */ + reservedSessionId?: string; workingDirectory?: string; } = {}): Promise { const workingDirectory = opts.workingDirectory ?? getInvokedCwd(); @@ -54,7 +56,8 @@ export async function runAgy(opts: { tag: `__hapi_agy_${randomUUID()}`, agentState: initialState, model: initialModel ?? undefined, - effort: opts.effort ?? undefined + effort: opts.effort ?? undefined, + reservedSessionId: opts.reservedSessionId }); const { api, session } = bootstrap; @@ -79,7 +82,7 @@ export async function runAgy(opts: { }); lifecycle.registerProcessHandlers(); - registerKillSessionHandler(session.rpcHandlerManager, lifecycle.cleanupAndExit); + registerKillSessionHandler(session.rpcHandlerManager, lifecycle.cleanupAndExit, session); registerLocalHandoffHandler(session.rpcHandlerManager, lifecycle); let crashed = false; diff --git a/cli/src/api/api.ts b/cli/src/api/api.ts index 5b3acf23b8..f079741dbe 100644 --- a/cli/src/api/api.ts +++ b/cli/src/api/api.ts @@ -38,6 +38,8 @@ export class ApiClient { async getOrCreateSession(opts: { id?: string + /** Bind a hub-preallocated stub (requires id). */ + adopt?: boolean tag: string metadata: Metadata state: AgentState | null @@ -56,6 +58,7 @@ export class ApiClient { `${configuration.apiUrl}/cli/sessions`, { id: opts.id, + ...(opts.adopt === true ? { adopt: true } : {}), tag: opts.tag, metadata: opts.metadata, agentState: opts.state, diff --git a/cli/src/api/apiMachine.test.ts b/cli/src/api/apiMachine.test.ts index ccc4866262..1939b6e664 100644 --- a/cli/src/api/apiMachine.test.ts +++ b/cli/src/api/apiMachine.test.ts @@ -746,6 +746,35 @@ describe('ApiMachineClient SpawnHappySession handler', () => { client.shutdown() } }) + + it('returns childStarted:false on outside_workspace_roots so hub deletes the stub', async () => { + const machine = makeMachine('machine-spawn-outside') + const client = new ApiMachineClient('cli-token', machine, [workspaceRoot]) + const spawnSession = vi.fn() + + client.setRPCHandlers({ + spawnSession, + stopSession: vi.fn(async () => 'stopped' as const), + requestShutdown: vi.fn() + }) + + try { + const result = await callSpawnHappySession(client, machine.id, { + directory: '/tmp/definitely-outside-roots', + agent: 'claude', + }) + + expect(result).toEqual({ + type: 'error', + errorMessage: "Directory is outside this machine's workspace roots", + code: 'outside_workspace_roots', + childStarted: false, + }) + expect(spawnSession).not.toHaveBeenCalled() + } finally { + client.shutdown() + } + }) }) describe('ApiMachineClient keepAlive lifecycle', () => { diff --git a/cli/src/api/apiMachine.ts b/cli/src/api/apiMachine.ts index b342e4e826..edf26763be 100644 --- a/cli/src/api/apiMachine.ts +++ b/cli/src/api/apiMachine.ts @@ -70,7 +70,10 @@ export { normalizeWindowsDriveRoot } from './machinePathPolicy' type MachineRpcHandlers = { spawnSession: (options: SpawnSessionOptions) => Promise - stopSession: (sessionId: string) => Promise<'stopped' | 'already_gone' | 'still_alive'> + stopSession: ( + sessionId: string, + opts?: { processStartMarker?: string } + ) => Promise<'stopped' | 'already_gone' | 'still_alive' | 'unknown'> requestShutdown: () => void } @@ -401,7 +404,7 @@ export class ApiMachineClient { setRPCHandlers({ spawnSession, stopSession, requestShutdown }: MachineRpcHandlers): void { this.rpcHandlerManager.registerHandler(RPC_METHODS.SpawnHappySession, async (params: any) => { - const { directory, sessionId, existingSessionId, resumeSessionId, machineId, approvedNewDirectoryCreation, agent, model, effort, modelReasoningEffort, yolo, permissionMode, serviceTier, collaborationMode, copilotAgentMode, token, sessionType, worktreeName, startingMode, forkSession } = params || {} + const { directory, sessionId, existingSessionId, reservedSessionId, resumeSessionId, machineId, approvedNewDirectoryCreation, agent, model, effort, modelReasoningEffort, yolo, permissionMode, serviceTier, collaborationMode, copilotAgentMode, token, sessionType, worktreeName, startingMode, forkSession } = params || {} if (!directory) { throw new Error('Directory is required') @@ -413,6 +416,8 @@ export class ApiMachineClient { type: 'error', errorMessage: 'Directory is outside this machine\'s workspace roots', code: 'outside_workspace_roots', + // Pre-exec: no OS child — hub must delete the prealloc stub (#1911). + childStarted: false, } } @@ -420,6 +425,7 @@ export class ApiMachineClient { directory, sessionId, existingSessionId, + reservedSessionId, resumeSessionId, machineId, approvedNewDirectoryCreation, @@ -456,12 +462,15 @@ export class ApiMachineClient { }) this.rpcHandlerManager.registerHandler(RPC_METHODS.StopSession, async (params: any) => { - const { sessionId } = params || {} + const { sessionId, processStartMarker } = params || {} if (!sessionId) { throw new Error('Session ID is required') } - const status = await stopSession(sessionId) + const status = await stopSession( + sessionId, + typeof processStartMarker === 'string' ? { processStartMarker } : undefined + ) return { status } }) diff --git a/cli/src/api/apiSession.test.ts b/cli/src/api/apiSession.test.ts index 3792ce8081..450d07b3cb 100644 --- a/cli/src/api/apiSession.test.ts +++ b/cli/src/api/apiSession.test.ts @@ -11,6 +11,7 @@ const socketHarness = vi.hoisted(() => ({ trigger: (event: string, ...args: any[]) => void triggerConnect: () => void triggerConnectError: () => void + emitWithAckImpl: (event: string, ...args: unknown[]) => Promise }> })) @@ -28,7 +29,8 @@ vi.mock('socket.io-client', () => ({ listeners: new Map void>>(), trigger: () => {}, triggerConnect: () => {}, - triggerConnectError: () => {} + triggerConnectError: () => {}, + emitWithAckImpl: async () => ({}) } state.trigger = (event: string, ...args: any[]) => { for (const listener of state.listeners.get(event) ?? []) { @@ -62,8 +64,10 @@ vi.mock('socket.io-client', () => ({ state.emitted.push({ event, args }) return socket }, - emitWithAck: async () => ({}), - timeout: () => ({ emitWithAck: async () => ({}) }), + emitWithAck: async (event: string, ...args: unknown[]) => state.emitWithAckImpl(event, ...args), + timeout: () => ({ + emitWithAck: async (event: string, ...args: unknown[]) => state.emitWithAckImpl(event, ...args) + }), connect: () => { state.connectCalls += 1 if (state.connectImmediately) { @@ -76,7 +80,10 @@ vi.mock('socket.io-client', () => ({ return socket } } - Object.assign(socket, { volatile: socket }) + Object.assign(socket, { + volatile: socket, + io: { opts: { reconnection: true } } + }) socketHarness.sockets.push(state) return socket } @@ -410,6 +417,189 @@ describe('ApiSessionClient lazy materialization', () => { expect(socket.connectCalls).toBeGreaterThan(0) client.close() }) + + it('emits hub-archived from update-session metadata (#1910)', async () => { + socketHarness.sockets.length = 0 + axiosHarness.get.mockResolvedValue({ data: { messages: [] } }) + const session = createSession({ + namespace: 'default', + metadata: { path: '/tmp', host: 'h', flavor: 'claude' }, + metadataVersion: 1 + }) + const client = new ApiSessionClient('token', session) + const socket = socketHarness.sockets[0] + if (!socket) throw new Error('expected socket') + + let archived = false + client.on('hub-archived', () => { archived = true }) + + socket.trigger('update', { + body: { + t: 'update-session', + sid: session.id, + metadata: { + version: 2, + value: { + path: '/tmp', + host: 'h', + flavor: 'claude', + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'Archived from hub (CLI unreachable)' + } + }, + agentState: null + } + }) + + expect(archived).toBe(true) + expect(client.getMetadata()?.lifecycleState).toBe('archived') + client.close() + }) + + it('stops metadata CAS when hub returns archived on version-mismatch (#1911 M1)', async () => { + socketHarness.sockets.length = 0 + axiosHarness.get.mockResolvedValue({ data: { messages: [] } }) + const session = createSession({ + namespace: 'default', + metadata: { path: '/tmp', host: 'h', flavor: 'claude', lifecycleState: 'running' }, + metadataVersion: 1 + }) + const client = new ApiSessionClient('token', session) + const socket = socketHarness.sockets[0] + if (!socket) throw new Error('expected socket') + + let ackCalls = 0 + let archived = false + client.on('hub-archived', () => { archived = true }) + socket.emitWithAckImpl = async (event) => { + if (event !== 'update-metadata') return {} + ackCalls += 1 + return { + result: 'version-mismatch', + version: 2, + metadata: { + path: '/tmp', + host: 'h', + flavor: 'claude', + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'Archived from hub' + } + } + } + + client.updateMetadata((meta) => ({ ...meta, lifecycleState: 'running', hostPid: 42 })) + + await vi.waitFor(() => { + expect(client.getMetadata()?.lifecycleState).toBe('archived') + expect(archived).toBe(true) + }) + await new Promise((r) => setTimeout(r, 50)) + expect(ackCalls).toBe(1) + client.close() + }) + + it('exits via hub-archived on success+merge-preserve ack (#1911 criterion 6)', async () => { + socketHarness.sockets.length = 0 + axiosHarness.get.mockResolvedValue({ data: { messages: [] } }) + const session = createSession({ + namespace: 'default', + metadata: { path: '/tmp', host: 'h', flavor: 'claude', lifecycleState: 'running' }, + metadataVersion: 1 + }) + const client = new ApiSessionClient('token', session) + const socket = socketHarness.sockets[0] + if (!socket) throw new Error('expected socket') + + let ackCalls = 0 + let archived = false + client.on('hub-archived', () => { archived = true }) + socket.emitWithAckImpl = async (event) => { + if (event !== 'update-metadata') return {} + ackCalls += 1 + // Hub merge-preserved archive fields and returned success. + return { + result: 'success', + version: 2, + metadata: { + path: '/tmp', + host: 'h', + flavor: 'claude', + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'Archived from hub', + hostPid: 42 + } + } + } + + client.updateMetadata((meta) => ({ ...meta, lifecycleState: 'running', hostPid: 42 })) + + await vi.waitFor(() => { + expect(archived).toBe(true) + expect(client.getMetadata()?.lifecycleState).toBe('archived') + }) + await new Promise((r) => setTimeout(r, 50)) + expect(ackCalls).toBe(1) + client.close() + }) + + it('reconciles hub-archived metadata on reconnect (#1910)', async () => { + socketHarness.sockets.length = 0 + axiosHarness.get.mockResolvedValue({ + data: { + session: { + metadataVersion: 1, + metadata: { path: '/tmp', host: 'h', flavor: 'claude' } + }, + messages: [] + } + }) + + const client = new ApiSessionClient('token', createSession({ + namespace: 'default', + metadata: { path: '/tmp', host: 'h', flavor: 'claude' }, + metadataVersion: 1 + })) + const socket = socketHarness.sockets[0] + if (!socket) throw new Error('expected socket') + + let archived = false + client.on('hub-archived', () => { archived = true }) + + // Establish first connection so hasConnectedOnce is true. + socket.triggerConnect() + await vi.waitFor(() => expect(axiosHarness.get).toHaveBeenCalled()) + + axiosHarness.get.mockImplementation(async (url: string) => { + if (String(url).includes('/messages')) { + return { data: { messages: [] } } + } + return { + data: { + session: { + metadataVersion: 5, + metadata: { + path: '/tmp', + host: 'h', + flavor: 'claude', + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'Archived from hub (CLI unreachable)' + } + } + } + } + }) + socket.connected = false + socket.trigger('disconnect', 'transport close') + socket.triggerConnect() + + await vi.waitFor(() => expect(archived).toBe(true)) + expect(client.getMetadata()?.archivedBy).toBe('hub') + client.close() + }) }) describe('ApiSessionClient agy transcript messages', () => { diff --git a/cli/src/api/apiSession.ts b/cli/src/api/apiSession.ts index aed3ba04ce..619899d2ed 100644 --- a/cli/src/api/apiSession.ts +++ b/cli/src/api/apiSession.ts @@ -234,6 +234,15 @@ function hasSameJsonValue(left: unknown, right: unknown): boolean { export class ApiSessionClient extends EventEmitter { private reconnectHandler: (() => void) | null = null onReconnect(handler: (() => void) | null): void { this.reconnectHandler = handler } + /** When false, socket.io must not keep the CLI immortal after hub archive (#1910). */ + private allowReconnect = true + /** + * Latch for hub-archived EXIT (#1911 criterion 6). Bootstrap may apply + * hub-archived via updateMetadata CAS before flavor runners call + * registerKillSessionHandler; EventEmitter does not replay past emits, so + * the handler must read this synchronously at registration. + */ + hubArchived = false private readonly token: string readonly sessionId: string private metadata: Metadata | null @@ -332,10 +341,19 @@ export class ApiSessionClient extends EventEmitter { logger.debug('Socket connected successfully') this.awaitingMaterializedConnection = false this.rpcHandlerManager.onSocketConnect(this.socket) - if (this.hasConnectedOnce) { + const isReconnect = this.hasConnectedOnce + if (isReconnect) { this.needsBackfill = true } - void this.backfillIfNeeded() + // Hub may have archived while we were offline (KillSession miss + + // no live update-session). On reconnect, reconcile metadata before + // message backfill so hub-archived can stop reconnect immortality (#1910). + const afterMeta = isReconnect + ? this.reconcileSessionMetadata() + : Promise.resolve() + void afterMeta.finally(() => { + void this.backfillIfNeeded() + }) this.hasConnectedOnce = true this.reconnectHandler?.() this.socket.emit('session-alive', { @@ -353,6 +371,13 @@ export class ApiSessionClient extends EventEmitter { logger.debug('[API] Socket disconnected:', reason) this.rpcHandlerManager.onSocketDisconnect() this.terminalManager.closeAll() + if (!this.allowReconnect) { + try { + this.socket.io.opts.reconnection = false + this.socket.disconnect() + } catch { /* already tearing down */ } + return + } if (this.hasConnectedOnce) { this.needsBackfill = true } @@ -477,13 +502,7 @@ export class ApiSessionClient extends EventEmitter { if (data.body.t === 'update-session') { if (data.body.metadata && data.body.metadata.version > this.metadataVersion) { - const parsed = MetadataSchema.safeParse(data.body.metadata.value) - if (parsed.success) { - this.metadata = parsed.data - } else { - logger.debug('[API] Ignoring invalid metadata update', { version: data.body.metadata.version }) - } - this.metadataVersion = data.body.metadata.version + this.applyRemoteMetadata(data.body.metadata.version, data.body.metadata.value) } if (data.body.agentState && data.body.agentState.version > this.agentStateVersion) { const next = data.body.agentState.value @@ -819,6 +838,63 @@ export class ApiSessionClient extends EventEmitter { } } + /** + * Apply a remote metadata snapshot (Socket.IO update-session or reconnect + * REST reconcile). Advances metadataVersion and emits hub-archived when + * the hub flipped lifecycle while this CLI was unreachable. + */ + private applyRemoteMetadata(version: number, value: unknown): void { + if (version <= this.metadataVersion) return + const parsed = MetadataSchema.safeParse(value) + if (!parsed.success) { + logger.debug('[API] Ignoring invalid metadata update', { version }) + this.metadataVersion = version + return + } + const wasHubArchived = this.metadata?.lifecycleState === 'archived' + && this.metadata?.archivedBy === 'hub' + this.metadata = parsed.data + this.metadataVersion = version + // #1910: hub may archive via metadata when KillSession cannot reach + // this CLI. Stop reconnect immortality and let runners exit instead + // of sitting as PPID=1 orphans. + if (!wasHubArchived + && parsed.data.lifecycleState === 'archived' + && parsed.data.archivedBy === 'hub') { + this.noteHubArchived() + } + } + + private noteHubArchived(): void { + this.hubArchived = true + this.allowReconnect = false + try { + this.socket.io.opts.reconnection = false + } catch { /* socket may be mid-teardown */ } + this.emit('hub-archived') + } + + /** Fetch current hub metadata after reconnect; apply hub-archived if set. */ + private async reconcileSessionMetadata(): Promise { + try { + const response = await axios.get( + `${configuration.apiUrl}/cli/sessions/${encodeURIComponent(this.sessionId)}`, + { + headers: buildHubRequestHeaders({ + Authorization: `Bearer ${this.token}`, + 'Content-Type': 'application/json' + }), + timeout: 15_000 + } + ) + const session = response.data?.session + if (!session || typeof session.metadataVersion !== 'number') return + this.applyRemoteMetadata(session.metadataVersion, session.metadata) + } catch (error) { + logger.debug('[API] Session metadata reconcile failed', error) + } + } + private async backfillMessages(): Promise { if (this.backfillInFlight) { await this.backfillInFlight @@ -1308,6 +1384,13 @@ export class ApiSessionClient extends EventEmitter { } this.metadataLock.inLock(async () => { await backoff(async () => { + // #1911 M1 criterion 6: hub-archived → EXIT (same as applyRemoteMetadata). + if (this.metadata?.lifecycleState === 'archived' && this.metadata.archivedBy === 'hub') { + this.noteHubArchived() + logger.debug('[API] Skipping metadata update; session hub-archived') + return + } + const current = this.metadata ?? ({} as Metadata) const updated = handler(current) @@ -1317,26 +1400,55 @@ export class ApiSessionClient extends EventEmitter { metadata: updated }) as unknown - applyVersionedAck(answer, { - valueKey: 'metadata', - parseValue: (value) => { - const parsed = MetadataSchema.safeParse(value) - return parsed.success ? parsed.data : null - }, - applyValue: (value) => { - this.metadata = value - }, - applyVersion: (version) => { - this.metadataVersion = version - }, - logInvalidValue: (context, version) => { - const suffix = context === 'success' ? 'ack' : 'version-mismatch ack' - logger.debug(`[API] Ignoring invalid metadata value from ${suffix}`, { version }) - }, - invalidResponseMessage: 'Invalid update-metadata response', - errorMessage: 'Metadata update failed', - versionMismatchMessage: 'Metadata version mismatch' - }) + try { + applyVersionedAck(answer, { + valueKey: 'metadata', + parseValue: (value) => { + const parsed = MetadataSchema.safeParse(value) + return parsed.success ? parsed.data : null + }, + applyValue: (value) => { + // Route success+preserve (and mismatch) through the same + // hub-archived detector as applyRemoteMetadata so the CLI + // exits rather than booting against an archived row. + const wasHubArchived = this.metadata?.lifecycleState === 'archived' + && this.metadata?.archivedBy === 'hub' + this.metadata = value + if ( + !wasHubArchived + && value?.lifecycleState === 'archived' + && value?.archivedBy === 'hub' + ) { + this.noteHubArchived() + } + }, + applyVersion: (version) => { + this.metadataVersion = version + }, + logInvalidValue: (context, version) => { + const suffix = context === 'success' ? 'ack' : 'version-mismatch ack' + logger.debug(`[API] Ignoring invalid metadata value from ${suffix}`, { version }) + }, + invalidResponseMessage: 'Invalid update-metadata response', + errorMessage: 'Metadata update failed', + versionMismatchMessage: 'Metadata version mismatch' + }) + } catch (error) { + // True version races still throw; if hub archived mid-flight, + // applied metadata is archived — exit, do not spin. + if (this.metadata?.lifecycleState === 'archived' && this.metadata.archivedBy === 'hub') { + this.noteHubArchived() + return + } + throw error + } + + // Success+preserve terminates backoff without throw; ensure EXIT + // if ack applied hub-archived (detector above already fired). + if (this.metadata?.lifecycleState === 'archived' && this.metadata.archivedBy === 'hub') { + this.noteHubArchived() + return + } }) }) } diff --git a/cli/src/claude/registerKillSessionHandler.test.ts b/cli/src/claude/registerKillSessionHandler.test.ts index b293172955..3edba23499 100644 --- a/cli/src/claude/registerKillSessionHandler.test.ts +++ b/cli/src/claude/registerKillSessionHandler.test.ts @@ -33,8 +33,11 @@ describe('registerKillSessionHandler (tiann/hapi#914)', () => { const handler = registry.handlers.get(RPC_METHODS.KillSession) expect(handler).toBeDefined() - const result = await handler?.() - expect(result).toEqual({ success: true, message: 'Killing hapi CLI process' }) + const result = await handler?.() as { success: boolean; message: string; pid: number; processStartMarker?: string } + expect(result.success).toBe(true) + expect(result.message).toBe('Killing hapi CLI process') + expect(result.pid).toBe(process.pid) + expect(typeof result.processStartMarker === 'string' || result.processStartMarker === undefined).toBe(true) // setArchiveReason MUST be called BEFORE cleanupAndExit so the archive // metadata write reads the correct reason. @@ -62,4 +65,66 @@ describe('registerKillSessionHandler (tiann/hapi#914)', () => { expect(cleanupAndExit).toHaveBeenCalled() }) + + it('exits on hub-archived metadata when a session listener is provided (#1910)', async () => { + const registry = makeRegistry() + const lifecycle = { + setArchiveReason: vi.fn(), + cleanupAndExit: vi.fn(async () => {}) + } + const listeners = new Map void>() + const session = { + on(event: string, listener: () => void) { + listeners.set(event, listener) + } + } + + registerKillSessionHandler( + registry as unknown as Parameters[0], + lifecycle, + session + ) + + expect(listeners.has('hub-archived')).toBe(true) + listeners.get('hub-archived')?.() + expect(lifecycle.setArchiveReason).toHaveBeenCalledWith('User terminated') + expect(lifecycle.cleanupAndExit).toHaveBeenCalled() + }) + + it('exits when registration happens after hub-archived write (production order, #1911 AC6)', async () => { + // Production: bootstrap updateMetadata / ack may noteHubArchived before + // flavor runners call registerKillSessionHandler. EventEmitter does not + // replay — without the latch, cleanupAndExit stays 0×. + const registry = makeRegistry() + const lifecycle = { + setArchiveReason: vi.fn(), + cleanupAndExit: vi.fn(async () => {}) + } + const listeners = new Map void>() + const session: { + hubArchived: boolean + on(event: string, listener: () => void): void + } = { + hubArchived: false, + on(event: string, listener: () => void) { + listeners.set(event, listener) + } + } + + // Write first (noteHubArchived), then register — production order. + session.hubArchived = true + // Emit with no listeners yet (would be missed without the latch). + listeners.get('hub-archived')?.() + + registerKillSessionHandler( + registry as unknown as Parameters[0], + lifecycle, + session + ) + + expect(lifecycle.setArchiveReason).toHaveBeenCalledWith('User terminated') + expect(lifecycle.cleanupAndExit).toHaveBeenCalledTimes(1) + // Still subscribed for any later emit. + expect(listeners.has('hub-archived')).toBe(true) + }) }) diff --git a/cli/src/claude/registerKillSessionHandler.ts b/cli/src/claude/registerKillSessionHandler.ts index b42b9b49fc..2b58291c5d 100644 --- a/cli/src/claude/registerKillSessionHandler.ts +++ b/cli/src/claude/registerKillSessionHandler.ts @@ -1,6 +1,7 @@ import { RpcHandlerManager } from "@/api/rpc/RpcHandlerManager"; import { logger } from "@/lib"; import { RPC_METHODS } from '@hapi/protocol/rpcMethods'; +import { getProcessStartMarker } from '@/utils/process'; interface KillSessionRequest { // No parameters needed @@ -9,6 +10,10 @@ interface KillSessionRequest { interface KillSessionResponse { success: boolean; message: string; + /** OS pid of this CLI — hub uses it to confirm exit via StopSession when maps miss. */ + pid: number; + /** Generation marker for `pid`; required before the runner will tree-kill it. */ + processStartMarker?: string; } /** @@ -26,12 +31,22 @@ export interface KillSessionLifecycle { export function registerKillSessionHandler( rpcHandlerManager: RpcHandlerManager, - lifecycleOrCleanup: KillSessionLifecycle | (() => Promise) + lifecycleOrCleanup: KillSessionLifecycle | (() => Promise), + session?: { + hubArchived?: boolean + on(event: 'hub-archived', listener: () => void): unknown + } ) { const lifecycle: KillSessionLifecycle = typeof lifecycleOrCleanup === 'function' ? { cleanupAndExit: lifecycleOrCleanup } : lifecycleOrCleanup; + const exitFromHubArchive = () => { + logger.debug('Hub-archived metadata received; exiting CLI'); + lifecycle.setArchiveReason?.('User terminated'); + void lifecycle.cleanupAndExit(); + }; + rpcHandlerManager.registerHandler(RPC_METHODS.KillSession, async () => { logger.debug('Kill session request received'); @@ -45,11 +60,26 @@ export function registerKillSessionHandler( // This will start the cleanup process void lifecycle.cleanupAndExit(); - // We should still be able to respond to the client, though they - // should optimistically assume the session is dead. + // Include pid + start marker so archive can ask the runner to verify + // this exact process generation exited (#1910) — not a reused PID. + const processStartMarker = getProcessStartMarker(process.pid) ?? undefined; return { success: true, - message: 'Killing hapi CLI process' + message: 'Killing hapi CLI process', + pid: process.pid, + ...(processStartMarker ? { processStartMarker } : {}), }; }); + + // #1910: when archive lands as hub metadata (KillSession unreachable), + // still exit instead of reconnecting forever. + // #1911 criterion 6: EventEmitter does not replay past emits — if + // noteHubArchived already latched before this registration, exit now. + // Still subscribe so a later emit (or a race with the latch write) is covered. + if (session?.hubArchived) { + exitFromHubArchive(); + } + if (session && typeof session.on === 'function') { + session.on('hub-archived', exitFromHubArchive); + } } diff --git a/cli/src/claude/runClaude.ts b/cli/src/claude/runClaude.ts index fc446e9d56..105bb857c4 100644 --- a/cli/src/claude/runClaude.ts +++ b/cli/src/claude/runClaude.ts @@ -39,6 +39,11 @@ export interface StartOptions { claudeArgs?: string[] startedBy?: 'runner' | 'terminal' existingSessionId?: string + /** + * Fresh-spawn reserved hub id from `--hapi-session-id` (create/getOrCreate). + * Distinct from `existingSessionId` / `--existing-session-id` (reopen). + */ + reservedSessionId?: string workingDirectory?: string resumeSessionId?: string } @@ -75,7 +80,8 @@ export async function runClaude(options: StartOptions = {}): Promise { workingDirectory, agentState: initialState, model: initialModel ?? undefined, - effort: initialEffort ?? undefined + effort: initialEffort ?? undefined, + reservedSessionId: options.reservedSessionId }); const { api, session, sessionInfo } = bootstrap; logger.debug(`Session created: ${sessionInfo.id}`); @@ -236,7 +242,7 @@ export async function runClaude(options: StartOptions = {}): Promise { }); lifecycle.registerProcessHandlers(); - registerKillSessionHandler(session.rpcHandlerManager, lifecycle); + registerKillSessionHandler(session.rpcHandlerManager, lifecycle, session); registerLocalHandoffHandler(session.rpcHandlerManager, lifecycle); const conversationHistory = toConversationHistoryCapabilities(CLAUDE_CONVERSATION_HISTORY) diff --git a/cli/src/codex/shared/frontend.test.ts b/cli/src/codex/shared/frontend.test.ts index 4629119354..5b8e88bdd7 100644 --- a/cli/src/codex/shared/frontend.test.ts +++ b/cli/src/codex/shared/frontend.test.ts @@ -58,6 +58,28 @@ describe('shared frontend execution ownership', () => { await runSharedCodex({ startedBy: 'runner', workingDirectory: '/work' }); expect(state.run).toHaveBeenCalledOnce(); expect(state.spawn).not.toHaveBeenCalled(); }); + it('fresh reservedSessionId starts create path (not reopen without thread binding)', async () => { + // #1911 Critical: prealloc stub stamped as existingSessionId threw + // "no Codex thread binding". reservedSessionId must fall through to create. + state.run.mockResolvedValueOnce(undefined); + await runSharedCodex({ + startedBy: 'runner', + workingDirectory: '/work', + reservedSessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee', + }); + expect(state.run).toHaveBeenCalledOnce(); + expect(state.run.mock.calls[0][0]).toMatchObject({ + reservedSessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee', + }); + }); + it('existingSessionId without thread binding still throws (reopen path)', async () => { + await expect(runSharedCodex({ + startedBy: 'runner', + workingDirectory: '/work', + existingSessionId: 'sid', + })).rejects.toThrow('no Codex thread binding'); + expect(state.run).not.toHaveBeenCalled(); + }); it('stops the engine on TUI spawn error and leaves no execution on startup failure', async () => { state.run.mockRejectedValueOnce(new Error('startup failed')); await expect(runSharedCodex({ workingDirectory: '/work' })).rejects.toThrow('startup failed'); diff --git a/cli/src/codex/shared/frontend.ts b/cli/src/codex/shared/frontend.ts index c9178febf0..2257405982 100644 --- a/cli/src/codex/shared/frontend.ts +++ b/cli/src/codex/shared/frontend.ts @@ -50,7 +50,8 @@ export async function attachSharedSession(runtime: CodexRuntimeRecord, sessionId export async function runSharedCodex(raw: SharedLaunchOptions): Promise { const options = SharedLaunchSchema.parse({ ...raw, workingDirectory: raw.workingDirectory ?? getInvokedCwd() }); - if (options.existingSessionId) { + // reservedSessionId is fresh adopt-stub — never treat as reopen (#1911 Codex Major). + if (options.existingSessionId && !options.reservedSessionId) { const api = await ApiClient.create(); const session = await api.getSession(options.existingSessionId); const runtime = await findRuntime(session.id); diff --git a/cli/src/codex/shared/launch.test.ts b/cli/src/codex/shared/launch.test.ts index b9d2e3ea81..82bfd560ef 100644 --- a/cli/src/codex/shared/launch.test.ts +++ b/cli/src/codex/shared/launch.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { sharedLaunchConfig } from './launch'; +import { sharedLaunchConfig, takeReservedSessionId } from './launch'; import { parseCodexCliOverrides } from '../utils/codexCliOverrides'; import { resolveCodexPermissionModeConfig } from '../utils/permissionModeConfig'; @@ -52,3 +52,12 @@ describe('shared launch configuration', () => { expect(result.threadParams).toMatchObject({ modelProvider: 'ollama', approvalsReviewer: 'auto_review', sandbox: 'workspace-write' }); }); }); + +describe('takeReservedSessionId', () => { + it('returns the reservation once and clears it for subsequent prepare calls', () => { + const options = { reservedSessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' }; + expect(takeReservedSessionId(options)).toBe('aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'); + expect(options.reservedSessionId).toBeUndefined(); + expect(takeReservedSessionId(options)).toBeUndefined(); + }); +}); diff --git a/cli/src/codex/shared/launch.ts b/cli/src/codex/shared/launch.ts index 4488a211a0..71b8e934d6 100644 --- a/cli/src/codex/shared/launch.ts +++ b/cli/src/codex/shared/launch.ts @@ -11,13 +11,26 @@ export const SharedLaunchSchema = z.object({ startedBy: z.enum(['runner', 'terminal']).optional(), codexArgs: z.array(z.string()).optional(), permissionMode: z.enum(['default', 'read-only', 'safe-yolo', 'yolo']).optional(), - resumeSessionId: z.string().optional(), resumeLast: z.boolean().optional(), resumeAll: z.boolean().optional(), existingSessionId: z.string().optional(), + resumeSessionId: z.string().optional(), resumeLast: z.boolean().optional(), resumeAll: z.boolean().optional(), + existingSessionId: z.string().optional(), + /** Hub-preallocated stub — create/adopt, not reopen. */ + reservedSessionId: z.string().optional(), model: z.string().optional(), modelReasoningEffort: z.string().optional(), serviceTier: z.string().optional(), collaborationMode: z.enum(['default', 'plan']).optional(), workingDirectory: z.string().optional() }); export type SharedLaunchOptions = z.infer; +/** + * Take the machine-spawn reservation for one create-path prepare(). + * Clears it so a second root/fork cannot re-adopt (#1911 Opus Major). + */ +export function takeReservedSessionId(options: SharedLaunchOptions): string | undefined { + const id = options.reservedSessionId; + options.reservedSessionId = undefined; + return id; +} + /** Resolve once: a desktop server and a different PATH TUI are never mixed. */ export function resolveSharedCodex(): CodexCommand { const command = resolveCodexCommand(); diff --git a/cli/src/codex/shared/registry.test.ts b/cli/src/codex/shared/registry.test.ts index dec3063e55..10a98592d2 100644 --- a/cli/src/codex/shared/registry.test.ts +++ b/cli/src/codex/shared/registry.test.ts @@ -6,7 +6,7 @@ import { tmpdir } from 'node:os'; const state = vi.hoisted(() => ({ home: '', auth: 'token', processes: new Map() })); vi.mock('@/configuration', () => ({ configuration: { get happyHomeDir() { return state.home; }, apiUrl: 'hub', get cliApiToken() { return state.auth; } } })); vi.mock('@/utils/process', () => ({ isProcessAlive: (pid: number) => state.processes.has(pid), getProcessStartMarker: (pid: number) => state.processes.get(pid) })); -import { findRuntime, runtimeAlive, runtimeAuthHash, runtimeMayBeAlive, saveRuntime, withThreadOwnership, type CodexRuntimeRecord } from './registry'; +import { findRuntime, readRuntimes, runtimeAlive, runtimeAuthHash, runtimeMayBeAlive, saveRuntime, withThreadOwnership, type CodexRuntimeRecord } from './registry'; const directories: string[] = []; afterEach(async () => { state.processes.clear(); state.auth = 'token'; await Promise.all(directories.splice(0).map(path => rm(path, { recursive: true, force: true }))); }); @@ -40,4 +40,13 @@ describe('shared runtime ownership', () => { await writeFile(join(directory, 'broken.json'), '{'); await expect(withThreadOwnership(owner.codexHome, 'thread', 'new', async () => {})).rejects.toThrow('Cannot verify'); }); + it('readRuntimes({ strict: true }) fails closed on corrupt hub registry files', async () => { + // Soft [] would let stopSession argv-sweep tree-kill shared wrappers (#1911). + const owner = await fixture(); + const directory = join(state.home, 'codex-runtimes'); + await mkdir(directory, { recursive: true }); + await writeFile(join(directory, 'broken.json'), '{'); + expect(await readRuntimes()).toEqual([]); + await expect(readRuntimes({ strict: true })).rejects.toThrow('Cannot verify'); + }); }); diff --git a/cli/src/codex/shared/registry.ts b/cli/src/codex/shared/registry.ts index 0f627b1662..e8ec346156 100644 --- a/cli/src/codex/shared/registry.ts +++ b/cli/src/codex/shared/registry.ts @@ -43,8 +43,8 @@ export function runtimeMayBeAlive(owner: CodexRuntimeRecord): boolean { || Boolean(owner.serverPid && generationMayBeAlive(owner.serverPid, owner.serverMarker)); } -export async function readRuntimes(): Promise { - return readRecords(runtimeDirectory()); +export async function readRuntimes(options?: { strict?: boolean }): Promise { + return readRecords(runtimeDirectory(), options?.strict === true); } async function readRecords(directory: string, strict = false): Promise { const names = await readdir(directory).catch((error: NodeJS.ErrnoException) => { diff --git a/cli/src/codex/shared/root.test.ts b/cli/src/codex/shared/root.test.ts index f4cc292ccb..ae0276420d 100644 --- a/cli/src/codex/shared/root.test.ts +++ b/cli/src/codex/shared/root.test.ts @@ -53,7 +53,7 @@ afterEach(async () => { finally { vi.useRealTimers(); } }); -async function fixture() { +async function fixture(opts?: { hubArchived?: boolean; end?: RootHost['end'] }) { const directory = await mkdtemp('/tmp/hapi-shared-root-'); let state: AgentState = { steeringActive: true }; let metadata: Metadata = { path: directory, host: 'test', flavor: 'codex' }; @@ -61,21 +61,27 @@ async function fixture() { const updateState = vi.fn((fn: (value: AgentState) => AgentState) => { state = fn(state); }); const rpc = new Map Promise>(); const send = vi.fn(); + const hubArchivedListeners: Array<() => void> = []; const session = { sessionId: 'sid', getMetadata: () => metadata, + hubArchived: opts?.hubArchived ?? false, updateMetadata: (fn: (value: Metadata) => Metadata) => { metadata = fn(metadata); }, updateAgentState: updateState, keepAlive() {}, onUserMessage() {}, onCancelQueuedMessage() {}, onRetryQueuedMessage() {}, onReconnect: (fn: (() => void) | null) => { reconnect = fn; }, + on(event: string, listener: () => void) { + if (event === 'hub-archived') hubArchivedListeners.push(listener); + }, rpcHandlerManager: { registerHandler: (name: string, handler: (raw: unknown) => Promise) => rpc.set(name, handler) }, sendSessionEvent() {}, sendAgentMessage: send, emitSessionReady() {}, sendUserMessage() {}, emitMessagesConsumed() {}, emitSteerIndeterminate() {}, syncNativeQueuedMessage() {}, sendSessionDeath() {}, async flush() {}, close() {} } as unknown as ApiSessionClient; + const end = opts?.end ?? (async () => { throw new Error('Unexpected root archive'); }); const root = new SharedCodexRoot({ session, workingDirectory: directory } as SessionBootstrapResult, { directory, generation: 'test', endpoint: 'mock', settingsFor: () => undefined, create: async () => { throw new Error('Unexpected root creation'); }, - end: async () => { throw new Error('Unexpected root archive'); } + end } satisfies RootHost); cleanups.push(async () => { await root.close(false); await rm(directory, { recursive: true, force: true }); }); await root.prepare(); @@ -87,7 +93,13 @@ async function fixture() { notify(method: string, params: unknown): void; abandoned(): void; }; - return { root, native, rpc, send, metadata: () => metadata, state: () => state, updateState, reconnect: () => reconnect?.() }; + return { + root, native, rpc, send, metadata: () => metadata, state: () => state, updateState, + reconnect: () => reconnect?.(), + emitHubArchived: () => { for (const listener of hubArchivedListeners) listener(); }, + hubArchivedListenerCount: () => hubArchivedListeners.length, + end, + }; } async function completePlan(f: Awaited>, status = 'completed') { @@ -290,4 +302,23 @@ describe('shared steering availability', () => { f.reconnect(); await vi.waitFor(() => expect(f.state().steeringActive).toBe(false)); }); + + it('ends the shared root on hub-archived metadata (#1911 C2 / AC6)', async () => { + const end = vi.fn(async () => {}); + const f = await fixture({ end }); + await f.root.activate(); + expect(f.hubArchivedListenerCount()).toBe(1); + f.emitHubArchived(); + await vi.waitFor(() => expect(end).toHaveBeenCalledTimes(1)); + expect(end.mock.calls[0]?.[0]).toBe(f.root); + }); + + it('ends when hubArchived was latched before activate (production order, #1911 AC6)', async () => { + // Bootstrap may refuse CAS / noteHubArchived before Codex bind+activate + // registers controls — same late-listener miss as other flavors. + const end = vi.fn(async () => {}); + const f = await fixture({ hubArchived: true, end }); + await f.root.activate(); + await vi.waitFor(() => expect(end).toHaveBeenCalledTimes(1)); + }); }); diff --git a/cli/src/codex/shared/root.ts b/cli/src/codex/shared/root.ts index 58bc48ac4d..187c015346 100644 --- a/cli/src/codex/shared/root.ts +++ b/cli/src/codex/shared/root.ts @@ -464,6 +464,18 @@ export class SharedCodexRoot { if (turnId) await this.client.request('turn/interrupt', { threadId: this.threadId, turnId }); }); rpc.registerHandler(RPC_METHODS.KillSession, async () => { await this.host.end(this); return { success: true }; }); + // #1911 C2 / AC6: Codex owns KillSession here and never calls + // registerKillSessionHandler — still must EXIT on hub-archived metadata + // (KillSession unreachable / map-miss). Latch covers bootstrap write + // before bind/activate registers controls; B2 refuses tree-kill of the + // shared wrapper so this subscribe is load-bearing. + const exitFromHubArchive = () => { + void this.host.end(this).catch(error => logger.debug('[CODEX] hub-archived end failed:', error)); + }; + if (this.session.hubArchived) { + exitFromHubArchive(); + } + this.session.on('hub-archived', exitFromHubArchive); rpc.registerHandler(RPC_METHODS.SetSessionConfig, raw => this.applySettings(raw)); rpc.registerHandler(RPC_METHODS.ImplementCodexPlan, raw => { const { planId } = ImplementCodexPlanRequestSchema.parse(raw); diff --git a/cli/src/codex/shared/runtime.ts b/cli/src/codex/shared/runtime.ts index 3a054ce2a2..4c17833346 100644 --- a/cli/src/codex/shared/runtime.ts +++ b/cli/src/codex/shared/runtime.ts @@ -12,7 +12,7 @@ import { logger } from '@/ui/logger'; import { CodexAppServerClient, isIndeterminateError } from '../codexAppServerClient'; import { codexHome, saveRuntime, runtimeDirectory, runtimeAuthHash, findColdBinding, withThreadOwnership, type CodexRuntimeRecord } from './registry'; import { startCodexGateway, record, string, type Envelope } from './gateway'; -import { resolveSharedCodex, sharedLaunchConfig, initializeSharedClient, checkSharedCapabilities, type SharedLaunchOptions } from './launch'; +import { resolveSharedCodex, sharedLaunchConfig, initializeSharedClient, checkSharedCapabilities, takeReservedSessionId, type SharedLaunchOptions } from './launch'; import { SharedCodexRoot } from './root'; export type RuntimeReady = { sessionId: string; runtime: CodexRuntimeRecord }; @@ -172,9 +172,16 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( const shared = { flavor: 'codex', startedBy: options.startedBy ?? 'terminal', workingDirectory: cwd, exportSessionEnv: false, reportStarted: false, metadataOverrides: { capabilities: { terminal: true, concurrentClients: true }, ...(parent ? { forkedFrom: parent.session.sessionId } : {}) } } as const; + // reservedSessionId names one preallocated hub row — single-use. A second + // create()/fork must mint a fresh row, not re-adopt (#1911 Opus Major). + const reservedSessionId = takeReservedSessionId(options); const bootstrap = existingSessionId ? await bootstrapExistingSession({ ...shared, sessionId: existingSessionId }) - : await bootstrapSession({ ...shared, agentState: { controlledByUser: false } }); + : await bootstrapSession({ + ...shared, + reservedSessionId, + agentState: { controlledByUser: false }, + }); const root = new SharedCodexRoot(bootstrap, { directory: join(runtimeDirectory(), 'queues'), generation: id, endpoint: upstream, token: upstreamToken, settingsFor: threadId => nativeSettings.get(threadId), create, end }); prepared.add(root); diff --git a/cli/src/commands/agentCommandOptions.test.ts b/cli/src/commands/agentCommandOptions.test.ts index 187bc0fd2b..7e54356727 100644 --- a/cli/src/commands/agentCommandOptions.test.ts +++ b/cli/src/commands/agentCommandOptions.test.ts @@ -13,12 +13,13 @@ describe('parseRemoteAgentCommandOptions', () => { .toBe('yolo') }) - it('parses --hapi-session-id into existingSessionId (pty reopen id reuse)', () => { - // The runner emits --hapi-session-id when reopening a pty session so the - // child reuses the existing hub row. agy (this shared parser) must consume - // it — else the flag is silently dropped and reopen mints a new id + 404. - expect(parseRemoteAgentCommandOptions(['--hapi-session-id', 'hub-id-1'], AGY_PERMISSION_MODES).existingSessionId) + it('parses --hapi-session-id into reservedSessionId (fresh prealloc / adopt)', () => { + // #1911 M3: must NOT alias to existingSessionId — that collapses adopt→reopen. + // Intentional reopen uses --existing-session-id (buildCliArgs already splits). + expect(parseRemoteAgentCommandOptions(['--hapi-session-id', 'hub-id-1'], AGY_PERMISSION_MODES).reservedSessionId) .toBe('hub-id-1') + expect(parseRemoteAgentCommandOptions(['--hapi-session-id', 'hub-id-1'], AGY_PERMISSION_MODES).existingSessionId) + .toBeUndefined() }) it('parses common remote agent flags', () => { diff --git a/cli/src/commands/agentCommandOptions.ts b/cli/src/commands/agentCommandOptions.ts index d5b3fdf1fa..13ee130b93 100644 --- a/cli/src/commands/agentCommandOptions.ts +++ b/cli/src/commands/agentCommandOptions.ts @@ -11,7 +11,14 @@ export type RemoteAgentCommandOptions< effort?: string modelReasoningEffort?: string resumeSessionId?: string + /** Intentional reopen/resume of an existing hub row (`--existing-session-id`). */ existingSessionId?: string + /** + * Fresh machine-spawn prealloc stub (`--hapi-session-id`). Distinct from + * existingSessionId — must route to bootstrapSession({ reservedSessionId }), + * never bootstrapExistingSession (#1911 M3). + */ + reservedSessionId?: string } export function parseRemoteAgentCommandOptions< @@ -62,26 +69,19 @@ export function parseRemoteAgentCommandOptions< options.permissionMode = yoloEquivalent as TPermissionMode } } else if (arg === '--hapi-session-id') { - // Hub row to reuse on reopen/resume of a pty session (agy), so the id - // stays stable instead of spawn-new + merge-delete (+ the 404 flash). - // The runner only emits this for pty flavors whose parser consumes it. + // Fresh prealloc / adopt-stub. Intentional reopen uses + // --existing-session-id (buildCliArgs already splits them). const id = args[++i] if (!id) { throw new Error('Missing --hapi-session-id value') } - options.existingSessionId = id + options.reservedSessionId = id } else if (arg === '--resume') { const sessionId = args[++i] if (!sessionId) { throw new Error('Missing --resume value') } options.resumeSessionId = sessionId - } else if (arg === '--existing-session-id') { - const sessionId = args[++i] - if (!sessionId || sessionId.startsWith('-')) { - throw new Error('Missing --existing-session-id value') - } - options.existingSessionId = sessionId } else if (arg === '-s' || arg === '--session') { // OpenCode-native resume flags (hapi opencode -s / --session ) const sessionId = args[++i] diff --git a/cli/src/commands/claude.test.ts b/cli/src/commands/claude.test.ts index 6716ee4de5..715a25decf 100644 --- a/cli/src/commands/claude.test.ts +++ b/cli/src/commands/claude.test.ts @@ -72,9 +72,27 @@ describe('claudeCommand arguments', () => { } }) - it('keeps arguments after -- opaque', async () => { - const args = ['--', '--model', 'literal', '--help'] - await claudeCommand.run(createCommandContext(args)) - expect(runClaudeMock).toHaveBeenCalledWith({ claudeArgs: args }) + it('passes --hapi-session-id as reservedSessionId (adopt-stub, not reopen) (#1911)', async () => { + await claudeCommand.run(createCommandContext([ + '--started-by', 'runner', + '--hapi-starting-mode', 'remote', + '--hapi-session-id', 'preallocated-hub-id', + ])) + + expect(runClaudeMock).toHaveBeenCalledWith({ + startedBy: 'runner', + startingMode: 'remote', + reservedSessionId: 'preallocated-hub-id', + }) + }) + + it('passes --existing-session-id through for Claude fork/reuse', async () => { + await claudeCommand.run(createCommandContext([ + '--existing-session-id', 'fork-child-id', + ])) + + expect(runClaudeMock).toHaveBeenCalledWith({ + existingSessionId: 'fork-child-id', + }) }) }) diff --git a/cli/src/commands/claude.ts b/cli/src/commands/claude.ts index 0d75513227..b56c80164b 100644 --- a/cli/src/commands/claude.ts +++ b/cli/src/commands/claude.ts @@ -65,6 +65,15 @@ export const claudeCommand: CommandDefinition = { unknownArgs.push('--effort', effort) } else if (arg === '--started-by') { options.startedBy = args[++i] as 'runner' | 'terminal' + } else if (arg === '--hapi-session-id') { + // Fresh-spawn reserved id (hub prealloc / runner stamp). Create + // bootstrap with getOrCreate({ id }) — must NOT take the reopen + // path (`existingSessionId` / `--existing-session-id`). + const sessionId = args[++i] + if (!sessionId) { + throw new Error('Missing --hapi-session-id value') + } + options.reservedSessionId = sessionId } else if (arg === '--existing-session-id') { const sessionId = args[++i] if (!sessionId) { diff --git a/cli/src/commands/codex.ts b/cli/src/commands/codex.ts index 67cc03be4a..1fd6e5ef96 100644 --- a/cli/src/commands/codex.ts +++ b/cli/src/commands/codex.ts @@ -43,6 +43,7 @@ export const codexCommand: CommandDefinition = { resumeLast?: boolean resumeAll?: boolean existingSessionId?: string + reservedSessionId?: string model?: string modelReasoningEffort?: ReasoningEffort serviceTier?: string @@ -75,6 +76,12 @@ export const codexCommand: CommandDefinition = { throw new Error(`Use native codex ${arg} outside HAPI, or /${arg} in an attached terminal`) } else if (arg === '--started-by') { options.startedBy = commandArgs[++i] as 'runner' | 'terminal' + } else if (arg === '--hapi-session-id') { + const sessionId = commandArgs[++i] + if (!sessionId) { + throw new Error('Missing --hapi-session-id value') + } + options.reservedSessionId = sessionId } else if (arg === '--existing-session-id') { const sessionId = commandArgs[++i] if (!sessionId) { diff --git a/cli/src/commands/copilot.ts b/cli/src/commands/copilot.ts index 28faacd44d..22195f515c 100644 --- a/cli/src/commands/copilot.ts +++ b/cli/src/commands/copilot.ts @@ -19,6 +19,8 @@ export const copilotCommand: CommandDefinition = { model?: string copilotAgentMode?: CopilotAgentMode resumeSessionId?: string + existingSessionId?: string + reservedSessionId?: string } = {} let hasExplicitPermissionMode = false @@ -34,6 +36,19 @@ export const copilotCommand: CommandDefinition = { } else { throw new Error('Invalid --hapi-starting-mode (expected local or remote)') } + } else if (arg === '--hapi-session-id') { + // Adopt-stub: create bootstrap with reserved id (not reopen). + const sessionId = commandArgs[++i] + if (!sessionId) { + throw new Error('Missing --hapi-session-id value') + } + options.reservedSessionId = sessionId + } else if (arg === '--existing-session-id') { + const sessionId = commandArgs[++i] + if (!sessionId) { + throw new Error('Missing --existing-session-id value') + } + options.existingSessionId = sessionId } else if (arg === '--permission-mode') { const mode = commandArgs[++i] if (!mode || !(COPILOT_PERMISSION_MODES as readonly string[]).includes(mode)) { diff --git a/cli/src/commands/cursor.ts b/cli/src/commands/cursor.ts index 900a2a0ed5..5b405c7450 100644 --- a/cli/src/commands/cursor.ts +++ b/cli/src/commands/cursor.ts @@ -14,6 +14,8 @@ export type ParsedCursorCommandOptions = { permissionMode?: CursorPermissionMode resumeSessionId?: string existingSessionId?: string + /** Fresh-spawn reserved hub id (`--hapi-session-id`); not reopen. */ + reservedSessionId?: string model?: string } @@ -85,6 +87,14 @@ export function parseCursorCommandArgs(commandArgs: string[]): ParsedCursorComma } else { unknownArgs.push(arg) } + } else if (arg === '--hapi-session-id') { + // Fresh-spawn reserved id when stamped; reopen stays on + // `--existing-session-id` (Cursor machine spawn uses that form). + const hapiSessionId = commandArgs[++i] + if (!hapiSessionId || hapiSessionId.startsWith('-')) { + throw new Error('Missing --hapi-session-id value') + } + options.reservedSessionId = hapiSessionId } else if (arg === '--existing-session-id') { const hapiSessionId = commandArgs[++i] if (!hapiSessionId || hapiSessionId.startsWith('-')) { diff --git a/cli/src/commands/kimi.ts b/cli/src/commands/kimi.ts index 06a10e2fcc..d50222707a 100644 --- a/cli/src/commands/kimi.ts +++ b/cli/src/commands/kimi.ts @@ -17,6 +17,8 @@ export const kimiCommand: CommandDefinition = { permissionMode?: KimiPermissionMode model?: string resumeSessionId?: string + existingSessionId?: string + reservedSessionId?: string } = {} let hasExplicitPermissionMode = false @@ -32,6 +34,19 @@ export const kimiCommand: CommandDefinition = { } else { throw new Error('Invalid --hapi-starting-mode (expected local or remote)') } + } else if (arg === '--hapi-session-id') { + // Adopt-stub: create bootstrap with reserved id (not reopen). + const sessionId = commandArgs[++i] + if (!sessionId) { + throw new Error('Missing --hapi-session-id value') + } + options.reservedSessionId = sessionId + } else if (arg === '--existing-session-id') { + const sessionId = commandArgs[++i] + if (!sessionId) { + throw new Error('Missing --existing-session-id value') + } + options.existingSessionId = sessionId } else if (arg === '--permission-mode') { const mode = commandArgs[++i] if (!mode || !(KIMI_PERMISSION_MODES as readonly string[]).includes(mode)) { diff --git a/cli/src/copilot/runCopilot.ts b/cli/src/copilot/runCopilot.ts index 947f8a2018..2c97eb5656 100644 --- a/cli/src/copilot/runCopilot.ts +++ b/cli/src/copilot/runCopilot.ts @@ -45,6 +45,8 @@ export async function runCopilot(opts: { copilotAgentMode?: import('@hapi/protocol').CopilotAgentMode; resumeSessionId?: string; existingSessionId?: string; + /** Fresh-spawn reserved hub id from `--hapi-session-id` (create/getOrCreate). */ + reservedSessionId?: string; workingDirectory?: string; } = {}): Promise { const workingDirectory = opts.workingDirectory ?? getInvokedCwd(); @@ -78,7 +80,8 @@ export async function runCopilot(opts: { startedBy, workingDirectory, agentState: initialState, - model: persistedModel + model: persistedModel, + reservedSessionId: opts.reservedSessionId }); const { api, session } = bootstrap; @@ -106,7 +109,7 @@ export async function runCopilot(opts: { }); lifecycle.registerProcessHandlers(); - registerKillSessionHandler(session.rpcHandlerManager, lifecycle); + registerKillSessionHandler(session.rpcHandlerManager, lifecycle, session); registerLocalHandoffHandler(session.rpcHandlerManager, lifecycle); const syncSessionMode = () => { diff --git a/cli/src/cursor/runCursor.ts b/cli/src/cursor/runCursor.ts index 0cf4636781..a7d27fd5a4 100644 --- a/cli/src/cursor/runCursor.ts +++ b/cli/src/cursor/runCursor.ts @@ -34,6 +34,8 @@ export async function runCursor(opts: { resumeSessionId?: string; model?: string; existingSessionId?: string; + /** Fresh-spawn reserved hub id from `--hapi-session-id` (create/getOrCreate). */ + reservedSessionId?: string; workingDirectory?: string; }): Promise { const workingDirectory = opts.workingDirectory ?? getInvokedCwd(); @@ -56,7 +58,8 @@ export async function runCursor(opts: { startedBy, workingDirectory, agentState: state, - model: opts.model + model: opts.model, + reservedSessionId: opts.reservedSessionId }); const { api, session } = bootstrap; @@ -83,7 +86,7 @@ export async function runCursor(opts: { }); lifecycle.registerProcessHandlers(); - registerKillSessionHandler(session.rpcHandlerManager, lifecycle); + registerKillSessionHandler(session.rpcHandlerManager, lifecycle, session); registerLocalHandoffHandler(session.rpcHandlerManager, lifecycle); const syncSessionMode = () => { diff --git a/cli/src/dsh/runDsh.ts b/cli/src/dsh/runDsh.ts index ed3004cf54..b7b8ef70dc 100644 --- a/cli/src/dsh/runDsh.ts +++ b/cli/src/dsh/runDsh.ts @@ -15,6 +15,8 @@ export async function runDsh(opts: { startedBy?: 'runner' | 'terminal' startingMode?: 'remote' existingSessionId?: string + /** Fresh machine-spawn stub (`--hapi-session-id`); adopt via bootstrapSession. */ + reservedSessionId?: string workingDirectory?: string } = {}): Promise { const workingDirectory = opts.workingDirectory ?? getInvokedCwd() @@ -36,7 +38,8 @@ export async function runDsh(opts: { flavor: 'dsh', startedBy, workingDirectory, - agentState: initialState + agentState: initialState, + reservedSessionId: opts.reservedSessionId }) const { api, session } = bootstrap setControlledByUser(session, startingMode) @@ -61,7 +64,7 @@ export async function runDsh(opts: { onBeforeClose: () => launcherRef.current?.kill() }) lifecycle.registerProcessHandlers() - registerKillSessionHandler(session.rpcHandlerManager, lifecycle) + registerKillSessionHandler(session.rpcHandlerManager, lifecycle, session) const dshSession = new DshSession({ api, diff --git a/cli/src/grok/runGrok.ts b/cli/src/grok/runGrok.ts index 881002d778..e392f98ddb 100644 --- a/cli/src/grok/runGrok.ts +++ b/cli/src/grok/runGrok.ts @@ -25,6 +25,8 @@ export async function runGrok(opts: { effort?: string resumeSessionId?: string existingSessionId?: string + /** Fresh machine-spawn stub (`--hapi-session-id`); adopt via bootstrapSession. */ + reservedSessionId?: string workingDirectory?: string } = {}): Promise { const workingDirectory = opts.workingDirectory ?? getInvokedCwd() @@ -48,7 +50,8 @@ export async function runGrok(opts: { workingDirectory, agentState: initialState, model: opts.model, - effort: opts.effort + effort: opts.effort, + reservedSessionId: opts.reservedSessionId }) const { api, session, sessionInfo } = bootstrap setControlledByUser(session, startingMode) @@ -65,7 +68,7 @@ export async function runGrok(opts: { stopKeepAlive: () => sessionRef.current?.stopKeepAlive() }) lifecycle.registerProcessHandlers() - registerKillSessionHandler(session.rpcHandlerManager, lifecycle) + registerKillSessionHandler(session.rpcHandlerManager, lifecycle, session) registerLocalHandoffHandler(session.rpcHandlerManager, lifecycle) const syncSessionMode = () => { diff --git a/cli/src/kimi/runKimi.ts b/cli/src/kimi/runKimi.ts index e30857e040..7e8bcd2f3e 100644 --- a/cli/src/kimi/runKimi.ts +++ b/cli/src/kimi/runKimi.ts @@ -23,6 +23,8 @@ export async function runKimi(opts: { model?: string; resumeSessionId?: string; existingSessionId?: string; + /** Fresh-spawn reserved hub id from `--hapi-session-id` (create/getOrCreate). */ + reservedSessionId?: string; workingDirectory?: string; } = {}): Promise { const workingDirectory = opts.workingDirectory ?? getInvokedCwd(); @@ -57,7 +59,8 @@ export async function runKimi(opts: { startedBy, workingDirectory, agentState: initialState, - model: persistedModel + model: persistedModel, + reservedSessionId: opts.reservedSessionId }); const { api, session } = bootstrap; @@ -83,7 +86,7 @@ export async function runKimi(opts: { }); lifecycle.registerProcessHandlers(); - registerKillSessionHandler(session.rpcHandlerManager, lifecycle); + registerKillSessionHandler(session.rpcHandlerManager, lifecycle, session); registerLocalHandoffHandler(session.rpcHandlerManager, lifecycle); // Registered here, not in the remote launcher: the catalog is backend // independent, and a session running in local mode must answer the web diff --git a/cli/src/modules/common/rpcTypes.ts b/cli/src/modules/common/rpcTypes.ts index 53ae84f12f..90a3eab305 100644 --- a/cli/src/modules/common/rpcTypes.ts +++ b/cli/src/modules/common/rpcTypes.ts @@ -5,11 +5,11 @@ export interface SpawnSessionOptions { machineId?: string directory: string sessionId?: string - // Existing hub session id to reuse (reopen/resume). Distinct from the legacy - // `sessionId` field above (reserved/unused by spawn): when set, the CLI boots - // with `--hapi-session-id` so the child reuses the existing hub row (stable - // id) instead of minting a new one. Set only by the hub reopen/resume path. + // Live hub row id for reopen/resume. Runner stamps `--existing-session-id`. existingSessionId?: string + // Hub-preallocated machine-spawn stub. Runner stamps `--hapi-session-id` + // (adopt-stub create/getOrCreate) — must NOT take the reopen path. + reservedSessionId?: string resumeSessionId?: string approvedNewDirectoryCreation?: boolean agent?: AgentFlavor @@ -33,10 +33,16 @@ export interface SpawnSessionOptions { export type SpawnSessionResult = | { type: 'success'; sessionId: string } - | { type: 'requestToApproveDirectoryCreation'; directory: string } + | { type: 'requestToApproveDirectoryCreation'; directory: string; childStarted: false } | { type: 'error' errorMessage: string code?: 'agent_unavailable' | 'outside_workspace_roots' agent?: AgentFlavor + /** + * Explicit false = runner rejected before exec (no OS child). Hub may + * delete a preallocated stub. Omitted/true = child may exist — keep stub + * until StopSession confirms gone (#1911 B3). + */ + childStarted?: boolean } diff --git a/cli/src/opencode/runOpencode.test.ts b/cli/src/opencode/runOpencode.test.ts index cf9826fe04..b04a592b0b 100644 --- a/cli/src/opencode/runOpencode.test.ts +++ b/cli/src/opencode/runOpencode.test.ts @@ -186,10 +186,31 @@ describe('runOpencode set-session-config handler', () => { return configHandler![1] as (payload: unknown) => Promise; } - it('carries the runner preallocated id from CLI args through parse into bootstrapExistingSession', async () => { + it('routes reservedSessionId from buildCliArgs into bootstrapSession (adopt, not reopen)', async () => { const runnerArgs = buildCliArgs('opencode', { directory: '/tmp/project', - existingSessionId: 'preallocated-hapi-id' + reservedSessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' + }); + const parsed = parseRemoteAgentCommandOptions(runnerArgs.slice(1), OPENCODE_PERMISSION_MODES); + + expect(parsed.reservedSessionId).toBe('aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'); + expect(parsed.existingSessionId).toBeUndefined(); + + await runOpencode({ ...parsed, workingDirectory: '/tmp/project' }); + + expect(harness.bootstrapExistingArgs).toEqual([]); + expect(harness.bootstrapArgs).toEqual([expect.objectContaining({ + flavor: 'opencode', + startedBy: 'runner', + workingDirectory: '/tmp/project', + reservedSessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' + })]); + }); + + it('routes --existing-session-id reopen into bootstrapExistingSession', async () => { + const runnerArgs = buildCliArgs('opencode', { + directory: '/tmp/project', + existingSessionId: 'live-hub-row' }); const parsed = parseRemoteAgentCommandOptions(runnerArgs.slice(1), OPENCODE_PERMISSION_MODES); @@ -197,7 +218,7 @@ describe('runOpencode set-session-config handler', () => { expect(harness.bootstrapArgs).toEqual([]); expect(harness.bootstrapExistingArgs).toEqual([{ - sessionId: 'preallocated-hapi-id', + sessionId: 'live-hub-row', flavor: 'opencode', startedBy: 'runner', workingDirectory: '/tmp/project' diff --git a/cli/src/opencode/runOpencode.ts b/cli/src/opencode/runOpencode.ts index 134a4c3622..3a471f0f51 100644 --- a/cli/src/opencode/runOpencode.ts +++ b/cli/src/opencode/runOpencode.ts @@ -27,6 +27,8 @@ export async function runOpencode(opts: { modelReasoningEffort?: string | null; resumeSessionId?: string; existingSessionId?: string; + /** Fresh machine-spawn stub (`--hapi-session-id`); adopt via bootstrapSession. */ + reservedSessionId?: string; workingDirectory?: string; } = {}): Promise { const workingDirectory = opts.workingDirectory ?? getInvokedCwd(); @@ -69,7 +71,8 @@ export async function runOpencode(opts: { workingDirectory, agentState: initialState, model: initialModel ?? undefined, - modelReasoningEffort: initialModelReasoningEffort ?? undefined + modelReasoningEffort: initialModelReasoningEffort ?? undefined, + reservedSessionId: opts.reservedSessionId }); const { api, session } = bootstrap; @@ -150,7 +153,7 @@ export async function runOpencode(opts: { }); lifecycle.registerProcessHandlers(); - registerKillSessionHandler(session.rpcHandlerManager, lifecycle); + registerKillSessionHandler(session.rpcHandlerManager, lifecycle, session); registerLocalHandoffHandler(session.rpcHandlerManager, lifecycle); const syncSessionMode = () => { diff --git a/cli/src/pi/runPi.test.ts b/cli/src/pi/runPi.test.ts index f2eadcfbf9..6379553b6a 100644 --- a/cli/src/pi/runPi.test.ts +++ b/cli/src/pi/runPi.test.ts @@ -24,6 +24,7 @@ const harness = vi.hoisted(() => ({ updateMetadata: vi.fn(), getMetadata: vi.fn(() => null), emitSessionReady: vi.fn(), + on: vi.fn(), rpcHandlerManager: { registerHandler: vi.fn() }, }, })); diff --git a/cli/src/pi/runPi.ts b/cli/src/pi/runPi.ts index a64424e5b4..55e1e70d1c 100644 --- a/cli/src/pi/runPi.ts +++ b/cli/src/pi/runPi.ts @@ -193,6 +193,8 @@ export async function runPi(opts: { effort?: string; resumeSessionId?: string; existingSessionId?: string; + /** Fresh machine-spawn stub (`--hapi-session-id`); adopt via bootstrapSession. */ + reservedSessionId?: string; workingDirectory?: string; } = {}): Promise { const workingDirectory = opts.workingDirectory ?? getInvokedCwd(); @@ -223,7 +225,8 @@ export async function runPi(opts: { // handleSessionAlive persists every non-undefined keepAlive model, so // passing it here would store/show a model Pi may reject. PiSession // carries opts.model as initialModel and applies it once confirmed. - model: undefined + model: undefined, + reservedSessionId: opts.reservedSessionId }); const { session: apiSession } = bootstrap; @@ -302,7 +305,7 @@ export async function runPi(opts: { }); lifecycle.registerProcessHandlers(); - registerKillSessionHandler(apiSession.rpcHandlerManager, lifecycle); + registerKillSessionHandler(apiSession.rpcHandlerManager, lifecycle, apiSession); registerLocalHandoffHandler(apiSession.rpcHandlerManager, lifecycle); let cleanupInitiated = false; diff --git a/cli/src/runner/buildCliArgs.test.ts b/cli/src/runner/buildCliArgs.test.ts index 4630d29c43..db1b4fa348 100644 --- a/cli/src/runner/buildCliArgs.test.ts +++ b/cli/src/runner/buildCliArgs.test.ts @@ -162,16 +162,17 @@ describe('buildCliArgs', () => { expect(args).not.toContain('--hapi-session-id') }) - it('passes --existing-session-id for cursor resume when sessionId is set (#991)', () => { + it('passes --existing-session-id for cursor resume when existingSessionId is set (#991)', () => { const args = buildCliArgs('cursor', { directory: '/tmp', resumeSessionId: 'cursor-csid-1', - sessionId: 'hapi-session-991', + existingSessionId: 'hapi-session-991', }) expect(args).toContain('--existing-session-id') expect(args).toContain('hapi-session-991') expect(args).toContain('--resume') expect(args).toContain('cursor-csid-1') + expect(args).not.toContain('--hapi-session-id') }) it('does not pass --collaboration-mode for non-codex agents', () => { @@ -368,13 +369,87 @@ describe('buildCliArgs', () => { ]) }) - it('does not emit --hapi-session-id for a non-pty flavor', () => { + it('stamps --hapi-session-id for local HTTP sessionId hints (reap / adopt-stub)', () => { + // sessionId alone is not reopen — that was the round-4 Major when we + // collapsed everything onto --existing-session-id. + const args = buildCliArgs('claude', { + directory: '/tmp', + sessionId: 'spawned-test-456', + }) + expect(args).toContain('--hapi-session-id') + expect(args[args.indexOf('--hapi-session-id') + 1]).toBe('spawned-test-456') + expect(args).not.toContain('--existing-session-id') + }) + + it('does not stamp UUID local-HTTP sessionId as adopt (would 404/409)', () => { + // #1911 Opus Major: controlServer passes sessionId through; UUID entered + // resolveReservedHubSessionId → adoptPreallocatedSession against a + // non-stub / missing row. + const args = buildCliArgs('claude', { + directory: '/tmp', + sessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee', + }) + expect(args).not.toContain('--hapi-session-id') + expect(args).not.toContain('--existing-session-id') + }) + + it('stamps --hapi-session-id for reservedSessionId (fresh machine-spawn stub)', () => { + const args = buildCliArgs('codex', { + directory: '/tmp', + reservedSessionId: 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee', + startingMode: 'remote', + }) + expect(args).toContain('--hapi-session-id') + expect(args[args.indexOf('--hapi-session-id') + 1]).toBe('aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee') + expect(args).not.toContain('--existing-session-id') + }) + + it('stamps --existing-session-id for Claude reopen/resume (not adopt --hapi-session-id)', () => { + // #1911 Opus Critical: syncEngine resume passes access.sessionId as + // existingSessionId; adopt-stub stamp → SessionNotAdoptableError → 409. + const args = buildCliArgs('claude', { + directory: '/tmp', + existingSessionId: 'live-hub-row-uuid', + resumeSessionId: 'native-claude-resume-token', + startingMode: 'remote', + }) + expect(args).toContain('--existing-session-id') + expect(args[args.indexOf('--existing-session-id') + 1]).toBe('live-hub-row-uuid') + expect(args).not.toContain('--hapi-session-id') + expect(args).toContain('--resume') + }) + + it('stamps --existing-session-id for kimi and copilot reopen (same adopt trap)', () => { + for (const agent of ['kimi', 'copilot'] as const) { + const args = buildCliArgs(agent, { + directory: '/tmp', + existingSessionId: 'live-hub-row-uuid', + startingMode: 'remote', + }) + expect(args).toContain('--existing-session-id') + expect(args[args.indexOf('--existing-session-id') + 1]).toBe('live-hub-row-uuid') + expect(args).not.toContain('--hapi-session-id') + } + }) + + it('prefers existingSessionId over reservedSessionId when both are set', () => { + const args = buildCliArgs('claude', { + directory: '/tmp', + existingSessionId: 'live-row', + reservedSessionId: 'stub-row', + }) + expect(args).toContain('--existing-session-id') + expect(args).not.toContain('--hapi-session-id') + }) + + it('does not emit --hapi-session-id for a non-pty flavor that already uses --existing-session-id', () => { const args = buildCliArgs('opencode', { directory: '/tmp', existingSessionId: 'existing-hub-id', startingMode: 'remote', }) expect(args).not.toContain('--hapi-session-id') + expect(args).toContain('--existing-session-id') }) }) diff --git a/cli/src/runner/controlClient.ts b/cli/src/runner/controlClient.ts index 3924a9edca..e3450ab62a 100644 --- a/cli/src/runner/controlClient.ts +++ b/cli/src/runner/controlClient.ts @@ -96,9 +96,9 @@ export async function listRunnerSessions(): Promise { return result.children || []; } -export async function stopRunnerSession(sessionId: string): Promise<'stopped' | 'already_gone' | 'still_alive'> { +export async function stopRunnerSession(sessionId: string): Promise<'stopped' | 'already_gone' | 'still_alive' | 'unknown'> { const result = await runnerPost('/stop-session', { sessionId }); - return result.status === 'stopped' || result.status === 'already_gone' || result.status === 'still_alive' + return result.status === 'stopped' || result.status === 'already_gone' || result.status === 'still_alive' || result.status === 'unknown' ? result.status : 'still_alive'; } diff --git a/cli/src/runner/controlServer.ts b/cli/src/runner/controlServer.ts index 358f863e2e..1a790cae63 100644 --- a/cli/src/runner/controlServer.ts +++ b/cli/src/runner/controlServer.ts @@ -19,7 +19,7 @@ export function startRunnerControlServer({ onHappySessionWebhook }: { getChildren: () => TrackedSession[]; - stopSession: (sessionId: string) => Promise<'stopped' | 'already_gone' | 'still_alive'>; + stopSession: (sessionId: string) => Promise<'stopped' | 'already_gone' | 'still_alive' | 'unknown'>; spawnSession: (options: SpawnSessionOptions) => Promise; requestShutdown: () => void; onHappySessionWebhook: (sessionId: string, metadata: Metadata) => void; @@ -91,7 +91,7 @@ export function startRunnerControlServer({ }), response: { 200: z.object({ - status: z.enum(['stopped', 'already_gone', 'still_alive']) + status: z.enum(['stopped', 'already_gone', 'still_alive', 'unknown']) }) } } diff --git a/cli/src/runner/lateRunnerWebhook.test.ts b/cli/src/runner/lateRunnerWebhook.test.ts new file mode 100644 index 0000000000..c8d993d083 --- /dev/null +++ b/cli/src/runner/lateRunnerWebhook.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { decideUntrackedRunnerWebhook } from './lateRunnerWebhook' + +describe('decideUntrackedRunnerWebhook', () => { + it('adopts shared Codex roots instead of killing (siblings)', () => { + expect(decideUntrackedRunnerWebhook({ + concurrentClients: true, + timedOutByThisRunner: false, + })).toBe('adopt') + expect(decideUntrackedRunnerWebhook({ + concurrentClients: true, + timedOutByThisRunner: true, + })).toBe('adopt') + }) + + it('kills nonshared CLIs that this runner timed out', () => { + expect(decideUntrackedRunnerWebhook({ + concurrentClients: false, + timedOutByThisRunner: true, + })).toBe('kill') + }) + + it('adopts nonshared CLIs after runner restart before webhook', () => { + // No timeout stamp on the new runner generation — ignoring would leave + // an unreapable process (no argv session id, no durable PID map). + expect(decideUntrackedRunnerWebhook({ + concurrentClients: false, + timedOutByThisRunner: false, + })).toBe('adopt') + }) +}) diff --git a/cli/src/runner/lateRunnerWebhook.ts b/cli/src/runner/lateRunnerWebhook.ts new file mode 100644 index 0000000000..311fffbc7f --- /dev/null +++ b/cli/src/runner/lateRunnerWebhook.ts @@ -0,0 +1,26 @@ +/** + * Decision for a runner-spawned session webhook whose PID is not in this + * runner's in-memory TrackedSession map. + * + * - Shared Codex: never kill (siblings); adopt so StopSession can find the PID. + * - Nonshared + timed out by this runner generation: terminate (ghost after + * webhook timeout). + * - Nonshared + not timed out (typical after runner restart mid-bootstrap): + * durably adopt — Claude often has no HAPI id on argv yet, so ignoring the + * webhook leaves an unreapable detached CLI (#1910 / #1911). + */ + +export type UntrackedRunnerWebhookDecision = 'kill' | 'adopt' + +export function decideUntrackedRunnerWebhook(opts: { + concurrentClients: boolean + timedOutByThisRunner: boolean +}): UntrackedRunnerWebhookDecision { + if (opts.concurrentClients) { + return 'adopt' + } + if (opts.timedOutByThisRunner) { + return 'kill' + } + return 'adopt' +} diff --git a/cli/src/runner/orphanReap.test.ts b/cli/src/runner/orphanReap.test.ts new file mode 100644 index 0000000000..89d5209162 --- /dev/null +++ b/cli/src/runner/orphanReap.test.ts @@ -0,0 +1,438 @@ +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { + commandMatchesRunnerSpawnedSession, + findRunnerSpawnedOrphanTargets, + findStopSessionOrphanTargets, + reapRunnerSpawnedOrphans, + selectOrphanPidsForSession, + selectOrphanTargetsForSession, +} from './orphanReap' +import { + WINDOWS_CIM_CREATION_DATE_MARKER_EXPR, + windowsProcessListCimCommand, + windowsProcessMarkerCimCommand, +} from '@/utils/process' + +describe('orphanReap argv matching', () => { + const sessionId = 'sess-abc-123' + + it('matches runner-spawned CLI with --existing-session-id', () => { + const cmd = `bun src/index.ts claude --hapi-starting-mode remote --started-by runner --existing-session-id ${sessionId}` + expect(commandMatchesRunnerSpawnedSession(cmd, sessionId)).toBe(true) + }) + + it('matches --hapi-session-id= form', () => { + const cmd = `hapi cursor --started-by=runner --hapi-session-id=${sessionId}` + expect(commandMatchesRunnerSpawnedSession(cmd, sessionId)).toBe(true) + }) + + it('rejects terminal-started sessions', () => { + const cmd = `bun src/index.ts claude --started-by terminal --existing-session-id ${sessionId}` + expect(commandMatchesRunnerSpawnedSession(cmd, sessionId)).toBe(false) + }) + + it('rejects --resume native id that merely equals the session string', () => { + // Guard: native agent resume ids must not be treated as HAPI row ids. + const cmd = `bun src/index.ts cursor --resume ${sessionId} --started-by runner` + expect(commandMatchesRunnerSpawnedSession(cmd, sessionId)).toBe(false) + }) + + it('rejects substring false positives', () => { + const cmd = `bun src/index.ts claude --started-by runner --existing-session-id ${sessionId}-extra` + expect(commandMatchesRunnerSpawnedSession(cmd, sessionId)).toBe(false) + }) + + it('treats empty Windows process-list stdout as scan_failed (not no-orphans)', async () => { + // listWindowsProcessesWithCommandLine throws on empty stdout; catch → + // scan_failed so archive does not fail-open as already_gone (#1911). + const found = await findRunnerSpawnedOrphanTargets(sessionId, async () => { + throw new Error('powershell Win32_Process returned empty stdout') + }) + expect(found).toBe('scan_failed') + }) + + it('treats signalled ps (status null) as scan_failed (#1911 Overseer B1)', async () => { + const found = await findRunnerSpawnedOrphanTargets(sessionId, async () => { + throw new Error('ps aborted (signal)') + }) + expect(found).toBe('scan_failed') + }) + + it('selectOrphanPidsForSession filters non-hapi and self', () => { + const pids = selectOrphanPidsForSession( + [ + { pid: 1, cmd: 'systemd', name: 'systemd' }, + { pid: 42, cmd: `bun src/index.ts claude --started-by runner --existing-session-id ${sessionId}`, name: 'bun' }, + { pid: 43, cmd: `bun src/index.ts claude --started-by runner --existing-session-id other`, name: 'bun' }, + { pid: process.pid, cmd: `bun src/index.ts claude --started-by runner --existing-session-id ${sessionId}`, name: 'bun' }, + ], + sessionId + ) + expect(pids).toEqual([42]) + }) + + it('selectOrphanPidsForSession coerces string PIDs from win32 ps-list', () => { + const pids = selectOrphanPidsForSession( + [ + { + // ps-list has returned string PIDs on Windows; Number.isFinite("42") is false + pid: '42' as unknown as number, + cmd: `hapi.exe cursor --started-by runner --existing-session-id ${sessionId}`, + name: 'hapi.exe', + }, + ], + sessionId + ) + expect(pids).toEqual([42]) + }) + + it('matches win32 CIM snapshots that include CommandLine', () => { + const pids = selectOrphanPidsForSession( + [ + { + pid: 99, + name: 'hapi.exe', + cmd: `"C:\\\\Temp\\\\hapi.exe" /c keep.cmd --started-by runner --existing-session-id ${sessionId}`, + }, + { + pid: 100, + name: 'hapi.exe', + cmd: '', // name-only (fastlist shape) cannot argv-match + }, + ], + sessionId + ) + expect(pids).toEqual([99]) + }) +}) + +describe('reapRunnerSpawnedOrphans (stopSession orphan path)', () => { + const stableMarker = (marker = 'gen-a') => { + let reads = 0 + return () => { + reads++ + // Capture + re-check both return the same generation. + void reads + return marker + } + } + + it('returns null when no argv orphans match', async () => { + const status = await reapRunnerSpawnedOrphans('missing-session', { + findOrphans: async () => [], + killTree: async () => { + throw new Error('should not kill') + }, + getStartMarker: () => 'unused', + }) + expect(status).toBeNull() + }) + + it('kills matching orphan PIDs and returns stopped when maps would have missed', async () => { + const killed: number[] = [] + const status = await reapRunnerSpawnedOrphans('sess-orphan-1', { + findOrphans: async (sessionId) => { + expect(sessionId).toBe('sess-orphan-1') + return [4242, 4243] + }, + killTree: async (pid) => { + killed.push(pid) + return true + }, + getStartMarker: stableMarker('gen-stable'), + }) + expect(status).toBe('stopped') + expect(killed).toEqual([4242, 4243]) + }) + + it('returns still_alive when the process scan fails', async () => { + const status = await reapRunnerSpawnedOrphans('sess-orphan-scan-fail', { + findOrphans: async () => 'scan_failed', + killTree: async () => { + throw new Error('should not kill') + }, + }) + expect(status).toBe('still_alive') + }) + + it('returns still_alive when tree-kill cannot prove death', async () => { + const status = await reapRunnerSpawnedOrphans('sess-orphan-2', { + findOrphans: async () => [9999], + killTree: async () => false, + getStartMarker: stableMarker(), + }) + expect(status).toBe('still_alive') + }) + + it('does not kill when start marker changes between capture and kill (PID reuse)', async () => { + const killed: number[] = [] + const markers = new Map([ + // First read = capture after argv match; second = pre-kill re-check + [4242, ['gen-orphan', 'gen-reused-unrelated']], + ]) + const status = await reapRunnerSpawnedOrphans('sess-orphan-reuse', { + findOrphans: async () => [4242], + killTree: async (pid) => { + killed.push(pid) + return true + }, + getStartMarker: (pid) => { + const queue = markers.get(pid) + if (!queue || queue.length === 0) return null + return queue.shift() ?? null + }, + isAlive: () => true, + }) + expect(killed).toEqual([]) + // Matched generation is gone (PID reused) — treat as resolved, not a kill. + expect(status).toBe('stopped') + }) + + it('returns still_alive when pre-kill marker probe fails while PID is alive', async () => { + const killed: number[] = [] + const markers = new Map>([ + [4242, ['gen-orphan', null]], + ]) + const status = await reapRunnerSpawnedOrphans('sess-orphan-probe-fail', { + findOrphans: async () => [4242], + killTree: async (pid) => { + killed.push(pid) + return true + }, + getStartMarker: (pid) => { + const queue = markers.get(pid) + if (!queue || queue.length === 0) return null + return queue.shift() ?? null + }, + isAlive: () => true, + }) + expect(killed).toEqual([]) + // Null re-check is not proof of death — must not claim stopped. + expect(status).toBe('still_alive') + }) + + it('returns still_alive without killing when marker cannot be read for a live orphan', async () => { + const killed: number[] = [] + const status = await reapRunnerSpawnedOrphans('sess-orphan-no-marker', { + findOrphans: async () => [5555], + killTree: async (pid) => { + killed.push(pid) + return true + }, + getStartMarker: () => null, + isAlive: () => true, + }) + expect(killed).toEqual([]) + expect(status).toBe('still_alive') + }) + + it('uses same-snapshot startMarker as expected (not a later probe) for PID-reuse guard', async () => { + const killed: number[] = [] + const status = await reapRunnerSpawnedOrphans('sess-orphan-snapshot', { + findTargets: async () => [ + { pid: 4242, startMarker: 'snapshot-gen' }, + ], + killTree: async (pid) => { + killed.push(pid) + return true + }, + // Re-check only — must not be consulted for the expected marker. + getStartMarker: () => 'snapshot-gen', + isAlive: () => true, + }) + expect(status).toBe('stopped') + expect(killed).toEqual([4242]) + }) + + it('continues other orphans when one live PID has an unreadable marker', async () => { + const killed: number[] = [] + const status = await reapRunnerSpawnedOrphans('sess-orphan-partial', { + findTargets: async () => [ + { pid: 1111, startMarker: null }, + { pid: 2222, startMarker: 'gen-ok' }, + ], + killTree: async (pid) => { + killed.push(pid) + return true + }, + getStartMarker: (pid) => (pid === 2222 ? 'gen-ok' : null), + isAlive: () => true, + }) + expect(killed).toEqual([2222]) + expect(status).toBe('still_alive') + }) + + it('PID-filter on findTargets retains same-snapshot markers (runner stopSession shape)', async () => { + // Regression: run.ts used findOrphans → number[], which forced a later + // getStartMarker probe and discarded the argv-snapshot marker (#1911 bot). + const killed: number[] = [] + const protectedPids = new Set([1111]) + const status = await reapRunnerSpawnedOrphans('sess-orphan-filter', { + findTargets: async () => { + const found = [ + { pid: 1111, startMarker: 'snap-protected' }, + { pid: 2222, startMarker: 'snap-orphan' }, + ] + return found.filter((t) => !protectedPids.has(t.pid)) + }, + killTree: async (pid) => { + killed.push(pid) + return true + }, + // Recheck only — expected marker must stay 'snap-orphan', not this. + getStartMarker: () => 'snap-orphan', + isAlive: () => true, + }) + expect(killed).toEqual([2222]) + expect(status).toBe('stopped') + }) + + it('production findStopSessionOrphanTargets→reap keeps list snapshot markers (not a later probe)', async () => { + // Drives the same helper run.ts uses — not an injected findOrphans seam. + const sessionId = 'sess-prod-wiring' + const snapshotIso = '2026-09-24T15:00:00.0000000Z' + const listProcesses = async () => [ + { + pid: 1111, + name: 'hapi', + cmd: `hapi cursor --started-by runner --existing-session-id ${sessionId}`, + startMarker: 'snap-protected', + }, + { + pid: 2222, + name: 'hapi', + cmd: `hapi cursor --started-by runner --existing-session-id ${sessionId}`, + startMarker: snapshotIso, + }, + ] + const protectedPids = new Set([1111]) + const killed: number[] = [] + let probeCalls = 0 + + const status = await reapRunnerSpawnedOrphans(sessionId, { + findTargets: (id) => findStopSessionOrphanTargets( + id, + (_sid, pid) => protectedPids.has(pid), + listProcesses + ), + killTree: async (pid) => { + killed.push(pid) + return true + }, + getStartMarker: (pid) => { + probeCalls++ + // Recheck only — if expected came from a post-find probe, a wrong + // generation could slip through. Snapshot must already be snapshotIso. + return pid === 2222 ? snapshotIso : 'snap-protected' + }, + isAlive: () => true, + }) + + expect(killed).toEqual([2222]) + expect(status).toBe('stopped') + // One recheck per remaining target (protected filtered out before reap). + expect(probeCalls).toBe(1) + }) +}) + +describe('run.ts stopSession orphan wiring (production callers)', () => { + it('imports findStopSessionOrphanTargets and does not call PID-only discovery', () => { + // Fails on 63bf8b25f (imported findRunnerSpawnedOrphanPids + findOrphans). + const runSrc = readFileSync( + join(dirname(fileURLToPath(import.meta.url)), 'run.ts'), + 'utf8' + ) + expect(runSrc).toContain('findStopSessionOrphanTargets') + expect(runSrc).not.toMatch(/\bfindRunnerSpawnedOrphanPids\b/) + expect(runSrc).not.toMatch(/findOrphans\s*:/) + // Both sweep sites must wire findTargets through the production helper. + const findTargetsSites = runSrc.match(/findTargets:\s*\(id\)\s*=>\s*findStopSessionOrphanTargets/g) ?? [] + expect(findTargetsSites.length).toBe(2) + }) +}) + +describe('Windows orphan startMarker format agreement', () => { + /** Pre-fix list command: raw DateTime → ConvertTo-Json (/Date(...)/ on WinPS 5.1). */ + const BROKEN_LIST_COMMAND = + 'Get-CimInstance Win32_Process | Select-Object ProcessId,Name,CommandLine,CreationDate | ConvertTo-Json -Compress' + /** Pre-fix probe: culture ToString / WMIC DMTF — not ISO 'o'. */ + const BROKEN_PROBE_COMMAND = + '(Get-CimInstance Win32_Process -Filter "ProcessId = 4242").CreationDate' + + it('list + single-probe CIM commands stringify CreationDate the same way', () => { + expect(WINDOWS_CIM_CREATION_DATE_MARKER_EXPR).toContain("ToString('o')") + expect(windowsProcessListCimCommand()).toContain(WINDOWS_CIM_CREATION_DATE_MARKER_EXPR) + expect(windowsProcessListCimCommand()).toContain("$ErrorActionPreference='Stop'") + expect(windowsProcessListCimCommand()).toContain('trap { exit 1 }') + const probe = windowsProcessMarkerCimCommand(4242) + expect(probe).toContain("CreationDate.ToUniversalTime().ToString('o')") + expect(probe).toContain('ProcessId = 4242') + expect(windowsProcessListCimCommand()).toMatch(/CreationDate.*ToString\('o'\)/) + }) + + it('broken WinPS shapes disagree; fixed list marker equals fixed probe marker', () => { + // Simulate the two producers Overseer called out: listing vs recheck. + // WinPS 5.1 ConvertTo-Json of DateTime: + const listFromBrokenConvertToJson = '/Date(1727182800000)/' + // Default DateTime stdout / culture ToString (not ISO 'o'): + const probeFromBrokenPropertyPrint = '9/24/2026 3:00:00 PM' + expect(listFromBrokenConvertToJson).not.toBe(probeFromBrokenPropertyPrint) + + const iso = '2026-09-24T15:00:00.0000000Z' + // Fixed path: both sides emit the same ToString('o') string. + expect(iso).toBe(iso) + expect(BROKEN_LIST_COMMAND).not.toContain("ToString('o')") + expect(BROKEN_PROBE_COMMAND).not.toContain("ToString('o')") + expect(windowsProcessListCimCommand()).toContain("ToString('o')") + expect(windowsProcessMarkerCimCommand(4242)).toContain("ToString('o')") + + // End-to-end: targets selected from a CIM-shaped list row, then reaped + // with a probe that returns the *same* ISO string (two code paths). + const sessionId = 'sess-win-marker' + const targets = selectOrphanTargetsForSession( + [ + { + pid: 4242, + name: 'hapi.exe', + cmd: `hapi.exe cursor --started-by runner --existing-session-id ${sessionId}`, + startMarker: iso, // as listWindowsProcessesWithCommandLine would set + }, + ], + sessionId + ) + expect(targets).toEqual([{ pid: 4242, startMarker: iso }]) + }) + + it('reap with list ISO expected + probe ISO current kills; mismatch skips', async () => { + const iso = '2026-09-24T15:00:00.0000000Z' + const killedMatch: number[] = [] + const ok = await reapRunnerSpawnedOrphans('sess-win-agree', { + findTargets: async () => [{ pid: 1, startMarker: iso }], + killTree: async (pid) => { + killedMatch.push(pid) + return true + }, + getStartMarker: () => iso, // windowsProcessMarkerCimCommand output + isAlive: () => true, + }) + expect(ok).toBe('stopped') + expect(killedMatch).toEqual([1]) + + const killedMismatch: number[] = [] + const skipped = await reapRunnerSpawnedOrphans('sess-win-disagree', { + findTargets: async () => [{ pid: 2, startMarker: iso }], + killTree: async (pid) => { + killedMismatch.push(pid) + return true + }, + // Broken probe shape vs ISO list → treat as generation gone, no kill. + getStartMarker: () => '/Date(1727182800000)/', + isAlive: () => true, + }) + expect(skipped).toBe('stopped') + expect(killedMismatch).toEqual([]) + }) +}) diff --git a/cli/src/runner/orphanReap.ts b/cli/src/runner/orphanReap.ts new file mode 100644 index 0000000000..831f9f2ff5 --- /dev/null +++ b/cli/src/runner/orphanReap.ts @@ -0,0 +1,332 @@ +/** + * Argv-based discovery of runner-spawned driver CLI processes that are no + * longer present in the runner's in-memory / resume-process maps. + * + * Detached children (PPID=1 after KillMode=process runner bounce) can survive + * with no tracking entry. `stopSession` must still be able to reap them when + * the hub archives by HAPI session id — matching `--started-by runner` plus + * an explicit `--existing-session-id` / `--hapi-session-id` flag. + * + * Windows: do not use ps-list. Its fastlist vendor binary is often missing from + * single-exe bundles and never returns CommandLine — argv matching needs CIM. + */ + +import spawn from 'cross-spawn' +import { getProcessStartMarker, isProcessAlive, windowsProcessListCimCommand } from '@/utils/process' + +export type ProcessSnapshot = { + pid: number + cmd?: string + name?: string + /** + * Generation identity from the *same* listing that produced argv match. + * POSIX: `ps` lstart (LC_ALL=C TZ=UTC) — same format as getProcessStartMarker. + * Windows: Win32_Process.CreationDate as UTC ISO 'o' — same as getProcessStartMarker. + */ + startMarker?: string | null +} + +export type OrphanTarget = { + pid: number + /** Marker captured in the argv-match snapshot (not a later probe). */ + startMarker: string | null +} + +/** True when cmd looks like a HAPI driver CLI (binary or bun/node src/index.ts). */ +export function isHapiDriverCliCommand(cmd: string, name = ''): boolean { + const isHappyBinary = name === 'hapi' || name === 'hapi.exe' || /\bhapi(\.exe)?\b/.test(cmd) + const isDevMode = cmd.includes('src/index.ts') + return ( + isHappyBinary + || isDevMode + || name.includes('happy') + || (name === 'node' && cmd.includes('happy-cli')) + || cmd.includes('happy-coder') + ) +} + +/** + * Token-aware match: session id must appear as its own argv token (or after + * `--existing-session-id=` / `--hapi-session-id=`), and the process must claim + * `--started-by runner` (or `--started-by=runner`). + */ +export function commandMatchesRunnerSpawnedSession(cmd: string, sessionId: string): boolean { + if (!sessionId || !cmd.includes('--started-by')) return false + if (!/(?:^|\s)--started-by(?:\s+|=)runner(?:\s|$)/.test(cmd)) return false + + const escaped = sessionId.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + // Require an explicit HAPI id flag. Do not match bare `--resume ` + // — that is the agent session, not the HAPI row id. + return new RegExp( + `(?:^|\\s)(?:--existing-session-id|--hapi-session-id)(?:\\s+|=)${escaped}(?:\\s|$)` + ).test(cmd) +} + +export function selectOrphanTargetsForSession( + processes: ProcessSnapshot[], + sessionId: string, + selfPid: number = process.pid +): OrphanTarget[] { + const targets: OrphanTarget[] = [] + const self = Number(selfPid) + for (const proc of processes) { + // Coerce: some listers return string PIDs; Number.isFinite("123") is false. + const pid = typeof proc.pid === 'number' ? proc.pid : Number(proc.pid) + if (!Number.isFinite(pid) || pid <= 0) continue + if (pid === self) continue + const cmd = proc.cmd || '' + const name = proc.name || '' + if (!isHapiDriverCliCommand(cmd, name)) continue + if (!commandMatchesRunnerSpawnedSession(cmd, sessionId)) continue + targets.push({ + pid, + startMarker: proc.startMarker === undefined ? null : proc.startMarker, + }) + } + return targets +} + +/** @deprecated Prefer selectOrphanTargetsForSession (keeps same-snapshot markers). */ +export function selectOrphanPidsForSession( + processes: ProcessSnapshot[], + sessionId: string, + selfPid: number = process.pid +): number[] { + return selectOrphanTargetsForSession(processes, sessionId, selfPid).map((t) => t.pid) +} + +/** + * POSIX orphan listing: one `ps` snapshot with pid + lstart + args. + * Uses the same LC_ALL/TZ as getProcessStartMarker so re-check compares equal. + */ +export function listPosixProcessesWithStartMarker(): ProcessSnapshot[] { + const result = spawn.sync( + 'ps', + ['awwxo', 'pid=,lstart=,args='], + { + encoding: 'utf8', + env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' }, + maxBuffer: 64 * 1024 * 1024, + timeout: 10_000, + } + ) + if (result.error || result.status !== 0) { + // status === null means the ps child was signal-killed; stdout is + // truncated and must not be parsed as a complete process table + // (#1911 Overseer B1 — twin of process.ts collectProcessTree). + throw result.error ?? new Error( + result.status === null ? 'ps aborted (signal)' : `ps exit ${result.status}` + ) + } + const stdout = (result.stdout ?? '').toString() + const snapshots: ProcessSnapshot[] = [] + for (const line of stdout.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + // pid, then lstart "Day Mon DD HH:MM:SS YYYY" (5 tokens), then args + const match = trimmed.match( + /^(\d+)\s+(\w{3}\s+\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.*)$/ + ) + if (!match) continue + const pid = Number(match[1]) + if (!Number.isFinite(pid) || pid <= 0) continue + snapshots.push({ + pid, + startMarker: match[2], + cmd: match[3] ?? '', + name: '', + }) + } + return snapshots +} + +/** + * Shared Win32 generation marker helpers live in `@/utils/process` so list + + * single-PID probe stay format-identical. Re-export for orphanReap tests. + */ +export { + WINDOWS_CIM_CREATION_DATE_MARKER_EXPR, + windowsProcessListCimCommand, + windowsProcessMarkerCimCommand, +} from '@/utils/process' + +/** Win32 process list with CommandLine + CreationDate for argv orphan matching. */ +export function listWindowsProcessesWithCommandLine(): ProcessSnapshot[] { + const result = spawn.sync( + 'powershell', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + windowsProcessListCimCommand(), + ], + { encoding: 'utf8', windowsHide: true, maxBuffer: 64 * 1024 * 1024, timeout: 10_000 } + ) + if (result.error || result.status !== 0) { + throw result.error ?? new Error(`powershell Win32_Process exit ${result.status}`) + } + const raw = (result.stdout ?? '').trim() + // Empty stdout is a failed scan, not "no processes" — a live Windows host + // always has System/Idle. Match process.ts collectWindowsProcessTree + // fail-closed semantics so orphan reap does not claim archive-ok (#1911). + if (!raw) { + throw new Error('powershell Win32_Process returned empty stdout') + } + const parsed = JSON.parse(raw) as + | Array<{ ProcessId?: number; Name?: string; CommandLine?: string; CreationDate?: string }> + | { ProcessId?: number; Name?: string; CommandLine?: string; CreationDate?: string } + const rows = Array.isArray(parsed) ? parsed : [parsed] + return rows + .map((row) => ({ + pid: Number(row.ProcessId), + name: row.Name ?? '', + cmd: row.CommandLine ?? '', + startMarker: typeof row.CreationDate === 'string' && row.CreationDate.length > 0 + ? row.CreationDate + : null, + })) + .filter((proc) => Number.isFinite(proc.pid) && proc.pid > 0) +} + +export async function listProcessesForOrphanScan(): Promise { + if (process.platform === 'win32') { + return listWindowsProcessesWithCommandLine() + } + return listPosixProcessesWithStartMarker() +} + +export async function findRunnerSpawnedOrphanTargets( + sessionId: string, + listProcesses: () => Promise = listProcessesForOrphanScan +): Promise { + try { + const processes = await listProcesses() + return selectOrphanTargetsForSession(processes, sessionId) + } catch { + return 'scan_failed' + } +} + +/** + * Production stopSession orphan discovery: argv match + same-snapshot markers, + * then optional PID skip (shared wrappers / sibling roots). Callers must use + * this (or an equivalent findTargets that keeps startMarker) — never strip to + * bare PIDs via findRunnerSpawnedOrphanPids / findOrphans (#1911 Overseer). + */ +export async function findStopSessionOrphanTargets( + sessionId: string, + shouldSkipPid: (sessionId: string, pid: number) => boolean, + listProcesses: () => Promise = listProcessesForOrphanScan +): Promise { + const found = await findRunnerSpawnedOrphanTargets(sessionId, listProcesses) + if (found === 'scan_failed') return found + return found.filter((t) => !shouldSkipPid(sessionId, t.pid)) +} + +export async function findRunnerSpawnedOrphanPids( + sessionId: string, + listProcesses: () => Promise = listProcessesForOrphanScan +): Promise { + const targets = await findRunnerSpawnedOrphanTargets(sessionId, listProcesses) + if (targets === 'scan_failed') return 'scan_failed' + return targets.map((t) => t.pid) +} + +/** + * Tree-kill every argv-matched orphan for `sessionId`. + * Returns null when none were found (caller continues to other stop paths). + * Returns still_alive when the process scan fails — empty is not proof gone. + * + * PID-reuse guard: expectedMarker comes from the *same* snapshot as the argv + * match (not a later getProcessStartMarker call). Re-check immediately before + * each killTree; skip when the marker changed or cannot be read for a live PID. + * + * When one target is unconfirmed, continue attempting the rest — return + * still_alive if any remain ambiguous (#1911 bot Minor). + */ +export async function reapRunnerSpawnedOrphans( + sessionId: string, + deps: { + findTargets?: (sessionId: string) => Promise + /** @deprecated Prefer findTargets (includes same-snapshot markers). */ + findOrphans?: (sessionId: string) => Promise + killTree?: (pid: number) => Promise + getStartMarker?: (pid: number) => string | null + isAlive?: (pid: number) => boolean + } = {} +): Promise<'stopped' | 'still_alive' | null> { + const killTree = deps.killTree ?? (async (pid: number) => { + const { killProcessTreeByPid } = await import('@/utils/process') + return killProcessTreeByPid(pid) + }) + const getStartMarker = deps.getStartMarker ?? getProcessStartMarker + const isAlive = deps.isAlive ?? isProcessAlive + + let targets: OrphanTarget[] + if (deps.findTargets) { + const found = await deps.findTargets(sessionId) + if (found === 'scan_failed') return 'still_alive' + targets = found + } else if (deps.findOrphans) { + // Legacy: pids only — capture markers in a separate probe (tests / older callers). + const orphanPids = await deps.findOrphans(sessionId) + if (orphanPids === 'scan_failed') return 'still_alive' + targets = orphanPids.map((pid) => ({ + pid, + startMarker: getStartMarker(pid), + })) + } else { + const found = await findRunnerSpawnedOrphanTargets(sessionId) + if (found === 'scan_failed') return 'still_alive' + targets = found + } + + if (targets.length === 0) return null + + // killProcessTreeByPid returns false if any collected descendant survives, + // even when the stamped root PID has already exited. Trust that result — + // do not downgrade to stopped based on root liveness alone (#1910 / #1911 B2). + let anyUnconfirmed = false + let resolved = 0 + for (const { pid: orphanPid, startMarker: expectedMarker } of targets) { + if (expectedMarker === null) { + // No generation identity from the match snapshot. If the PID is + // already dead the orphan is gone; if still alive, skip kill and + // continue other targets (do not abort the whole sweep). + if (!isAlive(orphanPid)) { + resolved++ + continue + } + anyUnconfirmed = true + continue + } + + const currentMarker = getStartMarker(orphanPid) + if (currentMarker === null) { + // Probe failed mid-flight. Null is not proof of PID reuse (psutil + // lesson: unknown create_time ≠ recycled). Skip kill; if still + // alive, mark unconfirmed and continue. + if (!isAlive(orphanPid)) { + resolved++ + continue + } + anyUnconfirmed = true + continue + } + if (currentMarker !== expectedMarker) { + // Generation changed (PID reuse) — never kill whatever process now + // holds this PID. The matched orphan generation is gone. + resolved++ + continue + } + + if (!(await killTree(orphanPid))) { + anyUnconfirmed = true + continue + } + resolved++ + } + + if (anyUnconfirmed) return 'still_alive' + return resolved === targets.length ? 'stopped' : 'still_alive' +} diff --git a/cli/src/runner/run.ts b/cli/src/runner/run.ts index c182febe30..eefe64a7f1 100644 --- a/cli/src/runner/run.ts +++ b/cli/src/runner/run.ts @@ -15,6 +15,19 @@ import { spawnHappyCLI } from '@/utils/spawnHappyCLI'; import { writeRunnerState, RunnerLocallyPersistedState, readRunnerState, acquireRunnerLock, releaseRunnerLock } from '@/persistence'; import { getCliArgs } from '@/utils/cliArgs'; import { getProcessStartMarker, isProcessAlive, isWindows, killProcess, killProcessByChildProcess, killProcessTreeByPid } from '@/utils/process'; +import { findStopSessionOrphanTargets, reapRunnerSpawnedOrphans } from '@/runner/orphanReap'; +import { decideUntrackedRunnerWebhook } from '@/runner/lateRunnerWebhook'; +import { + decideKeepWrapperArchive, + decideRawPidStop, + detachSharedRootFromWrapper, + keepWrapperForSharedSiblings, + pidHasActiveSharedRoots, + sessionRegistryBindingState, + sessionRuntimeHasActiveSiblings, + trackedSharedWrapperPidsWithSiblings, + wrapperHasActiveSiblingRoots, +} from '@/runner/sharedSessionStop'; import { PERMISSION_MODES } from '@hapi/protocol/modes'; import { RUNNER_CAPABILITIES } from '@hapi/protocol'; import { withRetry } from '@/utils/time'; @@ -274,6 +287,10 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): // tracking, so confirmed exit can be attributed to the requested HAPI row. const pidToRequestedSessionId = new Map(); const pidToConfirmedSessionId = new Map(); + // Generation-local: PIDs whose TrackedSession was dropped by webhook timeout. + // Late runner webhooks may kill only these — never recovered shared roots + // that merely appear in resume-processes after a runner restart (#1911). + const webhookTimeoutOrphanPids = new Set(); // Only actual observed child exits may create a stop-session tombstone. // Tracking loss (notably webhook timeout) is deliberately not evidence. const exitTombstoneFile = `${configuration.runnerStateFile}.verified-exits.json`; @@ -457,6 +474,21 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): if (persisted) { persisted.confirmedSessionId = sessionId; persistResumeProcesses(); + } else { + // Fresh Claude/etc. spawns often have no reserved HAPI id at spawn + // time, so nothing was persisted then. Once the webhook names the + // row, keep a durable PID mapping so stopSession can still reap + // after in-memory tracking is dropped (#1910). + const processStartMarker = getProcessStartMarker(pid); + if (processStartMarker) { + persistedResumeProcesses.set(pid, { + requestedSessionId: existingSession.requestedHappySessionId ?? sessionId, + confirmedSessionId: sessionId, + pid, + processStartMarker + }); + persistResumeProcesses(); + } } existingSession.happySessionMetadataFromLocalWebhook = sessionMetadata; logger.debug(`[RUNNER RUN] Updated runner-spawned session ${sessionId} with metadata`); @@ -481,22 +513,58 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): // anything claiming `'runner'` here must be the second case and // should be ignored + terminated instead of silently promoted. if (sessionMetadata.startedBy === 'runner') { - // A shared root can report /new after a Runner restart. Unknown is - // not proof of an orphan: never kill its sibling roots. Known spawn - // timeouts already terminate their ChildProcess tree at the source. - // No registry scan/adoption lifecycle is needed for live attachment. - if (sessionMetadata.capabilities?.concurrentClients) return; + // Untracked runner-spawned webhook: either this generation timed the + // spawn out, or the runner restarted before the webhook (no stamp). + // Shared Codex must never be killed here (siblings). Nonshared + // post-restart CLIs must be adopted so StopSession can find them — + // Claude often has no HAPI id on argv yet (#1910 / #1911). + const timedOutByThisRunner = webhookTimeoutOrphanPids.has(pid); + webhookTimeoutOrphanPids.delete(pid); + const decision = decideUntrackedRunnerWebhook({ + concurrentClients: Boolean(sessionMetadata.capabilities?.concurrentClients), + timedOutByThisRunner, + }); + if (decision === 'kill') { + logger.debug( + `[RUNNER RUN] Ignoring late webhook from orphaned runner-spawned PID ${pid} (session ${sessionId}). Terminating child.` + ); + // Use killProcess (SIGTERM → SIGKILL escalation) rather than a + // bare process.kill() so the orphan is reliably reaped even if + // it ignores SIGTERM. We don't have a ChildProcess reference + // here (tracking entry was already removed by the timeout + // handler), so tree-kill via killProcessByChildProcess is not + // available — but the timeout handler should have already + // tree-killed the process group; this is defence-in-depth. + void killProcess(pid); + return; + } + + const processStartMarker = getProcessStartMarker(pid); + const adopted: TrackedSession = { + ...(sessionMetadata.capabilities?.concurrentClients + ? { sharedSessions: { [sessionId]: sessionMetadata } } + : {}), + startedBy: 'runner', + happySessionId: sessionId, + happySessionMetadataFromLocalWebhook: sessionMetadata, + pid, + }; + invalidateVerifiedExit(sessionId); + invalidateVerifiedExit(`PID-${pid}`); + pidToTrackedSession.set(pid, adopted); + pidToConfirmedSessionId.set(pid, sessionId); + if (processStartMarker) { + persistedResumeProcesses.set(pid, { + requestedSessionId: sessionId, + confirmedSessionId: sessionId, + pid, + processStartMarker, + }); + persistResumeProcesses(); + } logger.debug( - `[RUNNER RUN] Ignoring late webhook from orphaned runner-spawned PID ${pid} (session ${sessionId}). Terminating child.` + `[RUNNER RUN] Adopted untracked runner-spawned session ${sessionId} (PID ${pid}) after restart or shared recovery` ); - // Use killProcess (SIGTERM → SIGKILL escalation) rather than a - // bare process.kill() so the orphan is reliably reaped even if - // it ignores SIGTERM. We don't have a ChildProcess reference - // here (tracking entry was already removed by the timeout - // handler), so tree-kill via killProcessByChildProcess is not - // available — but the timeout handler should have already - // tree-killed the process group; this is defence-in-depth. - void killProcess(pid); return; } @@ -534,14 +602,16 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): type: 'error', errorMessage, code: 'agent_unavailable', - agent + agent, + childStarted: false, }; } if (options.validateDirectory && !(await options.validateDirectory(directory))) { return { type: 'error', errorMessage: 'Directory is outside this machine\'s workspace roots', - code: 'outside_workspace_roots' + code: 'outside_workspace_roots', + childStarted: false, }; } const yolo = options.yolo === true; @@ -575,14 +645,16 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): logger.debug(`[RUNNER RUN] Directory creation not approved for: ${directory}`); return { type: 'requestToApproveDirectoryCreation', - directory + directory, + childStarted: false, }; } if (validation.type === 'error') { logger.debug(`[RUNNER RUN] Workspace directory validation failed: ${validation.errorMessage}`); return { type: 'error', - errorMessage: validation.errorMessage + errorMessage: validation.errorMessage, + childStarted: false, }; } directoryCreated = validation.created; @@ -599,7 +671,8 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): logger.debug(`[RUNNER RUN] Worktree base directory missing: ${directory}`); return { type: 'error', - errorMessage: `Worktree sessions require an existing Git repository. Directory not found: ${directory}` + errorMessage: `Worktree sessions require an existing Git repository. Directory not found: ${directory}`, + childStarted: false, }; } } @@ -611,7 +684,8 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): return { type: 'error', errorMessage: 'Directory is outside this machine\'s workspace roots', - code: 'outside_workspace_roots' + code: 'outside_workspace_roots', + childStarted: false, }; } @@ -638,7 +712,8 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): logger.debug(`[RUNNER RUN] Worktree creation failed: ${worktreeResult.error}`); return { type: 'error', - errorMessage: worktreeResult.error + errorMessage: worktreeResult.error, + childStarted: false, }; } worktreeInfo = worktreeResult.info; @@ -782,14 +857,15 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): // The OS process now exists, so this is the point where a new generation // invalidates exit evidence left by an older child with the same HAPI ID. - for (const id of [options.sessionId, options.existingSessionId]) { + for (const id of [options.sessionId, options.existingSessionId, options.reservedSessionId]) { if (id) invalidateVerifiedExit(id); } const pid = happyProcess.pid; - if (options.existingSessionId) { - existingSessionIdByChildPid.set(pid, options.existingSessionId); - spawnSession.markChildAlive(options.existingSessionId); + const trackHubId = options.existingSessionId ?? options.reservedSessionId; + if (trackHubId) { + existingSessionIdByChildPid.set(pid, trackHubId); + spawnSession.markChildAlive(trackHubId); } invalidateVerifiedExit(`PID-${pid}`); logger.debug(`[RUNNER RUN] Spawned process with PID ${pid}`); @@ -826,7 +902,7 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): const trackedSession: TrackedSession = { startedBy: 'runner', pid, - requestedHappySessionId: options.existingSessionId ?? options.sessionId, + requestedHappySessionId: options.existingSessionId ?? options.reservedSessionId ?? options.sessionId, childProcess: happyProcess, directoryCreated, message: directoryCreated ? `The path '${directory}' did not exist. We created a new folder and spawned a new session there.` : undefined @@ -883,43 +959,69 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): // HAPI_RUNNER_WEBHOOK_TIMEOUT_MS for users on slow models // (e.g. opus[1m] --resume). const timeout = setTimeout(() => { - pidToAwaiter.delete(pid); - pidToErrorAwaiter.delete(pid); + void (async () => { + pidToAwaiter.delete(pid); + pidToErrorAwaiter.delete(pid); + + // Remove the tracked session entry so a late-arriving webhook + // from this orphaned PID cannot be silently promoted into a + // ghost session by onHappySessionWebhook(). Keep durable + // resume-process / requested-id maps until the process is + // proven dead so StopSession can still target this PID (#1910). + pidToTrackedSession.delete(pid); + webhookTimeoutOrphanPids.add(pid); + + // Await tree-kill (wrapper + agent grandchildren). Do not fire- + // and-forget: under load an unawaited kill can fail silently and + // leave an immortal detached child. + let treeDead = false; + if (happyProcess) { + try { + treeDead = await killProcessByChildProcess(happyProcess); + } catch (error) { + logger.debug(`[RUNNER RUN] Webhook-timeout tree-kill failed for PID ${pid}:`, error); + } + } + if (!treeDead && isProcessAlive(pid)) { + try { + treeDead = await killProcessTreeByPid(pid); + } catch (error) { + logger.debug(`[RUNNER RUN] Webhook-timeout PID tree-kill failed for ${pid}:`, error); + } + } - // Remove the tracked session entry so a late-arriving webhook - // from this orphaned PID cannot be silently promoted into a - // ghost session by onHappySessionWebhook(). - pidToTrackedSession.delete(pid); - - // Terminate the entire process tree (wrapper + agent - // grandchildren). Using killProcessByChildProcess instead of - // a bare SIGTERM ensures that detached grandchild processes - // (the actual claude/codex agent) are also reaped, and that - // SIGTERM → SIGKILL escalation kicks in if needed. - if (happyProcess) { - void killProcessByChildProcess(happyProcess).finally(() => { - void cleanupCopiedCodexConfig('webhook-timeout'); - }); - } else { - void cleanupCopiedCodexConfig('webhook-timeout'); - } + if (!isProcessAlive(pid)) { + if (trackedSession.requestedHappySessionId) { + rememberVerifiedExit(trackedSession.requestedHappySessionId); + } + rememberVerifiedExit(`PID-${pid}`); + pidToRequestedSessionId.delete(pid); + pidToConfirmedSessionId.delete(pid); + webhookTimeoutOrphanPids.delete(pid); + if (persistedResumeProcesses.delete(pid)) persistResumeProcesses(); + releaseRecoveredSpawnDedupe(pid, existingSessionIdByChildPid, spawnSession); + } - // If this was a worktree session, the worktree can only be - // safely removed after the child has actually exited (the - // child may still be writing to it). Register a one-shot - // exit listener so cleanup happens once the tree-kill lands. - if (worktreeInfo && happyProcess) { - happyProcess.once('exit', () => { - void cleanupWorktree(); - }); - } + await cleanupCopiedCodexConfig('webhook-timeout'); + + // If this was a worktree session, the worktree can only be + // safely removed after the child has actually exited. + if (worktreeInfo && happyProcess) { + happyProcess.once('exit', () => { + void cleanupWorktree(); + }); + if (!isProcessAlive(pid)) { + void cleanupWorktree(); + } + } - logger.debug(`[RUNNER RUN] Session webhook timeout for PID ${pid}`); - logStderrTail(); - resolve({ - type: 'error', - errorMessage: buildWebhookFailureMessage('timeout') - }); + logger.debug(`[RUNNER RUN] Session webhook timeout for PID ${pid}`); + logStderrTail(); + resolve({ + type: 'error', + errorMessage: buildWebhookFailureMessage('timeout') + }); + })(); }, webhookTimeoutMs); // Register awaiter @@ -986,9 +1088,95 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): } // Stop a session by sessionId or PID fallback - const stopSession = async (sessionId: string): Promise<'stopped' | 'already_gone' | 'still_alive'> => { + const stopSession = async ( + sessionId: string, + opts?: { processStartMarker?: string } + ): Promise<'stopped' | 'already_gone' | 'still_alive' | 'unknown'> => { logger.debug(`[RUNNER RUN] Attempting to stop session ${sessionId}`); + // After a mapped/persisted PID path succeeds, still scan argv for other + // generations of the same HAPI id (untracked orphans from an earlier + // runner) before reporting stopped/already_gone (#1910). Skip only PIDs + // that still host active shared siblings — never skip the whole scan. + // Protect tracked wrappers even when the durable runtime registry is + // missing/unreadable (argv would otherwise match the primary session id). + const shouldSkipOrphanPid = ( + liveRuntimes: Parameters[0], + protectedTrackedPids: Set, + id: string, + pid: number + ): boolean => ( + protectedTrackedPids.has(pid) + || wrapperHasActiveSiblingRoots(liveRuntimes, id, pid) + ); + + // Strict registry read for sibling protection — soft [] after parse/readdir + // failure would tree-kill a shared wrapper hosting live roots (#1911 Opus). + // Fail-closed only for Codex stop contexts; other flavors must not become + // permanently un-archivable on a single corrupt runtime JSON (#1911 Major). + const isCodexStopContext = (): boolean => { + for (const [, session] of pidToTrackedSession) { + if (session.sharedSessions?.[sessionId]) return true + } + return false + } + + const readLiveRuntimesForStop = async () => { + try { + return (await readRuntimes({ strict: true })).filter(runtime => + runtime.hub === configuration.apiUrl + && runtime.authHash === runtimeAuthHash() + && runtimeMayBeAlive(runtime) + ); + } catch (error) { + logger.warn( + `[RUNNER RUN] Codex runtime registry unreadable during stop of ${sessionId}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + // KillSession PID-* fallback cannot prove the OS pid is not a shared + // Codex wrapper when the registry is unreadable — soft [] would + // tree-kill sibling roots (#1911 Overseer B2). + if (sessionId.startsWith('PID-')) return null; + // findRuntime also soft-fails; if a shared Codex root may still exist, + // refuse the orphan sweep. Non-Codex stops proceed with [] so archive + // is not machine-wide blocked by schema drift. + try { + const { findRuntime } = await import('@/codex/shared/registry'); + if (await findRuntime(sessionId) || isCodexStopContext()) return null; + } catch { + if (isCodexStopContext()) return null; + } + return []; + } + }; + + const finishWithOrphanSweep = async ( + base: 'stopped' | 'already_gone' | 'unknown' + ): Promise<'stopped' | 'already_gone' | 'still_alive' | 'unknown'> => { + const liveRuntimes = await readLiveRuntimesForStop(); + if (liveRuntimes === null) return 'still_alive'; + const protectedTrackedPids = trackedSharedWrapperPidsWithSiblings( + pidToTrackedSession.entries(), + sessionId + ); + const orphanStatus = await reapRunnerSpawnedOrphans(sessionId, { + findTargets: (id) => findStopSessionOrphanTargets( + id, + (sid, pid) => shouldSkipOrphanPid(liveRuntimes, protectedTrackedPids, sid, pid) + ), + }); + if (orphanStatus === 'still_alive') { + logger.debug(`[RUNNER RUN] Orphan argv sweep left live PIDs for session ${sessionId}`); + return 'still_alive'; + } + if (orphanStatus === 'stopped') { + rememberVerifiedExit(sessionId); + return 'stopped'; + } + return base; + }; + const { findRuntime } = await import('@/codex/shared/registry'); const sharedRuntime = await findRuntime(sessionId); if (sharedRuntime) { @@ -996,24 +1184,174 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): const { runtimeControl } = await import('@/codex/shared/frontend'); await runtimeControl(sharedRuntime, 'hapi/stopSession', sessionId); const tracked = pidToTrackedSession.get(sharedRuntime.pid); - if (tracked?.sharedSessions) delete tracked.sharedSessions[sessionId]; - return 'stopped'; + if (tracked) { + const detach = detachSharedRootFromWrapper(tracked, sessionId); + if (detach.kind === 'keep_wrapper' || keepWrapperForSharedSiblings(tracked, sessionId)) { + // App-server ended this root; sibling roots still need the wrapper. + // Still argv-sweep other generations; PID filter skips this wrapper. + logger.debug( + `[RUNNER RUN] Shared runtime stopped root ${sessionId}; wrapper PID ${sharedRuntime.pid} kept` + ); + return await finishWithOrphanSweep('stopped'); + } + } + // Post-restart: TrackedSession may be gone; registry still lists siblings. + const liveAfterStop = await readLiveRuntimesForStop(); + if (liveAfterStop === null) return 'still_alive'; + if (wrapperHasActiveSiblingRoots(liveAfterStop, sessionId, sharedRuntime.pid)) { + logger.debug( + `[RUNNER RUN] Shared runtime stopped root ${sessionId}; registry siblings keep PID ${sharedRuntime.pid}` + ); + return await finishWithOrphanSweep('stopped'); + } + return await finishWithOrphanSweep('stopped'); } catch { return 'still_alive'; } } - if ((await readRuntimes()).some(runtime => runtime.hub === configuration.apiUrl && runtime.authHash === runtimeAuthHash() - && runtime.sessions[sessionId]?.active && runtimeMayBeAlive(runtime))) return 'still_alive'; - // Missing registry is not permission to kill siblings in a live execution. - if ([...pidToTrackedSession.values()].some(session => session.sharedSessions?.[sessionId])) return 'still_alive'; + { + const probeRuntimes = await readLiveRuntimesForStop(); + if (probeRuntimes === null) return 'still_alive'; + if (probeRuntimes.some(runtime => runtime.sessions[sessionId]?.active)) return 'still_alive'; + } + + // Live Codex runtimes for this hub — used when in-memory sharedSessions + // only knows the root being archived (post-restart adoption of a new root + // while older roots remain active only in the durable registry). + const liveRegistryRuntimes = async () => readLiveRuntimesForStop(); + const registrySiblingsKeepPid = async (pid: number): Promise => { + const live = await liveRegistryRuntimes(); + if (live === null) return 'unreadable'; + return wrapperHasActiveSiblingRoots(live, sessionId, pid); + }; + + // KillSession pid fallback must verify the start marker BEFORE any tracked + // PID match can tree-kill a reused OS pid. + if (sessionId.startsWith('PID-')) { + const pid = parseInt(sessionId.slice(4), 10); + if (Number.isFinite(pid) && pid > 0) { + const liveForPid = await liveRegistryRuntimes(); + if (liveForPid === null) return 'still_alive'; + const decision = decideRawPidStop({ + alive: isProcessAlive(pid), + expectedMarker: opts?.processStartMarker, + currentMarker: getProcessStartMarker(pid), + hasActiveSharedRoots: pidHasActiveSharedRoots(liveForPid, pid), + }); + if (decision === 'already_gone') { + rememberVerifiedExit(sessionId); + return 'already_gone'; + } + if (decision === 'unknown') { + logger.debug( + `[RUNNER RUN] Raw PID ${pid} stop unconfirmed (missing/mismatched start marker or probe failed)` + ); + return 'unknown'; + } + if (decision === 'keep_shared') { + logger.debug( + `[RUNNER RUN] PID ${pid} still hosts active shared roots; not tree-killing` + ); + return await finishWithOrphanSweep('stopped'); + } + if (!(await killProcessTreeByPid(pid))) return 'still_alive'; + rememberVerifiedExit(sessionId); + return await finishWithOrphanSweep('stopped'); + } + return 'unknown'; + } + + // After KillSession, findRuntime may miss an inactive binding. Detach the + // root from sharedSessions without tree-killing siblings. An inactive + // registry binding is stop evidence (Codex KillSession already archived + // the root); absent evidence stays unknown across retries. + const finishKeepWrapperDetach = async (pid: number): Promise<'stopped' | 'already_gone' | 'still_alive' | 'unknown'> => { + const live = await liveRegistryRuntimes(); + if (live === null) return 'still_alive'; + const binding = sessionRegistryBindingState(live, sessionId, pid); + if (binding === 'active') return 'still_alive'; + // Base unknown so an argv orphan reap returning stopped is distinguishable + // from "no orphans" (which would otherwise echo a stopped base). + const orphan = await finishWithOrphanSweep('unknown'); + if (orphan === 'still_alive') return 'still_alive'; + if (orphan === 'stopped') return 'stopped'; + const decision = decideKeepWrapperArchive(binding); + if (decision === 'stopped') { + logger.debug( + `[RUNNER RUN] Detached shared root ${sessionId}; inactive registry binding confirms stop; PID ${pid} kept` + ); + } else { + logger.debug( + `[RUNNER RUN] Detached shared root ${sessionId} from PID ${pid}; stop unconfirmed without registry evidence` + ); + } + return decision; + }; + + for (const [pid, session] of pidToTrackedSession.entries()) { + if (!session.sharedSessions?.[sessionId]) continue; + if (detachSharedRootFromWrapper(session, sessionId).kind === 'keep_wrapper') { + return await finishKeepWrapperDetach(pid); + } + // In-memory map had only this root (typical after restart adoption of a + // newly reported root). Registry may still list older active siblings. + if (await registrySiblingsKeepPid(pid) !== false) { + return await finishKeepWrapperDetach(pid); + } + // Last shared entry removed — fall through so the wrapper can be stopped. + break; + } - // Try to find by sessionId first + // Try to find by sessionId first (never match raw PID- here — handled above). for (const [pid, session] of pidToTrackedSession.entries()) { if (session.happySessionId === sessionId || - session.requestedHappySessionId === sessionId || - (sessionId.startsWith('PID-') && pid === parseInt(sessionId.replace('PID-', '')))) { + session.requestedHappySessionId === sessionId) { + + // Primary match, but live shared siblings still use this wrapper + // (KillSession may already have cleared this id from sharedSessions). + if (keepWrapperForSharedSiblings(session, sessionId)) { + return await finishKeepWrapperDetach(pid); + } - if (session.startedBy === 'runner' && session.childProcess) { + // Post-restart: TrackedSession may only list the newly reported root + // while older roots remain active in the durable registry on this PID. + // Archiving the new root must not tree-kill those siblings. + if (await registrySiblingsKeepPid(pid) !== false) { + detachSharedRootFromWrapper(session, sessionId); + return await finishKeepWrapperDetach(pid); + } + + if (session.startedBy === 'runner') { + // Adopted post-restart sessions have no ChildProcess handle — still + // tree-kill so agent grandchildren cannot outlive the wrapper. + // Require a persisted start marker; without it (or on mismatch), do + // not kill by tracked PID — fall through to argv discovery. + if (!session.childProcess) { + const persisted = persistedResumeProcesses.get(pid); + if (!persisted?.processStartMarker) { + logger.debug( + `[RUNNER RUN] Adopted PID ${pid} has no start marker; refusing tracked kill for ${sessionId}` + ); + const orphan = await finishWithOrphanSweep('unknown'); + if (orphan === 'still_alive') return 'still_alive'; + if (orphan === 'stopped') return 'stopped'; + return 'unknown'; + } + const currentMarker = getProcessStartMarker(pid); + if (currentMarker === null || currentMarker !== persisted.processStartMarker) { + logger.debug( + `[RUNNER RUN] Adopted PID ${pid} generation mismatch; dropping stale tracking for ${sessionId}` + ); + pidToTrackedSession.delete(pid); + pidToRequestedSessionId.delete(pid); + pidToConfirmedSessionId.delete(pid); + if (persistedResumeProcesses.delete(pid)) persistResumeProcesses(); + releaseRecoveredSpawnDedupe(pid, existingSessionIdByChildPid, spawnSession); + continue; + } + } try { - const treeStopped = await killProcessByChildProcess(session.childProcess); + const treeStopped = session.childProcess + ? await killProcessByChildProcess(session.childProcess) + : await killProcessTreeByPid(pid); if (!treeStopped) { logger.debug(`[RUNNER RUN] Process tree for session ${sessionId} is still alive after stop request`); return 'still_alive'; @@ -1056,7 +1394,7 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): pidToConfirmedSessionId.delete(pid); if (persistedResumeProcesses.delete(pid)) persistResumeProcesses(); logger.debug(`[RUNNER RUN] Removed terminated session ${sessionId} from tracking`); - return 'stopped'; + return await finishWithOrphanSweep('stopped'); } } @@ -1078,14 +1416,32 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): const currentMarker = getProcessStartMarker(pid); if (currentMarker === null) return 'still_alive'; if (currentMarker !== persisted.processStartMarker) { + // PID reuse: drop the stale mapping, but do NOT claim already_gone + // or write a verified-exit tombstone — the HAPI CLI for this session + // may still be alive under a different PID (#1910). Continue so + // other fallback PIDs / argv orphan scan can still reap it. persistedResumeProcesses.delete(pid); persistResumeProcesses(); pidToRequestedSessionId.delete(pid); pidToConfirmedSessionId.delete(pid); - if (requestedSessionId) rememberVerifiedExit(requestedSessionId); - if (confirmedSessionId) rememberVerifiedExit(confirmedSessionId); releaseRecoveredSpawnDedupe(pid, existingSessionIdByChildPid, spawnSession); - return 'already_gone'; + continue; + } + const liveForPid = await liveRegistryRuntimes(); + if (liveForPid === null) return 'still_alive'; + if (wrapperHasActiveSiblingRoots(liveForPid, sessionId, pid)) { + // Keep this shared wrapper; siblings alone are not stop proof for + // this root — require an inactive registry binding (KillSession ack). + const binding = sessionRegistryBindingState(liveForPid, sessionId, pid); + if (binding === 'active') return 'still_alive'; + const orphan = await finishWithOrphanSweep('unknown'); + if (orphan === 'still_alive') return 'still_alive'; + if (orphan === 'stopped') return 'stopped'; + const decision = decideKeepWrapperArchive(binding); + logger.debug( + `[RUNNER RUN] Persisted PID ${pid} hosts active shared siblings for ${sessionId}; archive=${decision}` + ); + return decision; } if (!(await killProcessTreeByPid(pid))) return 'still_alive'; if (requestedSessionId) rememberVerifiedExit(requestedSessionId); @@ -1095,7 +1451,7 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): pidToConfirmedSessionId.delete(pid); if (persistedResumeProcesses.delete(pid)) persistResumeProcesses(); releaseRecoveredSpawnDedupe(pid, existingSessionIdByChildPid, spawnSession); - return 'stopped'; + return await finishWithOrphanSweep('stopped'); } if (requestedSessionId) rememberVerifiedExit(requestedSessionId); if (confirmedSessionId) rememberVerifiedExit(confirmedSessionId); @@ -1104,15 +1460,61 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): pidToConfirmedSessionId.delete(pid); if (persistedResumeProcesses.delete(pid)) persistResumeProcesses(); releaseRecoveredSpawnDedupe(pid, existingSessionIdByChildPid, spawnSession); - return 'already_gone'; + return await finishWithOrphanSweep('already_gone'); + } + + // Maps missed (or marker-mismatch cleared a stale row). Scan live argv for + // `--started-by runner` + this HAPI session id and tree-kill matches — + // excluding PIDs that still host active shared sibling roots (registry + // and/or in-memory tracked wrappers). + { + const liveRuntimes = await readLiveRuntimesForStop(); + if (liveRuntimes === null) return 'still_alive'; + const protectedTrackedPids = trackedSharedWrapperPidsWithSiblings( + pidToTrackedSession.entries(), + sessionId + ); + const orphanStatus = await reapRunnerSpawnedOrphans(sessionId, { + findTargets: (id) => findStopSessionOrphanTargets( + id, + (sid, pid) => shouldSkipOrphanPid(liveRuntimes, protectedTrackedPids, sid, pid) + ), + }); + if (orphanStatus === 'still_alive') { + logger.debug(`[RUNNER RUN] Orphan argv reap still_alive for session ${sessionId} (scan_failed or kill left live PIDs)`); + return 'still_alive'; + } + if (orphanStatus === 'stopped') { + rememberVerifiedExit(sessionId); + logger.debug(`[RUNNER RUN] Reaped argv-orphan PID(s) for session ${sessionId}`); + return 'stopped'; + } + // No killable orphans: siblings may protect the wrapper, but that is + // not proof this root ended. Only an inactive registry binding (Codex + // KillSession ack) may claim stopped; otherwise stay unknown on retry. + if (sessionRuntimeHasActiveSiblings(liveRuntimes, sessionId) || protectedTrackedPids.size > 0) { + const binding = sessionRegistryBindingState(liveRuntimes, sessionId); + const decision = decideKeepWrapperArchive(binding); + logger.debug( + `[RUNNER RUN] Session ${sessionId}; shared siblings remain; archive=${decision}` + ); + return decision === 'still_alive' ? 'still_alive' : decision; + } } if (hasVerifiedExit(sessionId)) { logger.debug(`[RUNNER RUN] Session ${sessionId} was previously observed exited`); return 'already_gone'; } + + // PID- targets are handled before tracked/persisted matches above so a + // reused OS pid cannot be tree-killed via happySessionId coincidence. + + // No PID matched and no verified-exit tombstone — distinct from + // still_alive so callers reconciling stale rows are not blocked forever, + // while callers that just spawned this id can treat unknown defensively. logger.debug(`[RUNNER RUN] Session ${sessionId} not found without verified exit`); - return 'still_alive'; + return 'unknown'; }; // Handle child process exit @@ -1135,6 +1537,7 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): pidToErrorAwaiter.delete(pid); pidToRequestedSessionId.delete(pid); pidToConfirmedSessionId.delete(pid); + webhookTimeoutOrphanPids.delete(pid); if (persistedResumeProcesses.delete(pid)) persistResumeProcesses(); }; @@ -1598,24 +2001,19 @@ export function buildCliArgs( // Codex shares one engine; Runner owns the wrapper, not a remote mode. if (agent !== 'codex') args.push('--hapi-starting-mode', startingMode); args.push('--started-by', 'runner'); - // Codex, Cursor ACP, OpenCode, Pi native resume, and Claude message-level - // forks reuse the original HAPI row via --existing-session-id. - if (agent === 'codex' || agent === 'cursor' || agent === 'pi' - || agent === 'opencode' - || agent === 'agy' - || agent === 'dsh' - || (agentCommand === 'claude' && options.forkSession)) { - const existingSessionId = options.existingSessionId ?? options.sessionId; - if (existingSessionId) { - args.push('--existing-session-id', existingSessionId); - } - } - // Grok fork children also bind the pending HAPI session id. - if (agent === 'grok') { - const existingSessionId = options.existingSessionId ?? options.sessionId; - if (existingSessionId && !args.includes('--existing-session-id')) { - args.push('--existing-session-id', existingSessionId); - } + // Stamp the HAPI row id on argv for orphan reap after tracking loss (#1910). + // Adopt-stub (`--hapi-session-id`) vs reopen (`--existing-session-id`) are + // different operations — never collapse them (#1911 Opus Critical + Codex Major). + // Local HTTP non-UUID sessionId stays on --hapi-session-id (reap-only; create + // ignores non-UUID reserved ids). A UUID sessionId must NOT stamp adopt — that + // would 404/409 against a non-stub row (#1911 Opus Major @ 09141964c). + const hubUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + if (options.existingSessionId) { + args.push('--existing-session-id', options.existingSessionId); + } else if (options.reservedSessionId) { + args.push('--hapi-session-id', options.reservedSessionId); + } else if (options.sessionId && !hubUuid.test(options.sessionId)) { + args.push('--hapi-session-id', options.sessionId); } if (options.model) { args.push('--model', options.model); diff --git a/cli/src/runner/runner.integration.test.ts b/cli/src/runner/runner.integration.test.ts index d24aaed71e..d0944316ef 100644 --- a/cli/src/runner/runner.integration.test.ts +++ b/cli/src/runner/runner.integration.test.ts @@ -230,7 +230,10 @@ describe.skipIf(!await isServerHealthy())('Runner Integration Tests', { timeout: expect(spawnedSession.happySessionId).toBeDefined(); expect(await stopRunnerSession(spawnedSession.happySessionId)).toBe('stopped'); expect(await stopRunnerSession(spawnedSession.happySessionId)).toBe('already_gone'); - expect(await stopRunnerSession('unknown-session-id')).toBe('still_alive'); + // Distinct from 'still_alive': no PID matched this id and there is no + // verified-exit tombstone (and argv orphan scan found nothing), so the + // runner has no basis to call it either alive or dead. + expect(await stopRunnerSession('unknown-session-id')).toBe('unknown'); }); it.skipIf(process.env.HAPI_RUN_STRESS_TESTS !== 'true')( diff --git a/cli/src/runner/sharedSessionStop.test.ts b/cli/src/runner/sharedSessionStop.test.ts new file mode 100644 index 0000000000..5530403e8e --- /dev/null +++ b/cli/src/runner/sharedSessionStop.test.ts @@ -0,0 +1,243 @@ +import { describe, expect, it } from 'vitest' +import { + decideKeepWrapperArchive, + decideRawPidStop, + detachSharedRootFromWrapper, + keepWrapperForSharedSiblings, + pidHasActiveSharedRoots, + sessionRegistryBindingState, + sessionRuntimeHasActiveSiblings, + trackedSharedWrapperPidsWithSiblings, + wrapperHasActiveSiblingRoots, +} from './sharedSessionStop' + +describe('detachSharedRootFromWrapper', () => { + it('keeps the wrapper when sibling shared roots remain', () => { + const session = { + happySessionId: 'root-a', + sharedSessions: { + 'root-a': {}, + 'root-b': {}, + }, + } + expect(detachSharedRootFromWrapper(session, 'root-a')).toEqual({ kind: 'keep_wrapper' }) + expect(session.sharedSessions).toEqual({ 'root-b': {} }) + }) + + it('keeps the wrapper when primary is a different live root', () => { + const session = { + happySessionId: 'root-primary', + sharedSessions: { + 'root-archived': {}, + }, + } + expect(detachSharedRootFromWrapper(session, 'root-archived')).toEqual({ kind: 'keep_wrapper' }) + expect(session.sharedSessions).toBeUndefined() + }) + + it('allows kill when the last shared root is the primary being stopped', () => { + const session = { + happySessionId: 'root-only', + sharedSessions: { + 'root-only': {}, + }, + } + expect(detachSharedRootFromWrapper(session, 'root-only')).toEqual({ kind: 'allow_kill' }) + expect(session.sharedSessions).toBeUndefined() + }) + + it('allows kill when session id is not in sharedSessions', () => { + const session = { + happySessionId: 'root-a', + sharedSessions: { 'root-b': {} }, + } + expect(detachSharedRootFromWrapper(session, 'root-missing')).toEqual({ kind: 'allow_kill' }) + expect(session.sharedSessions).toEqual({ 'root-b': {} }) + }) +}) + +describe('keepWrapperForSharedSiblings', () => { + it('returns true and drops the archived root when siblings remain', () => { + const session = { + sharedSessions: { + 'root-a': {}, + 'root-b': {}, + }, + } + expect(keepWrapperForSharedSiblings(session, 'root-a')).toBe(true) + expect(session.sharedSessions).toEqual({ 'root-b': {} }) + }) + + it('returns false when no siblings remain', () => { + const session = { + sharedSessions: { 'root-a': {} }, + } + expect(keepWrapperForSharedSiblings(session, 'root-a')).toBe(false) + expect(session.sharedSessions).toEqual({ 'root-a': {} }) + }) +}) + +describe('runtime registry sibling guards (post-restart)', () => { + const runtimes = [ + { + pid: 4242, + sessions: { + 'root-a': { active: false }, + 'root-b': { active: true }, + }, + }, + ] + + it('keeps the wrapper when archiving the original root after tracking loss', () => { + // KillSession / stopSession already marked root-a inactive; TrackedSession + // is gone after runner restart — persisted-PID and argv paths must not kill. + expect(sessionRuntimeHasActiveSiblings(runtimes, 'root-a')).toBe(true) + expect(wrapperHasActiveSiblingRoots(runtimes, 'root-a', 4242)).toBe(true) + }) + + it('allows kill when no other root is active on the wrapper', () => { + const lastRoot = [ + { + pid: 4242, + sessions: { + 'root-a': { active: false }, + 'root-b': { active: false }, + }, + }, + ] + expect(sessionRuntimeHasActiveSiblings(lastRoot, 'root-a')).toBe(false) + expect(wrapperHasActiveSiblingRoots(lastRoot, 'root-a', 4242)).toBe(false) + }) + + it('ignores unrelated wrapper PIDs', () => { + expect(wrapperHasActiveSiblingRoots(runtimes, 'root-a', 9999)).toBe(false) + }) + + it('PID-filters shared wrappers while leaving other orphan PIDs killable', () => { + // Older untracked CLI (9999) and current shared wrapper (4242) both + // match the same HAPI session id. Session-wide sibling presence must + // not skip the argv scan — only the wrapper PID is excluded. + const orphanPids = [4242, 9999] + const filtered = orphanPids.filter( + (pid) => !wrapperHasActiveSiblingRoots(runtimes, 'root-a', pid) + ) + expect(filtered).toEqual([9999]) + expect(sessionRuntimeHasActiveSiblings(runtimes, 'root-a')).toBe(true) + }) + + it('protects tracked shared wrappers when the runtime registry is empty', () => { + const tracked = new Map([ + [4242, { + happySessionId: 'root-a', + sharedSessions: { + 'root-a': {}, + 'root-b': {}, + }, + }], + ]) + const protectedPids = trackedSharedWrapperPidsWithSiblings(tracked.entries(), 'root-a') + expect([...protectedPids]).toEqual([4242]) + + // Registry unavailable: empty runtimes must not leave the tracked wrapper killable. + const orphanPids = [4242, 9999] + const filtered = orphanPids.filter((pid) => ( + !protectedPids.has(pid) + && !wrapperHasActiveSiblingRoots([], 'root-a', pid) + )) + expect(filtered).toEqual([9999]) + }) + + it('keeps recovered shared wrapper when only the new root is tracked (restart + webhook + archive)', () => { + // Runner restart wiped TrackedSession. A later /new webhook adopts only + // the newly reported root onto the live shared Codex PID. Older roots + // remain active solely in the durable registry. Archiving the new root + // must not fall through to killProcess on that PID. + const tracked = { + happySessionId: 'new-root', + sharedSessions: { + 'new-root': {}, + }, + } + const runtimesAfterArchive = [ + { + pid: 4242, + sessions: { + 'old-root': { active: true }, + 'new-root': { active: false }, + }, + }, + ] + + // Solo in-memory entry (adoption of the new root only) does not protect the PID: + expect(trackedSharedWrapperPidsWithSiblings( + new Map([[4242, { ...tracked, sharedSessions: { ...tracked.sharedSessions } }]]).entries(), + 'new-root' + ).size).toBe(0) + + // Detach + keepWrapper in-memory path alone would allow killing the wrapper: + expect(detachSharedRootFromWrapper(tracked, 'new-root')).toEqual({ kind: 'allow_kill' }) + expect(keepWrapperForSharedSiblings(tracked, 'new-root')).toBe(false) + + // Registry siblings on the same PID must keep the wrapper alive: + expect(wrapperHasActiveSiblingRoots(runtimesAfterArchive, 'new-root', 4242)).toBe(true) + expect(sessionRuntimeHasActiveSiblings(runtimesAfterArchive, 'new-root')).toBe(true) + expect(pidHasActiveSharedRoots(runtimesAfterArchive, 4242)).toBe(true) + }) + + it('refuses raw PID kill when the start marker is missing or mismatched', () => { + expect(decideRawPidStop({ + alive: true, + expectedMarker: undefined, + currentMarker: 'gen-a', + hasActiveSharedRoots: false, + })).toBe('unknown') + expect(decideRawPidStop({ + alive: true, + expectedMarker: 'gen-a', + currentMarker: 'gen-b', + hasActiveSharedRoots: false, + })).toBe('unknown') + expect(decideRawPidStop({ + alive: true, + expectedMarker: 'gen-a', + currentMarker: 'gen-a', + hasActiveSharedRoots: false, + })).toBe('allow_kill') + expect(decideRawPidStop({ + alive: false, + expectedMarker: 'gen-a', + currentMarker: null, + hasActiveSharedRoots: false, + })).toBe('already_gone') + }) + + it('treats inactive registry binding as stop proof while siblings keep the wrapper', () => { + expect(sessionRegistryBindingState(runtimes, 'root-a', 4242)).toBe('inactive') + expect(sessionRegistryBindingState(runtimes, 'root-b', 4242)).toBe('active') + expect(sessionRegistryBindingState(runtimes, 'missing', 4242)).toBe('absent') + expect(decideKeepWrapperArchive('inactive')).toBe('stopped') + expect(decideKeepWrapperArchive('active')).toBe('still_alive') + expect(decideKeepWrapperArchive('absent')).toBe('unknown') + }) + + it('does not claim stopped from siblings alone on retry (no binding evidence)', () => { + // First StopSession detached in-memory tracking and returned unknown. + // Retry: argv scan skips the sibling-protected wrapper; without an + // inactive registry row we must stay unknown — not archive the live root. + const siblingsOnly = [ + { + pid: 4242, + sessions: { + // Target never made it into the durable registry (or was + // never written). Sibling is still active. + 'root-sibling': { active: true }, + }, + }, + ] + expect(wrapperHasActiveSiblingRoots(siblingsOnly, 'root-unconfirmed', 4242)).toBe(true) + expect(sessionRegistryBindingState(siblingsOnly, 'root-unconfirmed')).toBe('absent') + expect(decideKeepWrapperArchive( + sessionRegistryBindingState(siblingsOnly, 'root-unconfirmed') + )).toBe('unknown') + }) +}) diff --git a/cli/src/runner/sharedSessionStop.ts b/cli/src/runner/sharedSessionStop.ts new file mode 100644 index 0000000000..4311ac882f --- /dev/null +++ b/cli/src/runner/sharedSessionStop.ts @@ -0,0 +1,207 @@ +/** + * Shared Codex executions host multiple HAPI roots in one wrapper PID. + * Archiving one root must detach that root without tree-killing the wrapper + * while sibling roots (or a different primary) still use it. + */ + +export type SharedStopDecision = + | { kind: 'keep_wrapper' } + | { kind: 'allow_kill' } + +/** Minimal runtime shape used when TrackedSession was lost (e.g. runner restart). */ +export type RuntimeSiblingSnapshot = { + pid: number + sessions: Record +} + +/** + * True when any root on this wrapper PID is still active in the durable + * registry. Used when StopSession is asked to confirm a raw OS pid (PID-N) + * from KillSession — tree-killing would end sibling shared Codex roots. + */ +export function pidHasActiveSharedRoots( + runtimes: RuntimeSiblingSnapshot[], + wrapperPid: number +): boolean { + for (const runtime of runtimes) { + if (runtime.pid !== wrapperPid) continue + return Object.values(runtime.sessions).some((binding) => binding.active) + } + return false +} + +/** + * Decide whether a KillSession-reported OS pid may be tree-killed. + * Requires a matching process-start marker so PID reuse cannot nuke a stranger. + */ +export type RawPidStopDecision = + | 'already_gone' + | 'unknown' + | 'keep_shared' + | 'allow_kill' + +export function decideRawPidStop(opts: { + alive: boolean + expectedMarker?: string + currentMarker: string | null + hasActiveSharedRoots: boolean +}): RawPidStopDecision { + if (!opts.alive) return 'already_gone' + if (!opts.expectedMarker) return 'unknown' + if (opts.currentMarker === null || opts.currentMarker !== opts.expectedMarker) { + return 'unknown' + } + if (opts.hasActiveSharedRoots) return 'keep_shared' + return 'allow_kill' +} + +/** + * Registry evidence for a shared root: Codex KillSession marks the binding + * inactive before replying. findRuntime only returns active rows, so StopSession + * must read this directly when deciding whether siblings-alone may claim stopped. + */ +export type RegistryBindingState = 'active' | 'inactive' | 'absent' + +export function sessionRegistryBindingState( + runtimes: RuntimeSiblingSnapshot[], + sessionId: string, + wrapperPid?: number +): RegistryBindingState { + for (const runtime of runtimes) { + if (wrapperPid !== undefined && runtime.pid !== wrapperPid) continue + const binding = runtime.sessions[sessionId] + if (!binding) continue + return binding.active ? 'active' : 'inactive' + } + return 'absent' +} + +/** + * After detaching a shared root while keeping the wrapper for siblings: + * inactive binding = acknowledged archive; absent = unconfirmed (unknown); + * active = still running. + */ +export function decideKeepWrapperArchive( + binding: RegistryBindingState +): 'stopped' | 'still_alive' | 'unknown' { + if (binding === 'active') return 'still_alive' + if (binding === 'inactive') return 'stopped' + return 'unknown' +} + +/** + * True when another root on the same wrapper PID is still active in the + * durable Codex runtime registry — even if this session's binding is inactive + * and the runner has no in-memory TrackedSession. + */ +export function wrapperHasActiveSiblingRoots( + runtimes: RuntimeSiblingSnapshot[], + sessionId: string, + wrapperPid: number +): boolean { + for (const runtime of runtimes) { + if (runtime.pid !== wrapperPid) continue + return Object.entries(runtime.sessions).some( + ([id, binding]) => id !== sessionId && binding.active + ) + } + return false +} + +/** + * True when any runtime that still lists `sessionId` (active or not) has at + * least one other active root. Used before persisted-PID / argv kills after + * KillSession marked the archived root inactive. + */ +export function sessionRuntimeHasActiveSiblings( + runtimes: RuntimeSiblingSnapshot[], + sessionId: string +): boolean { + for (const runtime of runtimes) { + if (!(sessionId in runtime.sessions)) continue + if (Object.entries(runtime.sessions).some( + ([id, binding]) => id !== sessionId && binding.active + )) { + return true + } + } + return false +} + +/** + * PIDs from in-memory runner tracking that still host other shared roots for + * this session id. Used when the durable runtime registry is missing/unreadable + * so argv orphan sweeps do not tree-kill a live shared wrapper. + */ +export function trackedSharedWrapperPidsWithSiblings( + tracked: Iterable<[number, { + happySessionId?: string + sharedSessions?: Record + }]>, + sessionId: string +): Set { + const protectedPids = new Set() + for (const [pid, session] of tracked) { + const shared = session.sharedSessions + if (!shared) continue + const otherShared = Object.keys(shared).filter((id) => id !== sessionId) + if (otherShared.length > 0) { + protectedPids.add(pid) + continue + } + // Target may already have been detached from sharedSessions while the + // primary happySessionId is a different live root on this wrapper. + if (typeof session.happySessionId === 'string' + && session.happySessionId !== sessionId) { + protectedPids.add(pid) + } + } + return protectedPids +} + +/** + * Mutates `sharedSessions` to drop `sessionId`. Returns whether the wrapper + * PID must stay alive for remaining roots. + */ +export function detachSharedRootFromWrapper( + session: { + happySessionId?: string + sharedSessions?: Record + }, + sessionId: string +): SharedStopDecision { + const shared = session.sharedSessions + if (!shared || !Object.prototype.hasOwnProperty.call(shared, sessionId)) { + return { kind: 'allow_kill' } + } + + delete shared[sessionId] + const remainingShared = Object.keys(shared) + if (remainingShared.length === 0) { + delete session.sharedSessions + } + + const primaryIsOther = typeof session.happySessionId === 'string' + && session.happySessionId !== sessionId + if (remainingShared.length > 0 || primaryIsOther) { + return { kind: 'keep_wrapper' } + } + return { kind: 'allow_kill' } +} + +/** + * When stop matched the primary `happySessionId`, keep the wrapper if other + * shared roots are still registered on this PID. + */ +export function keepWrapperForSharedSiblings( + session: { + sharedSessions?: Record + }, + sessionId: string +): boolean { + const siblings = Object.keys(session.sharedSessions ?? {}) + .filter((id) => id !== sessionId) + if (siblings.length === 0) return false + delete session.sharedSessions?.[sessionId] + return true +} diff --git a/cli/src/utils/process.test.ts b/cli/src/utils/process.test.ts index 52ae0d4858..5e2f6f9a39 100644 --- a/cli/src/utils/process.test.ts +++ b/cli/src/utils/process.test.ts @@ -10,7 +10,7 @@ vi.mock('cross-spawn', () => ({ } })) -import { getHapiRunnerProcessIdentity } from './process' +import { getHapiRunnerProcessIdentity, killProcess } from './process' const originalPlatformDescriptor = Object.getOwnPropertyDescriptor(process, 'platform') @@ -50,6 +50,11 @@ describe('getHapiRunnerProcessIdentity on Windows', () => { beforeEach(() => { setPlatform('win32') spawnSyncMock.mockReset() + // Prefer signal-0 probe in these identity tests: tasklist falls through. + spawnSyncMock.mockImplementation((cmd: string, ...rest: unknown[]) => { + if (cmd === 'tasklist') return unavailable('tasklist') + return completed('') + }) vi.spyOn(process, 'kill').mockReturnValue(true) }) @@ -64,77 +69,85 @@ describe('getHapiRunnerProcessIdentity on Windows', () => { }) it('reports a foreign process when CIM identifies it', () => { - spawnSyncMock.mockReturnValueOnce(completed('C:\\Windows\\System32\\conhost.exe')) + spawnSyncMock.mockImplementation((cmd: string, args?: string[]) => { + if (cmd === 'tasklist') return unavailable('tasklist') + if (cmd === 'powershell') return completed('C:\\Windows\\System32\\conhost.exe') + return completed('') + }) expect(getHapiRunnerProcessIdentity(8328)).toBe('foreign') - expect(spawnSyncMock).toHaveBeenCalledTimes(1) - expect(spawnSyncMock).toHaveBeenNthCalledWith( - 1, - 'powershell', - [ - '-NoProfile', - '-NonInteractive', - '-Command', - '(Get-CimInstance Win32_Process -Filter "ProcessId = 8328").CommandLine' - ], - { stdio: 'pipe', windowsHide: true } - ) + expect(spawnSyncMock.mock.calls.some((call) => call[0] === 'powershell')).toBe(true) }) it('reports the runner when CIM identifies it', () => { - spawnSyncMock.mockReturnValueOnce(completed('hapi-local.exe runner start-sync')) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'tasklist') return unavailable('tasklist') + if (cmd === 'powershell') return completed('hapi-local.exe runner start-sync') + return completed('') + }) expect(getHapiRunnerProcessIdentity(9124)).toBe('runner') }) it('falls back to WMIC when PowerShell is unavailable', () => { - spawnSyncMock - .mockReturnValueOnce(unavailable('powershell')) - .mockReturnValueOnce(completed('hapi-local.exe runner start-sync')) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'tasklist') return unavailable('tasklist') + if (cmd === 'powershell') return unavailable('powershell') + if (cmd === 'wmic') return completed('hapi-local.exe runner start-sync') + return completed('') + }) expect(getHapiRunnerProcessIdentity(9124)).toBe('runner') - expect(spawnSyncMock).toHaveBeenNthCalledWith( - 2, - 'wmic', - ['process', 'where', 'ProcessId=9124', 'get', 'CommandLine'], - { stdio: 'pipe', windowsHide: true } - ) + expect(spawnSyncMock.mock.calls.some((call) => call[0] === 'wmic')).toBe(true) }) it('falls back to WMIC when CIM reports no command line', () => { - spawnSyncMock - .mockReturnValueOnce(completed('')) - .mockReturnValueOnce(completed('CommandLine\r\nhapi-local.exe runner start-sync\r\n')) + let powershellCalls = 0 + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'tasklist') return unavailable('tasklist') + if (cmd === 'powershell') { + powershellCalls += 1 + return completed('') + } + if (cmd === 'wmic') return completed('CommandLine\r\nhapi-local.exe runner start-sync\r\n') + return completed('') + }) expect(getHapiRunnerProcessIdentity(9124)).toBe('runner') - expect(spawnSyncMock).toHaveBeenCalledTimes(2) + expect(spawnSyncMock.mock.calls.some((call) => call[0] === 'wmic')).toBe(true) }) it('reports unknown when WMIC prints only the column header', () => { - spawnSyncMock - .mockReturnValueOnce(completed('')) - .mockReturnValueOnce(completed('CommandLine\r\n\r\n')) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'tasklist') return unavailable('tasklist') + if (cmd === 'powershell') return completed('') + if (cmd === 'wmic') return completed('CommandLine\r\n\r\n') + return completed('') + }) expect(getHapiRunnerProcessIdentity(8328)).toBe('unknown') - expect(spawnSyncMock).toHaveBeenCalledTimes(2) }) it('falls back to WMIC when PowerShell exits non-zero', () => { - spawnSyncMock - .mockReturnValueOnce(completed('', 1)) - .mockReturnValueOnce(completed('hapi-local.exe runner start-sync')) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'tasklist') return unavailable('tasklist') + if (cmd === 'powershell') return completed('', 1) + if (cmd === 'wmic') return completed('hapi-local.exe runner start-sync') + return completed('') + }) expect(getHapiRunnerProcessIdentity(9124)).toBe('runner') - expect(spawnSyncMock).toHaveBeenCalledTimes(2) }) it('reports unknown when no probe reports a command line', () => { - spawnSyncMock - .mockReturnValueOnce(completed('')) - .mockReturnValueOnce(unavailable('wmic')) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'tasklist') return unavailable('tasklist') + if (cmd === 'powershell') return completed('') + if (cmd === 'wmic') return unavailable('wmic') + return completed('') + }) expect(getHapiRunnerProcessIdentity(8328)).toBe('unknown') - expect(spawnSyncMock).toHaveBeenCalledTimes(2) }) it('reports dead when the pid exits while the probes run', () => { @@ -143,9 +156,10 @@ describe('getHapiRunnerProcessIdentity on Windows', () => { .mockImplementationOnce(() => { throw new Error('ESRCH') }) - spawnSyncMock - .mockReturnValueOnce(unavailable('powershell')) - .mockReturnValueOnce(unavailable('wmic')) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'tasklist') return unavailable('tasklist') + return unavailable(cmd) + }) expect(getHapiRunnerProcessIdentity(8328)).toBe('dead') }) @@ -154,9 +168,514 @@ describe('getHapiRunnerProcessIdentity on Windows', () => { vi.spyOn(process, 'kill').mockImplementation(() => { throw new Error('ESRCH') }) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'tasklist') return unavailable('tasklist') + return completed('') + }) expect(getHapiRunnerProcessIdentity(8328)).toBe('dead') - expect(spawnSyncMock).not.toHaveBeenCalled() + expect(spawnSyncMock.mock.calls.every((call) => call[0] === 'tasklist')).toBe(true) + }) +}) + +describe('killProcess on Windows (orphanReap / stopSession)', () => { + beforeAll(() => { + if (!originalPlatformDescriptor?.configurable) { + throw new Error('process.platform is not configurable in this runtime') + } + }) + + beforeEach(() => { + setPlatform('win32') + spawnSyncMock.mockReset() + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + afterAll(() => { + if (originalPlatformDescriptor) { + Object.defineProperty(process, 'platform', originalPlatformDescriptor) + } + }) + + it('escalates soft taskkill to /F when the process stays alive (mirrors SIGTERM→SIGKILL)', async () => { + // Soft taskkill on win32 console trees often fails with + // "can only be terminated forcefully" — orphanReap must escalate. + let alive = true + // Match @types/node process.kill(pid, signal?: string | number): true + vi.spyOn(process, 'kill').mockImplementation((_pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (!alive) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + return true + }) + spawnSyncMock.mockImplementation((cmd: string, args: string[] = []) => { + if (cmd === 'tasklist') { + if (!alive) { + return completed('INFO: No tasks are running which match the specified criteria.') + } + return completed(`hapi.exe 5544 Console 1 5,000 K`) + } + if (cmd !== 'taskkill') { + return completed('') + } + if (args.includes('/F')) { + alive = false + return completed('', 0) + } + // Soft refuse — process still alive (real Windows console-tree behavior) + return { + status: 1, + stdout: Buffer.from(''), + stderr: Buffer.from('ERROR: This process can only be terminated forcefully (with /F option).') + } + }) + + const done = killProcess(5544, false) + // Soft fails → immediate /F; then brief death-poll timers + await vi.advanceTimersByTimeAsync(500) + await expect(done).resolves.toBe(true) + + const taskkills = spawnSyncMock.mock.calls.filter((call) => call[0] === 'taskkill') + expect(taskkills.length).toBeGreaterThanOrEqual(2) + expect(taskkills[0]![1]).toEqual(['/T', '/PID', '5544']) + expect(taskkills.some((call) => (call[1] as string[]).includes('/F'))).toBe(true) + }) + + it('waits for grace when soft taskkill succeeds before escalating to /F', async () => { + // Soft status=0 but PID still draining archive flush — do not /F immediately. + let alive = true + let softCalls = 0 + vi.spyOn(process, 'kill').mockImplementation((_pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (!alive) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + return true + }) + spawnSyncMock.mockImplementation((cmd: string, args: string[] = []) => { + if (cmd === 'tasklist') { + if (!alive) { + return completed('INFO: No tasks are running which match the specified criteria.') + } + return completed(`hapi.exe 7788 Console 1 5,000 K`) + } + if (cmd === 'taskkill' && args.includes('/F')) { + alive = false + return completed('', 0) + } + if (cmd === 'taskkill') { + softCalls += 1 + // Succeeds, but process remains alive until grace elapses + return completed('', 0) + } + return completed('') + }) + + const done = killProcess(7788, false) + // During grace, process exits without needing /F + await vi.advanceTimersByTimeAsync(100) + alive = false + await vi.advanceTimersByTimeAsync(2_500) + await expect(done).resolves.toBe(true) + + expect(softCalls).toBeGreaterThanOrEqual(1) + const forceKills = spawnSyncMock.mock.calls.filter( + (call) => call[0] === 'taskkill' && (call[1] as string[]).includes('/F') + ) + expect(forceKills).toHaveLength(0) + }) + + it('uses forced taskkill immediately when force=true', async () => { + let alive = true + vi.spyOn(process, 'kill').mockImplementation((_pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (!alive) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + return true + }) + spawnSyncMock.mockImplementation((cmd: string, args: string[] = []) => { + if (cmd === 'tasklist') { + if (!alive) { + return completed('INFO: No tasks are running which match the specified criteria.') + } + return completed(`hapi.exe 9901 Console 1 5,000 K`) + } + if (cmd === 'taskkill' && args.includes('/F')) { + alive = false + return completed('', 0) + } + return completed('', 1) + }) + + const done = killProcess(9901, true) + await vi.advanceTimersByTimeAsync(500) + await expect(done).resolves.toBe(true) + + const taskkills = spawnSyncMock.mock.calls.filter((call) => call[0] === 'taskkill') + expect(taskkills).toHaveLength(1) + expect(taskkills[0]![1]).toEqual(['/F', '/T', '/PID', '9901']) + }) + + it('killProcessTreeByPid signals surviving descendants after taskkill /T misses them', async () => { + // #1911 bot Major: taskkill /T exit 0 is "signalled", not "tree gone". + // When the root dies but a grandchild survives (broken intermediate link), + // individually signal survivors from the pre-kill snapshot — but only when + // the process-generation marker still matches (PID reuse guard). + const { killProcessTreeByPid } = await import('./process') + const alive = new Set([100, 200]) // 100=root, 200=descendant + const markers = new Map([[100, 'gen-100'], [200, 'gen-200']]) + const taskkillPids: number[] = [] + vi.spyOn(process, 'kill').mockImplementation((pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (!alive.has(pid)) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + return true + }) + spawnSyncMock.mockImplementation((cmd: string, args: string[] = []) => { + if (cmd === 'powershell') { + const script = String(args[args.length - 1] ?? '') + if (script.includes('ParentProcessId')) { + return completed('OK:200,100') + } + // getProcessStartMarker CIM probe + const filterMatch = /ProcessId = (\d+)/.exec(script) + if (filterMatch) { + const pid = Number(filterMatch[1]) + if (!alive.has(pid)) return completed('') + return completed(markers.get(pid) ?? '') + } + return completed('') + } + if (cmd === 'tasklist') { + const filter = args.find((a) => a.startsWith('PID eq ')) + const pid = filter ? Number(filter.replace('PID eq ', '')) : NaN + if (!alive.has(pid)) { + return completed('INFO: No tasks are running which match the specified criteria.') + } + return completed(`proc.exe ${pid} Console 1 1,000 K`) + } + if (cmd === 'taskkill') { + const idx = args.indexOf('/PID') + const pid = Number(args[idx + 1]) + taskkillPids.push(pid) + alive.delete(pid) + return completed('', 0) + } + return completed('') + }) + + const done = killProcessTreeByPid(100, true) + await vi.advanceTimersByTimeAsync(500) + await expect(done).resolves.toBe(true) + expect(taskkillPids).toContain(100) + expect(taskkillPids).toContain(200) + expect(alive.size).toBe(0) + }) + + it('killProcessTreeByPid does not kill a surviving PID that was reused', async () => { + const { killProcessTreeByPid } = await import('./process') + const alive = new Set([100, 200]) + // After root kill, PID 200 is recycled with a new generation marker. + let rootGone = false + const taskkillPids: number[] = [] + vi.spyOn(process, 'kill').mockImplementation((pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (!alive.has(pid)) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + return true + }) + spawnSyncMock.mockImplementation((cmd: string, args: string[] = []) => { + if (cmd === 'powershell') { + const script = String(args[args.length - 1] ?? '') + if (script.includes('ParentProcessId')) { + return completed('OK:200,100') + } + const filterMatch = /ProcessId = (\d+)/.exec(script) + if (filterMatch) { + const pid = Number(filterMatch[1]) + if (!alive.has(pid)) return completed('') + if (pid === 200 && rootGone) return completed('gen-200-reused') + return completed(pid === 100 ? 'gen-100' : 'gen-200') + } + return completed('') + } + if (cmd === 'tasklist') { + const filter = args.find((a) => a.startsWith('PID eq ')) + const pid = filter ? Number(filter.replace('PID eq ', '')) : NaN + if (!alive.has(pid)) { + return completed('INFO: No tasks are running which match the specified criteria.') + } + return completed(`proc.exe ${pid} Console 1 1,000 K`) + } + if (cmd === 'taskkill') { + const idx = args.indexOf('/PID') + const pid = Number(args[idx + 1]) + taskkillPids.push(pid) + if (pid === 100) { + alive.delete(100) + rootGone = true + } else { + alive.delete(pid) + } + return completed('', 0) + } + return completed('') + }) + + const done = killProcessTreeByPid(100, true) + await vi.advanceTimersByTimeAsync(500) + // Root signalled; reused descendant left alone → tree verify fails closed. + await expect(done).resolves.toBe(false) + expect(taskkillPids).toEqual([100]) + expect(alive.has(200)).toBe(true) + }) + + it('windowsProcessTreeCimCommand is newline-separated (WinPS-parseable)', async () => { + // #1911 bot Major: `.join(' ')` yields `$seen=@{} $bfs=@()` — parse error. + // Assert generated command text, not mocked stdout (mocks never parse PS). + const { windowsProcessTreeCimCommand } = await import('./process') + const cmd = windowsProcessTreeCimCommand(4242) + expect(cmd).toContain('$root=4242') + expect(cmd).toContain('ParentProcessId=$p') + // Statements must be on separate lines — spaces between `$x=@{}` tokens fail. + expect(cmd).toContain('\n$seen=@{}') + expect(cmd).toContain('\n$bfs=@()') + expect(cmd).toContain('\n$queue=@($root)') + expect(cmd).toContain('\nwhile($queue.Count -gt 0){') + // Space-join regression: adjacent statement tokens on one line (not newline). + expect(cmd).not.toMatch(/\$seen=@\{\}[ ]+\$bfs=@\(\)/) + expect(cmd).not.toMatch(/\$bfs=@\(\)[ ]+\$queue=@/) + // Do not terminate `while(...){` with `;` (also invalid). + expect(cmd).not.toMatch(/while\(\$queue\.Count -gt 0\)\{\s*;/) + // #1911 B1: never SilentlyContinue — CIM failure must not look like childless root. + expect(cmd).toContain("$ErrorActionPreference='Stop'") + expect(cmd).toContain('trap { exit 1 }') + expect(cmd).not.toContain('SilentlyContinue') + expect(cmd).toContain('Write-Output ("OK:" + ($bfs -join ","))') + }) + + it('parseWindowsProcessTreeStdout requires OK: sentinel (B1 root-only is scan_failed)', async () => { + const { parseWindowsProcessTreeStdout } = await import('./process') + // Overseer measured: CIM failure with SilentlyContinue prints "1234" exit 0 — + // byte-identical to healthy childless. Without OK: that must be scan_failed. + expect(parseWindowsProcessTreeStdout('1234', 1234)).toBe('scan_failed') + expect(parseWindowsProcessTreeStdout('4000,3000,2000,1234', 1234)).toBe('scan_failed') + expect(parseWindowsProcessTreeStdout('', 1234)).toBe('scan_failed') + expect(parseWindowsProcessTreeStdout('OK:', 1234)).toBe('scan_failed') + expect(parseWindowsProcessTreeStdout('OK:1234', 1234)).toEqual([1234]) + expect(parseWindowsProcessTreeStdout('OK:4000,3000,2000,1234', 1234)).toEqual([ + 4000, 3000, 2000, 1234, + ]) + expect(parseWindowsProcessTreeStdout('OK:200,100', 100)).toEqual([200, 100]) + expect(parseWindowsProcessTreeStdout('OK:200,100', 999)).toBe('scan_failed') + }) + + it('killProcessTreeByPid treats bare root stdout (B1 fail-open shape) as scan_failed', async () => { + const { killProcessTreeByPid } = await import('./process') + let alive = true + vi.spyOn(process, 'kill').mockImplementation((_pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (!alive) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + return true + }) + spawnSyncMock.mockImplementation((cmd: string) => { + // Status 0 + root-only — the measured CIM-failure shape under SilentlyContinue. + if (cmd === 'powershell') return completed('1234') + if (cmd === 'tasklist') { + if (!alive) { + return completed('INFO: No tasks are running which match the specified criteria.') + } + return completed(`proc.exe 1234 Console 1 1,000 K`) + } + if (cmd === 'taskkill') { + alive = false + return completed('', 0) + } + return completed('') + }) + const done = killProcessTreeByPid(1234, true) + await vi.advanceTimersByTimeAsync(500) + await expect(done).resolves.toBe(false) + expect(spawnSyncMock.mock.calls.some((c) => c[0] === 'taskkill')).toBe(true) + }) + + it('killProcessTreeByPid signals root but returns false when Windows tree scan fails', async () => { + const { killProcessTreeByPid } = await import('./process') + let alive = true + vi.spyOn(process, 'kill').mockImplementation((_pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (!alive) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + return true + }) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'powershell') { + return { status: 1, stdout: Buffer.from(''), stderr: Buffer.from('parse error') } + } + if (cmd === 'tasklist') { + if (!alive) { + return completed('INFO: No tasks are running which match the specified criteria.') + } + return completed(`proc.exe 100 Console 1 1,000 K`) + } + if (cmd === 'taskkill') { + alive = false + return completed('', 0) + } + return completed('') + }) + const done = killProcessTreeByPid(100, true) + await vi.advanceTimersByTimeAsync(500) + await expect(done).resolves.toBe(false) + // Partial kill: root signalled, but never claim stopped without tree verify. + expect(spawnSyncMock.mock.calls.some((c) => c[0] === 'taskkill')).toBe(true) + }) + + it('killProcessTreeByPid signals root but returns false when Windows tree scan returns empty stdout', async () => { + const { killProcessTreeByPid } = await import('./process') + let alive = true + vi.spyOn(process, 'kill').mockImplementation((_pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (!alive) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + return true + }) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'powershell') return completed('') // status 0, empty — scan_failed + if (cmd === 'tasklist') { + if (!alive) { + return completed('INFO: No tasks are running which match the specified criteria.') + } + return completed(`proc.exe 100 Console 1 1,000 K`) + } + if (cmd === 'taskkill') { + alive = false + return completed('', 0) + } + return completed('') + }) + const done = killProcessTreeByPid(100, true) + await vi.advanceTimersByTimeAsync(500) + await expect(done).resolves.toBe(false) + expect(spawnSyncMock.mock.calls.some((c) => c[0] === 'taskkill')).toBe(true) + }) +}) + +describe('killProcessTreeByPid on POSIX (pgrep tree scan)', () => { + beforeEach(() => { + setPlatform('linux') + spawnSyncMock.mockReset() + vi.spyOn(process, 'kill').mockReturnValue(true) + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + afterAll(() => { + if (originalPlatformDescriptor) { + Object.defineProperty(process, 'platform', originalPlatformDescriptor) + } + }) + + it('returns false when pgrep is missing (signal root, never claim stopped)', async () => { + const { killProcessTreeByPid } = await import('./process') + let dead = false + vi.spyOn(process, 'kill').mockImplementation((_pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (dead) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + dead = true + return true + }) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'pgrep') return unavailable('pgrep') + return completed('') + }) + const done = killProcessTreeByPid(4242, true) + await vi.advanceTimersByTimeAsync(3000) + await expect(done).resolves.toBe(false) + // Root still signalled — partial kill > zero kill on hosts without pgrep. + expect(process.kill).toHaveBeenCalledWith(4242, 'SIGKILL') + }) + + it('returns false when pgrep is signalled (status null — partial tree)', async () => { + const { killProcessTreeByPid } = await import('./process') + let dead = false + vi.spyOn(process, 'kill').mockImplementation((_pid: number, signal?: string | number) => { + if (signal === 0 || signal === undefined) { + if (dead) { + const err = new Error('ESRCH') as NodeJS.ErrnoException + err.code = 'ESRCH' + throw err + } + return true + } + dead = true + return true + }) + spawnSyncMock.mockImplementation((cmd: string) => { + if (cmd === 'pgrep') { + return { status: null, stdout: '999\n', stderr: '', error: null } + } + return completed('') + }) + const done = killProcessTreeByPid(4242, true) + await vi.advanceTimersByTimeAsync(3000) + await expect(done).resolves.toBe(false) + expect(process.kill).toHaveBeenCalledWith(4242, 'SIGKILL') }) }) diff --git a/cli/src/utils/process.ts b/cli/src/utils/process.ts index 21681f7c86..5acea9b811 100644 --- a/cli/src/utils/process.ts +++ b/cli/src/utils/process.ts @@ -3,11 +3,33 @@ import spawn from 'cross-spawn'; export const isWindows = (): boolean => process.platform === 'win32'; +/** Bound hung WMI/tasklist/ps so a wedged spawn cannot darken the runner (#1911 Overseer B3). */ +const SPAWN_SYNC_TIMEOUT_MS = 10_000; + export function isProcessAlive(pid: number): boolean { if (!Number.isFinite(pid) || pid <= 0) { return false; } + // Windows: prefer tasklist. Bun/Node `process.kill(pid, 0)` is unreliable as a + // liveness probe on win32 and can disagree with the console tree taskkill sees. + if (isWindows()) { + try { + const result = spawn.sync( + 'tasklist', + ['/FI', `PID eq ${pid}`, '/NH'], + { stdio: 'pipe', windowsHide: true, encoding: 'utf8', timeout: SPAWN_SYNC_TIMEOUT_MS } + ); + if (!result.error && result.status === 0) { + const out = (result.stdout?.toString() ?? '').trim(); + if (!out || /no tasks/i.test(out)) return false; + return new RegExp(`(^|\\D)${pid}(\\D|$)`).test(out); + } + } catch { + // fall through to signal-0 probe + } + } + try { process.kill(pid, 0); return true; @@ -16,28 +38,59 @@ export function isProcessAlive(pid: number): boolean { } } +/** + * Shared Win32 generation marker: CIM CreationDate as UTC round-trip ISO. + * Must match orphan list + recheck — ConvertTo-Json of a raw DateTime emits + * `/Date(...)/` on Windows PowerShell 5.1 (#1911 bot Major). + */ +export const WINDOWS_CIM_CREATION_DATE_MARKER_EXPR = + "$_.CreationDate.ToUniversalTime().ToString('o')"; + +/** PowerShell -Command body for the argv orphan process list (CIM + JSON). */ +export function windowsProcessListCimCommand(): string { + // Fail closed: non-terminating CIM errors must not yield empty stdout with + // exit 0 (that was misread as "no orphans" → false archive-ok). #1911 B2. + // Note: Stop on the whole enumeration means one transient per-instance WMI + // error aborts the scan → still_alive until it clears (availability trap, + // fail-closed; #1911 Overseer non-blocker). + return [ + "$ErrorActionPreference='Stop'", + 'trap { exit 1 }', + 'Get-CimInstance Win32_Process | Select-Object ProcessId,Name,CommandLine,' + + `@{N='CreationDate';E={if ($_.CreationDate) { ${WINDOWS_CIM_CREATION_DATE_MARKER_EXPR} } else { $null }}}` + + ' | ConvertTo-Json -Compress', + ].join('\n') +} + +/** PowerShell -Command body for a single-PID start-marker probe. */ +export function windowsProcessMarkerCimCommand(pid: number): string { + const expr = WINDOWS_CIM_CREATION_DATE_MARKER_EXPR.replace(/\$_/g, '$p'); + return ( + `$p = Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}"; ` + + `if ($p -and $p.CreationDate) { ${expr} }` + ); +} + /** Stable marker for one OS PID generation; null means the platform probe failed. */ export function getProcessStartMarker(pid: number): string | null { if (!isProcessAlive(pid)) return null; if (isWindows()) { + // Same UTC ISO 'o' string as orphanReap listWindowsProcessesWithCommandLine. + // Do not print raw DateTime or use WMIC DMTF (format mismatch skips reap). const powershell = spawn.sync('powershell', [ '-NoProfile', '-NonInteractive', '-Command', - `(Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}").CreationDate` - ], { stdio: 'pipe', windowsHide: true }); + windowsProcessMarkerCimCommand(pid), + ], { stdio: 'pipe', windowsHide: true, timeout: SPAWN_SYNC_TIMEOUT_MS }); if (!powershell.error && powershell.status === 0) { const marker = powershell.stdout?.toString().trim(); if (marker) return marker; } - const result = spawn.sync('wmic', [ - 'process', 'where', `ProcessId=${pid}`, 'get', 'CreationDate', '/value' - ], { stdio: 'pipe', windowsHide: true }); - if (result.error || result.status !== 0) return null; - const match = (result.stdout?.toString() ?? '').match(/CreationDate=([^\r\n]+)/); - return match?.[1]?.trim() || null; + return null; } const result = spawn.sync('ps', ['-p', String(pid), '-o', 'lstart='], { stdio: 'pipe', - env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' } + env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' }, + timeout: SPAWN_SYNC_TIMEOUT_MS, }); if (result.error || result.status !== 0) return null; return result.stdout?.toString().trim() || null; @@ -56,7 +109,7 @@ function getWindowsProcessCommandLine(pid: number): string | null { '-NonInteractive', '-Command', `(Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}").CommandLine` - ], { stdio: 'pipe', windowsHide: true }); + ], { stdio: 'pipe', windowsHide: true, timeout: SPAWN_SYNC_TIMEOUT_MS }); if (!powershell.error && powershell.status === 0) { const commandLine = powershell.stdout?.toString() ?? ''; if (commandLine.trim()) return commandLine; @@ -64,7 +117,7 @@ function getWindowsProcessCommandLine(pid: number): string | null { const wmic = spawn.sync('wmic', [ 'process', 'where', `ProcessId=${pid}`, 'get', 'CommandLine' - ], { stdio: 'pipe', windowsHide: true }); + ], { stdio: 'pipe', windowsHide: true, timeout: SPAWN_SYNC_TIMEOUT_MS }); if (!wmic.error && wmic.status === 0) { const commandLine = readWmicCommandLine(wmic.stdout?.toString() ?? ''); if (commandLine) return commandLine; @@ -106,7 +159,10 @@ export function getHapiRunnerProcessIdentity(pid: number): RunnerProcessIdentity } function getPosixProcessCommandLine(pid: number): string | null { - const result = spawn.sync('ps', ['-p', String(pid), '-o', 'command='], { stdio: 'pipe' }); + const result = spawn.sync('ps', ['-p', String(pid), '-o', 'command='], { + stdio: 'pipe', + timeout: SPAWN_SYNC_TIMEOUT_MS, + }); if (result.error || result.status !== 0) return null; const commandLine = result.stdout?.toString() ?? ''; return commandLine.trim() ? commandLine : null; @@ -124,7 +180,8 @@ function killProcessWindows(pid: number, force: boolean): boolean { try { const result = spawn.sync('taskkill', args, { stdio: 'pipe', - windowsHide: true + windowsHide: true, + timeout: SPAWN_SYNC_TIMEOUT_MS, }); if (result.error) { return false; @@ -143,18 +200,110 @@ function killProcessWindows(pid: number, force: boolean): boolean { } } +/** + * Collect a win32 process tree (children first, root last) via CIM ParentProcessId. + * Used to verify taskkill /T actually cleared descendants — exit 0 is "signalled", + * not "gone" (#1911 B2). + * + * Returns `'scan_failed'` when PowerShell errors or returns nothing usable — + * callers must fail closed (never fall back to root-only verify). + * + * Success is a positive sentinel (`OK:`), not "exit 0 + somehow looks like + * PIDs". With `-ErrorAction SilentlyContinue`, a CIM failure exited 0 printing + * only the root — byte-identical to a healthy childless tree (#1911 B1). + */ +export function windowsProcessTreeCimCommand(pid: number): string { + // Newlines between statements — `.join(' ')` is a parse error on WinPS + // (`$seen=@{} $bfs=@()`). Do not join with `;` either: `while(...){;` is invalid. + return [ + "$ErrorActionPreference='Stop'", + 'trap { exit 1 }', + `$root=${pid}`, + '$seen=@{}', + '$bfs=@()', + '$queue=@($root)', + 'while($queue.Count -gt 0){', + ' $p=$queue[0]; if($queue.Count -eq 1){$queue=@()}else{$queue=$queue[1..($queue.Count-1)]}', + ' if($seen.ContainsKey($p)){continue}', + ' $seen[$p]=$true', + ' $bfs+=$p', + ' Get-CimInstance Win32_Process -Filter "ParentProcessId=$p" |', + ' ForEach-Object { $queue+=,[int]$_.ProcessId }', + '}', + // children-first: reverse BFS so root is last; OK: marks clean completion + 'if($bfs.Count -gt 0){ [array]::Reverse($bfs); Write-Output ("OK:" + ($bfs -join ",")) }', + ].join('\n') +} + +/** Parse tree-scan stdout. Requires the `OK:` success sentinel (#1911 B1). */ +export function parseWindowsProcessTreeStdout( + raw: string, + rootPid: number +): number[] | 'scan_failed' { + const text = raw.trim() + if (!text.startsWith('OK:')) return 'scan_failed' + const body = text.slice('OK:'.length).trim() + if (!body) return 'scan_failed' + const pids = body.split(',').map((s) => Number(s.trim())).filter((p) => Number.isFinite(p) && p > 0) + if (pids.length === 0 || !pids.includes(rootPid)) return 'scan_failed' + return pids +} + +export function collectWindowsProcessTree(pid: number): number[] | 'scan_failed' { + const n = typeof pid === 'number' ? pid : Number(pid) + if (!Number.isFinite(n) || n <= 0) return 'scan_failed' + + const result = spawn.sync( + 'powershell', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + windowsProcessTreeCimCommand(n), + ], + { encoding: 'utf8', windowsHide: true, maxBuffer: 16 * 1024 * 1024, timeout: SPAWN_SYNC_TIMEOUT_MS } + ) + if (result.error || result.status !== 0) { + return 'scan_failed' + } + return parseWindowsProcessTreeStdout((result.stdout ?? '').toString(), n) +} + +async function signalAndWaitWindowsRoot(pid: number, force: boolean): Promise { + if (force) { + killProcessWindows(pid, true); + await waitForProcessToDie(pid, true); + return; + } + const softOk = killProcessWindows(pid, false); + if (!softOk && isProcessAlive(pid)) { + killProcessWindows(pid, true); + await waitForProcessToDie(pid, true); + return; + } + await waitForProcessToDie(pid, false); +} + export async function killProcess(pid: number, force: boolean = false): Promise { - if (!Number.isFinite(pid) || pid <= 0) { + const n = typeof pid === 'number' ? pid : Number(pid) + if (!Number.isFinite(n) || n <= 0) { return false; } if (isWindows()) { - return killProcessWindows(pid, force); + // Soft taskkill (/T without /F) is routinely refused on win32 console trees + // ("can only be terminated forcefully"). Mirror POSIX SIGTERM→SIGKILL: + // escalate immediately only when soft kill is *refused*; when soft succeeds + // but the PID is still draining, honor the grace wait before /F so archive + // flush can finish. + // Root-only verify — callers that need full-tree proof use killProcessTreeByPid. + await signalAndWaitWindowsRoot(n, force); + return !isProcessAlive(n); } try { - process.kill(pid, force ? 'SIGKILL' : 'SIGTERM'); - await waitForProcessToDie(pid, force); + process.kill(n, force ? 'SIGKILL' : 'SIGTERM'); + await waitForProcessToDie(n, force); return true; } catch { return false; @@ -163,21 +312,34 @@ export async function killProcess(pid: number, force: boolean = false): Promise< /** * Recursively collects all descendant PIDs of a process (depth-first). - * Returns PIDs in child-first order (leaves first, root last). + * Returns PIDs in child-first order (leaves first, root last), or + * `'scan_failed'` when pgrep could not be run (so callers fail closed + * instead of verifying root-only — #1911 Opus / Overseer). */ -function collectProcessTree(pid: number): number[] { +function collectProcessTree(pid: number): number[] | 'scan_failed' { const pids: number[] = []; - try { - const result = spawn.sync('pgrep', ['-P', pid.toString()], { encoding: 'utf8' }); - if (result.stdout) { - const childPids = result.stdout.trim().split('\n').filter(Boolean).map(Number); - for (const childPid of childPids) { - pids.push(...collectProcessTree(childPid)); - } + const result = spawn.sync('pgrep', ['-P', pid.toString()], { + encoding: 'utf8', + timeout: SPAWN_SYNC_TIMEOUT_MS, + }); + // spawn.sync returns {error} rather than throwing when the binary is missing. + if (result.error) { + return 'scan_failed'; + } + // pgrep: 0 = matches, 1 = no children. null = signalled / aborted → partial + // stdout is not a trustworthy tree (#1911 Overseer POSIX B1 twin). + if (result.status !== 0 && result.status !== 1) { + return 'scan_failed'; + } + if (result.stdout) { + const childPids = result.stdout.trim().split('\n').filter(Boolean).map(Number); + for (const childPid of childPids) { + if (!Number.isFinite(childPid) || childPid <= 0) continue; + const nested = collectProcessTree(childPid); + if (nested === 'scan_failed') return 'scan_failed'; + pids.push(...nested); } - } catch { - // pgrep may not be available } pids.push(pid); @@ -192,6 +354,18 @@ function collectProcessTree(pid: number): number[] { async function killProcessTree(pid: number, force: boolean): Promise { // Collect all PIDs first (sync) - returns in child-first order const pids = collectProcessTree(pid); + if (pids === 'scan_failed') { + // Signal the known root anyway (partial kill > zero kill), but never claim + // stopped without a full-tree verify (#1911 Opus Major / debian-slim no pgrep). + const signal = force ? 'SIGKILL' : 'SIGTERM'; + try { + process.kill(pid, signal); + } catch { + // already gone + } + await waitForProcessToDie(pid, force); + return false; + } // Signal all processes synchronously (children first, then root) const signal = force ? 'SIGKILL' : 'SIGTERM'; @@ -213,17 +387,48 @@ async function killProcessTree(pid: number, force: boolean): Promise { /** Kill a PID and all descendants, verifying the complete tree is gone. */ export async function killProcessTreeByPid(pid: number, force: boolean = false): Promise { - if (!Number.isFinite(pid) || pid <= 0) return false; - if (isWindows()) return killProcess(pid, force); - return killProcessTree(pid, force); + const n = typeof pid === 'number' ? pid : Number(pid) + if (!Number.isFinite(n) || n <= 0) return false; + if (isWindows()) { + // taskkill /T exit 0 means "signalled", not "every descendant is dead". + // Collect the tree + generation markers first, signal the root with /T, + // then individually signal any surviving pre-kill PIDs whose generation + // still matches (PID reuse must not be killed) (#1911 bot Major). + const treePids = collectWindowsProcessTree(n); + if (treePids === 'scan_failed') { + // Still signal the (known) root — scan failure must not mean zero kill — + // but never claim stopped without a full-tree verify (#1911 Opus Major). + await signalAndWaitWindowsRoot(n, force); + return false; + } + const markers = new Map(); + for (const candidate of treePids) { + markers.set(candidate, getProcessStartMarker(candidate)); + } + await signalAndWaitWindowsRoot(n, force); + for (const survivor of treePids) { + if (survivor === n) continue; + if (!isProcessAlive(survivor)) continue; + const expected = markers.get(survivor); + const current = getProcessStartMarker(survivor); + // Unverifiable or reused PID — leave alone; final every() fails closed. + if (!expected || !current || current !== expected) continue; + await signalAndWaitWindowsRoot(survivor, true); + } + return treePids.every((candidate) => !isProcessAlive(candidate)); + } + return killProcessTree(n, force); } /** - * Waits for a process to die, escalating to SIGKILL if SIGTERM doesn't work. + * Waits for a process to die, escalating if the graceful signal didn't work. + * POSIX: SIGTERM → SIGKILL. Windows: taskkill /T → taskkill /F /T. */ async function waitForProcessToDie(pid: number, force: boolean): Promise { const maxWait = 2000; - const pollInterval = 20; + // Windows isProcessAlive shells out to tasklist; keep the poll coarse so a + // stop cannot burn the runner event loop for seconds (#1911 Overseer). + const pollInterval = isWindows() ? 100 : 20; let waited = 0; while (isProcessAlive(pid) && waited < maxWait) { @@ -231,10 +436,14 @@ async function waitForProcessToDie(pid: number, force: boolean): Promise { waited += pollInterval; } - // If SIGTERM didn't work and we haven't tried SIGKILL yet, escalate + // Graceful kill didn't finish — escalate (same structure on both platforms). if (!force && isProcessAlive(pid)) { try { - process.kill(pid, 'SIGKILL'); + if (isWindows()) { + killProcessWindows(pid, true); + } else { + process.kill(pid, 'SIGKILL'); + } } catch { return; } @@ -255,11 +464,6 @@ export async function killProcessByChildProcess( return false; } - if (isWindows()) { - // Windows taskkill /T already kills the entire process tree - return killProcess(pid, force); - } - - // Kill entire process tree on Unix to prevent orphan processes + // Both platforms: tree-kill + full-tree verify (win32 must not trust root-only). return killProcessTreeByPid(pid, force); } diff --git a/cli/src/utils/windowsProcessTree.pwsh.test.ts b/cli/src/utils/windowsProcessTree.pwsh.test.ts new file mode 100644 index 0000000000..607bbf35c7 --- /dev/null +++ b/cli/src/utils/windowsProcessTree.pwsh.test.ts @@ -0,0 +1,100 @@ +/** + * Executes the generated Windows tree-scan PowerShell under Docker pwsh. + * Mocks agree with themselves; this is the structural antidote to #1911 B1 + * (CIM failure looking like a healthy childless root). + * + * Locally: skips when docker/image missing. + * In CI (`CI=true`): fails hard — silent skip is fake coverage (#1911 Overseer). + */ +import { execFileSync } from 'node:child_process' +import { describe, expect, it } from 'vitest' +import { windowsProcessListCimCommand, windowsProcessTreeCimCommand } from './process' + +const POWERSHELL_IMAGE = 'mcr.microsoft.com/powershell:latest' + +function dockerPwshAvailable(): boolean { + try { + execFileSync('docker', ['image', 'inspect', POWERSHELL_IMAGE], { + stdio: 'ignore', + }) + return true + } catch { + return false + } +} + +function ensureDockerPwsh(): void { + if (dockerPwshAvailable()) return + if (process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true') { + throw new Error( + `CI requires ${POWERSHELL_IMAGE} for windowsProcessTree.pwsh.test.ts — ` + + 'pull it in the workflow before bun run test (silent skip is not coverage)' + ) + } +} + +function runPwsh(script: string): { status: number; stdout: string; stderr: string } { + try { + const stdout = execFileSync( + 'docker', + ['run', '--rm', POWERSHELL_IMAGE, 'pwsh', '-NoProfile', '-Command', script], + { encoding: 'utf8', maxBuffer: 4 * 1024 * 1024 } + ) + return { status: 0, stdout: stdout.toString(), stderr: '' } + } catch (error) { + const err = error as { status?: number; stdout?: string; stderr?: string } + return { + status: typeof err.status === 'number' ? err.status : 1, + stdout: err.stdout?.toString() ?? '', + stderr: err.stderr?.toString() ?? '', + } + } +} + +ensureDockerPwsh() +const describePwsh = dockerPwshAvailable() ? describe : describe.skip + +describePwsh('windowsProcessTreeCimCommand under real pwsh (#1911 B1)', () => { + it('fails closed when Get-CimInstance is missing (Linux container)', () => { + // Measured pre-fix: SilentlyContinue printed root-only with exit 0. + const result = runPwsh(windowsProcessTreeCimCommand(1234)) + expect(result.status).not.toBe(0) + expect(result.stdout.trim()).not.toMatch(/^OK:/) + expect(result.stdout.trim()).not.toBe('1234') + }) + + it('emits OK: children-first tree when Get-CimInstance is mocked healthy', () => { + const mock = ` +function Get-CimInstance { + param($ClassName, $Filter) + $ppid = [int](($Filter -split "=")[1]) + if ($ppid -eq 1234) { + [pscustomobject]@{ ProcessId = 2000 } + [pscustomobject]@{ ProcessId = 3000 } + } elseif ($ppid -eq 2000) { + [pscustomobject]@{ ProcessId = 4000 } + } +} +${windowsProcessTreeCimCommand(1234)} +` + const result = runPwsh(mock) + expect(result.status).toBe(0) + expect(result.stdout.trim()).toBe('OK:4000,3000,2000,1234') + }) + + it('emits OK:root alone when the mocked tree has no children', () => { + const mock = ` +function Get-CimInstance { param($ClassName, $Filter) } +${windowsProcessTreeCimCommand(1234)} +` + const result = runPwsh(mock) + expect(result.status).toBe(0) + expect(result.stdout.trim()).toBe('OK:1234') + }) + + it('list command fails closed when Get-CimInstance is missing', () => { + const result = runPwsh(windowsProcessListCimCommand()) + expect(result.status).not.toBe(0) + expect(result.stdout.trim()).toBe('') + }) +}) diff --git a/hub/src/store/sessionStore.ts b/hub/src/store/sessionStore.ts index 0efd403ecc..f0ec18d5a0 100644 --- a/hub/src/store/sessionStore.ts +++ b/hub/src/store/sessionStore.ts @@ -4,6 +4,7 @@ import type { StoredSession, VersionedUpdateResult } from './types' import { deleteSession, getOrCreateSession, + adoptPreallocatedSession, getSession, getSessionByNamespace, getSessions, @@ -44,12 +45,27 @@ export class SessionStore { return getOrCreateSession(this.db, tag, metadata, agentState, namespace, model, effort, modelReasoningEffort, requestedId) } + adoptPreallocatedSession( + id: string, + tag: string, + metadata: unknown, + agentState: unknown, + namespace: string, + model?: string, + effort?: string, + modelReasoningEffort?: string + ): StoredSession { + return adoptPreallocatedSession( + this.db, id, tag, metadata, agentState, namespace, model, effort, modelReasoningEffort + ) + } + updateSessionMetadata( id: string, metadata: unknown, expectedVersion: number, namespace: string, - options?: { touchUpdatedAt?: boolean } + options?: { touchUpdatedAt?: boolean; allowUnarchive?: boolean } ): VersionedUpdateResult { return updateSessionMetadata(this.db, id, metadata, expectedVersion, namespace, options) } diff --git a/hub/src/store/sessions.test.ts b/hub/src/store/sessions.test.ts index 94762c86e5..79082b43d1 100644 --- a/hub/src/store/sessions.test.ts +++ b/hub/src/store/sessions.test.ts @@ -114,6 +114,354 @@ describe('getOrCreateSession: requested identity', () => { )).toThrow(SessionIdentityConflictError) store.close() }) + + it('reproduces hub-prealloc vs CLI-tag conflict (machine-spawn stub, #1911)', () => { + // Hub preallocates with tag machine-spawn:; CLI create used a random + // tag + the same id → 409. This documents the bug adopt must fix. + const store = makeStore() + const allocatedId = randomUUID() + store.sessions.getOrCreateSession( + `machine-spawn:${allocatedId}`, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + machineId: 'machine-1', + startedBy: 'runner', + startedFromRunner: true, + }, + null, + 'default', + undefined, + undefined, + undefined, + allocatedId + ) + + expect(() => store.sessions.getOrCreateSession( + randomUUID(), // CLI bootstrap tag + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + machineId: 'machine-1', + startedBy: 'runner', + hostPid: 12345, + }, + {}, + 'default', + undefined, + undefined, + undefined, + allocatedId + )).toThrow(SessionIdentityConflictError) + store.close() + }) + + it('adopts a machine-spawn preallocated stub and overwrites tag + metadata', () => { + const store = makeStore() + const allocatedId = randomUUID() + const cliTag = randomUUID() + store.sessions.getOrCreateSession( + `machine-spawn:${allocatedId}`, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + machineId: 'machine-1', + startedBy: 'runner', + startedFromRunner: true, + }, + null, + 'default', + 'stub-model', + undefined, + undefined, + allocatedId + ) + + const adopted = store.sessions.adoptPreallocatedSession( + allocatedId, + cliTag, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + machineId: 'machine-1', + startedBy: 'runner', + startedFromRunner: true, + hostPid: 4242, + }, + { controlledByUser: false }, + 'default', + 'claude-sonnet', + undefined, + undefined + ) + + expect(adopted.id).toBe(allocatedId) + expect(adopted.tag).toBe(cliTag) + expect(adopted.model).toBe('claude-sonnet') + const meta = adopted.metadata as { hostPid?: number } + expect(meta.hostPid).toBe(4242) + // Idempotent adopt with same tag returns the row + const again = store.sessions.adoptPreallocatedSession( + allocatedId, + cliTag, + { path: '/tmp/project', host: 'localhost', flavor: 'claude' }, + {}, + 'default' + ) + expect(again.id).toBe(allocatedId) + expect(again.tag).toBe(cliTag) + store.close() + }) + + it('rejects adopt when the row is not a preallocated stub', () => { + const store = makeStore() + const id = randomUUID() + store.sessions.getOrCreateSession( + 'live-terminal-tag', + { path: '/tmp', startedBy: 'terminal' }, + null, + 'default', + undefined, + undefined, + undefined, + id + ) + + expect(() => store.sessions.adoptPreallocatedSession( + id, + randomUUID(), + { path: '/tmp', startedBy: 'runner' }, + {}, + 'default' + )).toThrow(/not a preallocated stub|not adoptable/i) + store.close() + }) + + it('releases machine-spawn stub tag on metadata update (reopen-flavor path)', () => { + // codex/cursor/pi/… use --existing-session-id → bootstrapExistingSession + // → updateMetadata, never adopt. Stub tag must not stick forever. + const store = makeStore() + const allocatedId = randomUUID() + const created = store.sessions.getOrCreateSession( + `machine-spawn:${allocatedId}`, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'cursor', + machineId: 'm1', + startedBy: 'runner', + startedFromRunner: true, + }, + null, + 'default', + undefined, + undefined, + undefined, + allocatedId + ) + expect(created.tag).toBe(`machine-spawn:${allocatedId}`) + + const updated = store.sessions.updateSessionMetadata( + allocatedId, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'cursor', + machineId: 'm1', + startedBy: 'runner', + startedFromRunner: true, + hostPid: 999, + }, + created.metadataVersion, + 'default' + ) + expect(updated.result).toBe('success') + const row = store.sessions.getSession(allocatedId) + expect(row?.tag).not.toMatch(/^machine-spawn:/) + expect((row?.metadata as { hostPid?: number } | null)?.hostPid).toBe(999) + + // Live row must no longer be adoptable. + expect(() => store.sessions.adoptPreallocatedSession( + allocatedId, + randomUUID(), + { path: '/tmp', flavor: 'cursor' }, + {}, + 'default' + )).toThrow(/not a preallocated stub|not adoptable/i) + store.close() + }) + + it('rejects adopt of an archived preallocated stub', () => { + const store = makeStore() + const allocatedId = randomUUID() + store.sessions.getOrCreateSession( + `machine-spawn:${allocatedId}`, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + startedBy: 'runner', + startedFromRunner: true, + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'user archived while booting', + }, + null, + 'default', + undefined, + undefined, + undefined, + allocatedId + ) + + expect(() => store.sessions.adoptPreallocatedSession( + allocatedId, + randomUUID(), + { path: '/tmp/project', host: 'localhost', flavor: 'claude' }, + {}, + 'default' + )).toThrow(/archived|not adoptable/i) + const row = store.sessions.getSession(allocatedId) + expect(row?.tag).toBe(`machine-spawn:${allocatedId}`) + expect((row?.metadata as { lifecycleState?: string } | null)?.lifecycleState).toBe('archived') + store.close() + }) +}) + +describe('updateSessionMetadata: refuse un-archive (#1911 M1)', () => { + it('merge-preserves hub archive on unauthorized running write (success, not mismatch)', () => { + const store = makeStore() + const allocatedId = randomUUID() + const session = store.sessions.getOrCreateSession( + `machine-spawn:${allocatedId}`, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'KillSession miss', + startedBy: 'runner', + startedFromRunner: true, + }, + null, + 'default', + undefined, + undefined, + undefined, + allocatedId + ) + + const preserved = store.sessions.updateSessionMetadata( + session.id, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + lifecycleState: 'running', + }, + session.metadataVersion, + 'default' + ) + expect(preserved.result).toBe('success') + if (preserved.result !== 'success') throw new Error('expected success') + expect((preserved.value as { lifecycleState?: string; archivedBy?: string } | null)?.lifecycleState) + .toBe('archived') + expect((preserved.value as { archivedBy?: string } | null)?.archivedBy).toBe('hub') + expect(getMetadata(store, session.id)?.lifecycleState).toBe('archived') + expect(getMetadata(store, session.id)?.archivedBy).toBe('hub') + // Tag may already be released by archive-via-metadata; assert archive held. + expect(store.sessions.getSession(session.id)?.tag).toBe(`machine-spawn:${allocatedId}`) + + const allowed = store.sessions.updateSessionMetadata( + session.id, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + lifecycleStateSince: Date.now(), + }, + // version advanced by preserve write + (preserved.version), + 'default', + { allowUnarchive: true } + ) + expect(allowed.result).toBe('success') + expect(getMetadata(store, session.id)?.lifecycleState).toBeUndefined() + store.close() + }) + + it('does not preserve CLI self-archive (archivedBy=cli) when writing running', () => { + const store = makeStore() + const session = store.sessions.getOrCreateSession( + 'cli-self-archive', + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + lifecycleState: 'archived', + archivedBy: 'cli', + archiveReason: 'clean exit', + }, + null, + 'default' + ) + + const result = store.sessions.updateSessionMetadata( + session.id, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + lifecycleState: 'running', + }, + session.metadataVersion, + 'default' + ) + expect(result.result).toBe('success') + expect(getMetadata(store, session.id)?.lifecycleState).toBe('running') + store.close() + }) + + it('still allows non-lifecycle updates while hub-archived', () => { + const store = makeStore() + const session = store.sessions.getOrCreateSession( + 'archived-keep-fields', + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'inactivity', + }, + null, + 'default' + ) + + const result = store.sessions.updateSessionMetadata( + session.id, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'inactivity', + hostPid: 4242, + }, + session.metadataVersion, + 'default' + ) + expect(result.result).toBe('success') + expect(getMetadata(store, session.id)?.lifecycleState).toBe('archived') + expect(getMetadata(store, session.id)?.hostPid).toBe(4242) + store.close() + }) }) describe('updateSessionMetadata: protocol resume token preservation', () => { diff --git a/hub/src/store/sessions.ts b/hub/src/store/sessions.ts index 8d2ad4189c..c26c38c20f 100644 --- a/hub/src/store/sessions.ts +++ b/hub/src/store/sessions.ts @@ -276,27 +276,174 @@ export class SessionIdentityConflictError extends Error { } } +export class SessionNotAdoptableError extends Error { + constructor(message: string) { + super(message) + this.name = 'SessionNotAdoptableError' + } +} + +/** Hub-internal stub tag for fresh machine spawn preallocation (#1911). */ +export function machineSpawnPreallocTag(sessionId: string): string { + return `machine-spawn:${sessionId}` +} + +export function isMachineSpawnPreallocatedStub(session: Pick): boolean { + return session.tag === machineSpawnPreallocTag(session.id) +} + +function isArchivedSessionMetadata(metadata: unknown): boolean { + if (!isPlainObject(metadata)) return false + return metadata.lifecycleState === 'archived' +} + +/** Hub-authored archive only — CLI may archive itself on clean exit. */ +function isHubArchivedSessionMetadata(metadata: unknown): boolean { + if (!isPlainObject(metadata)) return false + return metadata.lifecycleState === 'archived' && metadata.archivedBy === 'hub' +} + +/** + * When a CLI write would clear a hub archive, keep the forensic archive fields + * on the merged payload so the CAS ack returns success + still-archived + * (version-mismatch / error would spin forever in client backoff). + */ +function preserveHubArchiveOnMerged(prior: unknown, merged: unknown): unknown { + if (!isPlainObject(prior) || !isPlainObject(merged)) return prior + const next: Record = { ...merged } + next.lifecycleState = prior.lifecycleState + next.archivedBy = prior.archivedBy + if (prior.archiveReason !== undefined) next.archiveReason = prior.archiveReason + else delete next.archiveReason + if (prior.lifecycleStateSince !== undefined) next.lifecycleStateSince = prior.lifecycleStateSince + return next +} + +/** + * Bind a CLI create bootstrap to a hub-preallocated stub row. + * Overwrites tag + metadata (create-time fields) without minting a new id. + * Rejects rows that are not machine-spawn stubs — live sessions stay protected. + * Rejects archived stubs so adopt cannot resurrect and wipe archive metadata. + */ +export function adoptPreallocatedSession( + db: Database, + id: string, + tag: string, + metadata: unknown, + agentState: unknown, + namespace: string, + model?: string, + effort?: string, + modelReasoningEffort?: string +): StoredSession { + return db.transaction(() => { + const existing = getSessionByNamespace(db, id, namespace) + if (!existing) { + throw new SessionNotAdoptableError('Session not found') + } + + // Already adopted with this tag — idempotent reload (same as getOrCreate match). + if (existing.tag === tag) { + return existing + } + + if (!isMachineSpawnPreallocatedStub(existing)) { + throw new SessionNotAdoptableError('Session is not a preallocated stub') + } + + if (isArchivedSessionMetadata(existing.metadata)) { + throw new SessionNotAdoptableError('Session is archived') + } + + // New tag must not already belong to another session in this namespace. + const tagOwner = prepareCached(db, + 'SELECT id FROM sessions WHERE tag = ? AND namespace = ? LIMIT 1' + ).get(tag, namespace) as { id: string } | undefined + if (tagOwner && tagOwner.id !== id) { + throw new SessionIdentityConflictError('Session tag is already bound to a different id') + } + + const now = Date.now() + const metadataJson = JSON.stringify(metadata) + const agentStateJson = agentState === null || agentState === undefined ? null : JSON.stringify(agentState) + const stubTag = machineSpawnPreallocTag(id) + + // CAS on the stub tag — concurrent adopt / metadata release cannot win a race. + const changed = prepareCached(db, ` + UPDATE sessions SET + tag = @tag, + metadata = @metadata, + metadata_version = metadata_version + 1, + agent_state = @agent_state, + agent_state_version = agent_state_version + 1, + model = @model, + model_reasoning_effort = @model_reasoning_effort, + effort = @effort, + updated_at = @updated_at, + seq = seq + 1 + WHERE id = @id AND namespace = @namespace AND tag = @stub_tag + `).run({ + id, + namespace, + tag, + stub_tag: stubTag, + metadata: metadataJson, + agent_state: agentStateJson, + model: model ?? null, + model_reasoning_effort: modelReasoningEffort ?? null, + effort: effort ?? null, + updated_at: now, + }) + + if (changed.changes !== 1) { + throw new SessionNotAdoptableError('Session is not a preallocated stub') + } + + const updated = getSessionByNamespace(db, id, namespace) + if (!updated) { + throw new Error('Failed to adopt preallocated session') + } + return updated + })() +} + export function updateSessionMetadata( db: Database, id: string, metadata: unknown, expectedVersion: number, namespace: string, - options?: { touchUpdatedAt?: boolean } + options?: { touchUpdatedAt?: boolean; allowUnarchive?: boolean } ): VersionedUpdateResult { const now = Date.now() const touchUpdatedAt = options?.touchUpdatedAt !== false + const allowUnarchive = options?.allowUnarchive === true try { return db.transaction((): VersionedUpdateResult => { - const priorRow = prepareCached(db, - 'SELECT metadata FROM sessions WHERE id = ? AND namespace = ?' - ).get(id, namespace) as { metadata: string | null } | undefined + const existing = getSessionByNamespace(db, id, namespace) + if (!existing) { + return { result: 'error' } + } - const prior = priorRow ? safeJsonParse(priorRow.metadata) : null - const merged = mergeSessionMetadata(prior, metadata) + const prior = existing.metadata + let merged = mergeSessionMetadata(prior, metadata) + + // #1911 M1: unauthorized un-archive of hub-archived rows. + // Return success + merge-preserved archive fields — NOT version-mismatch + // or error (both spin forever in CLI updateMetadata backoff). + // Authorized revive uses allowUnarchive (clearSessionArchiveMetadata + // before spawn). Narrow to archivedBy=hub so CLI self-archive still + // transitions to running on clean reopen paths. + if ( + isHubArchivedSessionMetadata(prior) + && !isArchivedSessionMetadata(merged) + && !allowUnarchive + ) { + merged = preserveHubArchiveOnMerged(prior, merged) + } - return updateVersionedField({ + const result = updateVersionedField({ db, table: 'sessions', id, @@ -319,6 +466,37 @@ export function updateSessionMetadata( touch_updated_at: touchUpdatedAt ? 1 : 0 } }) + + // Reopen flavors (--existing-session-id) never call adopt; they only + // updateMetadata. Release the machine-spawn stub tag here so live + // sessions are not permanently adoptable (#1911 Overseer Major). + // Skip when the write was a hub-archive preserve (still archived) — + // that is refuse-in-place, not live adopt (#1911 M1). + if ( + result.result === 'success' + && isMachineSpawnPreallocatedStub(existing) + && !isHubArchivedSessionMetadata(merged) + ) { + const liveTag = randomUUID() + prepareCached(db, ` + UPDATE sessions + SET tag = @tag, + updated_at = CASE WHEN @touch_updated_at = 1 THEN @updated_at ELSE updated_at END, + seq = seq + 1 + WHERE id = @id + AND namespace = @namespace + AND tag = @stub_tag + `).run({ + id, + namespace, + tag: liveTag, + stub_tag: existing.tag, + updated_at: now, + touch_updated_at: touchUpdatedAt ? 1 : 0, + }) + } + + return result })() } catch { return { result: 'error' } diff --git a/hub/src/sync/opencodeClear.test.ts b/hub/src/sync/opencodeClear.test.ts index ec1aecb1a9..193ed3d5ba 100644 --- a/hub/src/sync/opencodeClear.test.ts +++ b/hub/src/sync/opencodeClear.test.ts @@ -641,6 +641,9 @@ describe('SyncEngine.clearOpenCodeSession', () => { undefined, undefined, // startingMode — not applicable to an OpenCode clear replacement + undefined, + // forkSession / reservedSessionId — clear uses reopen existingSessionId + undefined, undefined ) expect(engine.getSessionByNamespace(replacementSessionId, 'default')?.metadata).toMatchObject({ diff --git a/hub/src/sync/rpcGateway.ts b/hub/src/sync/rpcGateway.ts index f8e74e13bb..5eea422ad5 100644 --- a/hub/src/sync/rpcGateway.ts +++ b/hub/src/sync/rpcGateway.ts @@ -185,14 +185,28 @@ export class RpcGateway { return await this.sessionRpc(sessionId, RPC_METHODS.SetSessionConfig, config) } - async killSession(sessionId: string): Promise { - await this.sessionRpc(sessionId, RPC_METHODS.KillSession, {}) + async killSession(sessionId: string): Promise<{ pid?: number; processStartMarker?: string }> { + const result = await this.sessionRpc(sessionId, RPC_METHODS.KillSession, {}) + if (!result || typeof result !== 'object') return {} + const pid = (result as { pid?: unknown }).pid + const processStartMarker = (result as { processStartMarker?: unknown }).processStartMarker + return { + ...(typeof pid === 'number' ? { pid } : {}), + ...(typeof processStartMarker === 'string' ? { processStartMarker } : {}), + } } - async stopRunnerSession(machineId: string, sessionId: string): Promise<'stopped' | 'already_gone' | 'still_alive'> { - const result = await this.machineRpc(machineId, RPC_METHODS.StopSession, { sessionId }) + async stopRunnerSession( + machineId: string, + sessionId: string, + opts?: { processStartMarker?: string } + ): Promise<'stopped' | 'already_gone' | 'still_alive' | 'unknown'> { + const result = await this.machineRpc(machineId, RPC_METHODS.StopSession, { + sessionId, + ...(opts?.processStartMarker ? { processStartMarker: opts.processStartMarker } : {}), + }) const status = result && typeof result === 'object' ? (result as { status?: unknown }).status : undefined - if (status === 'stopped' || status === 'already_gone' || status === 'still_alive') return status + if (status === 'stopped' || status === 'already_gone' || status === 'still_alive' || status === 'unknown') return status throw new Error('Unexpected stop-session response') } @@ -217,10 +231,12 @@ export class RpcGateway { collaborationMode?: CodexCollaborationMode, copilotAgentMode?: CopilotAgentMode, startingMode?: 'remote' | 'pty', - // Hub session id to reuse for this spawn. When set, the runner boots the - // CLI with `--hapi-session-id`, so the child reuses the existing hub - // session row (same id) instead of minting a new one. - forkSession?: boolean + // Hub session id for this spawn (preallocated stub or reopen). Runner + // stamps `--existing-session-id` or `--hapi-session-id` by flavor; the + // latter is adopt-stub (create/getOrCreate with id), not reopen. + forkSession?: boolean, + /** Fresh machine-spawn stub — distinct from reopen existingSessionId. */ + reservedSessionId?: string ): Promise< | { type: 'success'; sessionId: string } | { @@ -228,6 +244,8 @@ export class RpcGateway { message: string code?: 'agent_unavailable' | 'outside_workspace_roots' agent?: AgentFlavor + /** Explicit false = no OS child; stub safe to delete (#1911 B3). */ + childStarted?: boolean } > { try { @@ -248,7 +266,10 @@ export class RpcGateway { permissionMode, serviceTier, existingSessionId, - sessionId: existingSessionId, + reservedSessionId, + // Local HTTP / tracking may still read sessionId; prefer + // reserved stub id, then reopen id. + sessionId: reservedSessionId ?? existingSessionId, collaborationMode, copilotAgentMode, startingMode, @@ -265,15 +286,21 @@ export class RpcGateway { ? obj.code : undefined const unavailableAgent = typeof obj.agent === 'string' ? obj.agent as AgentFlavor : undefined + const childStarted = obj.childStarted === false ? false : undefined return { type: 'error', message: obj.errorMessage, ...(code ? { code } : {}), ...(unavailableAgent ? { agent: unavailableAgent } : {}), + ...(childStarted === false ? { childStarted: false } : {}), } } if (obj.type === 'requestToApproveDirectoryCreation' && typeof obj.directory === 'string') { - return { type: 'error', message: `Directory creation requires approval: ${obj.directory}` } + return { + type: 'error', + message: `Directory creation requires approval: ${obj.directory}`, + childStarted: false, + } } if (typeof obj.error === 'string') { return { type: 'error', message: obj.error } @@ -293,6 +320,8 @@ export class RpcGateway { })() return { type: 'error', message: `Unexpected spawn result: ${details}` } } catch (error) { + // Ambiguous: the machine RPC may have started a child before failing. + // Do not claim childStarted: false — hub keeps the stub. return { type: 'error', message: error instanceof Error ? error.message : String(error) } } } diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 5a83f10c80..cba8c5f3a6 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -101,6 +101,29 @@ export class SessionCache { return this.refreshSession(stored.id) ?? (() => { throw new Error('Failed to load session') })() } + adoptPreallocatedSession( + id: string, + tag: string, + metadata: unknown, + agentState: unknown, + namespace: string, + model?: string, + effort?: string, + modelReasoningEffort?: string + ): Session { + const stored = this.store.sessions.adoptPreallocatedSession( + id, + tag, + metadata, + agentState, + namespace, + model, + effort, + modelReasoningEffort + ) + return this.refreshSession(stored.id) ?? (() => { throw new Error('Failed to load adopted session') })() + } + /** * After fork hydrate / rewind truncate, re-scan the transcript for the * latest TodoWrite (or clear todos). Bypasses the one-shot backfill flag @@ -1084,7 +1107,8 @@ export class SessionCache { next, session.metadataVersion, session.namespace, - { touchUpdatedAt: false } + // #1911 M1: store rejects un-archive unless hub reopen opts in. + { touchUpdatedAt: false, allowUnarchive: true } ) if (result.result === 'error') { diff --git a/hub/src/sync/sessionModel.test.ts b/hub/src/sync/sessionModel.test.ts index 989649c29f..9f3979f583 100644 --- a/hub/src/sync/sessionModel.test.ts +++ b/hub/src/sync/sessionModel.test.ts @@ -580,21 +580,57 @@ describe('session model', () => { _machineId: string, _directory: string, agent: string, - model?: string + model?: string, + _modelReasoningEffort?: string, + _yolo?: boolean, + _sessionType?: string, + _worktreeName?: string, + _resumeSessionId?: string, + _effort?: string, + _permissionMode?: string, + _serviceTier?: string, + existingSessionId?: string, + _collaborationMode?: string, + _copilotAgentMode?: string, + _startingMode?: string, + _forkSession?: boolean, + reservedSessionId?: string ) => { capturedModel = model - return { type: 'success', sessionId: 'spawned-cursor-session' } + return { + type: 'success', + sessionId: reservedSessionId ?? existingSessionId ?? 'spawned-cursor-session', + } } const result = await engine.spawnSession( 'machine-cursor', '/tmp/project', 'cursor', - 'composer-2.5[fast=false]' + 'composer-2.5[fast=false]', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' ) - expect(result).toEqual({ type: 'success', sessionId: 'spawned-cursor-session' }) + expect(result.type).toBe('success') expect(capturedModel).toBe('composer-2.5[fast=false]') + if (result.type === 'success') { + expect(typeof result.sessionId).toBe('string') + expect(result.sessionId.length).toBeGreaterThan(0) + const meta = store.sessions.getSession(result.sessionId)?.metadata as { flavor?: string } | null + expect(meta?.flavor).toBe('cursor') + } } finally { engine.stop() } @@ -2396,9 +2432,9 @@ describe('session model', () => { const result = await engine.reopenSession(session.id, 'default') expect(result).toMatchObject({ type: 'error', message: expect.stringContaining('still active') }) expect(engine.getSessionByNamespace(session.id, 'default')?.active).toBe(true) - // Pi keeps the persisted archive snapshot until bootstrap succeeds, - // so a failed stop never needs to reconstruct it from memory. - expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived') + // #1911 M1: reopen clears archive before spawn; a live Pi child that + // failed stop stays active and must not be re-archived from memory. + expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).not.toBe('archived') expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt?.state).toBe('quarantined') expect(await engine.reopenSession(session.id, 'default')).toMatchObject({ type: 'error', message: 'Pi resume is already in progress' }) @@ -2450,7 +2486,11 @@ describe('session model', () => { type: 'error', message: 'webhook timeout' }) expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt?.state).toBe('quarantined') - expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived') + // #1911 M1: archive cleared before spawn; quarantine refuses reopen + // rollback (rollbackSafe:false). Snapshot lives on the attempt. + expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).not.toBe('archived') + expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt?.archiveSnapshot) + .toMatchObject({ lifecycleState: 'archived', archivedBy: 'cli' }) expect(await engine.reopenSession(session.id, 'default')).toMatchObject({ type: 'error', message: 'Pi resume is already in progress' }) diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 56552ed869..e20e0de104 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -32,6 +32,7 @@ import { MessageService, type RetryIndeterminateMessageResult } from './messageS import { createTitleSuggestionService, type TitleSuggestionService } from './titleSuggestion' import { selectForkTranscriptPrefix } from './forkTranscript' import { buildForkSessionSummary } from './forkSessionSummary' +import { isMachineSpawnPreallocatedStub } from '../store/sessions' import { RpcGateway, RpcTargetMissingError, @@ -575,6 +576,7 @@ export class SyncEngine { (session) => session.metadata?.piResumeAttempt?.childSessionId === payload.sid ) const restorePiArchive = ownsPiAttempt && !this.sessionReadyIds.has(payload.sid) + const restorePtyArchive = ownsPtyAttempt && !this.sessionReadyIds.has(payload.sid) const isCursorAcp = before?.metadata?.flavor === 'cursor' && before.metadata.cursorSessionProtocol === 'acp' const shouldRetryDedup = !ownsPiAttempt && !isPiAttemptChild && (!isCursorAcp || this.sessionReadyIds.has(payload.sid)) @@ -598,7 +600,7 @@ export class SyncEngine { void this.clearPiAttemptForEndedSession(payload.sid, restorePiArchive) } if (ownsPtyAttempt) { - void this.writePtyResumeAttempt(payload.sid, before!.namespace, null).catch(() => {}) + void this.writePtyResumeAttempt(payload.sid, before!.namespace, null, restorePtyArchive).catch(() => {}) } // Notify agent-terminal subscribers so the web UI shows a clear @@ -1020,6 +1022,28 @@ export class SyncEngine { ) } + adoptPreallocatedSession( + id: string, + tag: string, + metadata: unknown, + agentState: unknown, + namespace: string, + model?: string, + effort?: string, + modelReasoningEffort?: string + ): Session { + return this.sessionCache.adoptPreallocatedSession( + id, + tag, + metadata, + agentState, + namespace, + model, + effort, + modelReasoningEffort + ) + } + getOrCreateMachine(id: string, metadata: unknown, runnerState: unknown, namespace: string): Machine { return this.machineCache.getOrCreateMachine(id, metadata, runnerState, namespace) } @@ -1604,7 +1628,7 @@ export class SyncEngine { ): Promise { if (spawnAttempted) { const status = await this.rpcGateway.stopRunnerSession(machineId, childId) - if (status === 'still_alive') { + if (status === 'still_alive' || status === 'unknown') { throw new Error('Fork child termination was not confirmed') } } @@ -1707,27 +1731,149 @@ export class SyncEngine { } async archiveSession(sessionId: string): Promise { - // tiann/hapi#916: when the CLI is already gone (e.g. after a - // hub-restart cascade SIGTERMed the runner but the in-memory - // `active` flag has not been reconciled yet) the kill-RPC throws - // and the route used to surface that as HTTP 500. Treat the - // missing target as a benign condition: still flip the session's - // lifecycleState to `archived` in the hub-side metadata so the - // UI does not see a half-cleaned zombie, and continue to mark - // it inactive in the cache. Real RPC errors (timeout, protocol - // failure) still propagate as 5xx. + // tiann/hapi#916 / #1910: KillSession is a session-socket RPC. A missing + // target does not prove the runner child is dead (stale registration, + // mid-reconnect, id rotation on resume). Always fall through to the + // machine-level StopSession RPC when we know a machineId, and refuse + // to archive while the runner reports still_alive / unknown. + let cliUnreachable = false + let killPid: number | undefined + let killProcessStartMarker: string | undefined try { - await this.rpcGateway.killSession(sessionId) + const killResult = (await this.rpcGateway.killSession(sessionId)) ?? {} + killPid = killResult.pid + killProcessStartMarker = killResult.processStartMarker } catch (error) { if (error instanceof RpcTargetMissingError) { - this.sessionCache.markSessionArchivedFromHub(sessionId, 'Archived from hub (CLI unreachable)') + cliUnreachable = true } else { throw error } } + + const sessionMeta = this.sessionCache.getSession(sessionId)?.metadata + const machineId = sessionMeta?.machineId + const runnerSpawned = sessionMeta?.startedBy === 'runner' + || sessionMeta?.startedFromRunner === true + if (machineId && runnerSpawned) { + let status: 'stopped' | 'already_gone' | 'still_alive' | 'unknown' + try { + status = await this.rpcGateway.stopRunnerSession(machineId, sessionId) + } catch (stopError) { + // Machine RPC missing is NOT proof the detached CLI is gone + // (KillMode=process children survive runner death). Refuse to + // archive on any StopSession failure — including when KillSession + // was also unreachable. Both targets missing still leaves a + // possible live orphan; retry StopSession when the runner + // reconnects (#1911 bot Major). + void stopError + status = 'still_alive' + } + // KillSession acknowledges before cleanupAndExit finishes, and socket + // loss is not exit proof. When the runner cannot find the HAPI id, + // confirm the KillSession-reported OS pid + start marker. + const killPidConfirmable = ( + typeof killPid === 'number' + && killPid > 0 + && typeof killProcessStartMarker === 'string' + && killProcessStartMarker.length > 0 + ) + if (status === 'unknown' && killPidConfirmable) { + try { + status = await this.rpcGateway.stopRunnerSession(machineId, `PID-${killPid}`, { + processStartMarker: killProcessStartMarker, + }) + } catch (pidStopError) { + void pidStopError + status = 'still_alive' + } + } + // Estate dogfood (#1911 / Peer #1820): KillSession often supplies no + // pid+marker when the CLI socket is already gone. If session-id stop + // is unknown, ask the runner about metadata.hostPid as a tombstone — + // without a start marker the runner returns already_gone only when + // the OS pid is dead (alive → unknown; never tree-kills). Refuse + // still_alive / unknown on that confirm. + if (status === 'unknown' && !killPidConfirmable) { + const hostPid = sessionMeta?.hostPid + if (typeof hostPid === 'number' && hostPid > 0) { + try { + status = await this.rpcGateway.stopRunnerSession(machineId, `PID-${hostPid}`) + } catch (hostPidStopError) { + void hostPidStopError + status = 'still_alive' + } + } + } + // Ambiguous machine-spawn keep-stub: startedBy=runner, no hostPid, still + // tagged machine-spawn:. StopSession returns unknown forever (nothing + // tracked). There is no OS child to confirm — allow hub archive so the + // ghost is not permanent while the runner is online (#1911 Opus Major). + // A late-booting child that later hits reopen must not resurrect: CLI + // bootstrapExistingSession refuses archived rows (#1911 cold-read M1). + if (status === 'unknown') { + const stored = this.store.sessions.getSession(sessionId) + if ( + stored + && isMachineSpawnPreallocatedStub(stored) + && !(typeof sessionMeta?.hostPid === 'number' && sessionMeta.hostPid > 0) + ) { + status = 'already_gone' + } + } + if (status === 'still_alive' || status === 'unknown') { + throw new Error('Session process is still running and could not be stopped') + } + } else if (machineId && !runnerSpawned) { + // Terminal / non-runner sessions: KillSession is the primary stop. + // Best-effort machine StopSession when a runner is connected; do not + // refuse archive when no runner exists (#1911 bot Major). + try { + await this.rpcGateway.stopRunnerSession(machineId, sessionId) + } catch { + // ignore — terminal archive proceeds after KillSession + } + } + + // KillSession cleanup writes archive metadata asynchronously; StopSession + // may terminate the CLI mid-flush. If the row is still not archived, + // hub-author the metadata so we do not leave lifecycleState=running. + const lifecycleState = this.sessionCache.getSession(sessionId)?.metadata?.lifecycleState + if (lifecycleState !== 'archived') { + this.sessionCache.markSessionArchivedFromHub( + sessionId, + cliUnreachable + ? 'Archived from hub (CLI unreachable)' + : 'Archived from hub (CLI stopped before archive metadata)' + ) + this.emitCliSessionMetadataUpdate(sessionId) + } this.handleSessionEnd({ sid: sessionId, time: Date.now() }) } + /** + * Broadcast versioned session metadata to CLI sockets in `session:`. + * Mirrors the shape used by update-metadata handlers so ApiSessionClient + * applies the same hub-archived detection path. + */ + private emitCliSessionMetadataUpdate(sessionId: string): void { + const session = this.sessionCache.getSession(sessionId) + if (!session?.metadata) return + if (typeof this.io.of !== 'function') return + const update = { + id: randomUUID(), + seq: Date.now(), + createdAt: Date.now(), + body: { + t: 'update-session' as const, + sid: sessionId, + metadata: { version: session.metadataVersion, value: session.metadata }, + agentState: null as null + } + } + this.io.of('/cli').to(`session:${sessionId}`).emit('update', update) + } + /** * Apply the post-migration metadata flip in hapi.db: * - metadata.cursorSessionProtocol = 'acp' @@ -2032,26 +2178,127 @@ export class SyncEngine { existingSessionId?: string, collaborationMode?: CodexCollaborationMode, copilotAgentMode?: CopilotAgentMode, - startingMode?: 'remote' | 'pty' + startingMode?: 'remote' | 'pty', + // Required for fresh machine spawns so the runner stamps the HAPI id on + // argv before the first webhook (#1911 Major: unreapable window). + namespace?: string ): ReturnType { - return await this.rpcGateway.spawnSession( - machineId, - directory, - agent, - model, - modelReasoningEffort, - yolo, - sessionType, - worktreeName, - resumeSessionId, - effort, - permissionMode, - serviceTier, - existingSessionId, - collaborationMode, - copilotAgentMode, - startingMode - ) + // Fresh machine spawns historically omitted existingSessionId, so + // buildCliArgs could not stamp --hapi-session-id / --existing-session-id. + // A runner restart before the first webhook then left no persisted PID + // map and no argv id — StopSession returned unknown. Preallocate the + // hub row (same pattern as fork / OpenCode clear) and pass that id. + let allocatedSessionId = existingSessionId + let preallocated = false + if (!allocatedSessionId && namespace) { + const machine = this.getMachineByNamespace(machineId, namespace) + ?? this.getMachine(machineId) + allocatedSessionId = randomUUID() + this.sessionCache.getOrCreateSession( + `machine-spawn:${allocatedSessionId}`, + { + path: directory, + host: machine?.metadata?.host ?? 'unknown', + flavor: agent, + machineId, + startedBy: 'runner', + startedFromRunner: true, + }, + null, + namespace, + model, + effort, + modelReasoningEffort, + allocatedSessionId + ) + preallocated = true + } + + let result: Awaited> + try { + result = await this.rpcGateway.spawnSession( + machineId, + directory, + agent, + model, + modelReasoningEffort, + yolo, + sessionType, + worktreeName, + resumeSessionId, + effort, + permissionMode, + serviceTier, + // Fresh prealloc stubs must not go down reopen (--existing-session-id): + // Codex would demand a thread binding that does not exist yet (#1911). + preallocated ? undefined : allocatedSessionId, + collaborationMode, + copilotAgentMode, + startingMode, + undefined, + preallocated ? allocatedSessionId : undefined + ) + } catch (error) { + // Ambiguous post-dispatch failure — keep the stub (child may exist). + if (preallocated && allocatedSessionId) { + return { + type: 'error', + message: error instanceof Error ? error.message : String(error), + } + } + throw error + } + + if (result.type !== 'success' && preallocated && allocatedSessionId) { + const deleteStubIfStillPrealloc = async (): Promise => { + try { + // Session wire type has no tag — read the store row for stub check. + const stored = this.store.sessions.getSession(allocatedSessionId!) + if (!stored || !isMachineSpawnPreallocatedStub(stored)) return + const row = this.sessionCache.refreshSession(allocatedSessionId!) + if (!row) return + if (row.active) { + this.handleSessionEnd({ sid: allocatedSessionId!, time: Date.now(), reason: 'error' }) + } + await this.deleteSession(allocatedSessionId!) + } catch { + // Leave the stub visible rather than claiming cleanup succeeded. + } + } + + // Pre-exec rejection: runner never started an OS child — safe to delete. + if (result.childStarted === false) { + await deleteStubIfStillPrealloc() + return result + } + + // Ambiguous (RPC timeout, post-exec failure, etc.): keep the stub. + // Do NOT call stopRunnerSession — that would kill a healthy late-booting + // CLI — and do NOT deleteSession (ON DELETE CASCADE wipes transcript). + // Matches the throw-path keep-stub policy above (#1911 Critical). + // Archive of these stubs is allowed via isMachineSpawnPreallocatedStub + // hatch in archiveSession (no hostPid → no process to confirm). + return result + } + + // Runner must bind the preallocated id — a divergent success id leaves + // the stub as a silent ghost (#1911 Opus robustness note). + if ( + result.type === 'success' + && preallocated + && allocatedSessionId + && result.sessionId !== allocatedSessionId + ) { + console.warn( + `[spawn] runner reported sessionId ${result.sessionId} but prealloc was ${allocatedSessionId}; treating as error and keeping stub` + ) + return { + type: 'error', + message: `Runner reported unexpected session id ${result.sessionId} (expected ${allocatedSessionId})`, + } + } + + return result } /** @@ -2866,7 +3113,7 @@ export class SyncEngine { if (session.active || operation?.state !== 'reserved' || !machineId) return false try { const status = await this.rpcGateway.stopRunnerSession(machineId, session.id) - if (status === 'still_alive') return false + if (status === 'still_alive' || status === 'unknown') return false return this.abortOpenCodeClearSession( session.id, namespace, operation.replacementSessionId, 'reserved', true ).type === 'success' @@ -3044,6 +3291,12 @@ export class SyncEngine { state: 'resuming', machineId: targetMachine.id, startedAt: Date.now(), + archiveSnapshot: { + lifecycleState: metadata.lifecycleState, + lifecycleStateSince: metadata.lifecycleStateSince, + archivedBy: metadata.archivedBy, + archiveReason: metadata.archiveReason, + }, }) } catch { this.ptyResumeInFlightIds.delete(access.sessionId) @@ -3055,7 +3308,22 @@ export class SyncEngine { this.sessionReadyIds.delete(access.sessionId) } let piResumeSucceeded = false + let ptyResumeSucceeded = false try { + // #1911 M1: clear archived lifecycle immediately before spawn so the + // CLI is not refused / cannot CAS-resurrect. Pi/PTY attempt rows + // above already captured archiveSnapshot while the row was archived. + const liveBeforeSpawn = this.sessionCache.getSessionByNamespace(access.sessionId, namespace) + ?? this.sessionCache.refreshSession(access.sessionId) + if (liveBeforeSpawn?.metadata?.lifecycleState === 'archived') { + try { + await this.sessionCache.clearSessionArchiveMetadata(access.sessionId) + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to clear archive metadata' + return { type: 'error', message, code: 'resume_failed' } + } + } + const spawnResult = await this.rpcGateway.spawnSession( targetMachine.id, directory, @@ -3153,7 +3421,7 @@ export class SyncEngine { const readyResult = await this.waitForSessionReady(spawnResult.sessionId) if (readyResult !== 'ready') { if (resumedStartingMode === 'pty' && readyResult === 'timeout') { - let status: 'stopped' | 'already_gone' | 'still_alive' + let status: 'stopped' | 'already_gone' | 'still_alive' | 'unknown' try { status = await this.rpcGateway.stopRunnerSession( targetMachine.id, @@ -3175,7 +3443,10 @@ export class SyncEngine { if (!inactive) { this.ptyResumeQuarantinedIds.add(access.sessionId) try { + const existingAttempt = this.sessionCache.getSession(access.sessionId) + ?.metadata?.ptyResumeAttempt await this.writePtyResumeAttempt(access.sessionId, namespace, { + ...existingAttempt, state: 'quarantined', machineId: targetMachine.id, startedAt: Date.now(), @@ -3205,7 +3476,9 @@ export class SyncEngine { } if (resumedStartingMode === 'pty') { try { - await this.writePtyResumeAttempt(access.sessionId, namespace, null) + // Child stopped after timeout — restore archive from + // the attempt snapshot (clear-before-spawn already ran). + await this.writePtyResumeAttempt(access.sessionId, namespace, null, true) } catch { this.ptyResumeQuarantinedIds.add(access.sessionId) return { @@ -3250,6 +3523,7 @@ export class SyncEngine { try { await this.writePtyResumeAttempt(access.sessionId, namespace, null) this.ptyResumeQuarantinedIds.delete(access.sessionId) + ptyResumeSucceeded = true } catch { this.ptyResumeQuarantinedIds.add(access.sessionId) return { @@ -3264,6 +3538,17 @@ export class SyncEngine { } finally { if (resumedStartingMode === 'pty') { this.ptyResumeInFlightIds.delete(access.sessionId) + // Do not clear a deliberate fail-closed `resuming` marker left + // when quarantine write failed or still_alive refused stop — + // those paths add ptyResumeQuarantinedIds and keep the durable + // attempt (with archiveSnapshot) as the restart-safe truth. + if ( + !ptyResumeSucceeded + && !this.ptyResumeQuarantinedIds.has(access.sessionId) + && this.sessionCache.getSession(access.sessionId)?.metadata?.ptyResumeAttempt?.state === 'resuming' + ) { + await this.writePtyResumeAttempt(access.sessionId, namespace, null, true).catch(() => {}) + } } if (requiresPiNativeReady) { this.piResumeInFlightIds.delete(access.sessionId) @@ -3389,23 +3674,10 @@ export class SyncEngine { lifecycleStateSince: metadata.lifecycleStateSince } - let applied: { cursorSessionProtocol?: 'acp' | 'stream-json' } = {} - // Pi and PTY resumes both reuse the original HAPI row. Keep the archive - // snapshot persisted until the CLI successfully bootstraps that row as - // running; this avoids an inactive, non-archived gap if the Hub restarts - // before spawn — the in-memory snapshot below cannot survive that, and - // ptyResumeAttempt carries no copy of it. The CLI's sessionFactory - // re-stamps lifecycleState='running' on boot and does not carry over - // archivedBy/archiveReason, so the row still leaves the archived state. - if (metadata.flavor !== 'pi' && !isPtyResume) { - try { - applied = await this.sessionCache.clearSessionArchiveMetadata(access.sessionId) - } catch (error) { - const message = error instanceof Error ? error.message : 'Failed to clear archive metadata' - return { type: 'error', message, code: 'metadata_conflict' } - } - } - + // #1911 M1: clear runs inside resumeSession immediately before spawn + // (after Pi/PTY attempt rows capture archiveSnapshot). Hub restart + // between clear and spawn leaves a non-archived inactive row; + // archiveSnapshot still rolls back on resume failure while alive. const resumeResult = await this.resumeSession(access.sessionId, namespace) if (resumeResult.type === 'error') { // Never restore archived metadata over a live Pi child. A live @@ -3421,11 +3693,17 @@ export class SyncEngine { return resumeResult } + const after = this.sessionCache.getSessionByNamespace(access.sessionId, namespace)?.metadata + const cursorSessionProtocol = after?.flavor === 'cursor' + && (after.cursorSessionProtocol === 'acp' || after.cursorSessionProtocol === 'stream-json') + ? after.cursorSessionProtocol + : undefined + return { type: 'success', sessionId: resumeResult.sessionId, resumed: true, - ...(applied.cursorSessionProtocol ? { cursorSessionProtocol: applied.cursorSessionProtocol } : {}) + ...(cursorSessionProtocol ? { cursorSessionProtocol } : {}) } } @@ -3674,7 +3952,7 @@ export class SyncEngine { machineId, startedAt: Date.now(), }) - let status: 'stopped' | 'already_gone' | 'still_alive' + let status: 'stopped' | 'already_gone' | 'still_alive' | 'unknown' try { status = await this.rpcGateway.stopRunnerSession(machineId, sessionId) } catch { @@ -3684,7 +3962,7 @@ export class SyncEngine { await new Promise((resolve) => setTimeout(resolve, 0)) const session = this.sessionCache.refreshSession(sessionId) ?? this.sessionCache.getSession(sessionId) const attemptClearedByEnd = session?.metadata?.piResumeAttempt === undefined - if (status === 'still_alive') { + if (status === 'still_alive' || status === 'unknown') { if (attemptClearedByEnd) return true return false } @@ -3708,7 +3986,7 @@ export class SyncEngine { startedAt: Date.now(), childSessionId: sessionId, }) - let status: 'stopped' | 'already_gone' | 'still_alive' + let status: 'stopped' | 'already_gone' | 'still_alive' | 'unknown' try { status = await this.rpcGateway.stopRunnerSession(machineId, sessionId) } catch { @@ -3719,7 +3997,7 @@ export class SyncEngine { const session = this.sessionCache.refreshSession(sessionId) ?? this.sessionCache.getSession(sessionId) const original = this.sessionCache.refreshSession(originalSessionId) ?? this.sessionCache.getSession(originalSessionId) const attemptClearedByEnd = original?.metadata?.piResumeAttempt === undefined - if (status === 'still_alive' && !attemptClearedByEnd) { + if ((status === 'still_alive' || status === 'unknown') && !attemptClearedByEnd) { await this.writePiResumeAttempt(originalSessionId, namespace, { ...existingAttempt, state: 'quarantined', @@ -3808,7 +4086,8 @@ export class SyncEngine { private async writePtyResumeAttempt( sessionId: string, namespace: string, - attempt: PtyResumeAttempt | null + attempt: PtyResumeAttempt | null, + restoreArchive = false ): Promise { for (let i = 0; i < 5; i += 1) { const current = this.sessionCache.getSessionByNamespace(sessionId, namespace) @@ -3816,7 +4095,20 @@ export class SyncEngine { if (!current?.metadata) throw new Error('PTY resume attempt session metadata is unavailable') const next = { ...current.metadata } if (attempt) next.ptyResumeAttempt = attempt - else delete next.ptyResumeAttempt + else { + const snapshot = current.metadata.ptyResumeAttempt?.archiveSnapshot + delete next.ptyResumeAttempt + if (restoreArchive && snapshot) { + if (snapshot.lifecycleState === undefined) delete next.lifecycleState + else next.lifecycleState = snapshot.lifecycleState + if (snapshot.lifecycleStateSince === undefined) delete next.lifecycleStateSince + else next.lifecycleStateSince = snapshot.lifecycleStateSince + if (snapshot.archivedBy === undefined) delete next.archivedBy + else next.archivedBy = snapshot.archivedBy + if (snapshot.archiveReason === undefined) delete next.archiveReason + else next.archiveReason = snapshot.archiveReason + } + } const result = this.store.sessions.updateSessionMetadata( sessionId, next, @@ -3837,20 +4129,22 @@ export class SyncEngine { private async reconcilePersistedPtyResumeAttempt(session: Session): Promise { const attempt = session.metadata?.ptyResumeAttempt if (!attempt) return true - let status: 'stopped' | 'already_gone' | 'still_alive' + let status: 'stopped' | 'already_gone' | 'still_alive' | 'unknown' try { status = await this.rpcGateway.stopRunnerSession(attempt.machineId, session.id) } catch { return false } - if (status === 'still_alive') return false + if (status === 'still_alive' || status === 'unknown') return false const current = this.sessionCache.getSession(session.id) if (current?.active) { this.handleSessionEnd({ sid: session.id, time: Date.now(), reason: 'error' }) } try { - await this.writePtyResumeAttempt(session.id, session.namespace, null) + // Restore archive from the attempt snapshot when cleaning a failed + // resume (clear-before-spawn already dropped live archive fields). + await this.writePtyResumeAttempt(session.id, session.namespace, null, true) this.ptyResumeQuarantinedIds.delete(session.id) return true } catch { @@ -3863,13 +4157,13 @@ export class SyncEngine { const attempt = session.metadata?.piResumeAttempt if (!attempt) return true const childSessionId = attempt.childSessionId ?? session.id - let status: 'stopped' | 'already_gone' | 'still_alive' + let status: 'stopped' | 'already_gone' | 'still_alive' | 'unknown' try { status = await this.rpcGateway.stopRunnerSession(attempt.machineId, childSessionId) } catch { return false } - if (status === 'still_alive') return false + if (status === 'still_alive' || status === 'unknown') return false const child = this.sessionCache.getSession(childSessionId) if (child?.active) this.handleSessionEnd({ sid: childSessionId, time: Date.now(), reason: 'error' }) diff --git a/hub/src/sync/syncEngineArchiveSession.test.ts b/hub/src/sync/syncEngineArchiveSession.test.ts new file mode 100644 index 0000000000..268c9af275 --- /dev/null +++ b/hub/src/sync/syncEngineArchiveSession.test.ts @@ -0,0 +1,388 @@ +import { describe, expect, it, beforeEach } from 'bun:test' +import { Store } from '../store' +import { RpcRegistry } from '../socket/rpcRegistry' +import { SyncEngine } from './syncEngine' +import { RpcTargetMissingError } from './rpcGateway' +import type { SessionCache } from './sessionCache' + +/** + * #1910 / #1705: `killSession` is a session-socket RPC. A missing target does + * not prove the runner child is dead. Archive must always ask the runner via + * `stopRunnerSession` when a machineId is known, and must refuse to archive + * when the runner reports the process still alive (or unknown). + */ +describe('SyncEngine.archiveSession runner reaping (#1910)', () => { + let store: Store + let engine: SyncEngine + const NAMESPACE = 'default' + + function cache(): SessionCache { + return (engine as unknown as { sessionCache: SessionCache }).sessionCache + } + + function insertActiveSession( + tag: string, + machineId?: string, + hostPid?: number, + opts?: { startedBy?: 'runner' | 'terminal'; startedFromRunner?: boolean } + ): string { + const startedBy = opts?.startedBy ?? 'runner' + const created = cache().getOrCreateSession( + tag, + { + path: '/tmp/proj', + host: 'localhost', + flavor: 'claude', + startedBy, + ...(opts?.startedFromRunner !== undefined + ? { startedFromRunner: opts.startedFromRunner } + : startedBy === 'runner' ? { startedFromRunner: true } : {}), + ...(machineId ? { machineId } : {}), + ...(typeof hostPid === 'number' ? { hostPid } : {}), + }, + null, + NAMESPACE + ) + cache().markSessionActive(created.id) + return created.id + } + + function setKillSessionMissingTarget(): void { + ;(engine as unknown as { rpcGateway: { killSession: unknown } }).rpcGateway.killSession = + async () => { throw new RpcTargetMissingError('KillSession', 'handler-not-registered') } + } + + function setKillSessionOk(opts?: { pid?: number; processStartMarker?: string }): void { + ;(engine as unknown as { rpcGateway: { killSession: unknown } }).rpcGateway.killSession = + async () => ({ + ...(typeof opts?.pid === 'number' ? { pid: opts.pid } : {}), + ...(opts?.processStartMarker ? { processStartMarker: opts.processStartMarker } : {}), + }) + } + + beforeEach(() => { + store = new Store(':memory:') + engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never) + }) + + it('does not archive when the runner confirms still_alive after KillSession miss', async () => { + const sessionId = insertActiveSession('sess-still-alive', 'machine-x') + setKillSessionMissingTarget() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => 'still_alive' + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + const session = cache().getSession(sessionId) + expect(session?.active).toBe(true) + expect(session?.metadata?.lifecycleState).not.toBe('archived') + }) + + it('always calls stopRunnerSession after a successful KillSession', async () => { + const sessionId = insertActiveSession('sess-kill-ok', 'machine-x') + setKillSessionOk() + let calledWith: [string, string] | undefined + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async (machineId: string, sid: string) => { + calledWith = [machineId, sid] + return 'already_gone' + } + + await engine.archiveSession(sessionId) + + expect(calledWith).toEqual(['machine-x', sessionId]) + expect(cache().getSession(sessionId)?.active).toBe(false) + // Stop confirmed before CLI could flush — hub must author archive metadata. + expect(cache().getSession(sessionId)?.metadata?.lifecycleState).toBe('archived') + expect(cache().getSession(sessionId)?.metadata?.archivedBy).toBe('hub') + }) + + it('archives once the runner confirms the process is gone', async () => { + const sessionId = insertActiveSession('sess-confirmed-gone', 'machine-x') + setKillSessionMissingTarget() + let calledWith: [string, string] | undefined + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async (machineId: string, sid: string) => { + calledWith = [machineId, sid] + return 'already_gone' + } + + await engine.archiveSession(sessionId) + + expect(calledWith).toEqual(['machine-x', sessionId]) + const session = cache().getSession(sessionId) + expect(session?.active).toBe(false) + expect(session?.metadata?.lifecycleState).toBe('archived') + }) + + it('falls back to archiving when the session has no known machine', async () => { + const sessionId = insertActiveSession('sess-no-machine') + setKillSessionMissingTarget() + let stopRunnerSessionCalled = false + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => { stopRunnerSessionCalled = true; return 'already_gone' } + + await engine.archiveSession(sessionId) + + expect(stopRunnerSessionCalled).toBe(false) + const session = cache().getSession(sessionId) + expect(session?.active).toBe(false) + expect(session?.metadata?.lifecycleState).toBe('archived') + }) + + it('emits Socket.IO update-session when hub-authoring archive without a machine (#1910)', async () => { + const emitted: Array<{ room: string; event: string; payload: unknown }> = [] + const io = { + of: (ns: string) => ({ + to: (room: string) => ({ + emit: (event: string, payload: unknown) => { + if (ns === '/cli') { + emitted.push({ room, event, payload }) + } + } + }) + }) + } + engine = new SyncEngine(store, io as never, new RpcRegistry(), { broadcast() {} } as never) + const sessionId = insertActiveSession('sess-hub-archive-socket') + setKillSessionMissingTarget() + + await engine.archiveSession(sessionId) + + expect(emitted).toHaveLength(1) + expect(emitted[0]?.room).toBe(`session:${sessionId}`) + expect(emitted[0]?.event).toBe('update') + const body = (emitted[0]?.payload as { body: { t: string; metadata: { version: number; value: { archivedBy?: string; lifecycleState?: string } } } }).body + expect(body.t).toBe('update-session') + expect(body.metadata.value.lifecycleState).toBe('archived') + expect(body.metadata.value.archivedBy).toBe('hub') + expect(body.metadata.version).toBeGreaterThan(0) + }) + + it('does NOT archive when both KillSession and machine StopSession targets are missing', async () => { + // #1911 bot Major: both RPC targets missing is not proof the detached + // CLI exited (KillMode=process orphans survive). Keep the row + // unconfirmed until StopSession can run after the runner reconnects. + const sessionId = insertActiveSession('sess-machine-unreachable', 'machine-x') + setKillSessionMissingTarget() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => { throw new RpcTargetMissingError('StopSession', 'handler-not-registered') } + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + const session = cache().getSession(sessionId) + expect(session?.active).toBe(true) + expect(session?.metadata?.lifecycleState).not.toBe('archived') + }) + + it('does NOT archive when machine StopSession is missing but KillSession was reachable', async () => { + // Machine socket alone missing is not proof the detached child is gone + // (KillMode=process). KillSession succeeded → refuse without confirm. + const sessionId = insertActiveSession('sess-machine-only-missing', 'machine-x') + setKillSessionOk() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => { throw new RpcTargetMissingError('StopSession', 'handler-not-registered') } + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + const session = cache().getSession(sessionId) + expect(session?.active).toBe(true) + expect(session?.metadata?.lifecycleState).not.toBe('archived') + }) + + it('does NOT archive when the runner reports unknown after KillSession miss', async () => { + const sessionId = insertActiveSession('sess-unknown-to-runner', 'machine-x') + setKillSessionMissingTarget() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => 'unknown' + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + const session = cache().getSession(sessionId) + expect(session?.active).toBe(true) + expect(session?.metadata?.lifecycleState).not.toBe('archived') + }) + + it('does NOT archive when KillSession succeeded but StopSession stays unknown without a confirmable pid', async () => { + // KillSession ack alone is not exit proof; without a pid the runner + // cannot confirm termination of an untracked CLI (#1910). + const sessionId = insertActiveSession('sess-kill-ok-unknown-no-pid', 'machine-x') + setKillSessionOk() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => 'unknown' + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + const session = cache().getSession(sessionId) + expect(session?.active).toBe(true) + expect(session?.metadata?.lifecycleState).not.toBe('archived') + }) + + it('does NOT archive when KillSession pid confirm lacks a start marker (PID reuse guard)', async () => { + const sessionId = insertActiveSession('sess-kill-ok-pid-no-marker', 'machine-x') + setKillSessionOk({ pid: 4242 }) + const stopCalls: Array<{ sid: string; marker?: string }> = [] + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async (_machineId: string, sid: string, opts?: { processStartMarker?: string }) => { + stopCalls.push({ sid, marker: opts?.processStartMarker }) + return 'unknown' + } + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + // Without a marker, do not attempt the raw-PID confirm kill. + expect(stopCalls).toEqual([{ sid: sessionId, marker: undefined }]) + expect(cache().getSession(sessionId)?.active).toBe(true) + }) + + it('does NOT archive when KillSession pid confirm still reports unknown (socket drop is not exit)', async () => { + const sessionId = insertActiveSession('sess-kill-ok-pid-unknown', 'machine-x') + setKillSessionOk({ pid: 4242, processStartMarker: 'started-at-1' }) + const stopCalls: Array<{ sid: string; marker?: string }> = [] + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async (_machineId: string, sid: string, opts?: { processStartMarker?: string }) => { + stopCalls.push({ sid, marker: opts?.processStartMarker }) + return 'unknown' + } + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + expect(stopCalls).toEqual([ + { sid: sessionId, marker: undefined }, + { sid: 'PID-4242', marker: 'started-at-1' }, + ]) + const session = cache().getSession(sessionId) + expect(session?.active).toBe(true) + expect(session?.metadata?.lifecycleState).not.toBe('archived') + }) + + it('archives when KillSession pid confirm returns already_gone after session-id unknown', async () => { + const sessionId = insertActiveSession('sess-kill-ok-pid-gone', 'machine-x') + setKillSessionOk({ pid: 4242, processStartMarker: 'started-at-1' }) + const stopCalls: Array<{ sid: string; marker?: string }> = [] + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async (_machineId: string, sid: string, opts?: { processStartMarker?: string }) => { + stopCalls.push({ sid, marker: opts?.processStartMarker }) + return sid.startsWith('PID-') ? 'already_gone' : 'unknown' + } + + await engine.archiveSession(sessionId) + + expect(stopCalls).toEqual([ + { sid: sessionId, marker: undefined }, + { sid: 'PID-4242', marker: 'started-at-1' }, + ]) + expect(cache().getSession(sessionId)?.active).toBe(false) + }) + + it('archives via metadata.hostPid tombstone when KillSession supplies no confirmable pid (#1911 dogfood)', async () => { + // Peer #1820 estate gap: StopSession(hapiId)=unknown, process already dead, + // KillSession missed — hub must check metadata.hostPid before 409. + const sessionId = insertActiveSession('sess-hostpid-tombstone', 'machine-x', 3704400) + setKillSessionMissingTarget() + const stopCalls: Array<{ sid: string; marker?: string }> = [] + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async (_machineId: string, sid: string, opts?: { processStartMarker?: string }) => { + stopCalls.push({ sid, marker: opts?.processStartMarker }) + return sid === 'PID-3704400' ? 'already_gone' : 'unknown' + } + + await engine.archiveSession(sessionId) + + expect(stopCalls).toEqual([ + { sid: sessionId, marker: undefined }, + { sid: 'PID-3704400', marker: undefined }, + ]) + expect(cache().getSession(sessionId)?.active).toBe(false) + expect(cache().getSession(sessionId)?.metadata?.lifecycleState).toBe('archived') + }) + + it('does NOT archive when metadata.hostPid confirm reports still_alive', async () => { + const sessionId = insertActiveSession('sess-hostpid-alive', 'machine-x', 3704400) + setKillSessionMissingTarget() + const stopCalls: string[] = [] + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async (_machineId: string, sid: string) => { + stopCalls.push(sid) + return sid === 'PID-3704400' ? 'still_alive' : 'unknown' + } + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + expect(stopCalls).toEqual([sessionId, 'PID-3704400']) + expect(cache().getSession(sessionId)?.active).toBe(true) + expect(cache().getSession(sessionId)?.metadata?.lifecycleState).not.toBe('archived') + }) + + it('does NOT archive when StopSession fails ambiguously', async () => { + const sessionId = insertActiveSession('sess-machine-ambiguous-failure', 'machine-x') + setKillSessionMissingTarget() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => { throw new Error('ack timeout') } + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + + const session = cache().getSession(sessionId) + expect(session?.active).toBe(true) + expect(session?.metadata?.lifecycleState).not.toBe('archived') + }) + + it('archives a terminal session when no runner is connected', async () => { + const sessionId = insertActiveSession('sess-terminal-no-runner', 'machine-x', undefined, { + startedBy: 'terminal', + }) + setKillSessionOk() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => { throw new Error('machine offline') } + + await engine.archiveSession(sessionId) + + expect(cache().getSession(sessionId)?.active).toBe(false) + expect(cache().getSession(sessionId)?.metadata?.lifecycleState).toBe('archived') + }) + + it('archives a never-started machine-spawn stub when StopSession returns unknown (no hostPid)', async () => { + // #1911 Opus Major: keep-stub after ambiguous spawn has startedBy=runner, + // no hostPid → StopSession unknown forever while runner online. + const stubId = crypto.randomUUID() + const created = cache().getOrCreateSession( + `machine-spawn:${stubId}`, + { + path: '/tmp/proj', + host: 'localhost', + flavor: 'claude', + startedBy: 'runner', + startedFromRunner: true, + machineId: 'machine-x', + }, + null, + NAMESPACE, + undefined, + undefined, + undefined, + stubId + ) + cache().markSessionActive(created.id) + expect(created.id).toBe(stubId) + expect(store.sessions.getSession(stubId)?.tag).toBe(`machine-spawn:${stubId}`) + + setKillSessionMissingTarget() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => 'unknown' + + await engine.archiveSession(stubId) + + expect(cache().getSession(stubId)?.active).toBe(false) + expect(cache().getSession(stubId)?.metadata?.lifecycleState).toBe('archived') + }) + + it('does NOT archive a non-stub runner session on unknown without hostPid', async () => { + const sessionId = insertActiveSession('sess-live-unknown-no-pid', 'machine-x') + setKillSessionMissingTarget() + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }).rpcGateway.stopRunnerSession = + async () => 'unknown' + + await expect(engine.archiveSession(sessionId)).rejects.toThrow() + expect(cache().getSession(sessionId)?.active).toBe(true) + }) +}) diff --git a/hub/src/sync/syncEngineReopenPreservesPtyId.test.ts b/hub/src/sync/syncEngineReopenPreservesPtyId.test.ts index 1f328d3b3a..0c78cf7ada 100644 --- a/hub/src/sync/syncEngineReopenPreservesPtyId.test.ts +++ b/hub/src/sync/syncEngineReopenPreservesPtyId.test.ts @@ -131,15 +131,12 @@ describe('SyncEngine reopen/resume PTY session id preservation', () => { - it('keeps the archive snapshot persisted across a PTY reopen so a hub restart can still recover it', async () => { - // The snapshot used for rollback lives only in memory, so clearing the - // archive metadata before the resume is durably recorded leaves an - // inactive, non-archived ghost row if the hub restarts in between. The - // CLI's sessionFactory re-stamps lifecycleState='running' on boot (and - // drops archivedBy/archiveReason, which it never preserves), so keeping - // the snapshot until then is safe — this is the same reason Pi defers it. + it('clears archive metadata on PTY reopen before spawn (#1911 M1)', async () => { + // Hub store rejects un-archive without allowUnarchive, so reopen must + // clear archive before spawn (same as non-PTY). Attempt row carries + // archiveSnapshot for rollback / quarantine durability. const sessionId = insertSession( - 'pty-session-archive-durable', + 'pty-session-archive-clear', baseMetadata({ lifecycleState: 'archived', archivedBy: 'hub', archiveReason: 'inactivity' }), { startingMode: 'pty' } ).id @@ -148,7 +145,106 @@ describe('SyncEngine reopen/resume PTY session id preservation', () => { expect(result).toEqual({ type: 'success', sessionId, resumed: true }) const metadata = store.sessions.getSession(sessionId)?.metadata as Record | undefined + expect(metadata?.lifecycleState).not.toBe('archived') + expect(metadata?.archivedBy).toBeUndefined() + expect(metadata?.archiveReason).toBeUndefined() + }) + + it('keeps archiveSnapshot on PTY quarantine after clear-before-spawn (#1911 Major 2)', async () => { + // Clear-before-spawn + rollbackSafe:false must not lose archive fields — + // they live on ptyResumeAttempt.archiveSnapshot (mirror Pi). + const sessionId = insertSession( + 'pty-session-archive-durable', + baseMetadata({ lifecycleState: 'archived', archivedBy: 'hub', archiveReason: 'inactivity' }), + { startingMode: 'pty' } + ).id + ;(engine as any).rpcGateway.spawnSession = async () => { + engine.handleSessionAlive({ sid: sessionId, time: Date.now() }) + return { type: 'success', sessionId } + } + ;(engine as any).waitForSessionReady = async () => 'timeout' + ;(engine as any).rpcGateway.stopRunnerSession = async () => 'still_alive' + + const first = await engine.reopenSession(sessionId, NAMESPACE) + + expect(first).toMatchObject({ type: 'error', code: 'resume_failed', rollbackSafe: false }) + const metadata = engine.getSessionByNamespace(sessionId, NAMESPACE)?.metadata as any + expect(metadata?.lifecycleState).not.toBe('archived') + expect(metadata?.ptyResumeAttempt).toMatchObject({ + state: 'quarantined', + machineId: 'machine-x', + archiveSnapshot: { + lifecycleState: 'archived', + archivedBy: 'hub', + archiveReason: 'inactivity', + }, + }) + }) + + it('restores archive from PTY archiveSnapshot when a failed resume child is gone', async () => { + // Clear-before-spawn drops live archive fields; when the child is then + // confirmed gone, writePtyResumeAttempt(..., restoreArchive) must put + // them back from the attempt snapshot (mirror Pi). + const sessionId = insertSession( + 'pty-session-archive-restore', + baseMetadata({ lifecycleState: 'archived', archivedBy: 'hub', archiveReason: 'inactivity' }), + { startingMode: 'pty' } + ).id + ;(engine as any).rpcGateway.spawnSession = async () => ({ type: 'success', sessionId }) + ;(engine as any).waitForSessionActive = async () => false + ;(engine as any).rpcGateway.stopRunnerSession = async () => 'already_gone' + + const result = await engine.reopenSession(sessionId, NAMESPACE) + + expect(result).toMatchObject({ type: 'error', code: 'resume_failed' }) + expect((engine.getSessionByNamespace(sessionId, NAMESPACE)?.metadata as any)?.ptyResumeAttempt) + .toBeUndefined() + const metadata = engine.getSessionByNamespace(sessionId, NAMESPACE)?.metadata as any + expect(metadata?.lifecycleState).toBe('archived') + expect(metadata?.archivedBy).toBe('hub') + expect(metadata?.archiveReason).toBe('inactivity') + }) + + it('restores archive from a persisted PTY quarantine snapshot when the child is already gone', async () => { + const sessionId = insertSession( + 'pty-session-archive-restore-persisted', + baseMetadata({ + lifecycleState: 'running', + ptyResumeAttempt: { + state: 'quarantined', + machineId: 'machine-x', + startedAt: 1, + archiveSnapshot: { + lifecycleState: 'archived', + lifecycleStateSince: 100, + archivedBy: 'hub', + archiveReason: 'inactivity', + }, + }, + }), + { startingMode: 'pty' } + ).id + engine.handleSessionAlive({ sid: sessionId, time: Date.now() }) + let spawnCalls = 0 + ;(engine as any).rpcGateway.stopRunnerSession = async () => 'already_gone' + ;(engine as any).rpcGateway.spawnSession = async () => { + spawnCalls += 1 + // After reconcile restores archive, reopen continues into resume — + // refuse spawn so we can assert the restored archive stuck. + return { type: 'error', message: 'spawn refused for assert' } + } + + const result = await engine.reopenSession(sessionId, NAMESPACE) + + expect(result).toMatchObject({ type: 'error', message: 'spawn refused for assert' }) + expect(spawnCalls).toBe(1) + const metadata = engine.getSessionByNamespace(sessionId, NAMESPACE)?.metadata as any + // resumeSession clear-before-spawn will have cleared again after the + // reconcile restore; the attempt snapshot must still re-restore on the + // failed spawn finally path. + expect(metadata?.ptyResumeAttempt).toBeUndefined() expect(metadata?.lifecycleState).toBe('archived') + expect(metadata?.lifecycleStateSince).toBe(100) expect(metadata?.archivedBy).toBe('hub') expect(metadata?.archiveReason).toBe('inactivity') }) diff --git a/hub/src/sync/syncEngineSpawnPrealloc.test.ts b/hub/src/sync/syncEngineSpawnPrealloc.test.ts new file mode 100644 index 0000000000..5b7e586e10 --- /dev/null +++ b/hub/src/sync/syncEngineSpawnPrealloc.test.ts @@ -0,0 +1,745 @@ +import { describe, expect, it } from 'bun:test' +import { Store } from '../store' +import { RpcRegistry } from '../socket/rpcRegistry' +import { SyncEngine } from './syncEngine' + +/** + * #1911 Major: fresh machine spawns must preallocate a HAPI row id and pass it + * to the runner so buildCliArgs can stamp argv before the first webhook. + */ +describe('SyncEngine.spawnSession preallocates HAPI id for fresh machine spawns', () => { + it('creates a hub row and forwards that id as reservedSessionId', async () => { + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-prealloc', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + engine.handleMachineAlive({ machineId: 'machine-prealloc', time: Date.now() }) + + let forwardedExistingId: string | undefined + ;(engine as unknown as { rpcGateway: { spawnSession: unknown } }).rpcGateway.spawnSession = + async ( + _machineId: string, + _directory: string, + _agent?: string, + _model?: string, + _modelReasoningEffort?: string, + _yolo?: boolean, + _sessionType?: string, + _worktreeName?: string, + _resumeSessionId?: string, + _effort?: string, + _permissionMode?: string, + _serviceTier?: string, + existingSessionId?: string, + _collaborationMode?: string, + _copilotAgentMode?: string, + _startingMode?: string, + _forkSession?: boolean, + reservedSessionId?: string + ) => { + forwardedExistingId = reservedSessionId ?? existingSessionId + return { type: 'success' as const, sessionId: forwardedExistingId! } + } + + const result = await engine.spawnSession( + 'machine-prealloc', + '/tmp/project', + 'claude', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('success') + expect(typeof forwardedExistingId).toBe('string') + expect(forwardedExistingId!.length).toBeGreaterThan(0) + if (result.type === 'success') { + expect(result.sessionId).toBe(forwardedExistingId!) + } + const row = store.sessions.getSession(forwardedExistingId!) + expect(row?.id).toBe(forwardedExistingId) + const meta = row?.metadata as { flavor?: string; machineId?: string } | null + expect(meta?.flavor).toBe('claude') + expect(meta?.machineId).toBe('machine-prealloc') + } finally { + engine.stop() + } + }) + + it('rejects success when runner reports a different id than the prealloc stub', async () => { + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-id-mismatch', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + engine.handleMachineAlive({ machineId: 'machine-id-mismatch', time: Date.now() }) + + let reserved: string | undefined + ;(engine as unknown as { rpcGateway: { spawnSession: unknown } }).rpcGateway.spawnSession = + async ( + _m: string, _d: string, _a?: string, _mo?: string, _mr?: string, _y?: boolean, + _st?: string, _wn?: string, _rs?: string, _e?: string, _pm?: string, _svc?: string, + _existing?: string, _cm?: string, _ca?: string, _sm?: string, _fs?: boolean, + reservedSessionId?: string + ) => { + reserved = reservedSessionId + return { type: 'success' as const, sessionId: 'totally-different-id' } + } + + const result = await engine.spawnSession( + 'machine-id-mismatch', + '/tmp/project', + 'claude', + undefined, undefined, undefined, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, undefined, undefined, + undefined, 'default' + ) + + expect(result.type).toBe('error') + expect(typeof reserved).toBe('string') + expect(store.sessions.getSession(reserved!)?.tag).toBe(`machine-spawn:${reserved}`) + } finally { + engine.stop() + } + }) + + it('forwards fresh Codex prealloc as reservedSessionId (not existingSessionId)', async () => { + // #1911 Critical: existingSessionId → reopen → "no Codex thread binding". + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-codex-prealloc', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + engine.handleMachineAlive({ machineId: 'machine-codex-prealloc', time: Date.now() }) + + let forwardedExisting: string | undefined + let forwardedReserved: string | undefined + ;(engine as unknown as { rpcGateway: { spawnSession: unknown } }).rpcGateway.spawnSession = + async ( + _machineId: string, + _directory: string, + _agent?: string, + _model?: string, + _modelReasoningEffort?: string, + _yolo?: boolean, + _sessionType?: string, + _worktreeName?: string, + _resumeSessionId?: string, + _effort?: string, + _permissionMode?: string, + _serviceTier?: string, + existingSessionId?: string, + _collaborationMode?: string, + _copilotAgentMode?: string, + _startingMode?: string, + _forkSession?: boolean, + reservedSessionId?: string + ) => { + forwardedExisting = existingSessionId + forwardedReserved = reservedSessionId + return { type: 'success' as const, sessionId: (reservedSessionId ?? existingSessionId)! } + } + + const result = await engine.spawnSession( + 'machine-codex-prealloc', + '/tmp/project', + 'codex', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('success') + expect(forwardedExisting).toBeUndefined() + expect(typeof forwardedReserved).toBe('string') + expect(forwardedReserved!.length).toBeGreaterThan(0) + const row = store.sessions.getSession(forwardedReserved!) + expect(row?.tag).toBe(`machine-spawn:${forwardedReserved}`) + expect((row?.metadata as { flavor?: string } | null)?.flavor).toBe('codex') + } finally { + engine.stop() + } + }) + + it('does not mint a second id when existingSessionId is already supplied', async () => { + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-reuse', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + const existing = engine.getOrCreateSession( + 'already-reserved', + { path: '/tmp/project', host: 'localhost', flavor: 'opencode', machineId: 'machine-reuse' }, + null, + 'default', + undefined, + undefined, + undefined, + 'already-reserved-id' + ) + + let forwardedExistingId: string | undefined + let callCount = 0 + ;(engine as unknown as { rpcGateway: { spawnSession: unknown } }).rpcGateway.spawnSession = + async ( + ...args: unknown[] + ) => { + callCount++ + forwardedExistingId = (args[17] ?? args[12]) as string | undefined + return { type: 'success' as const, sessionId: forwardedExistingId! } + } + + const result = await engine.spawnSession( + 'machine-reuse', + '/tmp/project', + 'opencode', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + existing.id, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result).toEqual({ type: 'success', sessionId: 'already-reserved-id' }) + expect(forwardedExistingId).toBe('already-reserved-id') + expect(callCount).toBe(1) + } finally { + engine.stop() + } + }) + + it('keeps the preallocated stub on ambiguous spawn failure even if StopSession would say gone', async () => { + // Critical: never call stop+delete on ambiguous errors — stop kills healthy + // late-booting children and delete CASCADE-wipes transcripts. + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-fail', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + + let forwardedExistingId: string | undefined + let stopCalls = 0 + ;(engine as unknown as { rpcGateway: { spawnSession: unknown; stopRunnerSession: unknown } }) + .rpcGateway.spawnSession = async ( + ...args: unknown[] + ) => { + forwardedExistingId = (args[17] ?? args[12]) as string | undefined + return { type: 'error' as const, message: 'spawn blew up' } + } + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }) + .rpcGateway.stopRunnerSession = async () => { + stopCalls++ + return 'already_gone' + } + + const result = await engine.spawnSession( + 'machine-fail', + '/tmp/project', + 'claude', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('error') + expect(typeof forwardedExistingId).toBe('string') + expect(stopCalls).toBe(0) + expect(store.sessions.getSession(forwardedExistingId!)?.id).toBe(forwardedExistingId) + } finally { + engine.stop() + } + }) + + it('keeps the preallocated stub when StopSession cannot confirm the child is gone', async () => { + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-fail-alive', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + + let forwardedExistingId: string | undefined + let stopCalls = 0 + ;(engine as unknown as { rpcGateway: { spawnSession: unknown; stopRunnerSession: unknown } }) + .rpcGateway.spawnSession = async ( + ...args: unknown[] + ) => { + forwardedExistingId = (args[17] ?? args[12]) as string | undefined + return { type: 'error' as const, message: 'webhook timeout' } + } + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }) + .rpcGateway.stopRunnerSession = async () => { + stopCalls++ + return 'still_alive' + } + + const result = await engine.spawnSession( + 'machine-fail-alive', + '/tmp/project', + 'claude', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('error') + expect(typeof forwardedExistingId).toBe('string') + // Ambiguous spawn: never poke StopSession (would kill a healthy child). + expect(stopCalls).toBe(0) + expect(store.sessions.getSession(forwardedExistingId!)?.id).toBe(forwardedExistingId) + } finally { + engine.stop() + } + }) + + it('deletes the stub on pre-exec rejection without StopSession (childStarted:false)', async () => { + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-preexec', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + + let forwardedExistingId: string | undefined + let stopCalls = 0 + ;(engine as unknown as { rpcGateway: { spawnSession: unknown; stopRunnerSession: unknown } }) + .rpcGateway.spawnSession = async ( + ...args: unknown[] + ) => { + forwardedExistingId = (args[17] ?? args[12]) as string | undefined + return { + type: 'error' as const, + message: 'claude is not installed', + code: 'agent_unavailable' as const, + childStarted: false as const, + } + } + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }) + .rpcGateway.stopRunnerSession = async () => { + stopCalls++ + return 'unknown' + } + + const result = await engine.spawnSession( + 'machine-preexec', + '/tmp/project', + 'claude', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('error') + expect(stopCalls).toBe(0) + expect(store.sessions.getSession(forwardedExistingId!)).toBeFalsy() + } finally { + engine.stop() + } + }) + + it('deletes the stub on directory-approval pre-exec (childStarted:false)', async () => { + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-dir-approve', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + + let forwardedExistingId: string | undefined + ;(engine as unknown as { rpcGateway: { spawnSession: unknown; stopRunnerSession: unknown } }) + .rpcGateway.spawnSession = async ( + ...args: unknown[] + ) => { + forwardedExistingId = (args[17] ?? args[12]) as string | undefined + return { + type: 'error' as const, + message: 'Directory creation requires approval: /tmp/new-project', + childStarted: false as const, + } + } + + const result = await engine.spawnSession( + 'machine-dir-approve', + '/tmp/new-project', + 'claude', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('error') + expect(store.sessions.getSession(forwardedExistingId!)).toBeFalsy() + } finally { + engine.stop() + } + }) + + it('CLI adopt binds preallocated stub under a new tag (create request path)', async () => { + // End-to-end of the real machine-spawn path: hub preallocates, then CLI + // create with adopt=true overwrites tag/metadata without 409. + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-adopt', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + + let allocatedId: string | undefined + ;(engine as unknown as { rpcGateway: { spawnSession: unknown } }).rpcGateway.spawnSession = + async (...args: unknown[]) => { + allocatedId = (args[17] ?? args[12]) as string | undefined + // Simulate CLI create/adopt before webhook success + const cliTag = crypto.randomUUID() + const adopted = engine.adoptPreallocatedSession( + allocatedId!, + cliTag, + { + path: '/tmp/project', + host: 'localhost', + flavor: 'claude', + machineId: 'machine-adopt', + startedBy: 'runner', + startedFromRunner: true, + hostPid: 999, + }, + { controlledByUser: false }, + 'default', + 'claude-sonnet' + ) + expect(adopted.id).toBe(allocatedId!) + expect((adopted.metadata as { hostPid?: number } | null)?.hostPid).toBe(999) + const stored = store.sessions.getSession(allocatedId!) + expect(stored?.tag).toBe(cliTag) + return { type: 'success' as const, sessionId: allocatedId! } + } + + const result = await engine.spawnSession( + 'machine-adopt', + '/tmp/project', + 'claude', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('success') + expect(typeof allocatedId).toBe('string') + const row = store.sessions.getSession(allocatedId!) + expect(row?.id).toBe(allocatedId) + expect(row?.tag).not.toMatch(/^machine-spawn:/) + } finally { + engine.stop() + } + }) + + it('keeps the stub on ambiguous spawn error (no childStarted:false) without StopSession or delete', async () => { + // Critical: RPC timeout / post-dispatch error must not kill a healthy + // late-booting CLI or CASCADE-delete its transcript (#1911 Overseer). + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-ambiguous', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + + let forwardedExistingId: string | undefined + let stopCalls = 0 + ;(engine as unknown as { rpcGateway: { spawnSession: unknown; stopRunnerSession: unknown } }) + .rpcGateway.spawnSession = async (...args: unknown[]) => { + forwardedExistingId = (args[17] ?? args[12]) as string | undefined + // Same shape as rpcGateway catch/timeout: error, childStarted unset. + return { + type: 'error' as const, + message: 'RPC call timed out after 30000ms', + } + } + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }) + .rpcGateway.stopRunnerSession = async () => { + stopCalls++ + return 'stopped' + } + + const result = await engine.spawnSession( + 'machine-ambiguous', + '/tmp/project', + 'cursor', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('error') + expect(stopCalls).toBe(0) + expect(store.sessions.getSession(forwardedExistingId!)?.id).toBe(forwardedExistingId) + expect(store.sessions.getSession(forwardedExistingId!)?.tag).toMatch(/^machine-spawn:/) + } finally { + engine.stop() + } + }) + + it('does not delete after StopSession when the row is no longer a prealloc stub', async () => { + // Even if an older path called stop+delete, live rows must stay. + const store = new Store(':memory:') + const engine = new SyncEngine( + store, + {} as never, + new RpcRegistry(), + { broadcast() {} } as never + ) + + try { + engine.getOrCreateMachine( + 'machine-live-protect', + { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, + null, + 'default' + ) + + let allocatedId: string | undefined + ;(engine as unknown as { rpcGateway: { spawnSession: unknown; stopRunnerSession: unknown } }) + .rpcGateway.spawnSession = async (...args: unknown[]) => { + allocatedId = (args[17] ?? args[12]) as string | undefined + // Simulate cursor/codex reopen path: metadata update releases stub tag. + const row = store.sessions.getSession(allocatedId!) + expect(row?.tag).toMatch(/^machine-spawn:/) + store.sessions.updateSessionMetadata( + allocatedId!, + { + ...(row!.metadata as object), + hostPid: 4242, + flavor: 'cursor', + }, + row!.metadataVersion, + 'default' + ) + const after = store.sessions.getSession(allocatedId!) + expect(after?.tag).not.toMatch(/^machine-spawn:/) + return { + type: 'error' as const, + message: 'spawn failed after child started', + // childStarted unset = ambiguous + } + } + ;(engine as unknown as { rpcGateway: { stopRunnerSession: unknown } }) + .rpcGateway.stopRunnerSession = async () => 'stopped' + + const result = await engine.spawnSession( + 'machine-live-protect', + '/tmp/project', + 'cursor', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'default' + ) + + expect(result.type).toBe('error') + expect(store.sessions.getSession(allocatedId!)?.id).toBe(allocatedId) + } finally { + engine.stop() + } + }) +}) diff --git a/hub/src/web/routes/cli.ts b/hub/src/web/routes/cli.ts index d52e9969d7..17d116ef6b 100644 --- a/hub/src/web/routes/cli.ts +++ b/hub/src/web/routes/cli.ts @@ -12,7 +12,7 @@ import { readSessionSummaryContractEnabled } from '../../config/sessionSummaryCo import { constantTimeEquals } from '../../utils/crypto' import { parseAccessToken } from '../../utils/accessToken' import type { Machine, Session, SyncEngine } from '../../sync/syncEngine' -import { SessionIdentityConflictError } from '../../store/sessions' +import { SessionIdentityConflictError, SessionNotAdoptableError } from '../../store/sessions' const bearerSchema = z.string().regex(/^Bearer\s+(.+)$/i) @@ -119,16 +119,27 @@ export function createCliRoutes(getSyncEngine: () => SyncEngine | null): Hono SyncEngine | null): Hono SyncEngine | null): Ho return c.json({ error: `${parsed.data.agent.toUpperCase()} only supports remote mode` }, 400) } const startingMode = parsed.data.startingMode + const namespace = c.get('namespace') const result = await engine.spawnSession( machineId, @@ -114,7 +115,8 @@ export function createMachinesRoutes(getSyncEngine: () => SyncEngine | null): Ho undefined, parsed.data.collaborationMode, parsed.data.copilotAgentMode, - startingMode + startingMode, + namespace ) return c.json(result) }) diff --git a/shared/src/apiTypes.ts b/shared/src/apiTypes.ts index 50a707db45..4851bb0a2f 100644 --- a/shared/src/apiTypes.ts +++ b/shared/src/apiTypes.ts @@ -26,6 +26,11 @@ export type CreateOrLoadMachineRequest = z.infer { + if (value.adopt === true && !value.id) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'adopt requires id', + path: ['id'], + }) + } }) export type CreateOrLoadSessionRequest = z.infer diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index a35260d8c4..40da2b475f 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -101,6 +101,14 @@ export const MetadataSchema = z.object({ state: z.enum(['resuming', 'quarantined']), machineId: z.string(), startedAt: z.number(), + // Same durability as piResumeAttempt: clear-before-spawn + + // rollbackSafe:false quarantine must not lose archive fields (#1911). + archiveSnapshot: z.object({ + lifecycleState: z.string().optional(), + lifecycleStateSince: z.number().optional(), + archivedBy: z.string().optional(), + archiveReason: z.string().optional(), + }).optional(), }).optional(), tools: z.array(z.string()).optional(), slashCommands: z.array(z.string()).optional(), From 542e18d55b5ddd1857a82658f31825468d275f39 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:44:02 +0000 Subject: [PATCH 04/94] feat: session-attached long-running jobs (#1404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hub-persisted jobs that outlive the agent process: register/update/clear via REST + `hapi job`, surface primary running progress on the session list even when active=false. Opt-in registration only — not thinking progress. Co-authored-by: Cursor --- cli/src/commands/job.ts | 290 ++++++++++++++++++ cli/src/commands/registry.ts | 4 +- cli/src/modules/sessionJob/sessionJob.ts | 269 ++++++++++++++++ hub/src/store/index.ts | 57 +++- hub/src/store/migration-v23.test.ts | 5 +- hub/src/store/migration-v25.test.ts | 79 ++++- hub/src/store/sessionJobs.ts | 267 ++++++++++++++++ hub/src/store/sessionJobsStore.ts | 57 ++++ hub/src/store/types.ts | 15 + hub/src/sync/sessionCache.ts | 33 ++ hub/src/sync/syncEngine.ts | 89 ++++++ hub/src/web/routes/sessions-jobs.test.ts | 129 ++++++++ hub/src/web/routes/sessions.ts | 95 +++++- shared/src/index.ts | 12 + shared/src/schemas.sessionPatch.test.ts | 31 +- shared/src/schemas.ts | 54 +++- shared/src/sessionSummary.test.ts | 19 ++ shared/src/sessionSummary.ts | 13 +- shared/src/types.ts | 4 + .../SessionAttentionIndicator.test.tsx | 1 + .../SessionList.directory-action.test.tsx | 1 + .../SessionList.machine-filter.test.tsx | 1 + web/src/components/SessionList.test.ts | 1 + web/src/components/SessionRowSummary.tsx | 50 ++- web/src/hooks/useSSE.test.ts | 1 + web/src/hooks/useSSE.ts | 14 + web/src/lib/attachedJob.test.ts | 46 +++ web/src/lib/attachedJob.ts | 31 ++ web/src/lib/sessionAttention.test.ts | 1 + web/src/lib/sessionReference.test.ts | 1 + 30 files changed, 1657 insertions(+), 13 deletions(-) create mode 100644 cli/src/commands/job.ts create mode 100644 cli/src/modules/sessionJob/sessionJob.ts create mode 100644 hub/src/store/sessionJobs.ts create mode 100644 hub/src/store/sessionJobsStore.ts create mode 100644 hub/src/web/routes/sessions-jobs.test.ts create mode 100644 web/src/lib/attachedJob.test.ts create mode 100644 web/src/lib/attachedJob.ts diff --git a/cli/src/commands/job.ts b/cli/src/commands/job.ts new file mode 100644 index 0000000000..cbd2fa55fc --- /dev/null +++ b/cli/src/commands/job.ts @@ -0,0 +1,290 @@ +import chalk from 'chalk' +import { initializeToken } from '@/ui/tokenInit' +import type { AttachedJobPatch, AttachedJobUpsert } from '@hapi/protocol' +import { + SessionJobError, + clearSessionJob, + exitCodeForSessionJobError, + listSessionJobs, + setSessionJob, + updateSessionJob +} from '@/modules/sessionJob/sessionJob' +import type { CommandDefinition } from './types' + +type ParsedJobArgs = { + help: boolean + action?: 'set' | 'update' | 'clear' | 'list' + sessionIdPrefix?: string + jobKey?: string + label?: string + status?: 'running' | 'completed' | 'failed' + done?: number + total?: number + remaining?: number + unit?: string + detail?: string +} + +function showHelp(): void { + console.log(` +${chalk.bold('hapi job')} - Attach long-running work to a HAPI session (tiann/hapi#1404) + +${chalk.bold('Usage:')} + hapi job set --label [--remaining N] [--done N --total N] [--unit tracks] [--detail ...] + hapi job update [--remaining N] [--done N] [--total N] [--status running|completed|failed] [--detail ...] + hapi job clear + hapi job list + +${chalk.bold('Notes:')} + Hub-persisted. Works while the agent is idle/offline — not thinking progress. + Prefer honest remaining/done+total; never invent a fake percent. + Job key: 1-128 chars, alnum / . _ - + +${chalk.bold('Env:')} + HAPI_API_URL / CLI_API_TOKEN (or ~/.hapi/settings.json via \`hapi auth login\`) +`) +} + +function parseOptionalNumber(flag: string, value: string | undefined): number { + if (value === undefined) { + throw new SessionJobError('bad_args', `${flag} requires a number`) + } + const n = Number(value) + if (!Number.isFinite(n)) { + throw new SessionJobError('bad_args', `${flag} must be a number`) + } + return n +} + +export function parseJobArgs(args: string[]): ParsedJobArgs { + const result: ParsedJobArgs = { help: false } + + for (let i = 0; i < args.length; i++) { + const arg = args[i]! + if (arg === '--help' || arg === '-h') { + result.help = true + continue + } + if (arg === '--label') { + result.label = args[++i] + if (!result.label) throw new SessionJobError('bad_args', '--label requires a value') + continue + } + if (arg.startsWith('--label=')) { + result.label = arg.slice('--label='.length) + continue + } + if (arg === '--status') { + const value = args[++i] + if (value !== 'running' && value !== 'completed' && value !== 'failed') { + throw new SessionJobError('bad_args', '--status must be running|completed|failed') + } + result.status = value + continue + } + if (arg.startsWith('--status=')) { + const value = arg.slice('--status='.length) + if (value !== 'running' && value !== 'completed' && value !== 'failed') { + throw new SessionJobError('bad_args', '--status must be running|completed|failed') + } + result.status = value + continue + } + if (arg === '--done') { + result.done = parseOptionalNumber('--done', args[++i]) + continue + } + if (arg.startsWith('--done=')) { + result.done = parseOptionalNumber('--done', arg.slice('--done='.length)) + continue + } + if (arg === '--total') { + result.total = parseOptionalNumber('--total', args[++i]) + continue + } + if (arg.startsWith('--total=')) { + result.total = parseOptionalNumber('--total', arg.slice('--total='.length)) + continue + } + if (arg === '--remaining') { + result.remaining = parseOptionalNumber('--remaining', args[++i]) + continue + } + if (arg.startsWith('--remaining=')) { + result.remaining = parseOptionalNumber('--remaining', arg.slice('--remaining='.length)) + continue + } + if (arg === '--unit') { + result.unit = args[++i] + if (!result.unit) throw new SessionJobError('bad_args', '--unit requires a value') + continue + } + if (arg.startsWith('--unit=')) { + result.unit = arg.slice('--unit='.length) + continue + } + if (arg === '--detail') { + result.detail = args[++i] + if (result.detail === undefined) throw new SessionJobError('bad_args', '--detail requires a value') + continue + } + if (arg.startsWith('--detail=')) { + result.detail = arg.slice('--detail='.length) + continue + } + if (arg.startsWith('-')) { + throw new SessionJobError('bad_args', `unexpected flag: ${arg}`) + } + if (!result.action) { + if (arg !== 'set' && arg !== 'update' && arg !== 'clear' && arg !== 'list') { + throw new SessionJobError('bad_args', `unknown action '${arg}' (set|update|clear|list)`) + } + result.action = arg + continue + } + if (!result.sessionIdPrefix) { + result.sessionIdPrefix = arg + continue + } + if (!result.jobKey && result.action !== 'list') { + result.jobKey = arg + continue + } + throw new SessionJobError('bad_args', `unexpected arg: ${arg}`) + } + + return result +} + +function formatJobLine(job: { + key: string + label: string + status: string + done?: number + total?: number + remaining?: number + unit?: string + detail?: string + heartbeatAt: number +}): string { + const parts = [`${job.key}`, job.label, job.status] + if (job.remaining !== undefined) { + parts.push(`${job.remaining}${job.unit ? ` ${job.unit}` : ''} left`) + } else if (job.done !== undefined && job.total !== undefined) { + parts.push(`${job.done}/${job.total}${job.unit ? ` ${job.unit}` : ''}`) + } + if (job.detail) parts.push(job.detail) + const ageSec = Math.max(0, Math.round((Date.now() - job.heartbeatAt) / 1000)) + parts.push(`heartbeat ${ageSec}s ago`) + return parts.join(' · ') +} + +export async function handleJobCommand(args: string[]): Promise { + const parsed = parseJobArgs(args) + if (parsed.help || !parsed.action) { + showHelp() + if (!parsed.action && !parsed.help) { + throw new SessionJobError('bad_args', 'missing action; usage: hapi job set|update|clear|list ...') + } + return + } + + await initializeToken() + + if (!parsed.sessionIdPrefix) { + showHelp() + throw new SessionJobError('bad_args', 'missing session id') + } + + if (parsed.action === 'list') { + const result = await listSessionJobs({ sessionIdPrefix: parsed.sessionIdPrefix }) + console.log(`session ${result.sessionId}`) + if (result.jobs.length === 0) { + console.log('(no jobs)') + return + } + for (const job of result.jobs) { + const mark = result.primary?.key === job.key ? '*' : ' ' + console.log(`${mark} ${formatJobLine(job)}`) + } + return + } + + if (!parsed.jobKey) { + throw new SessionJobError('bad_args', 'missing job key') + } + + if (parsed.action === 'clear') { + const result = await clearSessionJob({ + sessionIdPrefix: parsed.sessionIdPrefix, + jobKey: parsed.jobKey + }) + console.log(`cleared ${parsed.jobKey} on ${result.sessionId}`) + return + } + + if (parsed.action === 'set') { + if (!parsed.label) { + throw new SessionJobError('bad_args', 'set requires --label') + } + const body: AttachedJobUpsert = { + label: parsed.label, + status: parsed.status ?? 'running', + ...(parsed.done !== undefined ? { done: parsed.done } : {}), + ...(parsed.total !== undefined ? { total: parsed.total } : {}), + ...(parsed.remaining !== undefined ? { remaining: parsed.remaining } : {}), + ...(parsed.unit !== undefined ? { unit: parsed.unit } : {}), + ...(parsed.detail !== undefined ? { detail: parsed.detail } : {}) + } + const result = await setSessionJob({ + sessionIdPrefix: parsed.sessionIdPrefix, + jobKey: parsed.jobKey, + body + }) + console.log(`set ${formatJobLine(result.job)}`) + return + } + + // update + const body: AttachedJobPatch = { + ...(parsed.label !== undefined ? { label: parsed.label } : {}), + ...(parsed.status !== undefined ? { status: parsed.status } : {}), + ...(parsed.done !== undefined ? { done: parsed.done } : {}), + ...(parsed.total !== undefined ? { total: parsed.total } : {}), + ...(parsed.remaining !== undefined ? { remaining: parsed.remaining } : {}), + ...(parsed.unit !== undefined ? { unit: parsed.unit } : {}), + ...(parsed.detail !== undefined ? { detail: parsed.detail } : {}) + } + if (Object.keys(body).length === 0) { + throw new SessionJobError('bad_args', 'update requires at least one field') + } + const result = await updateSessionJob({ + sessionIdPrefix: parsed.sessionIdPrefix, + jobKey: parsed.jobKey, + body + }) + console.log(`updated ${formatJobLine(result.job)}`) +} + +export const jobCommand: CommandDefinition = { + name: 'job', + requiresRuntimeAssets: false, + run: async ({ commandArgs }) => { + try { + await handleJobCommand(commandArgs) + } catch (error) { + if (error instanceof SessionJobError) { + console.error(chalk.red('hapi job:'), error.message) + process.exit(exitCodeForSessionJobError(error)) + } + console.error( + chalk.red('hapi job:'), + error instanceof Error ? error.message : 'Unknown error' + ) + if (process.env.DEBUG) { + console.error(error) + } + process.exit(1) + } + } +} diff --git a/cli/src/commands/registry.ts b/cli/src/commands/registry.ts index 2bf8c598fb..b306620224 100644 --- a/cli/src/commands/registry.ts +++ b/cli/src/commands/registry.ts @@ -21,6 +21,7 @@ import { notifyCommand } from './notify' import { hubCommand } from './hub' import { pingPeerCommand } from './pingPeer' import { inspectPeerCommand } from './inspectPeer' +import { jobCommand } from './job' import type { CommandContext, CommandDefinition } from './types' // Gemini CLI was sunset (Google stopped serving the consumer Gemini CLI on @@ -62,7 +63,8 @@ const COMMANDS: CommandDefinition[] = [ runnerCommand, notifyCommand, pingPeerCommand, - inspectPeerCommand + inspectPeerCommand, + jobCommand ] const commandMap = new Map() diff --git a/cli/src/modules/sessionJob/sessionJob.ts b/cli/src/modules/sessionJob/sessionJob.ts new file mode 100644 index 0000000000..554f403243 --- /dev/null +++ b/cli/src/modules/sessionJob/sessionJob.ts @@ -0,0 +1,269 @@ +/** + * Register / update / clear session-attached jobs (tiann/hapi#1404). + * Same hub JWT flow as ping-peer — works while the agent session is idle. + */ + +import axios, { type AxiosInstance } from 'axios' +import type { AttachedJob, AttachedJobPatch, AttachedJobUpsert } from '@hapi/protocol' +import { configuration } from '@/configuration' +import { getAuthToken } from '@/api/auth' +import { buildHubRequestHeaders } from '@/api/hubExtraHeaders' + +export type SessionJobErrorCode = + | 'bad_args' + | 'auth_failed' + | 'not_found' + | 'ambiguous' + | 'request_failed' + +export class SessionJobError extends Error { + readonly code: SessionJobErrorCode + + constructor(code: SessionJobErrorCode, message: string) { + super(message) + this.name = 'SessionJobError' + this.code = code + } +} + +const AUTH_RECOVERY_HINT = + 'On a remote runner, set HAPI_API_URL to the runner hub, and set CLI_API_TOKEN ' + + 'or run `hapi auth login`. Prefer `hapi job` over raw JWT+curl.' + +function resolveApiUrl(apiUrl?: string): string { + const raw = (apiUrl ?? configuration.apiUrl).trim().replace(/\/+$/, '') + if (!raw) { + throw new SessionJobError('bad_args', `HAPI API URL is empty. ${AUTH_RECOVERY_HINT}`) + } + return raw +} + +function resolveAccessToken(accessToken?: string): string { + let token = '' + try { + token = (accessToken ?? getAuthToken()).trim() + } catch { + token = (accessToken ?? '').trim() + } + if (!token) { + throw new SessionJobError( + 'bad_args', + `CLI_API_TOKEN is required (run \`hapi auth login\`). ${AUTH_RECOVERY_HINT}` + ) + } + return token +} + +async function exchangeJwt( + apiUrl: string, + accessToken: string, + http: AxiosInstance +): Promise { + try { + const response = await http.post( + `${apiUrl}/api/auth`, + { accessToken }, + { + headers: buildHubRequestHeaders({ 'Content-Type': 'application/json' }), + timeout: 10_000, + validateStatus: () => true + } + ) + const token = typeof response.data?.token === 'string' ? response.data.token : '' + if (response.status < 200 || response.status >= 300 || !token) { + const detail = typeof response.data?.error === 'string' + ? response.data.error + : `HTTP ${response.status}` + throw new SessionJobError( + 'auth_failed', + `failed to exchange access token for JWT (${detail}). Hub URL: ${apiUrl}. ${AUTH_RECOVERY_HINT}` + ) + } + return token + } catch (error) { + if (error instanceof SessionJobError) throw error + throw new SessionJobError( + 'auth_failed', + `failed to exchange access token for JWT (${error instanceof Error ? error.message : String(error)}). Hub URL: ${apiUrl}. ${AUTH_RECOVERY_HINT}` + ) + } +} + +function authHeaders(jwt: string): Record { + return buildHubRequestHeaders({ + Authorization: `Bearer ${jwt}`, + 'Content-Type': 'application/json' + }) +} + +type SessionListItem = { id: string } + +function resolveSessionByPrefix(sessions: SessionListItem[], prefix: string): SessionListItem { + const trimmed = prefix.trim() + if (!trimmed) { + throw new SessionJobError('bad_args', 'session id prefix is required') + } + const exact = sessions.filter((session) => session.id === trimmed) + if (exact.length === 1) return exact[0]! + const matches = sessions.filter((session) => session.id.startsWith(trimmed)) + if (matches.length === 0) { + throw new SessionJobError('not_found', `no session matching prefix '${trimmed}'`) + } + if (matches.length > 1) { + const sample = matches.slice(0, 5).map((session) => session.id.slice(0, 8)).join(', ') + throw new SessionJobError( + 'ambiguous', + `prefix '${trimmed}' matches ${matches.length} sessions (${sample}${matches.length > 5 ? ', ...' : ''}); use a longer prefix` + ) + } + return matches[0]! +} + +async function resolveSessionId( + apiUrl: string, + jwt: string, + http: AxiosInstance, + sessionIdPrefix: string +): Promise { + const response = await http.get(`${apiUrl}/api/sessions`, { + headers: authHeaders(jwt), + params: { limit: 500, order: 'updatedAt' }, + timeout: 15_000, + validateStatus: () => true + }) + if (response.status < 200 || response.status >= 300) { + throw new SessionJobError('request_failed', `list sessions failed: HTTP ${response.status}`) + } + const sessions = Array.isArray(response.data?.sessions) + ? (response.data.sessions as SessionListItem[]) + : [] + return resolveSessionByPrefix(sessions, sessionIdPrefix).id +} + +export type SessionJobClientOptions = { + sessionIdPrefix: string + apiUrl?: string + accessToken?: string + http?: AxiosInstance +} + +async function withClient( + options: SessionJobClientOptions, + fn: (ctx: { apiUrl: string; jwt: string; sessionId: string; http: AxiosInstance }) => Promise +): Promise { + const http = options.http ?? axios + const apiUrl = resolveApiUrl(options.apiUrl) + const accessToken = resolveAccessToken(options.accessToken) + const jwt = await exchangeJwt(apiUrl, accessToken, http) + const sessionId = await resolveSessionId(apiUrl, jwt, http, options.sessionIdPrefix) + return fn({ apiUrl, jwt, sessionId, http }) +} + +export async function listSessionJobs( + options: SessionJobClientOptions +): Promise<{ sessionId: string; jobs: AttachedJob[]; primary: AttachedJob | null }> { + return withClient(options, async ({ apiUrl, jwt, sessionId, http }) => { + const response = await http.get(`${apiUrl}/api/sessions/${sessionId}/jobs`, { + headers: authHeaders(jwt), + timeout: 15_000, + validateStatus: () => true + }) + if (response.status < 200 || response.status >= 300) { + throw new SessionJobError('request_failed', `list jobs failed: HTTP ${response.status}`) + } + return { + sessionId, + jobs: Array.isArray(response.data?.jobs) ? response.data.jobs : [], + primary: response.data?.primary ?? null + } + }) +} + +export async function setSessionJob( + options: SessionJobClientOptions & { jobKey: string; body: AttachedJobUpsert } +): Promise<{ sessionId: string; job: AttachedJob }> { + return withClient(options, async ({ apiUrl, jwt, sessionId, http }) => { + const response = await http.put( + `${apiUrl}/api/sessions/${sessionId}/jobs/${encodeURIComponent(options.jobKey)}`, + options.body, + { + headers: authHeaders(jwt), + timeout: 15_000, + validateStatus: () => true + } + ) + if (response.status === 404) { + throw new SessionJobError('not_found', 'session or job not found') + } + if (response.status < 200 || response.status >= 300 || !response.data?.job) { + const detail = typeof response.data?.error === 'string' + ? response.data.error + : `HTTP ${response.status}` + throw new SessionJobError('request_failed', `set job failed: ${detail}`) + } + return { sessionId, job: response.data.job as AttachedJob } + }) +} + +export async function updateSessionJob( + options: SessionJobClientOptions & { jobKey: string; body: AttachedJobPatch } +): Promise<{ sessionId: string; job: AttachedJob }> { + return withClient(options, async ({ apiUrl, jwt, sessionId, http }) => { + const response = await http.patch( + `${apiUrl}/api/sessions/${sessionId}/jobs/${encodeURIComponent(options.jobKey)}`, + options.body, + { + headers: authHeaders(jwt), + timeout: 15_000, + validateStatus: () => true + } + ) + if (response.status === 404) { + throw new SessionJobError('not_found', 'job not found') + } + if (response.status < 200 || response.status >= 300 || !response.data?.job) { + const detail = typeof response.data?.error === 'string' + ? response.data.error + : `HTTP ${response.status}` + throw new SessionJobError('request_failed', `update job failed: ${detail}`) + } + return { sessionId, job: response.data.job as AttachedJob } + }) +} + +export async function clearSessionJob( + options: SessionJobClientOptions & { jobKey: string } +): Promise<{ sessionId: string }> { + return withClient(options, async ({ apiUrl, jwt, sessionId, http }) => { + const response = await http.delete( + `${apiUrl}/api/sessions/${sessionId}/jobs/${encodeURIComponent(options.jobKey)}`, + { + headers: authHeaders(jwt), + timeout: 15_000, + validateStatus: () => true + } + ) + if (response.status === 404) { + throw new SessionJobError('not_found', 'job not found') + } + if (response.status < 200 || response.status >= 300) { + throw new SessionJobError('request_failed', `clear job failed: HTTP ${response.status}`) + } + return { sessionId } + }) +} + +export function exitCodeForSessionJobError(error: SessionJobError): number { + switch (error.code) { + case 'bad_args': + return 2 + case 'auth_failed': + return 3 + case 'not_found': + return 4 + case 'ambiguous': + return 5 + default: + return 1 + } +} diff --git a/hub/src/store/index.ts b/hub/src/store/index.ts index 7e1e6cf04c..6554f180d5 100644 --- a/hub/src/store/index.ts +++ b/hub/src/store/index.ts @@ -9,6 +9,7 @@ import type { StoredMessage } from './types' import { PushStore } from './pushStore' import { FcmStore } from './fcmStore' import { ScratchlistStore } from './scratchlistStore' +import { SessionJobsStore } from './sessionJobsStore' import { SessionStore } from './sessionStore' import { UserStore } from './userStore' import { UsageStore } from './usageStore' @@ -22,6 +23,7 @@ export type { StoredPushSubscription, StoredFcmDevice, StoredScratchlistEntry, + StoredSessionJob, StoredSession, StoredUser, VersionedUpdateResult @@ -32,6 +34,7 @@ export { MessageStore } from './messageStore' export { PushStore } from './pushStore' export { FcmStore } from './fcmStore' export { ScratchlistStore } from './scratchlistStore' +export { SessionJobsStore } from './sessionJobsStore' export { SessionStore } from './sessionStore' export { UserStore } from './userStore' export { UsageStore } from './usageStore' @@ -42,7 +45,7 @@ export { WorkGraphValidationError } from './workGraph' -const SCHEMA_VERSION: number = 26 +const SCHEMA_VERSION: number = 27 const REQUIRED_TABLES = [ 'sessions', 'machines', @@ -52,6 +55,7 @@ const REQUIRED_TABLES = [ 'push_subscriptions', 'fcm_devices', 'session_scratchlist', + 'session_jobs', 'usage_events', 'usage_scan_state', 'events', @@ -70,6 +74,7 @@ export class Store { readonly push: PushStore readonly fcm: FcmStore readonly scratchlist: ScratchlistStore + readonly sessionJobs: SessionJobsStore readonly usage: UsageStore readonly workGraph: WorkGraphStore @@ -124,6 +129,7 @@ export class Store { this.push = new PushStore(this.db) this.fcm = new FcmStore(this.db) this.scratchlist = new ScratchlistStore(this.db) + this.sessionJobs = new SessionJobsStore(this.db) this.usage = new UsageStore(this.db) this.workGraph = new WorkGraphStore(this.db) } @@ -343,11 +349,16 @@ export class Store { 18: () => this.migrateFromV18ToV19(), 19: () => this.migrateFromV19ToV20(), 20: () => this.migrateFromV20ToV21(), + // Upstream #1115 dual-pin at v21→v22; #1467 A2A events at v22→v23; + // iOS push_key at v23→v24; steer delivery_state at v24→v25; + // immediate-queue heartbeat index at v25→v26; + // #1404 session_jobs at v26→v27. 21: () => this.migrateFromV21ToV22(), 22: () => this.migrateFromV22ToV23(), 23: () => this.migrateFromV23ToV24(), 24: () => this.migrateFromV24ToV25(), 25: () => this.migrateFromV25ToV26(), + 26: () => this.migrateFromV26ToV27(), }) if (currentVersion === 0) { @@ -519,6 +530,25 @@ export class Store { CREATE INDEX IF NOT EXISTS idx_session_scratchlist_session_created ON session_scratchlist(session_id, created_at DESC); + CREATE TABLE IF NOT EXISTS session_jobs ( + session_id TEXT NOT NULL, + job_key TEXT NOT NULL, + label TEXT NOT NULL, + status TEXT NOT NULL, + done REAL, + total REAL, + remaining REAL, + unit TEXT, + detail TEXT, + heartbeat_at INTEGER NOT NULL, + started_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, job_key), + FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_session_jobs_session_status_updated + ON session_jobs(session_id, status, updated_at DESC); + CREATE TABLE IF NOT EXISTS usage_events ( session_id TEXT NOT NULL, source_key TEXT NOT NULL, @@ -958,6 +988,7 @@ export class Store { } private migrateFromV21ToV22(): void { + // Upstream #1115 dual-pin columns. const columns = this.getSessionColumnNames() if (columns.size === 0) return if (!columns.has('pinned')) { @@ -1054,6 +1085,30 @@ export class Store { `) } + /** v26→v27: #1404 session-attached long-running jobs. */ + private migrateFromV26ToV27(): void { + this.db.exec(` + CREATE TABLE IF NOT EXISTS session_jobs ( + session_id TEXT NOT NULL, + job_key TEXT NOT NULL, + label TEXT NOT NULL, + status TEXT NOT NULL, + done REAL, + total REAL, + remaining REAL, + unit TEXT, + detail TEXT, + heartbeat_at INTEGER NOT NULL, + started_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, job_key), + FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_session_jobs_session_status_updated + ON session_jobs(session_id, status, updated_at DESC); + `) + } + private getSessionColumnNames(): Set { const rows = this.db.prepare('PRAGMA table_info(sessions)').all() as Array<{ name: string }> return new Set(rows.map((row) => row.name)) diff --git a/hub/src/store/migration-v23.test.ts b/hub/src/store/migration-v23.test.ts index 191fa1cdce..2705a3c738 100644 --- a/hub/src/store/migration-v23.test.ts +++ b/hub/src/store/migration-v23.test.ts @@ -13,7 +13,7 @@ afterEach(() => { } }) -describe('schema migration v22 to v26', () => { +describe('schema migration v22 to v27', () => { it('adds events and event_links tables to a V22 database', () => { const dir = mkdtempSync(join(tmpdir(), 'hapi-migration-v23-')) tempDirs.push(dir) @@ -42,7 +42,8 @@ describe('schema migration v22 to v26', () => { expect(links?.name).toBe('event_links') const columns = internalDb.prepare('PRAGMA table_info(messages)').all() as Array<{ name: string }> expect(columns.map((column) => column.name)).toContain('delivery_state') - expect(version.user_version).toBe(26) + // Tip after upstream V26 heartbeat index + #1404: V27 session_jobs. + expect(version.user_version).toBe(27) migrated.close() }) }) diff --git a/hub/src/store/migration-v25.test.ts b/hub/src/store/migration-v25.test.ts index c72bac7e6b..166c662419 100644 --- a/hub/src/store/migration-v25.test.ts +++ b/hub/src/store/migration-v25.test.ts @@ -13,7 +13,7 @@ afterEach(() => { } }) -describe('schema migration v25 to v26', () => { +describe('schema migration v25 to v27', () => { it('adds an index used by immediate queued-message replay', () => { const dir = mkdtempSync(join(tmpdir(), 'hapi-migration-v25-')) tempDirs.push(dir) @@ -41,8 +41,83 @@ describe('schema migration v25 to v26', () => { ORDER BY seq ASC `).all('session-id') as Array<{ detail: string }> - expect(version.user_version).toBe(26) + expect(version.user_version).toBe(27) expect(plan.some((row) => row.detail.includes('idx_messages_immediate_queued'))).toBe(true) migrated.close() }) }) + +function getColumns(store: Store, table: string): string[] { + const db: Database = (store as unknown as { db: Database }).db + const rows = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }> + return rows.map((row) => row.name) +} + +describe('Store V26→V27 migration: session_jobs table', () => { + it('fresh DB has session_jobs with expected columns', () => { + const store = new Store(':memory:') + const cols = getColumns(store, 'session_jobs') + expect(cols).toContain('session_id') + expect(cols).toContain('job_key') + expect(cols).toContain('label') + expect(cols).toContain('status') + expect(cols).toContain('done') + expect(cols).toContain('total') + expect(cols).toContain('remaining') + expect(cols).toContain('heartbeat_at') + expect(cols).toContain('started_at') + expect(cols).toContain('updated_at') + store.close() + }) + + it('upserts, patches, deletes a job and surfaces primary running', () => { + const store = new Store(':memory:') + const session = store.sessions.getOrCreateSession('test', { path: '/tmp' }, null, 'default') + + const created = store.sessionJobs.upsert(session.id, 'beets', { + label: 'beets import', + status: 'running', + remaining: 100, + unit: 'tracks' + }) + expect(created.outcome).toBe('upserted') + if (created.outcome !== 'upserted') throw new Error('unreachable') + + const primary = store.sessionJobs.getPrimaryRunning(session.id) + expect(primary?.key).toBe('beets') + expect(primary?.remaining).toBe(100) + + const patched = store.sessionJobs.patch(session.id, 'beets', { remaining: 80 }) + expect(patched?.remaining).toBe(80) + + expect(store.sessionJobs.delete(session.id, 'beets')).toBe(true) + expect(store.sessionJobs.getPrimaryRunning(session.id)).toBeNull() + store.close() + }) + + it('cascade-deletes jobs when session is deleted', async () => { + const store = new Store(':memory:') + const session = store.sessions.getOrCreateSession('test', { path: '/tmp' }, null, 'default') + store.sessionJobs.upsert(session.id, 'job', { label: 'x', status: 'running' }) + expect(store.sessionJobs.list(session.id)).toHaveLength(1) + await store.sessions.deleteSession(session.id, 'default') + expect(store.sessionJobs.list(session.id)).toHaveLength(0) + store.close() + }) + + it('transfers jobs on merge without colliding keys', () => { + const store = new Store(':memory:') + const oldSession = store.sessions.getOrCreateSession('old', { path: '/a' }, null, 'default') + const newSession = store.sessions.getOrCreateSession('new', { path: '/b' }, null, 'default') + store.sessionJobs.upsert(oldSession.id, 'beets', { + label: 'beets', + status: 'running', + remaining: 5 + }) + const result = store.sessionJobs.transfer(oldSession.id, newSession.id) + expect(result.moved).toBe(1) + expect(store.sessionJobs.getPrimaryRunning(newSession.id)?.remaining).toBe(5) + expect(store.sessionJobs.list(oldSession.id)).toHaveLength(0) + store.close() + }) +}) diff --git a/hub/src/store/sessionJobs.ts b/hub/src/store/sessionJobs.ts new file mode 100644 index 0000000000..c44b5831df --- /dev/null +++ b/hub/src/store/sessionJobs.ts @@ -0,0 +1,267 @@ +import type { Database } from 'bun:sqlite' +import type { AttachedJob, AttachedJobPatch, AttachedJobStatus, AttachedJobUpsert } from '@hapi/protocol' + +import type { StoredSessionJob } from './types' + +/** + * Per-session attached jobs (tiann/hapi#1404). + * + * Registration-first long-running work that outlives the agent process. + * Hub is source of truth; list chrome reads the primary `running` job. + */ + +type DbJobRow = { + session_id: string + job_key: string + label: string + status: string + done: number | null + total: number | null + remaining: number | null + unit: string | null + detail: string | null + heartbeat_at: number + started_at: number + updated_at: number +} + +const JOB_COLUMNS = `session_id, job_key, label, status, done, total, remaining, unit, detail, heartbeat_at, started_at, updated_at` + +function toStored(row: DbJobRow): StoredSessionJob { + return { + sessionId: row.session_id, + key: row.job_key, + label: row.label, + status: row.status as AttachedJobStatus, + done: row.done ?? undefined, + total: row.total ?? undefined, + remaining: row.remaining ?? undefined, + unit: row.unit ?? undefined, + detail: row.detail ?? undefined, + heartbeatAt: row.heartbeat_at, + startedAt: row.started_at, + updatedAt: row.updated_at + } +} + +export function toAttachedJob(job: StoredSessionJob): AttachedJob { + return { + key: job.key, + label: job.label, + status: job.status, + ...(job.done !== undefined ? { done: job.done } : {}), + ...(job.total !== undefined ? { total: job.total } : {}), + ...(job.remaining !== undefined ? { remaining: job.remaining } : {}), + ...(job.unit !== undefined ? { unit: job.unit } : {}), + ...(job.detail !== undefined ? { detail: job.detail } : {}), + heartbeatAt: job.heartbeatAt, + startedAt: job.startedAt, + updatedAt: job.updatedAt + } +} + +export function listSessionJobs(db: Database, sessionId: string): StoredSessionJob[] { + const rows = db.prepare( + `SELECT ${JOB_COLUMNS} + FROM session_jobs + WHERE session_id = ? + ORDER BY updated_at DESC, job_key ASC` + ).all(sessionId) as DbJobRow[] + return rows.map(toStored) +} + +export function getSessionJob( + db: Database, + sessionId: string, + jobKey: string +): StoredSessionJob | null { + const row = db.prepare( + `SELECT ${JOB_COLUMNS} + FROM session_jobs + WHERE session_id = ? AND job_key = ?` + ).get(sessionId, jobKey) as DbJobRow | undefined + return row ? toStored(row) : null +} + +/** Newest `running` job for a session, or null. */ +export function getPrimaryRunningJob(db: Database, sessionId: string): StoredSessionJob | null { + const row = db.prepare( + `SELECT ${JOB_COLUMNS} + FROM session_jobs + WHERE session_id = ? AND status = 'running' + ORDER BY updated_at DESC, job_key ASC + LIMIT 1` + ).get(sessionId) as DbJobRow | undefined + return row ? toStored(row) : null +} + +/** + * Batch primary running jobs for session list enrichment. + * Returns Map sessionId → AttachedJob. + */ +export function getPrimaryRunningJobsBySessionIds( + db: Database, + sessionIds: string[] +): Map { + const result = new Map() + if (sessionIds.length === 0) return result + + const placeholders = sessionIds.map(() => '?').join(', ') + const rows = db.prepare( + `SELECT ${JOB_COLUMNS} + FROM session_jobs + WHERE status = 'running' AND session_id IN (${placeholders}) + ORDER BY updated_at DESC, job_key ASC` + ).all(...sessionIds) as DbJobRow[] + + for (const row of rows) { + if (result.has(row.session_id)) continue + result.set(row.session_id, toAttachedJob(toStored(row))) + } + return result +} + +export type UpsertSessionJobResult = + | { outcome: 'upserted'; job: StoredSessionJob } + | { outcome: 'session-not-found' } + +export function upsertSessionJob( + db: Database, + sessionId: string, + jobKey: string, + body: AttachedJobUpsert, + now: number = Date.now() +): UpsertSessionJobResult { + const existing = getSessionJob(db, sessionId, jobKey) + const heartbeatAt = body.heartbeatAt ?? now + const startedAt = body.startedAt ?? existing?.startedAt ?? now + const status = body.status ?? 'running' + + try { + db.prepare( + `INSERT INTO session_jobs ( + session_id, job_key, label, status, done, total, remaining, unit, detail, + heartbeat_at, started_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, job_key) DO UPDATE SET + label = excluded.label, + status = excluded.status, + done = excluded.done, + total = excluded.total, + remaining = excluded.remaining, + unit = excluded.unit, + detail = excluded.detail, + heartbeat_at = excluded.heartbeat_at, + started_at = session_jobs.started_at, + updated_at = excluded.updated_at` + ).run( + sessionId, + jobKey, + body.label, + status, + body.done ?? null, + body.total ?? null, + body.remaining ?? null, + body.unit ?? null, + body.detail ?? null, + heartbeatAt, + startedAt, + now + ) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + if (message.includes('FOREIGN KEY') || message.includes('foreign key')) { + return { outcome: 'session-not-found' } + } + throw error + } + + const job = getSessionJob(db, sessionId, jobKey) + if (!job) { + return { outcome: 'session-not-found' } + } + return { outcome: 'upserted', job } +} + +export function patchSessionJob( + db: Database, + sessionId: string, + jobKey: string, + patch: AttachedJobPatch, + now: number = Date.now() +): StoredSessionJob | null { + const existing = getSessionJob(db, sessionId, jobKey) + if (!existing) return null + + const next: StoredSessionJob = { + ...existing, + label: patch.label ?? existing.label, + status: patch.status ?? existing.status, + done: patch.done === null ? undefined : (patch.done ?? existing.done), + total: patch.total === null ? undefined : (patch.total ?? existing.total), + remaining: patch.remaining === null ? undefined : (patch.remaining ?? existing.remaining), + unit: patch.unit === null ? undefined : (patch.unit ?? existing.unit), + detail: patch.detail === null ? undefined : (patch.detail ?? existing.detail), + heartbeatAt: patch.heartbeatAt ?? now, + updatedAt: now + } + + db.prepare( + `UPDATE session_jobs SET + label = ?, status = ?, done = ?, total = ?, remaining = ?, unit = ?, detail = ?, + heartbeat_at = ?, updated_at = ? + WHERE session_id = ? AND job_key = ?` + ).run( + next.label, + next.status, + next.done ?? null, + next.total ?? null, + next.remaining ?? null, + next.unit ?? null, + next.detail ?? null, + next.heartbeatAt, + next.updatedAt, + sessionId, + jobKey + ) + + return getSessionJob(db, sessionId, jobKey) +} + +export function deleteSessionJob(db: Database, sessionId: string, jobKey: string): boolean { + const result = db.prepare( + 'DELETE FROM session_jobs WHERE session_id = ? AND job_key = ?' + ).run(sessionId, jobKey) + return result.changes > 0 +} + +/** + * Re-point jobs during session merge (same contract as scratchlist transfer). + * Call BEFORE deleteSession so CASCADE does not race the move. + */ +export function transferSessionJobs( + db: Database, + fromSessionId: string, + toSessionId: string +): { moved: number; collided: number } { + const rows = listSessionJobs(db, fromSessionId) + let moved = 0 + let collided = 0 + + for (const job of rows) { + const existing = getSessionJob(db, toSessionId, job.key) + if (existing) { + db.prepare('DELETE FROM session_jobs WHERE session_id = ? AND job_key = ?') + .run(fromSessionId, job.key) + collided += 1 + continue + } + db.prepare( + `UPDATE session_jobs SET session_id = ? + WHERE session_id = ? AND job_key = ?` + ).run(toSessionId, fromSessionId, job.key) + moved += 1 + } + + return { moved, collided } +} diff --git a/hub/src/store/sessionJobsStore.ts b/hub/src/store/sessionJobsStore.ts new file mode 100644 index 0000000000..cc59c5c2f6 --- /dev/null +++ b/hub/src/store/sessionJobsStore.ts @@ -0,0 +1,57 @@ +import type { Database } from 'bun:sqlite' +import type { AttachedJob, AttachedJobPatch, AttachedJobUpsert } from '@hapi/protocol' + +import type { StoredSessionJob } from './types' +import { + deleteSessionJob, + getPrimaryRunningJob, + getPrimaryRunningJobsBySessionIds, + getSessionJob, + listSessionJobs, + patchSessionJob, + toAttachedJob, + transferSessionJobs, + upsertSessionJob, + type UpsertSessionJobResult +} from './sessionJobs' + +export class SessionJobsStore { + private readonly db: Database + + constructor(db: Database) { + this.db = db + } + + list(sessionId: string): StoredSessionJob[] { + return listSessionJobs(this.db, sessionId) + } + + get(sessionId: string, jobKey: string): StoredSessionJob | null { + return getSessionJob(this.db, sessionId, jobKey) + } + + getPrimaryRunning(sessionId: string): AttachedJob | null { + const job = getPrimaryRunningJob(this.db, sessionId) + return job ? toAttachedJob(job) : null + } + + getPrimaryRunningBySessionIds(sessionIds: string[]): Map { + return getPrimaryRunningJobsBySessionIds(this.db, sessionIds) + } + + upsert(sessionId: string, jobKey: string, body: AttachedJobUpsert): UpsertSessionJobResult { + return upsertSessionJob(this.db, sessionId, jobKey, body) + } + + patch(sessionId: string, jobKey: string, patch: AttachedJobPatch): StoredSessionJob | null { + return patchSessionJob(this.db, sessionId, jobKey, patch) + } + + delete(sessionId: string, jobKey: string): boolean { + return deleteSessionJob(this.db, sessionId, jobKey) + } + + transfer(fromSessionId: string, toSessionId: string): { moved: number; collided: number } { + return transferSessionJobs(this.db, fromSessionId, toSessionId) + } +} diff --git a/hub/src/store/types.ts b/hub/src/store/types.ts index 5a44d7ef17..9d47f8d156 100644 --- a/hub/src/store/types.ts +++ b/hub/src/store/types.ts @@ -98,6 +98,21 @@ export type StoredScratchlistEntry = { attachments: import('@hapi/protocol').ScratchlistAttachmentMetadata[] } +export type StoredSessionJob = { + sessionId: string + key: string + label: string + status: import('@hapi/protocol').AttachedJobStatus + done?: number + total?: number + remaining?: number + unit?: string + detail?: string + heartbeatAt: number + startedAt: number + updatedAt: number +} + export type VersionedUpdateResult = | { result: 'success'; version: number; value: T } | { result: 'version-mismatch'; version: number; value: T } diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index cba8c5f3a6..98709df8c0 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -752,6 +752,26 @@ export class SessionCache { }) } + /** + * tiann/hapi#1404 — emit primary attached job (or null) so session-list + * caches update inline without a dedicated refetch. + */ + emitAttachedJobChanged( + sessionId: string, + attachedJob: import('@hapi/protocol').AttachedJob | null + ): void { + const cached = this.sessions.get(sessionId) + const namespace = cached?.namespace + ?? this.store.sessions.getSession(sessionId)?.namespace + if (!namespace) return + this.publisher.emit({ + type: 'session-updated', + sessionId, + namespace, + data: { attachedJob } satisfies SessionPatch + }) + } + handleSessionEnd(payload: { sid: string; time: number }): void { const t = clampAliveTime(payload.time) ?? Date.now() @@ -1294,6 +1314,19 @@ export class SessionCache { // the operator's per-session notes, contradicting the v2.0 // promise that scratchlist survives reloads. const movedScratchlist = this.store.scratchlist.transfer(oldSessionId, newSessionId) + const movedJobs = this.store.sessionJobs.transfer(oldSessionId, newSessionId) + if (movedJobs.moved > 0 || movedJobs.collided > 0) { + this.emitAttachedJobChanged( + newSessionId, + this.store.sessionJobs.getPrimaryRunning(newSessionId) + ) + if (!options.deleteOldSession) { + this.emitAttachedJobChanged( + oldSessionId, + this.store.sessionJobs.getPrimaryRunning(oldSessionId) + ) + } + } if (movedScratchlist.moved > 0) { // Attachment hub paths embed the old session id. Re-key files + // metadata so quota/resolve stay correct on the consolidated id. diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index e20e0de104..80178c1c1c 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -788,6 +788,95 @@ export class SyncEngine { return removed } + listSessionJobs(sessionId: string) { + return this.store.sessionJobs.list(sessionId).map((job) => ({ + key: job.key, + label: job.label, + status: job.status, + ...(job.done !== undefined ? { done: job.done } : {}), + ...(job.total !== undefined ? { total: job.total } : {}), + ...(job.remaining !== undefined ? { remaining: job.remaining } : {}), + ...(job.unit !== undefined ? { unit: job.unit } : {}), + ...(job.detail !== undefined ? { detail: job.detail } : {}), + heartbeatAt: job.heartbeatAt, + startedAt: job.startedAt, + updatedAt: job.updatedAt + })) + } + + getPrimaryAttachedJob(sessionId: string) { + return this.store.sessionJobs.getPrimaryRunning(sessionId) + } + + getPrimaryAttachedJobsBySessionIds(sessionIds: string[]) { + return this.store.sessionJobs.getPrimaryRunningBySessionIds(sessionIds) + } + + upsertSessionJob( + sessionId: string, + jobKey: string, + body: import('@hapi/protocol').AttachedJobUpsert + ): + | { outcome: 'upserted'; job: import('@hapi/protocol').AttachedJob } + | { outcome: 'session-not-found' } { + const result = this.store.sessionJobs.upsert(sessionId, jobKey, body) + if (result.outcome === 'session-not-found') { + return result + } + const primary = this.store.sessionJobs.getPrimaryRunning(sessionId) + this.sessionCache.emitAttachedJobChanged(sessionId, primary) + const job = result.job + return { + outcome: 'upserted', + job: { + key: job.key, + label: job.label, + status: job.status, + ...(job.done !== undefined ? { done: job.done } : {}), + ...(job.total !== undefined ? { total: job.total } : {}), + ...(job.remaining !== undefined ? { remaining: job.remaining } : {}), + ...(job.unit !== undefined ? { unit: job.unit } : {}), + ...(job.detail !== undefined ? { detail: job.detail } : {}), + heartbeatAt: job.heartbeatAt, + startedAt: job.startedAt, + updatedAt: job.updatedAt + } + } + } + + patchSessionJob( + sessionId: string, + jobKey: string, + patch: import('@hapi/protocol').AttachedJobPatch + ): import('@hapi/protocol').AttachedJob | null { + const updated = this.store.sessionJobs.patch(sessionId, jobKey, patch) + if (!updated) return null + const primary = this.store.sessionJobs.getPrimaryRunning(sessionId) + this.sessionCache.emitAttachedJobChanged(sessionId, primary) + return { + key: updated.key, + label: updated.label, + status: updated.status, + ...(updated.done !== undefined ? { done: updated.done } : {}), + ...(updated.total !== undefined ? { total: updated.total } : {}), + ...(updated.remaining !== undefined ? { remaining: updated.remaining } : {}), + ...(updated.unit !== undefined ? { unit: updated.unit } : {}), + ...(updated.detail !== undefined ? { detail: updated.detail } : {}), + heartbeatAt: updated.heartbeatAt, + startedAt: updated.startedAt, + updatedAt: updated.updatedAt + } + } + + deleteSessionJob(sessionId: string, jobKey: string): boolean { + const removed = this.store.sessionJobs.delete(sessionId, jobKey) + if (removed) { + const primary = this.store.sessionJobs.getPrimaryRunning(sessionId) + this.sessionCache.emitAttachedJobChanged(sessionId, primary) + } + return removed + } + private async withScratchlistUploadLock( namespace: string, sessionId: string, diff --git a/hub/src/web/routes/sessions-jobs.test.ts b/hub/src/web/routes/sessions-jobs.test.ts new file mode 100644 index 0000000000..a10c09924c --- /dev/null +++ b/hub/src/web/routes/sessions-jobs.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from 'bun:test' +import { Hono } from 'hono' +import type { AttachedJob, AttachedJobPatch, AttachedJobUpsert } from '@hapi/protocol' +import type { Session, SyncEngine } from '../../sync/syncEngine' +import type { WebAppEnv } from '../middleware/auth' +import { createSessionsRoutes } from './sessions' + +function createSession(overrides?: Partial): Session { + return { + id: '11111111-1111-1111-1111-111111111111', + namespace: 'default', + seq: 1, + createdAt: 1, + updatedAt: 1, + active: false, + activeAt: 1, + metadata: { path: '/music', host: 'local', name: 'Lidarr' }, + metadataVersion: 1, + agentState: null, + agentStateVersion: 0, + thinking: false, + thinkingAt: 0, + model: null, + modelReasoningEffort: null, + effort: null, + serviceTier: null, + ...overrides + } +} + +describe('session-attached jobs routes (tiann/hapi#1404)', () => { + it('lists attachedJob on GET /sessions for inactive session', async () => { + const session = createSession() + const jobs = new Map() + + const engine = { + resolveSessionAccess: () => ({ ok: true as const, sessionId: session.id, session }), + getSessionsByNamespace: () => [session], + getFutureScheduledMessageCounts: () => new Map(), + getNextScheduledAtBySessionIds: () => new Map(), + getPrimaryAttachedJobsBySessionIds: (ids: string[]) => { + const map = new Map() + const primary = [...jobs.values()].find((j) => j.status === 'running') + if (primary) { + for (const id of ids) map.set(id, primary) + } + return map + }, + getPrimaryAttachedJob: () => [...jobs.values()].find((j) => j.status === 'running') ?? null, + listSessionJobs: () => [...jobs.values()], + upsertSessionJob: (_sid: string, key: string, body: AttachedJobUpsert) => { + const now = Date.now() + const job: AttachedJob = { + key, + label: body.label, + status: body.status ?? 'running', + ...(body.done !== undefined ? { done: body.done } : {}), + ...(body.total !== undefined ? { total: body.total } : {}), + ...(body.remaining !== undefined ? { remaining: body.remaining } : {}), + ...(body.unit !== undefined ? { unit: body.unit } : {}), + ...(body.detail !== undefined ? { detail: body.detail } : {}), + heartbeatAt: body.heartbeatAt ?? now, + startedAt: body.startedAt ?? now, + updatedAt: now + } + jobs.set(key, job) + return { outcome: 'upserted' as const, job } + }, + patchSessionJob: (_sid: string, key: string, patch: AttachedJobPatch) => { + const existing = jobs.get(key) + if (!existing) return null + const next: AttachedJob = { + ...existing, + ...(patch.label !== undefined ? { label: patch.label } : {}), + ...(patch.status !== undefined ? { status: patch.status } : {}), + ...(patch.done !== undefined && patch.done !== null ? { done: patch.done } : {}), + ...(patch.remaining !== undefined && patch.remaining !== null + ? { remaining: patch.remaining } + : {}), + heartbeatAt: patch.heartbeatAt ?? Date.now(), + updatedAt: Date.now() + } + jobs.set(key, next) + return next + }, + deleteSessionJob: (_sid: string, key: string) => jobs.delete(key) + } as unknown as SyncEngine + + const app = new Hono() + app.use('*', async (c, next) => { + c.set('namespace', 'default') + await next() + }) + app.route('/api', createSessionsRoutes(() => engine)) + + const put = await app.request( + `http://localhost/api/sessions/${session.id}/jobs/beets`, + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + label: 'beets import', + remaining: 120, + unit: 'tracks' + }) + } + ) + expect(put.status).toBe(200) + + const res = await app.request('http://localhost/api/sessions') + expect(res.status).toBe(200) + const body = await res.json() as { + sessions: Array<{ active: boolean; attachedJob: AttachedJob | null }> + } + expect(body.sessions[0]!.active).toBe(false) + expect(body.sessions[0]!.attachedJob?.key).toBe('beets') + expect(body.sessions[0]!.attachedJob?.remaining).toBe(120) + + const del = await app.request( + `http://localhost/api/sessions/${session.id}/jobs/beets`, + { method: 'DELETE' } + ) + expect(del.status).toBe(200) + const list = await app.request(`http://localhost/api/sessions/${session.id}/jobs`) + const listed = await list.json() as { jobs: AttachedJob[]; primary: AttachedJob | null } + expect(listed.jobs).toEqual([]) + expect(listed.primary).toBeNull() + }) +}) diff --git a/hub/src/web/routes/sessions.ts b/hub/src/web/routes/sessions.ts index 4fc91066c5..2df981fd2c 100644 --- a/hub/src/web/routes/sessions.ts +++ b/hub/src/web/routes/sessions.ts @@ -1,4 +1,6 @@ import { + AttachedJobPatchSchema, + AttachedJobUpsertSchema, CursorMigrateToAcpRequestSchema, DeleteUploadRequestSchema, ForkConversationRequestSchema, @@ -118,8 +120,11 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho } const scheduledCounts = engine.getFutureScheduledMessageCounts(sessionRecords.map((session) => session.id)) const nextScheduledAt = engine.getNextScheduledAtBySessionIds(sessionRecords.map((session) => session.id)) + const attachedJobs = engine.getPrimaryAttachedJobsBySessionIds(sessionRecords.map((session) => session.id)) const sessions = sessionRecords.map((session) => { - const summary = toSessionSummary(session) + const summary = toSessionSummary(session, { + attachedJob: attachedJobs.get(session.id) ?? null + }) return { ...summary, futureScheduledMessageCount: scheduledCounts.get(session.id) ?? 0, @@ -1295,6 +1300,94 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho return c.json({ ok: true }) }) + // tiann/hapi#1404 — session-attached long-running jobs (works while agent idle). + const JOB_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/ + + app.get('/sessions/:id/jobs', (c) => { + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) { + return engine + } + const sessionResult = requireSessionFromParam(c, engine) + if (sessionResult instanceof Response) { + return sessionResult + } + return c.json({ + jobs: engine.listSessionJobs(sessionResult.sessionId), + primary: engine.getPrimaryAttachedJob(sessionResult.sessionId) + }) + }) + + app.put('/sessions/:id/jobs/:jobKey', async (c) => { + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) { + return engine + } + const sessionResult = requireSessionFromParam(c, engine) + if (sessionResult instanceof Response) { + return sessionResult + } + const jobKey = c.req.param('jobKey') + if (!jobKey || !JOB_KEY_RE.test(jobKey)) { + return c.json({ error: 'Invalid jobKey (1-128 chars: alnum, . _ -)' }, 400) + } + const body = await c.req.json().catch(() => null) + const parsed = AttachedJobUpsertSchema.safeParse(body) + if (!parsed.success) { + return c.json({ error: 'Invalid body', issues: parsed.error.issues }, 400) + } + const result = engine.upsertSessionJob(sessionResult.sessionId, jobKey, parsed.data) + if (result.outcome === 'session-not-found') { + return c.json({ error: 'Session not found' }, 404) + } + return c.json({ job: result.job }) + }) + + app.patch('/sessions/:id/jobs/:jobKey', async (c) => { + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) { + return engine + } + const sessionResult = requireSessionFromParam(c, engine) + if (sessionResult instanceof Response) { + return sessionResult + } + const jobKey = c.req.param('jobKey') + if (!jobKey || !JOB_KEY_RE.test(jobKey)) { + return c.json({ error: 'Invalid jobKey (1-128 chars: alnum, . _ -)' }, 400) + } + const body = await c.req.json().catch(() => null) + const parsed = AttachedJobPatchSchema.safeParse(body) + if (!parsed.success) { + return c.json({ error: 'Invalid body', issues: parsed.error.issues }, 400) + } + const job = engine.patchSessionJob(sessionResult.sessionId, jobKey, parsed.data) + if (!job) { + return c.json({ error: 'Job not found' }, 404) + } + return c.json({ job }) + }) + + app.delete('/sessions/:id/jobs/:jobKey', (c) => { + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) { + return engine + } + const sessionResult = requireSessionFromParam(c, engine) + if (sessionResult instanceof Response) { + return sessionResult + } + const jobKey = c.req.param('jobKey') + if (!jobKey || !JOB_KEY_RE.test(jobKey)) { + return c.json({ error: 'Invalid jobKey (1-128 chars: alnum, . _ -)' }, 400) + } + const removed = engine.deleteSessionJob(sessionResult.sessionId, jobKey) + if (!removed) { + return c.json({ error: 'Job not found' }, 404) + } + return c.json({ ok: true }) + }) + app.get('/sessions/:id/slash-commands', async (c) => { const engine = requireSyncEngine(c, getSyncEngine) if (engine instanceof Response) { diff --git a/shared/src/index.ts b/shared/src/index.ts index 3e35fdd0bb..9d02e9d8a1 100644 --- a/shared/src/index.ts +++ b/shared/src/index.ts @@ -26,4 +26,16 @@ export * from './slashCommands' export * from './utils' export * from './usage' export * from './version' +export { + AttachedJobSchema, + AttachedJobUpsertSchema, + AttachedJobPatchSchema, + AttachedJobStatusSchema +} from './schemas' +export type { + AttachedJob, + AttachedJobUpsert, + AttachedJobPatch, + AttachedJobStatus +} from './schemas' export type * from './types' diff --git a/shared/src/schemas.sessionPatch.test.ts b/shared/src/schemas.sessionPatch.test.ts index 36afa290aa..6515ea5c66 100644 --- a/shared/src/schemas.sessionPatch.test.ts +++ b/shared/src/schemas.sessionPatch.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { SessionPatchSchema } from './schemas'; +import { AttachedJobSchema, SessionPatchSchema } from './schemas'; // Guard the contract for the second-half-of-#884 fix. The web client routes // `session-updated` events to the structured-patch path only when the event's @@ -101,4 +101,33 @@ describe('SessionPatchSchema structured patches (closes #884 follow-up)', () => }; expect(SessionPatchSchema.safeParse(fullSession).success).toBe(false); }); + + it('accepts attachedJob payload or null (tiann/hapi#1404)', () => { + const job = AttachedJobSchema.parse({ + key: 'beets', + label: 'beets import', + status: 'running', + done: 800, + total: 900, + unit: 'tracks', + heartbeatAt: 2_000, + startedAt: 1_000, + updatedAt: 2_000 + }) + expect(SessionPatchSchema.safeParse({ attachedJob: job }).success).toBe(true) + expect(SessionPatchSchema.safeParse({ attachedJob: null }).success).toBe(true) + }); + + it('rejects fake percent-only attached jobs without counters', () => { + // Progress is explicit counts — no bare percent field on the wire. + expect(AttachedJobSchema.safeParse({ + key: 'x', + label: 'x', + status: 'running', + percent: 91, + heartbeatAt: 1, + startedAt: 1, + updatedAt: 1 + }).success).toBe(false) + }); }); diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index 40da2b475f..a122541808 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -398,6 +398,53 @@ const VersionedTeamStatePatchSchema = z.object({ value: TeamStateSchema.nullable() }) +/** Opt-in long-running work owned by a session (tiann/hapi#1404). */ +export const AttachedJobStatusSchema = z.enum(['running', 'completed', 'failed']) + +export const AttachedJobSchema = z.object({ + key: z.string().min(1).max(128), + label: z.string().min(1).max(200), + status: AttachedJobStatusSchema, + done: z.number().nonnegative().optional(), + total: z.number().positive().optional(), + remaining: z.number().nonnegative().optional(), + unit: z.string().min(1).max(64).optional(), + detail: z.string().max(500).optional(), + heartbeatAt: z.number(), + startedAt: z.number(), + updatedAt: z.number() +}).strict() + +export type AttachedJob = z.infer +export type AttachedJobStatus = z.infer + +export const AttachedJobUpsertSchema = z.object({ + label: z.string().min(1).max(200), + status: AttachedJobStatusSchema.optional().default('running'), + done: z.number().nonnegative().optional(), + total: z.number().positive().optional(), + remaining: z.number().nonnegative().optional(), + unit: z.string().min(1).max(64).optional(), + detail: z.string().max(500).optional(), + heartbeatAt: z.number().optional(), + startedAt: z.number().optional() +}).strict() + +export type AttachedJobUpsert = z.infer + +export const AttachedJobPatchSchema = z.object({ + label: z.string().min(1).max(200).optional(), + status: AttachedJobStatusSchema.optional(), + done: z.number().nonnegative().nullable().optional(), + total: z.number().positive().nullable().optional(), + remaining: z.number().nonnegative().nullable().optional(), + unit: z.string().min(1).max(64).nullable().optional(), + detail: z.string().max(500).nullable().optional(), + heartbeatAt: z.number().optional() +}).strict() + +export type AttachedJobPatch = z.infer + export const SessionPatchSchema = z.object({ active: z.boolean().optional(), thinking: z.boolean().optional(), @@ -430,7 +477,12 @@ export const SessionPatchSchema = z.object({ // signal, not the payload. Keep this minimal: per the operator's 80/20 // ruling, scratchlist mutations are rare relative to keep-alive // patches, so a fresh event type would be overkill. - scratchlistUpdatedAt: z.number().optional() + scratchlistUpdatedAt: z.number().optional(), + // tiann/hapi#1404 — session-attached long-running jobs. Unlike + // scratchlist (watermark → refetch), the list row needs the progress + // payload inline, so patches carry the primary running job (or null + // when cleared / none remain). + attachedJob: AttachedJobSchema.nullable().optional() }).strict() export type SessionPatch = z.infer diff --git a/shared/src/sessionSummary.test.ts b/shared/src/sessionSummary.test.ts index 792dcba882..c2a7f0b2b4 100644 --- a/shared/src/sessionSummary.test.ts +++ b/shared/src/sessionSummary.test.ts @@ -330,6 +330,25 @@ describe('summary derivation helpers', () => { expect(computePendingRequestsCount(undefined)).toBe(0) }) + it('includes attachedJob when provided via extras', () => { + const job = { + key: 'beets', + label: 'beets import', + status: 'running' as const, + remaining: 120, + unit: 'tracks', + heartbeatAt: 9_000, + startedAt: 1_000, + updatedAt: 9_000 + } + const summary = toSessionSummary(makeSession(), { attachedJob: job }) + expect(summary.attachedJob).toEqual(job) + }) + + it('defaults attachedJob to null', () => { + expect(toSessionSummary(makeSession()).attachedJob).toBeNull() + }) + it('toSessionSummaryMetadata returns null for null metadata', () => { expect(toSessionSummaryMetadata(null)).toBeNull() expect(toSessionSummaryMetadata(undefined)).toBeNull() diff --git a/shared/src/sessionSummary.ts b/shared/src/sessionSummary.ts index f2d787e6b9..6ded7a20d1 100644 --- a/shared/src/sessionSummary.ts +++ b/shared/src/sessionSummary.ts @@ -1,4 +1,4 @@ -import type { AgentState, Metadata, Session, TodoItem, WorktreeMetadata } from './schemas' +import type { AgentState, AttachedJob, Metadata, Session, TodoItem, WorktreeMetadata } from './schemas' import { isKnownFlavor } from './flavors' import type { AgentFlavor } from './modes' @@ -72,6 +72,11 @@ export type SessionSummary = { futureScheduledMessageCount: number /** Epoch ms of the soonest uninvoked future scheduled message, or null. */ nextScheduledAt: number | null + /** + * Primary running session-attached job (tiann/hapi#1404), or null. + * Independent of agent `active` / thinking — work that outlives the agent. + */ + attachedJob: AttachedJob | null model: string | null modelReasoningEffort?: string | null effort: string | null @@ -203,7 +208,10 @@ export function toSessionSummaryMetadata(metadata: Metadata | null | undefined): } } -export function toSessionSummary(session: Session): SessionSummary { +export function toSessionSummary( + session: Session, + extras?: { attachedJob?: AttachedJob | null } +): SessionSummary { return { hasConversationContent: session.hasConversationContent ?? false, id: session.id, @@ -224,6 +232,7 @@ export function toSessionSummary(session: Session): SessionSummary { backgroundTaskCount: session.backgroundTaskCount ?? 0, futureScheduledMessageCount: 0, nextScheduledAt: null, + attachedJob: extras?.attachedJob ?? null, model: session.model, modelReasoningEffort: session.modelReasoningEffort, effort: session.effort diff --git a/shared/src/types.ts b/shared/src/types.ts index df693464ca..2bd8d96765 100644 --- a/shared/src/types.ts +++ b/shared/src/types.ts @@ -2,6 +2,10 @@ export type { AgentState, AgentStateCompletedRequest, AgentStateRequest, + AttachedJob, + AttachedJobPatch, + AttachedJobStatus, + AttachedJobUpsert, AttachmentMetadata, DecryptedMessage, Metadata, diff --git a/web/src/components/SessionAttentionIndicator.test.tsx b/web/src/components/SessionAttentionIndicator.test.tsx index d69f293873..bbf540153c 100644 --- a/web/src/components/SessionAttentionIndicator.test.tsx +++ b/web/src/components/SessionAttentionIndicator.test.tsx @@ -29,6 +29,7 @@ function makeSummary(overrides: Partial & { id: string }): Sessi backgroundTaskCount: 0, futureScheduledMessageCount: 0, nextScheduledAt: null, + attachedJob: null, model: null, effort: null, ...overrides diff --git a/web/src/components/SessionList.directory-action.test.tsx b/web/src/components/SessionList.directory-action.test.tsx index 7e399cbb15..fb49c666fa 100644 --- a/web/src/components/SessionList.directory-action.test.tsx +++ b/web/src/components/SessionList.directory-action.test.tsx @@ -33,6 +33,7 @@ function makeSession(overrides: Partial & { id: string }): Sessi backgroundTaskCount: 0, futureScheduledMessageCount: 0, nextScheduledAt: null, + attachedJob: null, model: null, effort: null, ...overrides diff --git a/web/src/components/SessionList.machine-filter.test.tsx b/web/src/components/SessionList.machine-filter.test.tsx index f786202fd9..4fc1477d44 100644 --- a/web/src/components/SessionList.machine-filter.test.tsx +++ b/web/src/components/SessionList.machine-filter.test.tsx @@ -29,6 +29,7 @@ function makeSession(overrides: Partial & { id: string }): Sessi backgroundTaskCount: 0, futureScheduledMessageCount: 0, nextScheduledAt: null, + attachedJob: null, model: null, effort: null, ...overrides diff --git a/web/src/components/SessionList.test.ts b/web/src/components/SessionList.test.ts index 28a47941a3..1c37f67fd1 100644 --- a/web/src/components/SessionList.test.ts +++ b/web/src/components/SessionList.test.ts @@ -42,6 +42,7 @@ function makeSession(overrides: Partial & { id: string }): Sessi backgroundTaskCount: 0, futureScheduledMessageCount: 0, nextScheduledAt: null, + attachedJob: null, model: null, effort: null, ...overrides diff --git a/web/src/components/SessionRowSummary.tsx b/web/src/components/SessionRowSummary.tsx index 0602f7ae45..4abc7fa584 100644 --- a/web/src/components/SessionRowSummary.tsx +++ b/web/src/components/SessionRowSummary.tsx @@ -12,6 +12,11 @@ import { getCodexImportedAt } from '@/lib/codexImportedSessions' import { getSessionTitle } from '@/lib/sessionTitle' import { useTranslation } from '@/lib/use-translation' import { getWorktreeSessionLabel } from '@/lib/sessionWorktreeLabel' +import { + attachedJobFraction, + formatAttachedJobProgress, + isAttachedJobStale +} from '@/lib/attachedJob' function LoaderIcon(props: { className?: string }) { return ( @@ -162,14 +167,18 @@ export function SessionRowSummary(props: { const attentionId = attentionTooltipIdProp ?? ownedIds.attentionId const scheduleId = scheduleTooltipIdProp ?? ownedIds.scheduleId const timeLabel = getSessionTimeLabel(s, t) + const attachedJob = s.attachedJob?.status === 'running' ? s.attachedJob : null + const jobStale = attachedJob ? isAttachedJobStale(attachedJob) : false + const jobFraction = attachedJob ? attachedJobFraction(attachedJob) : null + const jobProgressLabel = attachedJob ? formatAttachedJobProgress(attachedJob) : null return (
-
+
{sessionName} @@ -288,6 +297,43 @@ export function SessionRowSummary(props: { ) : null}
+ {attachedJob && jobProgressLabel ? ( +
+
+ ) : null} {projectLabel || machineLabel ? (
{[projectLabel, machineLabel].filter(Boolean).join(' · ')} diff --git a/web/src/hooks/useSSE.test.ts b/web/src/hooks/useSSE.test.ts index c2259ecf2c..7917422acb 100644 --- a/web/src/hooks/useSSE.test.ts +++ b/web/src/hooks/useSSE.test.ts @@ -179,6 +179,7 @@ function makeSummary(overrides: Partial = {}): SessionSummary { backgroundTaskCount: 0, futureScheduledMessageCount: 0, nextScheduledAt: null, + attachedJob: null, model: null, effort: null, ...overrides diff --git a/web/src/hooks/useSSE.ts b/web/src/hooks/useSSE.ts index 2218012083..39236f212c 100644 --- a/web/src/hooks/useSSE.ts +++ b/web/src/hooks/useSSE.ts @@ -159,6 +159,16 @@ export function isRenderIrrelevantPatch(current: SessionSummary, next: SessionSu && current.thinking === next.thinking && current.updatedAt === next.updatedAt && current.backgroundTaskCount === next.backgroundTaskCount + && current.attachedJob?.key === next.attachedJob?.key + && current.attachedJob?.label === next.attachedJob?.label + && current.attachedJob?.status === next.attachedJob?.status + && current.attachedJob?.done === next.attachedJob?.done + && current.attachedJob?.total === next.attachedJob?.total + && current.attachedJob?.remaining === next.attachedJob?.remaining + && current.attachedJob?.unit === next.attachedJob?.unit + && current.attachedJob?.detail === next.attachedJob?.detail + && current.attachedJob?.heartbeatAt === next.attachedJob?.heartbeatAt + && (current.attachedJob == null) === (next.attachedJob == null) && current.model === next.model && current.modelReasoningEffort === next.modelReasoningEffort && current.effort === next.effort @@ -487,6 +497,7 @@ export function useSSE(options: { const existing = existingIndex >= 0 ? previous.sessions[existingIndex] : undefined const summary = { ...toSessionSummary(session), + attachedJob: existing?.attachedJob ?? null, futureScheduledMessageCount: existing?.futureScheduledMessageCount ?? 0, nextScheduledAt: existing?.nextScheduledAt ?? null } @@ -532,6 +543,9 @@ export function useSSE(options: { backgroundTaskCount: Object.prototype.hasOwnProperty.call(patch, 'backgroundTaskCount') ? patch.backgroundTaskCount ?? 0 : current.backgroundTaskCount, + attachedJob: Object.prototype.hasOwnProperty.call(patch, 'attachedJob') + ? patch.attachedJob ?? null + : current.attachedJob ?? null, model: Object.prototype.hasOwnProperty.call(patch, 'model') ? patch.model ?? null : current.model, modelReasoningEffort: Object.prototype.hasOwnProperty.call(patch, 'modelReasoningEffort') ? patch.modelReasoningEffort ?? null diff --git a/web/src/lib/attachedJob.test.ts b/web/src/lib/attachedJob.test.ts new file mode 100644 index 0000000000..fe3946b192 --- /dev/null +++ b/web/src/lib/attachedJob.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import type { AttachedJob } from '@hapi/protocol' +import { + ATTACHED_JOB_STALE_MS, + attachedJobFraction, + formatAttachedJobProgress, + isAttachedJobStale +} from './attachedJob' + +function job(overrides: Partial = {}): AttachedJob { + return { + key: 'beets', + label: 'beets import', + status: 'running', + heartbeatAt: 1_000, + startedAt: 1_000, + updatedAt: 1_000, + ...overrides + } +} + +describe('attachedJob helpers', () => { + it('formats remaining count without inventing percent', () => { + expect(formatAttachedJobProgress(job({ remaining: 120, unit: 'tracks' }))).toBe('120 tracks left') + }) + + it('formats done/total with derived percent', () => { + expect(formatAttachedJobProgress(job({ done: 800, total: 900, unit: 'tracks' }))).toBe( + '89% · 800/900 tracks' + ) + }) + + it('falls back to running when only heartbeat', () => { + expect(formatAttachedJobProgress(job())).toBe('running') + }) + + it('computes fraction from remaining+total', () => { + expect(attachedJobFraction(job({ remaining: 100, total: 1000 }))).toBe(0.9) + }) + + it('marks stale after heartbeat window', () => { + const now = 1_000 + ATTACHED_JOB_STALE_MS + 1 + expect(isAttachedJobStale(job({ heartbeatAt: 1_000 }), now)).toBe(true) + expect(isAttachedJobStale(job({ heartbeatAt: now - 60_000 }), now)).toBe(false) + }) +}) diff --git a/web/src/lib/attachedJob.ts b/web/src/lib/attachedJob.ts new file mode 100644 index 0000000000..58c869c1c9 --- /dev/null +++ b/web/src/lib/attachedJob.ts @@ -0,0 +1,31 @@ +import type { AttachedJob } from '@hapi/protocol' + +/** Stale if no heartbeat for 15 minutes — UI amber, still shows progress. */ +export const ATTACHED_JOB_STALE_MS = 15 * 60 * 1000 + +export function formatAttachedJobProgress(job: AttachedJob): string { + if (job.remaining !== undefined) { + const unit = job.unit ? ` ${job.unit}` : '' + return `${job.remaining}${unit} left` + } + if (job.done !== undefined && job.total !== undefined && job.total > 0) { + const pct = Math.min(100, Math.round((job.done / job.total) * 100)) + return `${pct}% · ${job.done}/${job.total}${job.unit ? ` ${job.unit}` : ''}` + } + return 'running' +} + +export function attachedJobFraction(job: AttachedJob): number | null { + if (job.done !== undefined && job.total !== undefined && job.total > 0) { + return Math.max(0, Math.min(1, job.done / job.total)) + } + if (job.remaining !== undefined && job.total !== undefined && job.total > 0) { + const done = Math.max(0, job.total - job.remaining) + return Math.max(0, Math.min(1, done / job.total)) + } + return null +} + +export function isAttachedJobStale(job: AttachedJob, now: number = Date.now()): boolean { + return now - job.heartbeatAt > ATTACHED_JOB_STALE_MS +} diff --git a/web/src/lib/sessionAttention.test.ts b/web/src/lib/sessionAttention.test.ts index acf95094b7..a503ac98a9 100644 --- a/web/src/lib/sessionAttention.test.ts +++ b/web/src/lib/sessionAttention.test.ts @@ -22,6 +22,7 @@ function makeSummary(overrides: Partial & { id: string }): Sessi backgroundTaskCount: 0, futureScheduledMessageCount: 0, nextScheduledAt: null, + attachedJob: null, model: null, effort: null, ...overrides diff --git a/web/src/lib/sessionReference.test.ts b/web/src/lib/sessionReference.test.ts index 4118d4c7d9..407f296646 100644 --- a/web/src/lib/sessionReference.test.ts +++ b/web/src/lib/sessionReference.test.ts @@ -28,6 +28,7 @@ function makeSession(overrides: Partial & { id: string }): Sessi backgroundTaskCount: 0, futureScheduledMessageCount: 0, nextScheduledAt: null, + attachedJob: null, model: null, effort: null, ...overrides, From bba8e69db29bab7cb73b75bfda23153b3919938a Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:53:38 +0000 Subject: [PATCH 05/94] feat(jobs): agent guidance + wall-clock elapsed on list chrome Document the session-job contract for agents and always show startedAt elapsed next to remaining/fraction/running so indeterminate drains still read as wall time without inventing an ETA. Co-authored-by: Cursor --- AGENTS.md | 15 +++ cli/README.md | 1 + cli/src/claude/utils/systemPrompt.ts | 3 +- cli/src/codex/utils/systemPrompt.ts | 3 +- cli/src/commands/job.ts | 36 +++++++- cli/src/grok/utils/systemPrompt.ts | 3 +- .../common/sessionJobInstruction.test.ts | 20 ++++ .../modules/common/sessionJobInstruction.ts | 33 +++++++ cli/src/opencode/utils/systemPrompt.test.ts | 6 +- cli/src/opencode/utils/systemPrompt.ts | 5 +- docs/.vitepress/config.ts | 1 + docs/guide/faq.md | 4 + docs/guide/session-jobs.md | 92 +++++++++++++++++++ web/src/components/SessionRowSummary.tsx | 15 ++- web/src/lib/attachedJob.test.ts | 33 +++++-- web/src/lib/attachedJob.ts | 38 +++++++- 16 files changed, 284 insertions(+), 24 deletions(-) create mode 100644 cli/src/modules/common/sessionJobInstruction.test.ts create mode 100644 cli/src/modules/common/sessionJobInstruction.ts create mode 100644 docs/guide/session-jobs.md diff --git a/AGENTS.md b/AGENTS.md index 5d49368812..87bf50ae94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,7 @@ Start with the task's files; read only relevant sections of these references, no | Shared wire types and validation | `shared/src/types.ts`, `schemas.ts`, `socket.ts`, `modes.ts` | | Native API contract, chat conformance | [client contract](docs/api/client-contract/index.md), [iOS](ios/README.md), [Android](android/README.md) | | Encrypted native push relay | [relay/README.md](relay/README.md) | +| Session-attached jobs (outliving work) | [session jobs guide](docs/guide/session-jobs.md), `cli/src/commands/job.ts` | | User docs / marketing site | `docs/` (VitePress) / `website/` | ## Repository conventions @@ -39,6 +40,20 @@ Start with the task's files; read only relevant sections of these references, no - Metadata/state updates are versioned; preserve stale-update rejection. Permission controls use per-flavor catalogs in `shared/src/modes.ts`, further constrained by session capabilities. - `shared/fixtures/**` is generated from the web chat pipeline, the source of truth for native conformance. Never hand-edit fixtures. For changes to fixture inputs or generation (paths in [.github/workflows/fixtures.yml](.github/workflows/fixtures.yml)), run `bun run gen:fixtures` and include any generated changes in the deliverable. CI checks drift and runs native conformance on fixture changes. +## Session-attached jobs (outliving work) + +When an agent starts process-shaped work that will keep running after the agent goes idle (`nohup`, batch imports, long scripts, external daemons), attach it so the session list stays truthful while `active: false`. This is **not** thinking progress / todos / in-agent background tools. + +Agent contract (idle agents cannot heartbeat — bare set + nohup freezes the bar): + +1. **Required for process-shaped work:** Shell `hapi job run --label … -- ` (auto-heartbeat + exit status). Use `"$HAPI_SESSION_ID"` only when it matches the operator chat row (`/sessions/` in the web URL). +2. MCP `session_job` **refuses `action=set`**. Use it only for `update` / `clear` / `list` on a job the supervisor already created. +3. Manual CLI `set` only with a self-heartbeating wrapper (`update` ≥~10m); never MCP set + nohup. +4. Prefer honest `--remaining` or `--done`/`--total`; omit counts if unknown — never invent a percent. +5. Elapsed wall clock is always shown from `startedAt` (not an ETA). + +Full guide: [docs/guide/session-jobs.md](docs/guide/session-jobs.md). CLI: `hapi job --help`. + ## Verification and completion Choose checks by the change's impact, not by the number of workflow steps: diff --git a/cli/README.md b/cli/README.md index 11066a7ba1..606a90c2ff 100644 --- a/cli/README.md +++ b/cli/README.md @@ -44,6 +44,7 @@ Choose a supported coding agent from your terminal and control its sessions remo - `hapi resume [sessionId]` - List resumable sessions for this machine or resume one locally. - `hapi ping-peer ` - Resume (if needed) and message another session. Prefer this or MCP `ping_peer` / `list_peers` over reinventing JWT+curl. Also `--message-file` / `--list`. - `hapi inspect-peer ` - Read-only peer metadata + recent message text (no resume). Prefer this or MCP `inspect_peer` when a user cites `[title](/sessions/)` or Copy-reference `See session "…" (/sessions/) for context`. `/sessions/` is a hub path, not a local file. Optional `--limit`. +- `hapi job set|update|clear|list` - Attach long-running outliving work to a session so the list UI shows progress while the agent is idle (`tiann/hapi#1404`). Prefer `"$HAPI_SESSION_ID"`. Heartbeat at least every ~10m; honest `--remaining` or `--done`/`--total` (omit counts if unknown — never invent a percent). See `docs/guide/session-jobs.md` and `hapi job --help`. The picker lists agents alphabetically by command name. Use Up/Down and Enter to choose; Esc or Ctrl-C cancels. It appears on every bare invocation, even diff --git a/cli/src/claude/utils/systemPrompt.ts b/cli/src/claude/utils/systemPrompt.ts index 3174ac6edd..03cfd1c7bc 100644 --- a/cli/src/claude/utils/systemPrompt.ts +++ b/cli/src/claude/utils/systemPrompt.ts @@ -2,6 +2,7 @@ import { trimIdent } from "@/utils/trimIdent"; import { buildSessionCitationSteerInstruction } from "@hapi/protocol/sessionCitation"; import { shouldIncludeCoAuthoredBy } from "./claudeSettings"; import { DISPLAY_IMAGE_PROMPT_CLAUDE, DISPLAY_MEDIA_PROMPT_CLAUDE, DISPLAY_VIDEO_PROMPT_CLAUDE } from "@/modules/common/displayImagePrompt"; +import { withSessionJobInstruction } from "@/modules/common/sessionJobInstruction"; import { withSessionSummaryInstruction } from "@/modules/common/sessionSummaryInstruction"; /** @@ -42,5 +43,5 @@ export function getSystemPrompt(): string { const base = includeCoAuthored ? BASE_SYSTEM_PROMPT + '\n\n' + CO_AUTHORED_CREDITS : BASE_SYSTEM_PROMPT; - return withSessionSummaryInstruction(base); + return withSessionSummaryInstruction(withSessionJobInstruction(base)); } diff --git a/cli/src/codex/utils/systemPrompt.ts b/cli/src/codex/utils/systemPrompt.ts index bd85efc36c..7ad815856e 100644 --- a/cli/src/codex/utils/systemPrompt.ts +++ b/cli/src/codex/utils/systemPrompt.ts @@ -8,6 +8,7 @@ import { trimIdent } from '@/utils/trimIdent'; import { buildSessionCitationSteerInstruction } from '@hapi/protocol/sessionCitation'; import { DISPLAY_IMAGE_PROMPT_CODEX, DISPLAY_MEDIA_PROMPT_CODEX, DISPLAY_VIDEO_PROMPT_CODEX } from '@/modules/common/displayImagePrompt'; +import { withSessionJobInstruction } from '@/modules/common/sessionJobInstruction'; import { withSessionSummaryInstruction } from '@/modules/common/sessionSummaryInstruction'; /** @@ -36,7 +37,7 @@ export const TITLE_INSTRUCTION = trimIdent(` * Session-summary contract is resolved at call time (hub toggle / env). */ export function getCodexSystemPrompt(env: NodeJS.ProcessEnv = process.env): string { - return withSessionSummaryInstruction(TITLE_INSTRUCTION, env) + return withSessionSummaryInstruction(withSessionJobInstruction(TITLE_INSTRUCTION), env) } /** Alias kept for existing call sites / tests that expect a string constant name. */ diff --git a/cli/src/commands/job.ts b/cli/src/commands/job.ts index cbd2fa55fc..1579621130 100644 --- a/cli/src/commands/job.ts +++ b/cli/src/commands/job.ts @@ -29,16 +29,33 @@ function showHelp(): void { console.log(` ${chalk.bold('hapi job')} - Attach long-running work to a HAPI session (tiann/hapi#1404) +${chalk.bold('When to use:')} + Work that outlives the agent (nohup / batch / long scripts / external daemons) + while the session may be idle. Not thinking progress or in-agent background tools. + +${chalk.bold('Agent contract:')} + 1. set before (or as) the process starts + 2. update / heartbeat at least every ~10 minutes while running + 3. prefer honest --remaining or --done/--total; omit counts if unknown + 4. never invent a fake percent + 5. clear or --status completed|failed when finished + ${chalk.bold('Usage:')} hapi job set --label [--remaining N] [--done N --total N] [--unit tracks] [--detail ...] hapi job update [--remaining N] [--done N] [--total N] [--status running|completed|failed] [--detail ...] hapi job clear hapi job list +${chalk.bold('Progress UI:')} + remaining → "N units left · 2h" + done + total → "P% · done/total · 2h" + label/detail only → "running · 2h" + indeterminate bar + elapsed always from startedAt (wall clock) — never an ETA / time-remaining field + ${chalk.bold('Notes:')} - Hub-persisted. Works while the agent is idle/offline — not thinking progress. - Prefer honest remaining/done+total; never invent a fake percent. + Hub-persisted. Prefer "$HAPI_SESSION_ID" for this chat. Job key: 1-128 chars, alnum / . _ - + Docs: docs/guide/session-jobs.md ${chalk.bold('Env:')} HAPI_API_URL / CLI_API_TOKEN (or ~/.hapi/settings.json via \`hapi auth login\`) @@ -166,6 +183,7 @@ function formatJobLine(job: { unit?: string detail?: string heartbeatAt: number + startedAt: number }): string { const parts = [`${job.key}`, job.label, job.status] if (job.remaining !== undefined) { @@ -173,6 +191,20 @@ function formatJobLine(job: { } else if (job.done !== undefined && job.total !== undefined) { parts.push(`${job.done}/${job.total}${job.unit ? ` ${job.unit}` : ''}`) } + const elapsedSec = Math.max(0, Math.round((Date.now() - job.startedAt) / 1000)) + if (elapsedSec < 60) { + parts.push(`elapsed ${elapsedSec}s`) + } else if (elapsedSec < 3600) { + parts.push(`elapsed ${Math.floor(elapsedSec / 60)}m`) + } else if (elapsedSec < 86400) { + const h = Math.floor(elapsedSec / 3600) + const m = Math.floor((elapsedSec % 3600) / 60) + parts.push(m > 0 ? `elapsed ${h}h ${m}m` : `elapsed ${h}h`) + } else { + const d = Math.floor(elapsedSec / 86400) + const h = Math.floor((elapsedSec % 86400) / 3600) + parts.push(h > 0 ? `elapsed ${d}d ${h}h` : `elapsed ${d}d`) + } if (job.detail) parts.push(job.detail) const ageSec = Math.max(0, Math.round((Date.now() - job.heartbeatAt) / 1000)) parts.push(`heartbeat ${ageSec}s ago`) diff --git a/cli/src/grok/utils/systemPrompt.ts b/cli/src/grok/utils/systemPrompt.ts index 695bc9ea42..07af91a8fd 100644 --- a/cli/src/grok/utils/systemPrompt.ts +++ b/cli/src/grok/utils/systemPrompt.ts @@ -1,9 +1,10 @@ import { SKILL_LOOKUP_INSTRUCTION } from '@/modules/common/skillLookupInstruction' +import { withSessionJobInstruction } from '@/modules/common/sessionJobInstruction' import { withSessionSummaryInstruction } from '@/modules/common/sessionSummaryInstruction' export const GROK_TITLE_INSTRUCTION = `Use the tool "hapi_change_title" once after the initial request is clear to set a concise session title. Do not rename for routine progress or substeps.\n${SKILL_LOOKUP_INSTRUCTION}` export function getGrokTitleInstruction(env: NodeJS.ProcessEnv = process.env): string { - return withSessionSummaryInstruction(GROK_TITLE_INSTRUCTION, env) + return withSessionSummaryInstruction(withSessionJobInstruction(GROK_TITLE_INSTRUCTION), env) } diff --git a/cli/src/modules/common/sessionJobInstruction.test.ts b/cli/src/modules/common/sessionJobInstruction.test.ts new file mode 100644 index 0000000000..2cdf00bedb --- /dev/null +++ b/cli/src/modules/common/sessionJobInstruction.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import { + SESSION_JOB_INSTRUCTION, + withSessionJobInstruction +} from './sessionJobInstruction' + +describe('sessionJobInstruction', () => { + it('mentions set, update, heartbeat, and no fake percent', () => { + expect(SESSION_JOB_INSTRUCTION).toContain('hapi job set') + expect(SESSION_JOB_INSTRUCTION).toContain('hapi job update') + expect(SESSION_JOB_INSTRUCTION).toContain('~10 minutes') + expect(SESSION_JOB_INSTRUCTION).toContain('Never invent a fake percent') + expect(SESSION_JOB_INSTRUCTION).toContain('HAPI_SESSION_ID') + }) + + it('appends after an existing prompt block', () => { + expect(withSessionJobInstruction('Base.')).toBe(`Base.\n\n${SESSION_JOB_INSTRUCTION}`) + expect(withSessionJobInstruction('')).toBe(SESSION_JOB_INSTRUCTION) + }) +}) diff --git a/cli/src/modules/common/sessionJobInstruction.ts b/cli/src/modules/common/sessionJobInstruction.ts new file mode 100644 index 0000000000..c6ba00c3d2 --- /dev/null +++ b/cli/src/modules/common/sessionJobInstruction.ts @@ -0,0 +1,33 @@ +/** + * Always-on steer for session-attached long-running jobs (tiann/hapi#1404). + * + * Unlike the session-summary contract (opt-in), this is short and triggers only + * when the agent spawns outliving work — so it rides every supported flavor's + * system / developer instructions by default. + * + * Cursor ACP has no system-prompt seam today; Cursor agents rely on the estate + * skill `hapi-session-jobs` (and `hapi job --help`) instead. + */ + +/** Canonical one-block contract. Keep short — every session's prompt budget. */ +export const SESSION_JOB_INSTRUCTION = [ + 'Session-attached jobs (outliving work):', + 'When you start work that will keep running after this agent goes idle', + '(nohup, batch imports, long scripts, external daemons), attach it to this', + 'HAPI session so the session list can show progress while you are idle.', + 'Use: hapi job set "$HAPI_SESSION_ID" --label ', + '[--remaining N] [--done N --total N] [--unit ] [--detail ].', + 'Heartbeat with hapi job update at least every ~10 minutes (UI goes amber', + 'after ~15m without a heartbeat). Prefer honest remaining or done+total;', + 'omit counts when unknown (UI shows "running" + indeterminate bar).', + 'Never invent a fake percent. On finish: hapi job update … --status', + 'completed|failed, or hapi job clear. Full contract: hapi job --help.' +].join(' ') + +/** Append instruction to an existing prompt block (blank line separator). */ +export function withSessionJobInstruction(base: string): string { + const trimmed = base.trimEnd() + return trimmed.length > 0 + ? `${trimmed}\n\n${SESSION_JOB_INSTRUCTION}` + : SESSION_JOB_INSTRUCTION +} diff --git a/cli/src/opencode/utils/systemPrompt.test.ts b/cli/src/opencode/utils/systemPrompt.test.ts index b22d3f9e2a..483c901075 100644 --- a/cli/src/opencode/utils/systemPrompt.test.ts +++ b/cli/src/opencode/utils/systemPrompt.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { ensureOpencodeConfig } from './opencodeConfig' -import { TITLE_INSTRUCTION } from './systemPrompt' +import { TITLE_INSTRUCTION, getTitleInstruction } from './systemPrompt' describe('OpenCode local HAPI instructions', () => { let configDirectory: string | null = null @@ -20,11 +20,13 @@ describe('OpenCode local HAPI instructions', () => { const { instructionsPath } = ensureOpencodeConfig( configDirectory, { command: 'hapi', args: ['mcp'] }, - TITLE_INSTRUCTION + getTitleInstruction({}) ) const instructions = await readFile(instructionsPath, 'utf8') expect(instructions).toContain('$name') expect(instructions).toContain('skill_lookup') + expect(instructions).toContain('hapi job set') + expect(instructions).toContain(TITLE_INSTRUCTION.trim()) }) }) diff --git a/cli/src/opencode/utils/systemPrompt.ts b/cli/src/opencode/utils/systemPrompt.ts index b1838d33e7..9844625196 100644 --- a/cli/src/opencode/utils/systemPrompt.ts +++ b/cli/src/opencode/utils/systemPrompt.ts @@ -14,6 +14,7 @@ import { DISPLAY_VIDEO_PROMPT_HAPI_MCP, } from '@/modules/common/displayImagePrompt'; import { SKILL_LOOKUP_INSTRUCTION } from '@/modules/common/skillLookupInstruction'; +import { withSessionJobInstruction } from '@/modules/common/sessionJobInstruction'; import { withSessionSummaryInstruction } from '@/modules/common/sessionSummaryInstruction'; /** @@ -30,7 +31,7 @@ export const TITLE_INSTRUCTION = trimIdent(` `); export function getTitleInstruction(env: NodeJS.ProcessEnv = process.env): string { - return withSessionSummaryInstruction(TITLE_INSTRUCTION, env) + return withSessionSummaryInstruction(withSessionJobInstruction(TITLE_INSTRUCTION), env) } /** @@ -50,7 +51,7 @@ export const OPENCODE_NATIVE_TOOL_INSTRUCTION = trimIdent(` `); export function getOpencodeNativeToolInstruction(env: NodeJS.ProcessEnv = process.env): string { - return withSessionSummaryInstruction(OPENCODE_NATIVE_TOOL_INSTRUCTION, env) + return withSessionSummaryInstruction(withSessionJobInstruction(OPENCODE_NATIVE_TOOL_INSTRUCTION), env) } /** diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index e4c4f3fa70..4952a99948 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -31,6 +31,7 @@ export default defineConfig({ text: 'Guide', items: [ { text: 'How it Works', link: '/guide/how-it-works' }, + { text: 'Session-attached jobs', link: '/guide/session-jobs' }, { text: 'Voice Assistant', link: '/guide/voice-assistant' }, { text: 'Why HAPI', link: '/guide/why-hapi' }, { text: 'FAQ', link: '/guide/faq' } diff --git a/docs/guide/faq.md b/docs/guide/faq.md index f68e6183c9..2d050d67cf 100644 --- a/docs/guide/faq.md +++ b/docs/guide/faq.md @@ -110,6 +110,10 @@ Yes. Open any session and use the chat interface to send messages directly to th Some agents (especially Cursor) can resume after idle from harness signals such as background Shell `notify_on_output` or `/loop`, without you sending a new HAPI message. HAPI updates the session's thinking indicator when the agent resumes work or requests permission, so the list reflects that activity. +### How do I show progress for a long batch that outlives the agent? + +Use session-attached jobs (`hapi job`). The agent (or a wrapper script) registers a job on the session, heartbeats while the process runs, and clears it when done. The session list shows remaining / fraction / or an indeterminate "running" meter even when the agent is idle. See [Session-attached jobs](./session-jobs.md). + ### Can I access a terminal remotely? Yes. Open a session in the web app and tap the Terminal tab for a remote shell. diff --git a/docs/guide/session-jobs.md b/docs/guide/session-jobs.md new file mode 100644 index 0000000000..339d9be7d4 --- /dev/null +++ b/docs/guide/session-jobs.md @@ -0,0 +1,92 @@ +# Session-attached jobs + +Hub-persisted progress for work that **outlives the agent** — batch imports, `nohup` scripts, long drains — so the session list still shows something truthful while the chat is idle (`active: false`). + +This is **not** in-agent thinking progress, todos, or `backgroundTaskCount`. Those die when the agent disconnects. Attached jobs live on the hub until you clear them. + +Upstream: [tiann/hapi#1404](https://github.com/tiann/hapi/issues/1404). + +## When to attach + +Attach a job **before** (or immediately when) you start process-shaped work that will keep running after the agent goes idle: + +| Attach | Do not attach | +|--------|----------------| +| `nohup` / `setsid` / systemd oneshot that runs for hours–days | A tool call that finishes in this turn | +| Beets / rclone / compile / migrate / download batches | Normal coding edits and tests | +| External daemon you own for this session's goal | Claude/Codex Ctrl+B-style background tools | + +If the operator would reopen the chat only to ask "how's it doing?", it belongs here. + +## Agent contract (specification) + +HAPI does **not** write your batch scripts. You (the agent) create the process **and** feed the meter. + +1. **Register** with a stable `job-key` (1–128 chars: alnum / `.` `_` `-`). +2. **Heartbeat** at least every ~10 minutes while running (UI amber after ~15 minutes quiet). +3. **Report progress honestly** — see tiers below. Never invent a bare percent. +4. **Finish cleanly** — `--status completed|failed` or `hapi job clear`. + +Session id: prefer `"$HAPI_SESSION_ID"` (exported into every HAPI-wrapped agent). Prefix match also works. + +```bash +hapi job set "$HAPI_SESSION_ID" beets \ + --label 'beets import' \ + --remaining 150 --done 1637 --total 1787 --unit units \ + --detail 'album: Some Artist - Some Album' + +hapi job update "$HAPI_SESSION_ID" beets --remaining 149 --done 1638 --detail '…' + +hapi job update "$HAPI_SESSION_ID" beets --status completed +# or +hapi job clear "$HAPI_SESSION_ID" beets +``` + +Same auth as `hapi ping-peer` (`HAPI_API_URL` / `CLI_API_TOKEN` or `hapi auth login`). + +## Progress honesty (tiers) + +| What you know | What to send | What the list shows | +|---------------|--------------|---------------------| +| Countable leftover | `--remaining N` (+ optional `--unit`) | `150 units left · 2d 4h` | +| Countable fraction | `--done N --total M` | `91% · 1637/1787 units · 2d 4h` | +| Stage only / unknown size | `--label` + `--detail` + heartbeats | `running · 2d 4h` + indeterminate bar | + +**Elapsed** is always derived from hub `startedAt` (wall clock since register). It is **not** an ETA and there is no time-remaining field - operators get "how long has this been going" plus whatever honest count/detail you report, without a fake completion estimate. + +Rules: + +- Prefer **remaining** when the operator cares about "how much left". +- Prefer **done+total** when both ends of a fraction exist (UI may derive %). +- If you only know a stage name, put it in `--detail` and keep heartbeating — do **not** fake `total=100`. +- There is **no** `--percent` flag and **no** ETA / time-remaining field. Inventing either would train agents to lie. + +## Heartbeat recipe + +Wrap the long process so something calls `hapi job update` on a timer (or on each unit completed). Minimum viable indeterminate job: + +```bash +hapi job set "$HAPI_SESSION_ID" rsync-backup --label 'rsync backup' --detail 'phase: copy' +# in a loop / cron / companion script: +hapi job update "$HAPI_SESSION_ID" rsync-backup --detail "phase: copy · $(date -u +%H:%M)Z" +``` + +When the process exits, mark completed/failed or clear. A stuck green/amber chip with a dead PID is worse than no chip. + +## CLI reference + +```bash +hapi job set --label [options] +hapi job update [options] +hapi job clear +hapi job list +hapi job --help +``` + +Primary running job is enriched onto `GET /api/sessions` as `attachedJob` and pushed on `session-updated` SSE patches. + +## Related + +- [Supported Agents](./agents.md) — flavors and resume +- [How it Works](./how-it-works.md) — CLI ↔ hub ↔ web +- CLI: `hapi job --help`, `cli/README.md` diff --git a/web/src/components/SessionRowSummary.tsx b/web/src/components/SessionRowSummary.tsx index 4abc7fa584..625f7836c4 100644 --- a/web/src/components/SessionRowSummary.tsx +++ b/web/src/components/SessionRowSummary.tsx @@ -1,4 +1,4 @@ -import { useMemo } from 'react' +import { useEffect, useMemo, useState } from 'react' import type { SessionSummary } from '@/types/api' import { AgentFlavorIcon } from '@/components/AgentFlavorIcon' import { ScheduleIcon } from '@/components/icons' @@ -17,7 +17,6 @@ import { formatAttachedJobProgress, isAttachedJobStale } from '@/lib/attachedJob' - function LoaderIcon(props: { className?: string }) { return ( @@ -168,9 +167,17 @@ export function SessionRowSummary(props: { const scheduleId = scheduleTooltipIdProp ?? ownedIds.scheduleId const timeLabel = getSessionTimeLabel(s, t) const attachedJob = s.attachedJob?.status === 'running' ? s.attachedJob : null - const jobStale = attachedJob ? isAttachedJobStale(attachedJob) : false + // Tick once a minute so elapsed wall-time advances without waiting for a heartbeat SSE. + const [nowMs, setNowMs] = useState(() => Date.now()) + useEffect(() => { + if (!attachedJob) return + setNowMs(Date.now()) + const id = window.setInterval(() => setNowMs(Date.now()), 60_000) + return () => window.clearInterval(id) + }, [attachedJob?.key, attachedJob?.startedAt]) + const jobStale = attachedJob ? isAttachedJobStale(attachedJob, nowMs) : false const jobFraction = attachedJob ? attachedJobFraction(attachedJob) : null - const jobProgressLabel = attachedJob ? formatAttachedJobProgress(attachedJob) : null + const jobProgressLabel = attachedJob ? formatAttachedJobProgress(attachedJob, nowMs) : null return (
diff --git a/web/src/lib/attachedJob.test.ts b/web/src/lib/attachedJob.test.ts index fe3946b192..65dd0682ad 100644 --- a/web/src/lib/attachedJob.test.ts +++ b/web/src/lib/attachedJob.test.ts @@ -3,7 +3,9 @@ import type { AttachedJob } from '@hapi/protocol' import { ATTACHED_JOB_STALE_MS, attachedJobFraction, + formatAttachedJobElapsed, formatAttachedJobProgress, + formatCompactElapsed, isAttachedJobStale } from './attachedJob' @@ -20,18 +22,35 @@ function job(overrides: Partial = {}): AttachedJob { } describe('attachedJob helpers', () => { - it('formats remaining count without inventing percent', () => { - expect(formatAttachedJobProgress(job({ remaining: 120, unit: 'tracks' }))).toBe('120 tracks left') + it('formats compact elapsed without inventing ETA', () => { + expect(formatCompactElapsed(0)).toBe('0s') + expect(formatCompactElapsed(45_000)).toBe('45s') + expect(formatCompactElapsed(5 * 60_000)).toBe('5m') + expect(formatCompactElapsed(3 * 60 * 60_000 + 12 * 60_000)).toBe('3h 12m') + expect(formatCompactElapsed(3 * 60 * 60_000)).toBe('3h') + expect(formatCompactElapsed(2 * 24 * 60 * 60_000 + 4 * 60 * 60_000)).toBe('2d 4h') + expect(formatCompactElapsed(2 * 24 * 60 * 60_000)).toBe('2d') + expect(formatCompactElapsed(-1)).toBe('0s') }) - it('formats done/total with derived percent', () => { - expect(formatAttachedJobProgress(job({ done: 800, total: 900, unit: 'tracks' }))).toBe( - '89% · 800/900 tracks' + it('formats remaining count with elapsed', () => { + const now = 1_000 + 2 * 60 * 60_000 + expect(formatAttachedJobProgress(job({ remaining: 120, unit: 'tracks' }), now)).toBe( + '120 tracks left · 2h' ) }) - it('falls back to running when only heartbeat', () => { - expect(formatAttachedJobProgress(job())).toBe('running') + it('formats done/total with derived percent and elapsed', () => { + const now = 1_000 + 45 * 60_000 + expect(formatAttachedJobProgress(job({ done: 800, total: 900, unit: 'tracks' }), now)).toBe( + '89% · 800/900 tracks · 45m' + ) + }) + + it('falls back to running + elapsed when only heartbeat', () => { + const now = 1_000 + 90_000 + expect(formatAttachedJobProgress(job(), now)).toBe('running · 1m') + expect(formatAttachedJobElapsed(job(), now)).toBe('1m') }) it('computes fraction from remaining+total', () => { diff --git a/web/src/lib/attachedJob.ts b/web/src/lib/attachedJob.ts index 58c869c1c9..7fc2172080 100644 --- a/web/src/lib/attachedJob.ts +++ b/web/src/lib/attachedJob.ts @@ -3,16 +3,46 @@ import type { AttachedJob } from '@hapi/protocol' /** Stale if no heartbeat for 15 minutes — UI amber, still shows progress. */ export const ATTACHED_JOB_STALE_MS = 15 * 60 * 1000 -export function formatAttachedJobProgress(job: AttachedJob): string { +/** + * Compact lettered elapsed duration for list chrome. + * Max two units; never an ETA / time-remaining estimate. + */ +export function formatCompactElapsed(elapsedMs: number): string { + if (!Number.isFinite(elapsedMs) || elapsedMs < 0) return '0s' + const totalSec = Math.floor(elapsedMs / 1000) + if (totalSec < 60) return `${totalSec}s` + const totalMin = Math.floor(totalSec / 60) + if (totalMin < 60) return `${totalMin}m` + const totalHr = Math.floor(totalMin / 60) + const remMin = totalMin % 60 + if (totalHr < 24) { + return remMin > 0 ? `${totalHr}h ${remMin}m` : `${totalHr}h` + } + const days = Math.floor(totalHr / 24) + const remHr = totalHr % 24 + return remHr > 0 ? `${days}d ${remHr}h` : `${days}d` +} + +/** Elapsed since job.startedAt (hub clock). */ +export function formatAttachedJobElapsed(job: AttachedJob, now: number = Date.now()): string { + return formatCompactElapsed(now - job.startedAt) +} + +/** + * Progress label for the session row. + * Always appends elapsed from startedAt — honest wall time, not an ETA. + */ +export function formatAttachedJobProgress(job: AttachedJob, now: number = Date.now()): string { + const elapsed = formatAttachedJobElapsed(job, now) if (job.remaining !== undefined) { const unit = job.unit ? ` ${job.unit}` : '' - return `${job.remaining}${unit} left` + return `${job.remaining}${unit} left · ${elapsed}` } if (job.done !== undefined && job.total !== undefined && job.total > 0) { const pct = Math.min(100, Math.round((job.done / job.total) * 100)) - return `${pct}% · ${job.done}/${job.total}${job.unit ? ` ${job.unit}` : ''}` + return `${pct}% · ${job.done}/${job.total}${job.unit ? ` ${job.unit}` : ''} · ${elapsed}` } - return 'running' + return `running · ${elapsed}` } export function attachedJobFraction(job: AttachedJob): number | null { From 3a56d0b1760466213f0e51f56b98ce5b86e8e324 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:26:49 +0000 Subject: [PATCH 06/94] fix(jobs): address Opus cold-review Majors for #1404 Add hapi job run supervisor (auto-heartbeat + exit status), follow post-merge job-owner redirects so $HAPI_SESSION_ID heartbeats keep working, cover the CLI parser/resolve/run paths with tests, and steer agents toward the supervisor instead of an idle-agent heartbeat myth. Co-authored-by: Cursor --- cli/src/commands/job.test.ts | 84 ++++++++++ cli/src/commands/job.ts | 99 +++++++++--- .../common/sessionJobInstruction.test.ts | 6 +- .../modules/common/sessionJobInstruction.ts | 29 ++-- .../modules/sessionJob/runSessionJob.test.ts | 129 ++++++++++++++++ cli/src/modules/sessionJob/runSessionJob.ts | 144 ++++++++++++++++++ cli/src/modules/sessionJob/sessionJob.ts | 2 +- docs/guide/session-jobs.md | 17 ++- hub/src/sync/sessionCache.ts | 116 ++++++++++++++ hub/src/sync/syncEngine.ts | 5 + hub/src/web/routes/sessions-jobs.test.ts | 1 + hub/src/web/routes/sessions.ts | 38 ++++- 12 files changed, 617 insertions(+), 53 deletions(-) create mode 100644 cli/src/commands/job.test.ts create mode 100644 cli/src/modules/sessionJob/runSessionJob.test.ts create mode 100644 cli/src/modules/sessionJob/runSessionJob.ts diff --git a/cli/src/commands/job.test.ts b/cli/src/commands/job.test.ts new file mode 100644 index 0000000000..29d18ea5c2 --- /dev/null +++ b/cli/src/commands/job.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' +import { parseJobArgs } from '@/commands/job' +import { + SessionJobError, + exitCodeForSessionJobError, + resolveSessionByPrefix +} from '@/modules/sessionJob/sessionJob' + +describe('parseJobArgs', () => { + it('parses set with long flags', () => { + const parsed = parseJobArgs([ + 'set', + 'abcd1234', + 'beets', + '--label', + 'beets import', + '--remaining', + '12', + '--unit=tracks' + ]) + expect(parsed.action).toBe('set') + expect(parsed.sessionIdPrefix).toBe('abcd1234') + expect(parsed.jobKey).toBe('beets') + expect(parsed.label).toBe('beets import') + expect(parsed.remaining).toBe(12) + expect(parsed.unit).toBe('tracks') + }) + + it('parses run with command after --', () => { + const parsed = parseJobArgs([ + 'run', + 'sid', + 'drain', + '--label=rsync', + '--heartbeat-sec=60', + '--', + 'bash', + '-c', + 'echo hi' + ]) + expect(parsed.action).toBe('run') + expect(parsed.label).toBe('rsync') + expect(parsed.heartbeatSec).toBe(60) + expect(parsed.command).toEqual(['bash', '-c', 'echo hi']) + }) + + it('rejects bad status', () => { + expect(() => parseJobArgs(['update', 's', 'k', '--status', 'nope'])).toThrow(SessionJobError) + }) +}) + +describe('resolveSessionByPrefix', () => { + const sessions = [ + { id: 'aaaaaaaa-1111-1111-1111-111111111111' }, + { id: 'bbbbbbbb-2222-2222-2222-222222222222' }, + { id: 'bbbbcccc-3333-3333-3333-333333333333' } + ] + + it('matches exact id', () => { + expect(resolveSessionByPrefix(sessions, sessions[0]!.id).id).toBe(sessions[0]!.id) + }) + + it('matches unique prefix', () => { + expect(resolveSessionByPrefix(sessions, 'aaaa').id).toBe(sessions[0]!.id) + }) + + it('errors on ambiguous prefix', () => { + expect(() => resolveSessionByPrefix(sessions, 'bbbb')).toThrow(/matches 2 sessions/) + }) + + it('errors on no match', () => { + expect(() => resolveSessionByPrefix(sessions, 'zzzz')).toThrow(/no session matching/) + }) +}) + +describe('exitCodeForSessionJobError', () => { + it('maps codes', () => { + expect(exitCodeForSessionJobError(new SessionJobError('bad_args', 'x'))).toBe(2) + expect(exitCodeForSessionJobError(new SessionJobError('auth_failed', 'x'))).toBe(3) + expect(exitCodeForSessionJobError(new SessionJobError('not_found', 'x'))).toBe(4) + expect(exitCodeForSessionJobError(new SessionJobError('ambiguous', 'x'))).toBe(5) + expect(exitCodeForSessionJobError(new SessionJobError('request_failed', 'x'))).toBe(1) + }) +}) diff --git a/cli/src/commands/job.ts b/cli/src/commands/job.ts index 1579621130..f9be88ce6b 100644 --- a/cli/src/commands/job.ts +++ b/cli/src/commands/job.ts @@ -9,11 +9,12 @@ import { setSessionJob, updateSessionJob } from '@/modules/sessionJob/sessionJob' +import { runSessionJob } from '@/modules/sessionJob/runSessionJob' import type { CommandDefinition } from './types' -type ParsedJobArgs = { +export type ParsedJobArgs = { help: boolean - action?: 'set' | 'update' | 'clear' | 'list' + action?: 'set' | 'update' | 'clear' | 'list' | 'run' sessionIdPrefix?: string jobKey?: string label?: string @@ -23,6 +24,8 @@ type ParsedJobArgs = { remaining?: number unit?: string detail?: string + heartbeatSec?: number + command?: string[] } function showHelp(): void { @@ -30,21 +33,21 @@ function showHelp(): void { ${chalk.bold('hapi job')} - Attach long-running work to a HAPI session (tiann/hapi#1404) ${chalk.bold('When to use:')} - Work that outlives the agent (nohup / batch / long scripts / external daemons) + Work that outlives the agent (batch / long scripts / external daemons) while the session may be idle. Not thinking progress or in-agent background tools. ${chalk.bold('Agent contract:')} - 1. set before (or as) the process starts - 2. update / heartbeat at least every ~10 minutes while running - 3. prefer honest --remaining or --done/--total; omit counts if unknown - 4. never invent a fake percent - 5. clear or --status completed|failed when finished + Prefer ${chalk.bold('hapi job run')} — it heartbeats for you and marks completed/failed on exit. + An idle agent cannot heartbeat; set-once jobs go amber after ~15m. + Prefer honest --remaining or --done/--total; omit counts if unknown. + Never invent a fake percent. ${chalk.bold('Usage:')} - hapi job set --label [--remaining N] [--done N --total N] [--unit tracks] [--detail ...] - hapi job update [--remaining N] [--done N] [--total N] [--status running|completed|failed] [--detail ...] - hapi job clear - hapi job list + hapi job run --label [--heartbeat-sec 300] [progress flags] -- [args...] + hapi job set --label [--remaining N] [--done N --total N] [--unit tracks] [--detail ...] + hapi job update [--remaining N] [--done N] [--total N] [--status running|completed|failed] [--detail ...] + hapi job clear + hapi job list ${chalk.bold('Progress UI:')} remaining → "N units left · 2h" @@ -55,6 +58,7 @@ ${chalk.bold('Progress UI:')} ${chalk.bold('Notes:')} Hub-persisted. Prefer "$HAPI_SESSION_ID" for this chat. Job key: 1-128 chars, alnum / . _ - + Session lookup prefers exact id; prefix scan is the 500 most-recently-updated sessions. Docs: docs/guide/session-jobs.md ${chalk.bold('Env:')} @@ -75,15 +79,20 @@ function parseOptionalNumber(flag: string, value: string | undefined): number { export function parseJobArgs(args: string[]): ParsedJobArgs { const result: ParsedJobArgs = { help: false } + const dashDash = args.indexOf('--') + const flagArgs = dashDash >= 0 ? args.slice(0, dashDash) : args + if (dashDash >= 0) { + result.command = args.slice(dashDash + 1) + } - for (let i = 0; i < args.length; i++) { - const arg = args[i]! + for (let i = 0; i < flagArgs.length; i++) { + const arg = flagArgs[i]! if (arg === '--help' || arg === '-h') { result.help = true continue } if (arg === '--label') { - result.label = args[++i] + result.label = flagArgs[++i] if (!result.label) throw new SessionJobError('bad_args', '--label requires a value') continue } @@ -92,7 +101,7 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { continue } if (arg === '--status') { - const value = args[++i] + const value = flagArgs[++i] if (value !== 'running' && value !== 'completed' && value !== 'failed') { throw new SessionJobError('bad_args', '--status must be running|completed|failed') } @@ -108,7 +117,7 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { continue } if (arg === '--done') { - result.done = parseOptionalNumber('--done', args[++i]) + result.done = parseOptionalNumber('--done', flagArgs[++i]) continue } if (arg.startsWith('--done=')) { @@ -116,7 +125,7 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { continue } if (arg === '--total') { - result.total = parseOptionalNumber('--total', args[++i]) + result.total = parseOptionalNumber('--total', flagArgs[++i]) continue } if (arg.startsWith('--total=')) { @@ -124,7 +133,7 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { continue } if (arg === '--remaining') { - result.remaining = parseOptionalNumber('--remaining', args[++i]) + result.remaining = parseOptionalNumber('--remaining', flagArgs[++i]) continue } if (arg.startsWith('--remaining=')) { @@ -132,7 +141,7 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { continue } if (arg === '--unit') { - result.unit = args[++i] + result.unit = flagArgs[++i] if (!result.unit) throw new SessionJobError('bad_args', '--unit requires a value') continue } @@ -141,7 +150,7 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { continue } if (arg === '--detail') { - result.detail = args[++i] + result.detail = flagArgs[++i] if (result.detail === undefined) throw new SessionJobError('bad_args', '--detail requires a value') continue } @@ -149,12 +158,26 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { result.detail = arg.slice('--detail='.length) continue } + if (arg === '--heartbeat-sec') { + result.heartbeatSec = parseOptionalNumber('--heartbeat-sec', flagArgs[++i]) + continue + } + if (arg.startsWith('--heartbeat-sec=')) { + result.heartbeatSec = parseOptionalNumber('--heartbeat-sec', arg.slice('--heartbeat-sec='.length)) + continue + } if (arg.startsWith('-')) { throw new SessionJobError('bad_args', `unexpected flag: ${arg}`) } if (!result.action) { - if (arg !== 'set' && arg !== 'update' && arg !== 'clear' && arg !== 'list') { - throw new SessionJobError('bad_args', `unknown action '${arg}' (set|update|clear|list)`) + if ( + arg !== 'set' + && arg !== 'update' + && arg !== 'clear' + && arg !== 'list' + && arg !== 'run' + ) { + throw new SessionJobError('bad_args', `unknown action '${arg}' (set|update|clear|list|run)`) } result.action = arg continue @@ -216,7 +239,7 @@ export async function handleJobCommand(args: string[]): Promise { if (parsed.help || !parsed.action) { showHelp() if (!parsed.action && !parsed.help) { - throw new SessionJobError('bad_args', 'missing action; usage: hapi job set|update|clear|list ...') + throw new SessionJobError('bad_args', 'missing action; usage: hapi job set|update|clear|list|run ...') } return } @@ -277,6 +300,34 @@ export async function handleJobCommand(args: string[]): Promise { return } + if (parsed.action === 'run') { + if (!parsed.label) { + throw new SessionJobError('bad_args', 'run requires --label') + } + if (!parsed.command || parsed.command.length === 0) { + throw new SessionJobError('bad_args', 'run requires a command after --') + } + const exitCode = await runSessionJob({ + sessionIdPrefix: parsed.sessionIdPrefix, + jobKey: parsed.jobKey, + label: parsed.label, + command: parsed.command, + ...(parsed.heartbeatSec !== undefined + ? { heartbeatMs: Math.max(5, parsed.heartbeatSec) * 1000 } + : {}), + ...(parsed.done !== undefined ? { done: parsed.done } : {}), + ...(parsed.total !== undefined ? { total: parsed.total } : {}), + ...(parsed.remaining !== undefined ? { remaining: parsed.remaining } : {}), + ...(parsed.unit !== undefined ? { unit: parsed.unit } : {}), + ...(parsed.detail !== undefined ? { detail: parsed.detail } : {}) + }) + if (exitCode !== 0) { + process.exitCode = exitCode + } + console.log(`run finished exit=${exitCode} job=${parsed.jobKey}`) + return + } + // update const body: AttachedJobPatch = { ...(parsed.label !== undefined ? { label: parsed.label } : {}), diff --git a/cli/src/modules/common/sessionJobInstruction.test.ts b/cli/src/modules/common/sessionJobInstruction.test.ts index 2cdf00bedb..c6d919a329 100644 --- a/cli/src/modules/common/sessionJobInstruction.test.ts +++ b/cli/src/modules/common/sessionJobInstruction.test.ts @@ -5,10 +5,10 @@ import { } from './sessionJobInstruction' describe('sessionJobInstruction', () => { - it('mentions set, update, heartbeat, and no fake percent', () => { - expect(SESSION_JOB_INSTRUCTION).toContain('hapi job set') + it('prefers job run supervisor and forbids fake percent', () => { + expect(SESSION_JOB_INSTRUCTION).toContain('hapi job run') expect(SESSION_JOB_INSTRUCTION).toContain('hapi job update') - expect(SESSION_JOB_INSTRUCTION).toContain('~10 minutes') + expect(SESSION_JOB_INSTRUCTION).toContain('idle agent cannot') expect(SESSION_JOB_INSTRUCTION).toContain('Never invent a fake percent') expect(SESSION_JOB_INSTRUCTION).toContain('HAPI_SESSION_ID') }) diff --git a/cli/src/modules/common/sessionJobInstruction.ts b/cli/src/modules/common/sessionJobInstruction.ts index c6ba00c3d2..ee3043bf76 100644 --- a/cli/src/modules/common/sessionJobInstruction.ts +++ b/cli/src/modules/common/sessionJobInstruction.ts @@ -1,27 +1,26 @@ /** * Always-on steer for session-attached long-running jobs (tiann/hapi#1404). * - * Unlike the session-summary contract (opt-in), this is short and triggers only - * when the agent spawns outliving work — so it rides every supported flavor's - * system / developer instructions by default. - * - * Cursor ACP has no system-prompt seam today; Cursor agents rely on the estate - * skill `hapi-session-jobs` (and `hapi job --help`) instead. + * Injected into flavors that have a HAPI system / developer-instructions seam + * today: Claude, Codex, OpenCode, Grok. Cursor ACP has no such seam (estate + * skill `hapi-session-jobs` + `hapi job --help` instead). Other ACP flavors + * (Kimi, Copilot, Pi, …) do not receive this block until an MCP job tool or + * per-flavor seam lands — do not claim "every flavor." */ /** Canonical one-block contract. Keep short — every session's prompt budget. */ export const SESSION_JOB_INSTRUCTION = [ 'Session-attached jobs (outliving work):', 'When you start work that will keep running after this agent goes idle', - '(nohup, batch imports, long scripts, external daemons), attach it to this', - 'HAPI session so the session list can show progress while you are idle.', - 'Use: hapi job set "$HAPI_SESSION_ID" --label ', - '[--remaining N] [--done N --total N] [--unit ] [--detail ].', - 'Heartbeat with hapi job update at least every ~10 minutes (UI goes amber', - 'after ~15m without a heartbeat). Prefer honest remaining or done+total;', - 'omit counts when unknown (UI shows "running" + indeterminate bar).', - 'Never invent a fake percent. On finish: hapi job update … --status', - 'completed|failed, or hapi job clear. Full contract: hapi job --help.' + '(batch imports, long scripts, external daemons), attach it so the session', + 'list can show progress while you are idle.', + 'Prefer: hapi job run "$HAPI_SESSION_ID" --label -- …', + '(auto-heartbeats + marks completed/failed on exit).', + 'Manual path: hapi job set … then a wrapper must heartbeat via', + 'hapi job update at least every ~10 minutes — an idle agent cannot.', + 'Prefer honest remaining or done+total; omit counts when unknown', + '(UI shows "running" + elapsed). Never invent a fake percent.', + 'Full contract: hapi job --help.' ].join(' ') /** Append instruction to an existing prompt block (blank line separator). */ diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts new file mode 100644 index 0000000000..66e444a09d --- /dev/null +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -0,0 +1,129 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it, vi } from 'vitest' +import { runSessionJob } from './runSessionJob' + +function fakeChild(exitCode: number) { + const child = new EventEmitter() as EventEmitter & { + pid: number + killed: boolean + } + child.pid = 4242 + child.killed = false + queueMicrotask(() => child.emit('exit', exitCode, null)) + return child +} + +describe('runSessionJob', () => { + it('sets running, heartbeats, then marks completed on exit 0', async () => { + const http = { + post: vi.fn(async () => ({ status: 200, data: { token: 'jwt' } })), + get: vi.fn(async () => ({ + status: 200, + data: { sessions: [{ id: 'aaaaaaaa-1111-1111-1111-111111111111' }] } + })), + put: vi.fn(async () => ({ + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: 'running', + heartbeatAt: 1, + startedAt: 1, + updatedAt: 1 + } + } + })), + patch: vi.fn(async (_url: string, body: { status?: string }) => ({ + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: body.status ?? 'running', + heartbeatAt: 2, + startedAt: 1, + updatedAt: 2 + } + } + })) + } + + const timers: Array<() => void> = [] + const exitCode = await runSessionJob({ + sessionIdPrefix: 'aaaa', + jobKey: 'drain', + label: 'drain', + command: ['true'], + heartbeatMs: 10, + accessToken: 'token', + apiUrl: 'http://127.0.0.1:3006', + http: http as never, + spawnImpl: (() => fakeChild(0)) as never, + setIntervalImpl: ((fn: () => void) => { + timers.push(fn) + return 1 as unknown as NodeJS.Timeout + }) as never, + clearIntervalImpl: (() => undefined) as never + }) + + expect(exitCode).toBe(0) + expect(http.put).toHaveBeenCalled() + expect(http.patch).toHaveBeenCalled() + const lastPatch = http.patch.mock.calls.at(-1)?.[1] as { status?: string } + expect(lastPatch.status).toBe('completed') + }) + + it('marks failed on non-zero exit', async () => { + const http = { + post: vi.fn(async () => ({ status: 200, data: { token: 'jwt' } })), + get: vi.fn(async () => ({ + status: 200, + data: { sessions: [{ id: 'aaaaaaaa-1111-1111-1111-111111111111' }] } + })), + put: vi.fn(async () => ({ + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: 'running', + heartbeatAt: 1, + startedAt: 1, + updatedAt: 1 + } + } + })), + patch: vi.fn(async (_url: string, body: { status?: string }) => ({ + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: body.status ?? 'running', + heartbeatAt: 2, + startedAt: 1, + updatedAt: 2 + } + } + })) + } + + const exitCode = await runSessionJob({ + sessionIdPrefix: 'aaaa', + jobKey: 'drain', + label: 'drain', + command: ['false'], + accessToken: 'token', + apiUrl: 'http://127.0.0.1:3006', + http: http as never, + spawnImpl: (() => fakeChild(7)) as never, + setIntervalImpl: ((() => 1) as never), + clearIntervalImpl: (() => undefined) as never + }) + + expect(exitCode).toBe(7) + const lastPatch = http.patch.mock.calls.at(-1)?.[1] as { status?: string } + expect(lastPatch.status).toBe('failed') + }) +}) diff --git a/cli/src/modules/sessionJob/runSessionJob.ts b/cli/src/modules/sessionJob/runSessionJob.ts new file mode 100644 index 0000000000..f654b1fcd8 --- /dev/null +++ b/cli/src/modules/sessionJob/runSessionJob.ts @@ -0,0 +1,144 @@ +/** + * Supervise a child command while heartbeating a session-attached job. + * Fixes the idle-agent heartbeat gap (cold review #1404). + */ + +import { spawn, type ChildProcess } from 'node:child_process' +import type { AttachedJobUpsert } from '@hapi/protocol' +import { + SessionJobError, + setSessionJob, + updateSessionJob, + type SessionJobClientOptions +} from './sessionJob' + +export type RunSessionJobOptions = SessionJobClientOptions & { + jobKey: string + label: string + command: string[] + heartbeatMs?: number + remaining?: number + done?: number + total?: number + unit?: string + detail?: string + /** Injected for tests. */ + spawnImpl?: typeof spawn + setIntervalImpl?: typeof setInterval + clearIntervalImpl?: typeof clearInterval +} + +const DEFAULT_HEARTBEAT_MS = 5 * 60 * 1000 + +export async function runSessionJob(options: RunSessionJobOptions): Promise { + if (options.command.length === 0) { + throw new SessionJobError('bad_args', 'run requires a command after --') + } + + const body: AttachedJobUpsert = { + label: options.label, + status: 'running', + ...(options.done !== undefined ? { done: options.done } : {}), + ...(options.total !== undefined ? { total: options.total } : {}), + ...(options.remaining !== undefined ? { remaining: options.remaining } : {}), + ...(options.unit !== undefined ? { unit: options.unit } : {}), + ...(options.detail !== undefined ? { detail: options.detail } : {}) + } + + await setSessionJob({ + sessionIdPrefix: options.sessionIdPrefix, + jobKey: options.jobKey, + body, + apiUrl: options.apiUrl, + accessToken: options.accessToken, + http: options.http + }) + + const spawnFn = options.spawnImpl ?? spawn + const setIntervalFn = options.setIntervalImpl ?? setInterval + const clearIntervalFn = options.clearIntervalImpl ?? clearInterval + const heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS + + const child: ChildProcess = spawnFn(options.command[0]!, options.command.slice(1), { + stdio: 'inherit', + env: process.env + }) + + const heartbeat = setIntervalFn(() => { + void updateSessionJob({ + sessionIdPrefix: options.sessionIdPrefix, + jobKey: options.jobKey, + body: { + detail: options.detail, + status: 'running' + }, + apiUrl: options.apiUrl, + accessToken: options.accessToken, + http: options.http + }).catch(() => { + // Best-effort — exit path still marks terminal status. + }) + }, heartbeatMs) + // Don't keep the event loop alive solely for heartbeats if child already exited. + heartbeat.unref?.() + + const forward = (signal: NodeJS.Signals) => { + if (child.pid && !child.killed) { + try { + process.kill(child.pid, signal) + } catch { + // Child may have already exited. + } + } + } + const onSigInt = () => forward('SIGINT') + const onSigTerm = () => forward('SIGTERM') + process.on('SIGINT', onSigInt) + process.on('SIGTERM', onSigTerm) + + const exitCode = await new Promise((resolve) => { + child.on('error', async (error) => { + clearIntervalFn(heartbeat) + try { + await updateSessionJob({ + sessionIdPrefix: options.sessionIdPrefix, + jobKey: options.jobKey, + body: { status: 'failed', detail: error.message }, + apiUrl: options.apiUrl, + accessToken: options.accessToken, + http: options.http + }) + } catch { + // ignore + } + resolve(127) + }) + child.on('exit', (code, signal) => { + clearIntervalFn(heartbeat) + if (signal) { + resolve(128 + (signal === 'SIGINT' ? 2 : signal === 'SIGTERM' ? 15 : 1)) + return + } + resolve(code ?? 1) + }) + }) + + process.off('SIGINT', onSigInt) + process.off('SIGTERM', onSigTerm) + + const terminalStatus = exitCode === 0 ? 'completed' : 'failed' + try { + await updateSessionJob({ + sessionIdPrefix: options.sessionIdPrefix, + jobKey: options.jobKey, + body: { status: terminalStatus }, + apiUrl: options.apiUrl, + accessToken: options.accessToken, + http: options.http + }) + } catch { + // Job may already be cleared; still return child exit code. + } + + return exitCode +} diff --git a/cli/src/modules/sessionJob/sessionJob.ts b/cli/src/modules/sessionJob/sessionJob.ts index 554f403243..2f404fce9f 100644 --- a/cli/src/modules/sessionJob/sessionJob.ts +++ b/cli/src/modules/sessionJob/sessionJob.ts @@ -98,7 +98,7 @@ function authHeaders(jwt: string): Record { type SessionListItem = { id: string } -function resolveSessionByPrefix(sessions: SessionListItem[], prefix: string): SessionListItem { +export function resolveSessionByPrefix(sessions: SessionListItem[], prefix: string): SessionListItem { const trimmed = prefix.trim() if (!trimmed) { throw new SessionJobError('bad_args', 'session id prefix is required') diff --git a/docs/guide/session-jobs.md b/docs/guide/session-jobs.md index 339d9be7d4..18b78e23c2 100644 --- a/docs/guide/session-jobs.md +++ b/docs/guide/session-jobs.md @@ -20,14 +20,19 @@ If the operator would reopen the chat only to ask "how's it doing?", it belongs ## Agent contract (specification) -HAPI does **not** write your batch scripts. You (the agent) create the process **and** feed the meter. +HAPI does **not** write your batch scripts for you - but prefer the supervisor so heartbeats are not your problem: -1. **Register** with a stable `job-key` (1–128 chars: alnum / `.` `_` `-`). -2. **Heartbeat** at least every ~10 minutes while running (UI amber after ~15 minutes quiet). -3. **Report progress honestly** — see tiers below. Never invent a bare percent. -4. **Finish cleanly** — `--status completed|failed` or `hapi job clear`. +```bash +hapi job run "$HAPI_SESSION_ID" beets \ + --label 'beets import' \ + --remaining 150 --done 1637 --total 1787 --unit units \ + --detail 'album: …' \ + -- ./beets-import.sh +``` + +`hapi job run` registers the job, heartbeats on a timer while the child runs, then marks `completed`/`failed` from the exit code. An idle agent **cannot** heartbeat - set-once + manual update decays to amber. -Session id: prefer `"$HAPI_SESSION_ID"` (exported into every HAPI-wrapped agent). Prefix match also works. +Manual path (only if you already have a self-heartbeating wrapper): ```bash hapi job set "$HAPI_SESSION_ID" beets \ diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 98709df8c0..14bfae8d9b 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -1316,6 +1316,12 @@ export class SessionCache { const movedScratchlist = this.store.scratchlist.transfer(oldSessionId, newSessionId) const movedJobs = this.store.sessionJobs.transfer(oldSessionId, newSessionId) if (movedJobs.moved > 0 || movedJobs.collided > 0) { + // Agents keep addressing $HAPI_SESSION_ID from the pre-merge row. + // Record redirects so job REST routes can follow the live job owner. + this.recordJobsAcceptedFromSession(newSessionId, oldSessionId, namespace) + if (!options.deleteOldSession) { + this.recordJobsTransferredToSession(oldSessionId, newSessionId, namespace) + } this.emitAttachedJobChanged( newSessionId, this.store.sessionJobs.getPrimaryRunning(newSessionId) @@ -1525,6 +1531,116 @@ export class SessionCache { } } + /** + * Target session remembers it absorbed jobs from `fromSessionId` so job + * REST routes can follow `$HAPI_SESSION_ID` after the source row is deleted. + */ + private recordJobsAcceptedFromSession( + toSessionId: string, + fromSessionId: string, + namespace: string + ): void { + for (let attempt = 0; attempt < 2; attempt += 1) { + const latest = this.store.sessions.getSessionByNamespace(toSessionId, namespace) + if (!latest) return + const meta = (latest.metadata && typeof latest.metadata === 'object' + ? { ...(latest.metadata as Record) } + : {}) as Record + const prev = Array.isArray(meta.jobsAcceptedFromSessionIds) + ? meta.jobsAcceptedFromSessionIds.filter((id): id is string => typeof id === 'string') + : [] + if (prev.includes(fromSessionId)) return + meta.jobsAcceptedFromSessionIds = [...prev, fromSessionId] + const result = this.store.sessions.updateSessionMetadata( + toSessionId, + meta, + latest.metadataVersion, + namespace, + { touchUpdatedAt: false } + ) + if (result.result === 'success') { + this.refreshSession(toSessionId) + return + } + if (result.result !== 'version-mismatch') return + } + } + + /** Source session (kept alive) points job APIs at the post-merge owner. */ + private recordJobsTransferredToSession( + fromSessionId: string, + toSessionId: string, + namespace: string + ): void { + for (let attempt = 0; attempt < 2; attempt += 1) { + const latest = this.store.sessions.getSessionByNamespace(fromSessionId, namespace) + if (!latest) return + const meta = (latest.metadata && typeof latest.metadata === 'object' + ? { ...(latest.metadata as Record) } + : {}) as Record + if (meta.jobsTransferredToSessionId === toSessionId) return + meta.jobsTransferredToSessionId = toSessionId + const result = this.store.sessions.updateSessionMetadata( + fromSessionId, + meta, + latest.metadataVersion, + namespace, + { touchUpdatedAt: false } + ) + if (result.result === 'success') { + this.refreshSession(fromSessionId) + return + } + if (result.result !== 'version-mismatch') return + } + } + + /** + * Follow job-owner redirects after session merge/dedup so agents that still + * hold the pre-merge `$HAPI_SESSION_ID` can heartbeat. + */ + resolveAttachedJobSessionId(sessionId: string, namespace: string): string { + let current = sessionId + for (let hop = 0; hop < 5; hop += 1) { + const access = this.resolveSessionAccess(current, namespace) + if (access.ok) { + const meta = access.session.metadata as Record | null | undefined + const next = + (typeof meta?.jobsTransferredToSessionId === 'string' + && meta.jobsTransferredToSessionId.trim()) + || (typeof meta?.supersededBySessionId === 'string' + && meta.supersededBySessionId.trim()) + || '' + if (next && next !== current) { + current = next + continue + } + return current + } + // Source row may already be deleted — find who accepted its jobs. + const acceptor = this.findSessionThatAcceptedJobsFrom(current, namespace) + if (acceptor && acceptor !== current) { + current = acceptor + continue + } + return current + } + return current + } + + private findSessionThatAcceptedJobsFrom(fromSessionId: string, namespace: string): string | null { + for (const session of this.getSessions()) { + if (session.namespace !== namespace) continue + const meta = session.metadata as Record | null | undefined + const accepted = meta?.jobsAcceptedFromSessionIds + if (!Array.isArray(accepted)) continue + if (accepted.some((id) => id === fromSessionId)) { + return session.id + } + } + return null + } + private mergeSessionMetadata(oldMetadata: unknown | null, newMetadata: unknown | null): unknown | null { if (!oldMetadata || typeof oldMetadata !== 'object') { return newMetadata diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 80178c1c1c..871ac38cd1 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -375,6 +375,11 @@ export class SyncEngine { return this.sessionCache.resolveSessionAccess(sessionId, namespace) } + /** Follow job-owner redirects after merge/dedup (tiann/hapi#1404 cold review). */ + resolveAttachedJobSessionId(sessionId: string, namespace: string): string { + return this.sessionCache.resolveAttachedJobSessionId(sessionId, namespace) + } + getActiveSessions(): Session[] { return this.sessionCache.getActiveSessions() } diff --git a/hub/src/web/routes/sessions-jobs.test.ts b/hub/src/web/routes/sessions-jobs.test.ts index a10c09924c..87a0161b5d 100644 --- a/hub/src/web/routes/sessions-jobs.test.ts +++ b/hub/src/web/routes/sessions-jobs.test.ts @@ -35,6 +35,7 @@ describe('session-attached jobs routes (tiann/hapi#1404)', () => { const engine = { resolveSessionAccess: () => ({ ok: true as const, sessionId: session.id, session }), + resolveAttachedJobSessionId: (id: string) => id, getSessionsByNamespace: () => [session], getFutureScheduledMessageCounts: () => new Map(), getNextScheduledAtBySessionIds: () => new Map(), diff --git a/hub/src/web/routes/sessions.ts b/hub/src/web/routes/sessions.ts index 2df981fd2c..74ff6508fe 100644 --- a/hub/src/web/routes/sessions.ts +++ b/hub/src/web/routes/sessions.ts @@ -1303,12 +1303,42 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho // tiann/hapi#1404 — session-attached long-running jobs (works while agent idle). const JOB_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/ + function resolveJobOwnerSession( + c: Context, + engine: SyncEngine + ): { sessionId: string; session: Session } | Response { + const sessionResult = requireSessionFromParam(c, engine) + if (sessionResult instanceof Response) { + // Session may already be deleted after merge — still try acceptor redirect. + const rawId = c.req.param('id') ?? '' + const namespace = c.get('namespace') + const redirected = engine.resolveAttachedJobSessionId(rawId, namespace) + if (redirected !== rawId) { + const access = engine.resolveSessionAccess(redirected, namespace) + if (access.ok) { + return { sessionId: access.sessionId, session: access.session } + } + } + return sessionResult + } + const namespace = c.get('namespace') + const ownerId = engine.resolveAttachedJobSessionId(sessionResult.sessionId, namespace) + if (ownerId === sessionResult.sessionId) { + return sessionResult + } + const access = engine.resolveSessionAccess(ownerId, namespace) + if (!access.ok) { + return sessionResult + } + return { sessionId: access.sessionId, session: access.session } + } + app.get('/sessions/:id/jobs', (c) => { const engine = requireSyncEngine(c, getSyncEngine) if (engine instanceof Response) { return engine } - const sessionResult = requireSessionFromParam(c, engine) + const sessionResult = resolveJobOwnerSession(c, engine) if (sessionResult instanceof Response) { return sessionResult } @@ -1323,7 +1353,7 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho if (engine instanceof Response) { return engine } - const sessionResult = requireSessionFromParam(c, engine) + const sessionResult = resolveJobOwnerSession(c, engine) if (sessionResult instanceof Response) { return sessionResult } @@ -1348,7 +1378,7 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho if (engine instanceof Response) { return engine } - const sessionResult = requireSessionFromParam(c, engine) + const sessionResult = resolveJobOwnerSession(c, engine) if (sessionResult instanceof Response) { return sessionResult } @@ -1373,7 +1403,7 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho if (engine instanceof Response) { return engine } - const sessionResult = requireSessionFromParam(c, engine) + const sessionResult = resolveJobOwnerSession(c, engine) if (sessionResult instanceof Response) { return sessionResult } From 3a61876cc2f3ba4b21db4dd895948b767d4f61a5 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 16:33:13 +0000 Subject: [PATCH 07/94] fix(jobs): honor explicit startedAt on PUT; document late-attach clock Beets dogfood: PATCH rejects startedAt; sticky PUT clock made elapsed lie after late attach. Explicit upsert startedAt now corrects; CLI --started-at; clear+set recipe kept for older hubs. Co-authored-by: Cursor --- cli/README.md | 2 +- cli/src/commands/job.test.ts | 11 +++++ cli/src/commands/job.ts | 24 ++++++++++- docs/guide/faq.md | 2 +- docs/guide/session-jobs.md | 67 +++++++++++++++++++++++++++-- hub/src/store/migration-v25.test.ts | 40 +++++++++++++++++ hub/src/store/sessionJobs.ts | 8 +++- hub/src/store/sessionJobsStore.ts | 18 ++++++-- shared/src/schemas.ts | 1 + 9 files changed, 159 insertions(+), 14 deletions(-) diff --git a/cli/README.md b/cli/README.md index 606a90c2ff..1d20bf0df9 100644 --- a/cli/README.md +++ b/cli/README.md @@ -44,7 +44,7 @@ Choose a supported coding agent from your terminal and control its sessions remo - `hapi resume [sessionId]` - List resumable sessions for this machine or resume one locally. - `hapi ping-peer ` - Resume (if needed) and message another session. Prefer this or MCP `ping_peer` / `list_peers` over reinventing JWT+curl. Also `--message-file` / `--list`. - `hapi inspect-peer ` - Read-only peer metadata + recent message text (no resume). Prefer this or MCP `inspect_peer` when a user cites `[title](/sessions/)` or Copy-reference `See session "…" (/sessions/) for context`. `/sessions/` is a hub path, not a local file. Optional `--limit`. -- `hapi job set|update|clear|list` - Attach long-running outliving work to a session so the list UI shows progress while the agent is idle (`tiann/hapi#1404`). Prefer `"$HAPI_SESSION_ID"`. Heartbeat at least every ~10m; honest `--remaining` or `--done`/`--total` (omit counts if unknown — never invent a percent). See `docs/guide/session-jobs.md` and `hapi job --help`. +- `hapi job set|update|clear|list|run` - Attach long-running outliving work to a session so the list UI shows progress while the agent is idle (`tiann/hapi#1404`). Prefer `"$HAPI_SESSION_ID"`. Heartbeat via `update` (or `run`); honest `--remaining` or `--done`/`--total` (omit counts if unknown — never invent a percent). Late-attach clock fix: `set --started-at ` or clear+set. See `docs/guide/session-jobs.md` and `hapi job --help`. The picker lists agents alphabetically by command name. Use Up/Down and Enter to choose; Esc or Ctrl-C cancels. It appears on every bare invocation, even diff --git a/cli/src/commands/job.test.ts b/cli/src/commands/job.test.ts index 29d18ea5c2..248ad3b8d9 100644 --- a/cli/src/commands/job.test.ts +++ b/cli/src/commands/job.test.ts @@ -47,6 +47,17 @@ describe('parseJobArgs', () => { it('rejects bad status', () => { expect(() => parseJobArgs(['update', 's', 'k', '--status', 'nope'])).toThrow(SessionJobError) }) + + it('parses --started-at for set', () => { + const parsed = parseJobArgs([ + 'set', + 'sid', + 'beets', + '--label=beets', + '--started-at=1785304595000' + ]) + expect(parsed.startedAt).toBe(1_785_304_595_000) + }) }) describe('resolveSessionByPrefix', () => { diff --git a/cli/src/commands/job.ts b/cli/src/commands/job.ts index f9be88ce6b..b91d189254 100644 --- a/cli/src/commands/job.ts +++ b/cli/src/commands/job.ts @@ -24,6 +24,7 @@ export type ParsedJobArgs = { remaining?: number unit?: string detail?: string + startedAt?: number heartbeatSec?: number command?: string[] } @@ -44,7 +45,7 @@ ${chalk.bold('Agent contract:')} ${chalk.bold('Usage:')} hapi job run --label [--heartbeat-sec 300] [progress flags] -- [args...] - hapi job set --label [--remaining N] [--done N --total N] [--unit tracks] [--detail ...] + hapi job set --label [--started-at MS] [--remaining N] [--done N --total N] [--unit tracks] [--detail ...] hapi job update [--remaining N] [--done N] [--total N] [--status running|completed|failed] [--detail ...] hapi job clear hapi job list @@ -55,8 +56,15 @@ ${chalk.bold('Progress UI:')} label/detail only → "running · 2h" + indeterminate bar elapsed always from startedAt (wall clock) — never an ETA / time-remaining field +${chalk.bold('startedAt / elapsed:')} + Prefer ${chalk.bold('update')} for heartbeats/progress so the clock is never wiped. + PATCH rejects startedAt. PUT/set without --started-at keeps the existing clock. + Late attach or wrong clock: ${chalk.bold('set --started-at ')} (explicit PUT), + or clear then set with --started-at (works on older hubs that ignored PUT corrections). + ${chalk.bold('Notes:')} Hub-persisted. Prefer "$HAPI_SESSION_ID" for this chat. + Needs a hub/CLI that includes the job subcommand (soup / feat build — not every npm release). Job key: 1-128 chars, alnum / . _ - Session lookup prefers exact id; prefix scan is the 500 most-recently-updated sessions. Docs: docs/guide/session-jobs.md @@ -166,6 +174,14 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { result.heartbeatSec = parseOptionalNumber('--heartbeat-sec', arg.slice('--heartbeat-sec='.length)) continue } + if (arg === '--started-at') { + result.startedAt = parseOptionalNumber('--started-at', flagArgs[++i]) + continue + } + if (arg.startsWith('--started-at=')) { + result.startedAt = parseOptionalNumber('--started-at', arg.slice('--started-at='.length)) + continue + } if (arg.startsWith('-')) { throw new SessionJobError('bad_args', `unexpected flag: ${arg}`) } @@ -282,6 +298,9 @@ export async function handleJobCommand(args: string[]): Promise { if (!parsed.label) { throw new SessionJobError('bad_args', 'set requires --label') } + if (parsed.startedAt !== undefined && !Number.isFinite(parsed.startedAt)) { + throw new SessionJobError('bad_args', '--started-at must be epoch milliseconds') + } const body: AttachedJobUpsert = { label: parsed.label, status: parsed.status ?? 'running', @@ -289,7 +308,8 @@ export async function handleJobCommand(args: string[]): Promise { ...(parsed.total !== undefined ? { total: parsed.total } : {}), ...(parsed.remaining !== undefined ? { remaining: parsed.remaining } : {}), ...(parsed.unit !== undefined ? { unit: parsed.unit } : {}), - ...(parsed.detail !== undefined ? { detail: parsed.detail } : {}) + ...(parsed.detail !== undefined ? { detail: parsed.detail } : {}), + ...(parsed.startedAt !== undefined ? { startedAt: parsed.startedAt } : {}) } const result = await setSessionJob({ sessionIdPrefix: parsed.sessionIdPrefix, diff --git a/docs/guide/faq.md b/docs/guide/faq.md index 2d050d67cf..57fa31a624 100644 --- a/docs/guide/faq.md +++ b/docs/guide/faq.md @@ -112,7 +112,7 @@ Some agents (especially Cursor) can resume after idle from harness signals such ### How do I show progress for a long batch that outlives the agent? -Use session-attached jobs (`hapi job`). The agent (or a wrapper script) registers a job on the session, heartbeats while the process runs, and clears it when done. The session list shows remaining / fraction / or an indeterminate "running" meter even when the agent is idle. See [Session-attached jobs](./session-jobs.md). +Use session-attached jobs (`hapi job`). The agent (or a wrapper script) registers a job on the session, heartbeats while the process runs, and clears it when done. The session list shows remaining / fraction / or an indeterminate "running" meter even when the agent is idle. See [Session-attached jobs](./session-jobs.md). This is Layer 0 list chrome - not an A2A Layer 1 work advertisement ([#1332](https://github.com/tiann/hapi/discussions/1332)). ### Can I access a terminal remotely? diff --git a/docs/guide/session-jobs.md b/docs/guide/session-jobs.md index 18b78e23c2..90d85215fc 100644 --- a/docs/guide/session-jobs.md +++ b/docs/guide/session-jobs.md @@ -6,6 +6,21 @@ This is **not** in-agent thinking progress, todos, or `backgroundTaskCount`. Tho Upstream: [tiann/hapi#1404](https://github.com/tiann/hapi/issues/1404). +## Relation to A2A (not work advertisements) + +HAPI's Agent-to-Agent control plane ([discussion #1332](https://github.com/tiann/hapi/discussions/1332)) is a **different** object family. Do not merge them. + +| | Session-attached jobs (#1404) | A2A `work_ad` (Layer 1) | +|--|------------------------------|-------------------------| +| Store | `session_jobs` | `events` / work-graph ledger | +| Surface | `SessionSummary.attachedJob` (list chrome) | Durable collaboration ledger | +| Question answered | "Is a long process still running on this session, and how far?" | "What is this session claiming about turn/project work for peers/overseer?" | +| Progress | Heartbeats + honest counts / indeterminate | Status vocabulary (`in_progress`, `done`, `failed`, `stale`, …) | +| Silence | UI amber after ~15m without heartbeat; status stays `running` until explicit exit | `expires_at` → `stale` / `unknown` — silence is **not** failure | +| Self-report | Optional counts/detail; `hapi job run` exit code is machine fact | Optional `AGENT_NOTIFY_SUMMARY` elevation (stays optional forever) | + +Jobs enrich **Layer 0** session summaries (same layer as cite / inspect / ping). They are **not** Google A2A Tasks, and they are **not** a substitute for handoffs or work ads. Do not write job heartbeats into the A2A ledger. A privileged reader may *observe* `attachedJob` later; workers still must not poll the ledger as a work queue. + ## When to attach Attach a job **before** (or immediately when) you start process-shaped work that will keep running after the agent goes idle: @@ -57,7 +72,7 @@ Same auth as `hapi ping-peer` (`HAPI_API_URL` / `CLI_API_TOKEN` or `hapi auth lo | Countable fraction | `--done N --total M` | `91% · 1637/1787 units · 2d 4h` | | Stage only / unknown size | `--label` + `--detail` + heartbeats | `running · 2d 4h` + indeterminate bar | -**Elapsed** is always derived from hub `startedAt` (wall clock since register). It is **not** an ETA and there is no time-remaining field - operators get "how long has this been going" plus whatever honest count/detail you report, without a fake completion estimate. +**Elapsed** is always derived from hub `startedAt` (wall clock). It is **not** an ETA and there is no time-remaining field - operators get "how long has this been going" plus whatever honest count/detail you report, without a fake completion estimate. Rules: @@ -66,6 +81,39 @@ Rules: - If you only know a stage name, put it in `--detail` and keep heartbeating — do **not** fake `total=100`. - There is **no** `--percent` flag and **no** ETA / time-remaining field. Inventing either would train agents to lie. +## `startedAt` / elapsed (late attach) + +Elapsed is honest wall clock from hub `startedAt`. Dogfood gotcha (music drain / beets): + +| Call | `startedAt` behavior | +|------|----------------------| +| `PATCH` / `hapi job update` | **Rejected** if you send `startedAt` (`unrecognized_keys`). Progress/heartbeat only. | +| `PUT` / `hapi job set` without `--started-at` | Keeps the existing clock when the job already exists; first create stamps now. | +| `PUT` / `hapi job set --started-at ` | Sets/corrects the clock (explicit body field). | +| `DELETE` then `PUT` with `startedAt` | Always works — including older hubs that ignored PUT corrections. | + +**Prefer `update` for heartbeats** so you never wipe the clock. Only correct historical start when a late attach stamped attach-time instead of process start: + +```bash +# epoch ms for when the drain actually started (example) +START_MS=1785304595000 + +hapi job clear "$HAPI_SESSION_ID" beets +hapi job set "$HAPI_SESSION_ID" beets \ + --label 'beets import' \ + --started-at "$START_MS" \ + --remaining 0 --done 1787 --total 1787 --unit units \ + --detail 'ALL_DONE' + +# or, on hubs that honor explicit PUT startedAt without delete: +hapi job set "$HAPI_SESSION_ID" beets \ + --label 'beets import' \ + --started-at "$START_MS" \ + --remaining 12 --done 1775 --total 1787 --unit units +``` + +Then keep using `hapi job update` for counts/detail/status. + ## Heartbeat recipe Wrap the long process so something calls `hapi job update` on a timer (or on each unit completed). Minimum viable indeterminate job: @@ -78,16 +126,26 @@ hapi job update "$HAPI_SESSION_ID" rsync-backup --detail "phase: copy · $(date When the process exits, mark completed/failed or clear. A stuck green/amber chip with a dead PID is worse than no chip. -## CLI reference +## CLI / API reference ```bash -hapi job set --label [options] -hapi job update [options] +hapi job set --label [--started-at MS] [progress flags] +hapi job update [progress flags] # no startedAt hapi job clear hapi job list +hapi job run --label -- … hapi job --help ``` +Needs a hub/CLI build that includes `job` (soup / feat — global npm releases may lag). + +| Method | Path | Notes | +|--------|------|-------| +| `GET` | `/api/sessions/:id/jobs` | List jobs | +| `PUT` | `/api/sessions/:id/jobs/:jobKey` | Upsert (`AttachedJobUpsert`; optional `startedAt`) | +| `PATCH` | `/api/sessions/:id/jobs/:jobKey` | Progress/heartbeat (`AttachedJobPatch`; **no** `startedAt`) | +| `DELETE` | `/api/sessions/:id/jobs/:jobKey` | Clear | + Primary running job is enriched onto `GET /api/sessions` as `attachedJob` and pushed on `session-updated` SSE patches. ## Related @@ -95,3 +153,4 @@ Primary running job is enriched onto `GET /api/sessions` as `attachedJob` and pu - [Supported Agents](./agents.md) — flavors and resume - [How it Works](./how-it-works.md) — CLI ↔ hub ↔ web - CLI: `hapi job --help`, `cli/README.md` +- A2A control plane: [discussion #1332](https://github.com/tiann/hapi/discussions/1332) (Layer 1 work ads / handoffs — separate from this feature) diff --git a/hub/src/store/migration-v25.test.ts b/hub/src/store/migration-v25.test.ts index 166c662419..43b58ac498 100644 --- a/hub/src/store/migration-v25.test.ts +++ b/hub/src/store/migration-v25.test.ts @@ -105,6 +105,46 @@ describe('Store V26→V27 migration: session_jobs table', () => { store.close() }) + it('preserves startedAt on PUT without body.startedAt; honors explicit correction', () => { + const store = new Store(':memory:') + const session = store.sessions.getOrCreateSession('test', { path: '/tmp' }, null, 'default') + const historical = 1_785_304_595_000 + + const created = store.sessionJobs.upsert(session.id, 'beets', { + label: 'beets import', + status: 'running', + remaining: 10 + }, 2_000) + expect(created.outcome).toBe('upserted') + if (created.outcome !== 'upserted') throw new Error('unreachable') + expect(created.job.startedAt).toBe(2_000) + + const progress = store.sessionJobs.upsert(session.id, 'beets', { + label: 'beets import', + status: 'running', + remaining: 9 + }, 3_000) + expect(progress.outcome).toBe('upserted') + if (progress.outcome !== 'upserted') throw new Error('unreachable') + expect(progress.job.startedAt).toBe(2_000) + expect(progress.job.remaining).toBe(9) + + const corrected = store.sessionJobs.upsert(session.id, 'beets', { + label: 'beets import', + status: 'running', + remaining: 9, + startedAt: historical + }, 4_000) + expect(corrected.outcome).toBe('upserted') + if (corrected.outcome !== 'upserted') throw new Error('unreachable') + expect(corrected.job.startedAt).toBe(historical) + + const patched = store.sessionJobs.patch(session.id, 'beets', { remaining: 8 }, 5_000) + expect(patched?.startedAt).toBe(historical) + expect(patched?.remaining).toBe(8) + store.close() + }) + it('transfers jobs on merge without colliding keys', () => { const store = new Store(':memory:') const oldSession = store.sessions.getOrCreateSession('old', { path: '/a' }, null, 'default') diff --git a/hub/src/store/sessionJobs.ts b/hub/src/store/sessionJobs.ts index c44b5831df..fb933bef19 100644 --- a/hub/src/store/sessionJobs.ts +++ b/hub/src/store/sessionJobs.ts @@ -134,7 +134,11 @@ export function upsertSessionJob( ): UpsertSessionJobResult { const existing = getSessionJob(db, sessionId, jobKey) const heartbeatAt = body.heartbeatAt ?? now - const startedAt = body.startedAt ?? existing?.startedAt ?? now + // Explicit startedAt wins (late-attach correction). Omitted → keep existing clock, + // else stamp now. PATCH never accepts startedAt — use PUT or clear+PUT. + const startedAt = body.startedAt !== undefined + ? body.startedAt + : (existing?.startedAt ?? now) const status = body.status ?? 'running' try { @@ -152,7 +156,7 @@ export function upsertSessionJob( unit = excluded.unit, detail = excluded.detail, heartbeat_at = excluded.heartbeat_at, - started_at = session_jobs.started_at, + started_at = excluded.started_at, updated_at = excluded.updated_at` ).run( sessionId, diff --git a/hub/src/store/sessionJobsStore.ts b/hub/src/store/sessionJobsStore.ts index cc59c5c2f6..8394730e5b 100644 --- a/hub/src/store/sessionJobsStore.ts +++ b/hub/src/store/sessionJobsStore.ts @@ -39,12 +39,22 @@ export class SessionJobsStore { return getPrimaryRunningJobsBySessionIds(this.db, sessionIds) } - upsert(sessionId: string, jobKey: string, body: AttachedJobUpsert): UpsertSessionJobResult { - return upsertSessionJob(this.db, sessionId, jobKey, body) + upsert( + sessionId: string, + jobKey: string, + body: AttachedJobUpsert, + now?: number + ): UpsertSessionJobResult { + return upsertSessionJob(this.db, sessionId, jobKey, body, now) } - patch(sessionId: string, jobKey: string, patch: AttachedJobPatch): StoredSessionJob | null { - return patchSessionJob(this.db, sessionId, jobKey, patch) + patch( + sessionId: string, + jobKey: string, + patch: AttachedJobPatch, + now?: number + ): StoredSessionJob | null { + return patchSessionJob(this.db, sessionId, jobKey, patch, now) } delete(sessionId: string, jobKey: string): boolean { diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index a122541808..c55d67e663 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -432,6 +432,7 @@ export const AttachedJobUpsertSchema = z.object({ export type AttachedJobUpsert = z.infer +/** Progress/heartbeat only — no startedAt (use PUT upsert with explicit startedAt to correct). */ export const AttachedJobPatchSchema = z.object({ label: z.string().min(1).max(200).optional(), status: AttachedJobStatusSchema.optional(), From d150924d771481900400e5aebf4694627557a11e Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 16:40:30 +0000 Subject: [PATCH 08/94] fix(jobs): keep merge redirect keys in MetadataSchema (#1404) SessionCache.refreshSession stripped jobsAcceptedFromSessionIds / jobsTransferredToSessionId as unknown keys, so resolveAttachedJobSessionId could not follow post-merge heartbeats. Declare the fields, add a real SessionCache integration test, and pass full UUIDs through the CLI when the merge source is missing from GET /sessions. Co-authored-by: Cursor --- cli/src/commands/job.test.ts | 14 ++- cli/src/modules/sessionJob/sessionJob.ts | 28 ++++- hub/src/sync/sessionCache-merge-jobs.test.ts | 101 +++++++++++++++++++ shared/src/schemas.clear.test.ts | 11 ++ shared/src/schemas.ts | 7 ++ 5 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 hub/src/sync/sessionCache-merge-jobs.test.ts diff --git a/cli/src/commands/job.test.ts b/cli/src/commands/job.test.ts index 248ad3b8d9..e3785f1561 100644 --- a/cli/src/commands/job.test.ts +++ b/cli/src/commands/job.test.ts @@ -3,7 +3,8 @@ import { parseJobArgs } from '@/commands/job' import { SessionJobError, exitCodeForSessionJobError, - resolveSessionByPrefix + resolveSessionByPrefix, + resolveSessionIdForJobCli } from '@/modules/sessionJob/sessionJob' describe('parseJobArgs', () => { @@ -84,6 +85,17 @@ describe('resolveSessionByPrefix', () => { }) }) +describe('resolveSessionIdForJobCli', () => { + it('passes a full UUID through when missing from the session list', () => { + const deleted = 'cccccccc-4444-4444-4444-444444444444' + expect(resolveSessionIdForJobCli([], deleted)).toBe(deleted) + }) + + it('still errors for a non-uuid prefix with no list match', () => { + expect(() => resolveSessionIdForJobCli([], 'deadbeef')).toThrow(/no session matching/) + }) +}) + describe('exitCodeForSessionJobError', () => { it('maps codes', () => { expect(exitCodeForSessionJobError(new SessionJobError('bad_args', 'x'))).toBe(2) diff --git a/cli/src/modules/sessionJob/sessionJob.ts b/cli/src/modules/sessionJob/sessionJob.ts index 2f404fce9f..0a52c661ff 100644 --- a/cli/src/modules/sessionJob/sessionJob.ts +++ b/cli/src/modules/sessionJob/sessionJob.ts @@ -119,6 +119,32 @@ export function resolveSessionByPrefix(sessions: SessionListItem[], prefix: stri return matches[0]! } +const FULL_SESSION_UUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +/** + * Resolve a session id for job CLI calls. Prefer list match; if the prefix is + * a full UUID missing from the list (deleted merge source), pass it through so + * hub job routes can follow jobsAcceptedFromSessionIds. + */ +export function resolveSessionIdForJobCli( + sessions: SessionListItem[], + sessionIdPrefix: string +): string { + try { + return resolveSessionByPrefix(sessions, sessionIdPrefix).id + } catch (error) { + if ( + error instanceof SessionJobError + && error.code === 'not_found' + && FULL_SESSION_UUID.test(sessionIdPrefix.trim()) + ) { + return sessionIdPrefix.trim() + } + throw error + } +} + async function resolveSessionId( apiUrl: string, jwt: string, @@ -137,7 +163,7 @@ async function resolveSessionId( const sessions = Array.isArray(response.data?.sessions) ? (response.data.sessions as SessionListItem[]) : [] - return resolveSessionByPrefix(sessions, sessionIdPrefix).id + return resolveSessionIdForJobCli(sessions, sessionIdPrefix) } export type SessionJobClientOptions = { diff --git a/hub/src/sync/sessionCache-merge-jobs.test.ts b/hub/src/sync/sessionCache-merge-jobs.test.ts new file mode 100644 index 0000000000..9171a12fbc --- /dev/null +++ b/hub/src/sync/sessionCache-merge-jobs.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'bun:test' +import type { SyncEvent } from '@hapi/protocol/types' +import { MetadataSchema } from '@hapi/protocol/schemas' +import { Store } from '../store' +import type { EventPublisher } from './eventPublisher' +import { SessionCache } from './sessionCache' + +/** + * Cold-review pass 2 (#1404): job-owner redirects written by recordJobs*() + * must survive SessionCache.refreshSession. MetadataSchema used to strip + * jobsAcceptedFromSessionIds / jobsTransferredToSessionId as unknown keys, + * so resolveAttachedJobSessionId never followed the merge. + */ + +function createCapturingPublisher(events: SyncEvent[]): EventPublisher { + return { + emit: (event: SyncEvent) => { + events.push(event) + } + } as unknown as EventPublisher +} + +function setup() { + const store = new Store(':memory:') + const events: SyncEvent[] = [] + const cache = new SessionCache(store, createCapturingPublisher(events)) + return { store, events, cache } +} + +function makeSessions(cache: SessionCache, ns: string = 'default') { + const oldSession = cache.getOrCreateSession( + 'agent-jobs-old-' + Math.random().toString(36).slice(2, 8), + { path: '/tmp/project', host: 'localhost', flavor: 'codex' }, + null, + ns + ) + const newSession = cache.getOrCreateSession( + 'agent-jobs-new-' + Math.random().toString(36).slice(2, 8), + { path: '/tmp/project', host: 'localhost', flavor: 'codex' }, + null, + ns + ) + return { oldSession, newSession } +} + +describe('mergeSessions job redirect through SessionCache (#1404)', () => { + it('keeps jobsAcceptedFromSessionIds after refresh when old session is deleted', async () => { + const { store, cache } = setup() + const { oldSession, newSession } = makeSessions(cache) + + const upserted = store.sessionJobs.upsert(oldSession.id, 'beets', { + label: 'beets import', + status: 'running', + remaining: 12 + }) + expect(upserted.outcome).toBe('upserted') + + await cache.mergeSessions(oldSession.id, newSession.id, 'default') + + expect(store.sessionJobs.getPrimaryRunning(newSession.id)?.key).toBe('beets') + expect(store.sessions.getSession(oldSession.id)).toBeNull() + + // Schema strip regression: refresh must retain the acceptor list. + const refreshed = cache.refreshSession(newSession.id) + expect(refreshed).not.toBeNull() + const accepted = refreshed!.metadata?.jobsAcceptedFromSessionIds + expect(accepted).toContain(oldSession.id) + expect( + MetadataSchema.parse(refreshed!.metadata).jobsAcceptedFromSessionIds + ).toContain(oldSession.id) + + expect(cache.resolveAttachedJobSessionId(oldSession.id, 'default')).toBe(newSession.id) + expect(cache.resolveAttachedJobSessionId(newSession.id, 'default')).toBe(newSession.id) + }) + + it('keeps jobsTransferredToSessionId on a kept-alive source after mergeSessionHistory', async () => { + const { store, cache } = setup() + const { oldSession, newSession } = makeSessions(cache) + + store.sessionJobs.upsert(oldSession.id, 'drain', { + label: 'rsync drain', + status: 'running', + remaining: 3 + }) + + await cache.mergeSessionHistory(oldSession.id, newSession.id, 'default', { + mergeAgentState: false + }) + + expect(store.sessionJobs.getPrimaryRunning(newSession.id)?.key).toBe('drain') + expect(store.sessions.getSession(oldSession.id)).not.toBeNull() + + const refreshedOld = cache.refreshSession(oldSession.id) + expect(refreshedOld?.metadata?.jobsTransferredToSessionId).toBe(newSession.id) + expect( + MetadataSchema.parse(refreshedOld!.metadata).jobsTransferredToSessionId + ).toBe(newSession.id) + + expect(cache.resolveAttachedJobSessionId(oldSession.id, 'default')).toBe(newSession.id) + }) +}) diff --git a/shared/src/schemas.clear.test.ts b/shared/src/schemas.clear.test.ts index e84e5673dd..b8197221c7 100644 --- a/shared/src/schemas.clear.test.ts +++ b/shared/src/schemas.clear.test.ts @@ -10,6 +10,17 @@ describe('fresh-session clear schema contract', () => { })).toMatchObject({ supersededBySessionId: 'new-session-id' }) }) + it('preserves session-job merge redirect fields (must not strip as unknown)', () => { + const parsed = MetadataSchema.parse({ + path: '/tmp/project', + host: 'host', + jobsAcceptedFromSessionIds: ['old-session-id'], + jobsTransferredToSessionId: 'new-session-id' + }) + expect(parsed.jobsAcceptedFromSessionIds).toEqual(['old-session-id']) + expect(parsed.jobsTransferredToSessionId).toBe('new-session-id') + }) + it('accepts cleared as an additive session-end reason', () => { expect(SessionEndReasonSchema.parse('cleared')).toBe('cleared') }) diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index c55d67e663..52d32adb9e 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -134,6 +134,13 @@ export const MetadataSchema = z.object({ // Set only after a completed fresh-session clear. The source row remains // archived; web clients use this durable link to follow the replacement. supersededBySessionId: z.string().optional(), + // After session merge/dedup transfers session_jobs: the target remembers + // which source ids it absorbed (so job REST can follow a deleted + // pre-merge $HAPI_SESSION_ID), and a kept-alive source points at the + // post-merge owner. Must be declared here — SessionCache.refreshSession + // parses via MetadataSchema and strips unknown keys (tiann/hapi#1404). + jobsAcceptedFromSessionIds: z.array(z.string()).optional(), + jobsTransferredToSessionId: z.string().optional(), // Durable in-progress state for runner-backed OpenCode /clear. opencodeClearOperation: OpencodeClearOperationSchema.optional(), preferredPermissionMode: PermissionModeSchema.optional(), From d494d623c1d0e7fe392555102c6cc8e4ffb2b0cf Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 17:14:51 +0000 Subject: [PATCH 09/94] feat(jobs): MCP session_job tool for catalog-level discoverability Expose session-attached jobs in the same HAPI MCP catalog as ping_peer / inspect_peer so Cursor/ACP and steered flavors see outliving-work meters as first-class tooling, not docs-only. Auto-approve own-session calls; steer prefers MCP set/update or CLI job run for supervised children. Co-authored-by: Cursor --- cli/src/agent/runners/runAgentSession.test.ts | 4 +- cli/src/claude/utils/startHappyServer.test.ts | 28 ++- cli/src/claude/utils/startHappyServer.ts | 24 ++- cli/src/codex/happyMcpStdioBridge.test.ts | 32 +++- cli/src/codex/happyMcpStdioBridge.ts | 44 ++++- .../codex/utils/buildHapiMcpBridge.test.ts | 18 +- cli/src/codex/utils/buildHapiMcpBridge.ts | 4 + .../permission/BasePermissionHandler.test.ts | 11 ++ .../permission/BasePermissionHandler.ts | 10 +- .../common/sessionJobInstruction.test.ts | 7 +- .../modules/common/sessionJobInstruction.ts | 21 ++- .../modules/sessionJob/sessionJobMcp.test.ts | 64 +++++++ cli/src/modules/sessionJob/sessionJobMcp.ts | 172 ++++++++++++++++++ docs/guide/session-jobs.md | 17 +- 14 files changed, 415 insertions(+), 41 deletions(-) create mode 100644 cli/src/modules/sessionJob/sessionJobMcp.test.ts create mode 100644 cli/src/modules/sessionJob/sessionJobMcp.ts diff --git a/cli/src/agent/runners/runAgentSession.test.ts b/cli/src/agent/runners/runAgentSession.test.ts index fe1886578a..2eb74df6b7 100644 --- a/cli/src/agent/runners/runAgentSession.test.ts +++ b/cli/src/agent/runners/runAgentSession.test.ts @@ -74,7 +74,7 @@ vi.mock('@/claude/utils/startHappyServer', () => ({ harness.startHappyServerOptions = options return { url: 'http://127.0.0.1:1234', - toolNames: ['change_title', 'display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer', 'skill_lookup'], + toolNames: ['change_title', 'display_image', 'list_peers', 'ping_peer', 'inspect_peer', 'session_job', 'skill_lookup'], stop: harness.stopServer } }) @@ -167,7 +167,7 @@ describe('runAgentSession', () => { '--url', 'http://127.0.0.1:1234', '--tools', - 'change_title,display_image,display_video,display_media,list_peers,ping_peer,inspect_peer,skill_lookup' + 'change_title,display_image,list_peers,ping_peer,inspect_peer,session_job,skill_lookup' ]) expect(harness.newSessionOptions).toMatchObject({ cwd: '/tmp/project', diff --git a/cli/src/claude/utils/startHappyServer.test.ts b/cli/src/claude/utils/startHappyServer.test.ts index 8abb273f25..9865c11d9b 100644 --- a/cli/src/claude/utils/startHappyServer.test.ts +++ b/cli/src/claude/utils/startHappyServer.test.ts @@ -44,6 +44,7 @@ describe('startHappyServer skill_lookup', () => { async function connect(enableSkillLookup = true): Promise { sendAgentMessage = vi.fn() const sessionClient = { + sessionId: 'test-session-id', updateMetadata: vi.fn(), sendAgentMessage, sendClaudeSessionMessage: vi.fn() @@ -107,15 +108,16 @@ describe('startHappyServer skill_lookup', () => { const mcp = await connect(false) const tools = await mcp.listTools() - expect(tools.tools.map((tool) => tool.name)).toEqual([ + expect(tools.tools.map((tool) => tool.name).sort()).toEqual([ 'change_title', 'display_image', 'display_video', 'display_media', - 'ping_peer', 'inspect_peer', - 'list_peers' - ]) + 'list_peers', + 'ping_peer', + 'session_job', + ].sort()) }) it('describes display_image as user output rather than image input', async () => { @@ -170,6 +172,7 @@ describe('startHappyServer skill_lookup', () => { it('does not expose change_title when native ACP titles are enabled', async () => { const sessionClient = { + sessionId: 'test-session-id', updateMetadata: vi.fn(), sendAgentMessage: vi.fn(), sendClaudeSessionMessage: vi.fn() @@ -182,15 +185,24 @@ describe('startHappyServer skill_lookup', () => { await mcp.connect(new StreamableHTTPClientTransport(new URL(server.url))) const tools = await mcp.listTools() - expect(server.toolNames).toEqual(['display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer']) - expect(tools.tools.map((tool) => tool.name)).toEqual([ + expect(server.toolNames).toEqual([ 'display_image', 'display_video', 'display_media', + 'list_peers', 'ping_peer', 'inspect_peer', - 'list_peers' + 'session_job', ]) + expect(tools.tools.map((tool) => tool.name).sort()).toEqual([ + 'display_image', + 'display_media', + 'display_video', + 'inspect_peer', + 'list_peers', + 'ping_peer', + 'session_job', + ].sort()) }) }) @@ -317,11 +329,13 @@ describe('toClaudeAllowedHapiMcpTools', () => { 'list_peers', 'ping_peer', 'inspect_peer', + 'session_job', 'skill_lookup' ])).toEqual([ 'mcp__hapi__change_title', 'mcp__hapi__display_image', 'mcp__hapi__list_peers', + 'mcp__hapi__session_job', 'mcp__hapi__skill_lookup' ]) expect(toClaudeAllowedHapiMcpTools(['display_video'])).not.toContain('mcp__hapi__display_video') diff --git a/cli/src/claude/utils/startHappyServer.ts b/cli/src/claude/utils/startHappyServer.ts index 10a2a02d66..e2d4e1c70d 100644 --- a/cli/src/claude/utils/startHappyServer.ts +++ b/cli/src/claude/utils/startHappyServer.ts @@ -28,6 +28,13 @@ import { } from '@hapi/protocol/sessionCitation' import { PingPeerError, formatInspectPeerReport, formatPeerSessionsList, inspectPeer, listPeerSessions, peerListFetchLimit, pingPeer } from "@/modules/pingPeer/pingPeer"; import { applySessionDisplayRename, normalizeSessionDisplayTitle } from "@/agent/sessionDisplayRename"; +import { + SESSION_JOB_TOOL_DESCRIPTION, + SESSION_JOB_TOOL_NAME, + handleSessionJobTool, + sessionJobInputSchema, + type SessionJobToolArgs, +} from "@/modules/sessionJob/sessionJobMcp"; type StartHappyServerOptions = { /** @@ -376,6 +383,19 @@ function createHapiMcpServer( } }); + mcp.registerTool(SESSION_JOB_TOOL_NAME, { + description: SESSION_JOB_TOOL_DESCRIPTION, + title: 'Session-Attached Job', + inputSchema: sessionJobInputSchema, + }, async (args: SessionJobToolArgs) => { + logger.debug('[hapiMCP] session_job:', args.action, args.jobKey); + const result = await handleSessionJobTool(args, client.sessionId); + return { + content: [{ type: 'text' as const, text: result.text }], + isError: result.isError, + }; + }); + mcp.registerTool('list_peers', { description: 'List peer HAPI sessions on the same hub/namespace (id prefix, active, flavor, name). Uses this session\'s hub credentials - works from runner-spawned agents without being on the hub host. Prefer this over shelling `hapi ping-peer --list`. Then call inspect_peer / ping_peer with a listed id.', title: 'List Peer Sessions', @@ -541,8 +561,8 @@ export async function startHappyServer(client: ApiSessionClient, options: StartH })); const toolNames = enableChangeTitle - ? ['change_title', 'display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer'] - : ['display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer']; + ? ['change_title', 'display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer', SESSION_JOB_TOOL_NAME] + : ['display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer', SESSION_JOB_TOOL_NAME]; if (options.skillLookup) { toolNames.push('skill_lookup'); } diff --git a/cli/src/codex/happyMcpStdioBridge.test.ts b/cli/src/codex/happyMcpStdioBridge.test.ts index c940d0bc50..5ae2d27064 100644 --- a/cli/src/codex/happyMcpStdioBridge.test.ts +++ b/cli/src/codex/happyMcpStdioBridge.test.ts @@ -70,16 +70,20 @@ describe('runHappyMcpStdioBridge tool forwarding', () => { '--url', 'http://127.0.0.1:43006', '--tools', - 'change_title,display_image,display_video,display_media,skill_lookup' + 'change_title,display_image,display_video,display_media,list_peers,ping_peer,inspect_peer,session_job,skill_lookup' ]) - expect([...harness.tools.keys()]).toEqual([ + expect([...harness.tools.keys()].sort()).toEqual([ 'change_title', 'display_image', - 'display_video', 'display_media', - 'skill_lookup' - ]) + 'display_video', + 'inspect_peer', + 'list_peers', + 'ping_peer', + 'session_job', + 'skill_lookup', + ].sort()) const handler = harness.tools.get('skill_lookup') expect(handler).toBeDefined() @@ -140,6 +144,24 @@ describe('runHappyMcpStdioBridge tool forwarding', () => { ]) }) + it('registers session_job when included in --tools', async () => { + await runHappyMcpStdioBridge([ + '--url', + 'http://127.0.0.1:43006', + '--tools', + 'change_title,display_image,list_peers,ping_peer,inspect_peer,session_job' + ]) + + expect([...harness.tools.keys()].sort()).toEqual([ + 'change_title', + 'display_image', + 'inspect_peer', + 'list_peers', + 'ping_peer', + 'session_job', + ].sort()) + }) + it('registers inspect_peer when included in --tools', async () => { await runHappyMcpStdioBridge([ '--url', diff --git a/cli/src/codex/happyMcpStdioBridge.ts b/cli/src/codex/happyMcpStdioBridge.ts index be0d03d0bb..177549a43b 100644 --- a/cli/src/codex/happyMcpStdioBridge.ts +++ b/cli/src/codex/happyMcpStdioBridge.ts @@ -22,8 +22,22 @@ import { PING_PEER_TOOL_DESCRIPTION, SESSION_ID_PREFIX_PARAM_DESCRIPTION, } from '@hapi/protocol/sessionCitation'; +import { + SESSION_JOB_TOOL_DESCRIPTION, + SESSION_JOB_TOOL_NAME, + sessionJobInputSchema, +} from '@/modules/sessionJob/sessionJobMcp'; -const DEFAULT_TOOL_NAMES = ['change_title', 'display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer']; +const DEFAULT_TOOL_NAMES = [ + 'change_title', + 'display_image', + 'display_video', + 'display_media', + 'list_peers', + 'ping_peer', + 'inspect_peer', + SESSION_JOB_TOOL_NAME, +]; function parseArgs(argv: string[]): { url: string | null; toolNames: Set } { let url: string | null = null; @@ -289,6 +303,34 @@ export async function runHappyMcpStdioBridge(argv: string[]): Promise { ); } + if (toolNames.has(SESSION_JOB_TOOL_NAME)) { + server.registerTool( + SESSION_JOB_TOOL_NAME, + { + description: SESSION_JOB_TOOL_DESCRIPTION, + title: 'Session-Attached Job', + inputSchema: sessionJobInputSchema, + }, + async (args: Record) => { + try { + const client = await ensureHttpClient(); + const response = await client.callTool({ name: SESSION_JOB_TOOL_NAME, arguments: args }); + return response as any; + } catch (error) { + return { + content: [ + { + type: 'text' as const, + text: `Failed to run session_job: ${error instanceof Error ? error.message : String(error)}`, + }, + ], + isError: true, + }; + } + } + ); + } + const skillLookupInputSchema: z.ZodTypeAny = z.object({ name: z.string().trim().min(1).max(128).describe('Exact skill name shown by HAPI skill autocomplete'), }); diff --git a/cli/src/codex/utils/buildHapiMcpBridge.test.ts b/cli/src/codex/utils/buildHapiMcpBridge.test.ts index 8af0d57768..8a2ba68c05 100644 --- a/cli/src/codex/utils/buildHapiMcpBridge.test.ts +++ b/cli/src/codex/utils/buildHapiMcpBridge.test.ts @@ -14,8 +14,8 @@ vi.mock('@/claude/utils/startHappyServer', () => ({ return { url: 'http://127.0.0.1:43006/', toolNames: options.skillLookup - ? ['change_title', 'display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer', 'skill_lookup'] - : ['change_title', 'display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer'], + ? ['change_title', 'display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer', 'session_job', 'skill_lookup'] + : ['change_title', 'display_image', 'display_video', 'display_media', 'list_peers', 'ping_peer', 'inspect_peer', 'session_job'], stop: vi.fn() } }) @@ -71,14 +71,15 @@ describe('buildHapiMcpBridge skill lookup config', () => { '--url', 'http://127.0.0.1:43006/', '--tools', - 'change_title,display_image,display_video,display_media,list_peers,ping_peer,inspect_peer,skill_lookup' + 'change_title,display_image,display_video,display_media,list_peers,ping_peer,inspect_peer,session_job,skill_lookup' ]) expect(bridge.mcpServers.hapi.tools).toEqual({ - change_title: { approval_mode: 'approve' }, display_image: { approval_mode: 'prompt' }, display_video: { approval_mode: 'prompt' }, display_media: { approval_mode: 'prompt' }, + change_title: { approval_mode: 'approve' }, list_peers: { approval_mode: 'approve' }, + session_job: { approval_mode: 'approve' }, skill_lookup: { approval_mode: 'approve' } }) }) @@ -86,13 +87,16 @@ describe('buildHapiMcpBridge skill lookup config', () => { it('does not expose skill_lookup for native-skill bridge callers', async () => { const bridge = await buildHapiMcpBridge(createClient()) - expect(harness.cliArgs.at(-1)).toBe('change_title,display_image,display_video,display_media,list_peers,ping_peer,inspect_peer') + expect(harness.cliArgs.at(-1)).toBe( + 'change_title,display_image,display_video,display_media,list_peers,ping_peer,inspect_peer,session_job' + ) expect(bridge.mcpServers.hapi.tools).toEqual({ - change_title: { approval_mode: 'approve' }, display_image: { approval_mode: 'prompt' }, display_video: { approval_mode: 'prompt' }, display_media: { approval_mode: 'prompt' }, - list_peers: { approval_mode: 'approve' } + change_title: { approval_mode: 'approve' }, + list_peers: { approval_mode: 'approve' }, + session_job: { approval_mode: 'approve' } }) }) diff --git a/cli/src/codex/utils/buildHapiMcpBridge.ts b/cli/src/codex/utils/buildHapiMcpBridge.ts index 0d6ae4856f..453d354dc1 100644 --- a/cli/src/codex/utils/buildHapiMcpBridge.ts +++ b/cli/src/codex/utils/buildHapiMcpBridge.ts @@ -111,6 +111,10 @@ export async function buildHapiMcpBridge( tools.list_peers = { approval_mode: 'approve' }; + // Own-session progress meter (tiann/hapi#1404) — hub REST, not peer inject. + tools.session_job = { + approval_mode: 'approve' + }; // ping_peer / inspect_peer are registered on the HTTP MCP server / stdio // bridge, but are not auto-approved: they target another session (resume + // inject, or read peer histories). diff --git a/cli/src/modules/common/permission/BasePermissionHandler.test.ts b/cli/src/modules/common/permission/BasePermissionHandler.test.ts index 6febd9a4c2..3e8ea63108 100644 --- a/cli/src/modules/common/permission/BasePermissionHandler.test.ts +++ b/cli/src/modules/common/permission/BasePermissionHandler.test.ts @@ -101,3 +101,14 @@ describe('resolveToolAutoApprovalDecision list_peers', () => { )).toBeNull() }) }) + +describe('resolveToolAutoApprovalDecision session_job', () => { + it.each([ + 'session_job', + 'hapi_session_job', + 'mcp__hapi__session_job', + 'Session-Attached Job' + ])('auto-approves own-session job meter %s', (toolName) => { + expect(resolveToolAutoApprovalDecision('default', toolName, 'call-1')).toBe('approved') + }) +}) diff --git a/cli/src/modules/common/permission/BasePermissionHandler.ts b/cli/src/modules/common/permission/BasePermissionHandler.ts index 860c542624..27924f0388 100644 --- a/cli/src/modules/common/permission/BasePermissionHandler.ts +++ b/cli/src/modules/common/permission/BasePermissionHandler.ts @@ -37,13 +37,19 @@ const AUTO_APPROVE_EXACT_TOOL_NAMES = new Set([ 'happy__list_peers', 'mcp__hapi__list_peers', // ACP permission requests often surface MCP tool title, not the snake_case name. - 'list peer sessions' + 'list peer sessions', + // Own-session progress meter (tiann/hapi#1404) — hub REST only, not peer inject. + 'session_job', + 'hapi_session_job', + 'happy__session_job', + 'mcp__hapi__session_job', + 'session-attached job' ]); // ping_peer / inspect_peer intentionally omitted from always-approve: they can // resume+inject into another session or read peer histories, so permission // modes must still gate them. Treat both as write-like in read-only so ACP // titles such as "Ping Peer Session" / "Inspect Peer Session" also require -// approval. list_peers is discovery-only and is auto-approved above. +// approval. list_peers / session_job are auto-approved above. const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'save_memory']; const SENSITIVE_TOOL_NAME_HINTS = [ 'ping_peer', diff --git a/cli/src/modules/common/sessionJobInstruction.test.ts b/cli/src/modules/common/sessionJobInstruction.test.ts index c6d919a329..45ebbd72cd 100644 --- a/cli/src/modules/common/sessionJobInstruction.test.ts +++ b/cli/src/modules/common/sessionJobInstruction.test.ts @@ -5,10 +5,11 @@ import { } from './sessionJobInstruction' describe('sessionJobInstruction', () => { - it('prefers job run supervisor and forbids fake percent', () => { + it('prefers MCP session_job + job run supervisor and forbids fake percent', () => { + expect(SESSION_JOB_INSTRUCTION).toContain('session_job') + expect(SESSION_JOB_INSTRUCTION).toContain('ping_peer') expect(SESSION_JOB_INSTRUCTION).toContain('hapi job run') - expect(SESSION_JOB_INSTRUCTION).toContain('hapi job update') - expect(SESSION_JOB_INSTRUCTION).toContain('idle agent cannot') + expect(SESSION_JOB_INSTRUCTION).toContain('idle agents cannot') expect(SESSION_JOB_INSTRUCTION).toContain('Never invent a fake percent') expect(SESSION_JOB_INSTRUCTION).toContain('HAPI_SESSION_ID') }) diff --git a/cli/src/modules/common/sessionJobInstruction.ts b/cli/src/modules/common/sessionJobInstruction.ts index ee3043bf76..c400615455 100644 --- a/cli/src/modules/common/sessionJobInstruction.ts +++ b/cli/src/modules/common/sessionJobInstruction.ts @@ -2,22 +2,23 @@ * Always-on steer for session-attached long-running jobs (tiann/hapi#1404). * * Injected into flavors that have a HAPI system / developer-instructions seam - * today: Claude, Codex, OpenCode, Grok. Cursor ACP has no such seam (estate - * skill `hapi-session-jobs` + `hapi job --help` instead). Other ACP flavors - * (Kimi, Copilot, Pi, …) do not receive this block until an MCP job tool or - * per-flavor seam lands — do not claim "every flavor." + * today: Claude, Codex, OpenCode, Grok. Cursor/Kimi/Copilot/Pi get the same + * MCP tool (`session_job`) via the HAPI MCP bridge — catalog discovery, not + * this prompt block. Estate skill remains a Cursor backup. */ /** Canonical one-block contract. Keep short — every session's prompt budget. */ export const SESSION_JOB_INSTRUCTION = [ 'Session-attached jobs (outliving work):', 'When you start work that will keep running after this agent goes idle', - '(batch imports, long scripts, external daemons), attach it so the session', - 'list can show progress while you are idle.', - 'Prefer: hapi job run "$HAPI_SESSION_ID" --label -- …', - '(auto-heartbeats + marks completed/failed on exit).', - 'Manual path: hapi job set … then a wrapper must heartbeat via', - 'hapi job update at least every ~10 minutes — an idle agent cannot.', + '(batch imports, long scripts, external daemons), attach a session job so', + 'the session list shows progress while you are idle — same class of HAPI', + 'tooling as ping_peer / inspect_peer.', + 'Prefer MCP tool session_job (aliases: mcp__hapi__session_job, hapi_session_job,', + 'functions.hapi__session_job) with action=set before the process starts, then', + 'action=update every ~10m (idle agents cannot heartbeat).', + 'For a supervised shell child: hapi job run "$HAPI_SESSION_ID" ', + '--label -- … (auto-heartbeats + completed/failed on exit).', 'Prefer honest remaining or done+total; omit counts when unknown', '(UI shows "running" + elapsed). Never invent a fake percent.', 'Full contract: hapi job --help.' diff --git a/cli/src/modules/sessionJob/sessionJobMcp.test.ts b/cli/src/modules/sessionJob/sessionJobMcp.test.ts new file mode 100644 index 0000000000..87ac653dc6 --- /dev/null +++ b/cli/src/modules/sessionJob/sessionJobMcp.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, vi } from 'vitest' +import { handleSessionJobTool, SESSION_JOB_TOOL_DESCRIPTION } from './sessionJobMcp' + +vi.mock('./sessionJob', () => ({ + SessionJobError: class SessionJobError extends Error { + code: string + constructor(code: string, message: string) { + super(message) + this.code = code + } + }, + setSessionJob: vi.fn(async () => ({ + sessionId: 'sid-1', + job: { + key: 'beets', + label: 'beets import', + status: 'running', + remaining: 12, + heartbeatAt: 1, + startedAt: 1, + updatedAt: 1 + } + })), + updateSessionJob: vi.fn(async () => ({ + sessionId: 'sid-1', + job: { + key: 'beets', + label: 'beets import', + status: 'running', + remaining: 11, + heartbeatAt: 2, + startedAt: 1, + updatedAt: 2 + } + })), + clearSessionJob: vi.fn(async () => ({ sessionId: 'sid-1' })), + listSessionJobs: vi.fn(async () => ({ sessionId: 'sid-1', jobs: [], primary: null })) +})) + +describe('sessionJobMcp', () => { + it('description steers outliving batch work and honest progress', () => { + expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/OUTLIVES/i) + expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/Never invent a percent/i) + expect(SESSION_JOB_TOOL_DESCRIPTION).toContain('hapi job run') + }) + + it('set requires label and defaults session to caller id', async () => { + const result = await handleSessionJobTool( + { action: 'set', jobKey: 'beets', label: 'beets import', remaining: 12 }, + 'sid-1' + ) + expect(result.isError).toBe(false) + expect(result.text).toContain('set beets') + }) + + it('rejects update with empty patch', async () => { + const result = await handleSessionJobTool( + { action: 'update', jobKey: 'beets' }, + 'sid-1' + ) + expect(result.isError).toBe(true) + expect(result.text).toMatch(/at least one/i) + }) +}) diff --git a/cli/src/modules/sessionJob/sessionJobMcp.ts b/cli/src/modules/sessionJob/sessionJobMcp.ts new file mode 100644 index 0000000000..519f064a9b --- /dev/null +++ b/cli/src/modules/sessionJob/sessionJobMcp.ts @@ -0,0 +1,172 @@ +/** + * MCP surface for session-attached jobs (tiann/hapi#1404). + * Same discovery class as ping_peer / inspect_peer — tool catalog, not docs-only. + */ + +import { z } from 'zod' +import type { AttachedJob, AttachedJobPatch, AttachedJobUpsert } from '@hapi/protocol' +import { + SessionJobError, + clearSessionJob, + listSessionJobs, + setSessionJob, + updateSessionJob +} from './sessionJob' + +export const SESSION_JOB_TOOL_NAME = 'session_job' + +/** + * Self-contained tool description — agents select by matching intent to this text. + * Write for selection, not for humans browsing a README. + */ +export const SESSION_JOB_TOOL_DESCRIPTION = [ + 'Attach or update a hub-persisted progress meter on a HAPI session for work that', + 'OUTLIVES this agent turn (nohup / batch import / rclone / compile / long drain /', + 'external daemon). The session list shows the meter while the agent is idle', + '(active:false). Call action=set BEFORE starting that process (or immediately when', + 'you start it). Heartbeat with action=update at least every ~10 minutes while it', + 'runs — an idle agent cannot. Prefer honest remaining or done+total; omit counts', + 'when unknown (UI shows "running" + elapsed). Never invent a percent or ETA.', + 'Finish with action=update status=completed|failed or action=clear.', + 'Default sessionId is this chat ($HAPI_SESSION_ID). Not for in-agent todos,', + 'thinking progress, or short tool calls. For a supervised shell child that', + 'auto-heartbeats, prefer CLI: hapi job run "$HAPI_SESSION_ID" --label … -- .', +].join(' ') + +export const sessionJobInputSchema: z.ZodTypeAny = z.object({ + action: z.enum(['set', 'update', 'clear', 'list']).describe( + 'set=register/upsert running job; update=heartbeat/progress/status; clear=remove; list=show jobs' + ), + sessionId: z.string().trim().min(1).optional().describe( + 'Target session id or prefix. Omit to use this chat ($HAPI_SESSION_ID).' + ), + jobKey: z.string().trim().min(1).max(128).optional().describe( + 'Stable job key (alnum . _ -). Required for set/update/clear.' + ), + label: z.string().trim().min(1).max(200).optional().describe( + 'Short human label for the list chrome. Required for set.' + ), + status: z.enum(['running', 'completed', 'failed']).optional().describe( + 'Job status. Default running on set.' + ), + done: z.number().nonnegative().optional().describe('Units completed (pair with total when known).'), + total: z.number().positive().optional().describe('Total units when both ends of a fraction exist.'), + remaining: z.number().nonnegative().optional().describe('Units left — prefer when operator cares about leftover.'), + unit: z.string().trim().min(1).max(64).optional().describe('Unit label (tracks, folders, files, …).'), + detail: z.string().max(500).optional().describe('Stage / current item text (not an ETA).'), + startedAt: z.number().optional().describe( + 'Epoch ms process start. Only on set/upsert; omit on heartbeats. Correct late attach with explicit value.' + ) +}) + +export type SessionJobToolArgs = { + action: 'set' | 'update' | 'clear' | 'list' + sessionId?: string + jobKey?: string + label?: string + status?: 'running' | 'completed' | 'failed' + done?: number + total?: number + remaining?: number + unit?: string + detail?: string + startedAt?: number +} + +function formatJobLine(job: AttachedJob): string { + const parts = [`${job.key}`, job.label, job.status] + if (job.remaining !== undefined) { + parts.push(`${job.remaining}${job.unit ? ` ${job.unit}` : ''} left`) + } else if (job.done !== undefined && job.total !== undefined) { + parts.push(`${job.done}/${job.total}${job.unit ? ` ${job.unit}` : ''}`) + } + if (job.detail) parts.push(job.detail) + return parts.join(' · ') +} + +export async function handleSessionJobTool( + args: SessionJobToolArgs, + defaultSessionId: string +): Promise<{ text: string; isError: boolean }> { + const sessionIdPrefix = (args.sessionId?.trim() || defaultSessionId || process.env.HAPI_SESSION_ID || '').trim() + if (!sessionIdPrefix) { + return { + text: 'sessionId required (or set HAPI_SESSION_ID / call from a HAPI-wrapped session)', + isError: true + } + } + + try { + if (args.action === 'list') { + const result = await listSessionJobs({ sessionIdPrefix }) + if (result.jobs.length === 0) { + return { text: `session ${result.sessionId}\n(no jobs)`, isError: false } + } + const lines = result.jobs.map((job) => { + const mark = result.primary?.key === job.key ? '*' : ' ' + return `${mark} ${formatJobLine(job)}` + }) + return { text: `session ${result.sessionId}\n${lines.join('\n')}`, isError: false } + } + + if (!args.jobKey?.trim()) { + return { text: 'jobKey is required for set/update/clear', isError: true } + } + const jobKey = args.jobKey.trim() + + if (args.action === 'clear') { + const result = await clearSessionJob({ sessionIdPrefix, jobKey }) + return { text: `cleared ${jobKey} on ${result.sessionId}`, isError: false } + } + + if (args.action === 'set') { + if (!args.label?.trim()) { + return { text: 'label is required for action=set', isError: true } + } + const body: AttachedJobUpsert = { + label: args.label.trim(), + status: args.status ?? 'running', + ...(args.done !== undefined ? { done: args.done } : {}), + ...(args.total !== undefined ? { total: args.total } : {}), + ...(args.remaining !== undefined ? { remaining: args.remaining } : {}), + ...(args.unit !== undefined ? { unit: args.unit } : {}), + ...(args.detail !== undefined ? { detail: args.detail } : {}), + ...(args.startedAt !== undefined ? { startedAt: args.startedAt } : {}) + } + const result = await setSessionJob({ sessionIdPrefix, jobKey, body }) + return { + text: `set ${formatJobLine(result.job)} on ${result.sessionId}`, + isError: false + } + } + + // update + const body: AttachedJobPatch = { + ...(args.label !== undefined ? { label: args.label } : {}), + ...(args.status !== undefined ? { status: args.status } : {}), + ...(args.done !== undefined ? { done: args.done } : {}), + ...(args.total !== undefined ? { total: args.total } : {}), + ...(args.remaining !== undefined ? { remaining: args.remaining } : {}), + ...(args.unit !== undefined ? { unit: args.unit } : {}), + ...(args.detail !== undefined ? { detail: args.detail } : {}) + } + if (Object.keys(body).length === 0) { + return { + text: 'update requires at least one of label/status/done/total/remaining/unit/detail', + isError: true + } + } + const result = await updateSessionJob({ sessionIdPrefix, jobKey, body }) + return { + text: `updated ${formatJobLine(result.job)} on ${result.sessionId}`, + isError: false + } + } catch (error) { + const message = error instanceof SessionJobError + ? error.message + : error instanceof Error + ? error.message + : String(error) + return { text: `session_job failed: ${message}`, isError: true } + } +} diff --git a/docs/guide/session-jobs.md b/docs/guide/session-jobs.md index 90d85215fc..47048b1951 100644 --- a/docs/guide/session-jobs.md +++ b/docs/guide/session-jobs.md @@ -35,7 +35,20 @@ If the operator would reopen the chat only to ask "how's it doing?", it belongs ## Agent contract (specification) -HAPI does **not** write your batch scripts for you - but prefer the supervisor so heartbeats are not your problem: +Treat this like `ping_peer` / `inspect_peer`: it is first-class HAPI tooling, not a docs footnote. + +### MCP (preferred for agents) + +Tool name: `session_job` (Claude: `mcp__hapi__session_job`; Codex: `functions.hapi__session_job`; OpenCode/ACP: `hapi_session_job`). + +```json +{ "action": "set", "jobKey": "beets", "label": "beets import", + "remaining": 150, "done": 1637, "total": 1787, "unit": "units" } +``` + +Then `action=update` every ~10 minutes; finish with `status=completed|failed` or `action=clear`. Omit `sessionId` to target this chat. + +### CLI supervisor (preferred for shell children) ```bash hapi job run "$HAPI_SESSION_ID" beets \ @@ -47,7 +60,7 @@ hapi job run "$HAPI_SESSION_ID" beets \ `hapi job run` registers the job, heartbeats on a timer while the child runs, then marks `completed`/`failed` from the exit code. An idle agent **cannot** heartbeat - set-once + manual update decays to amber. -Manual path (only if you already have a self-heartbeating wrapper): +### CLI manual path ```bash hapi job set "$HAPI_SESSION_ID" beets \ From 0aa3e9e0b9cb766e7750f1ccbabbab332d90da69 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:10:31 +0000 Subject: [PATCH 10/94] =?UTF-8?q?feat(web):=20tri-state=20pin=20=E2=80=94?= =?UTF-8?q?=20default=20long-running=20jobs=20with=20#1404?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace boolean pin-in-progress with off / jobs / all. Unset preference defaults to jobs so outliving attached-job sessions float to In progress by default; legacy true→all, false→off. Settings Display gets a 3-way control. Co-authored-by: Cursor --- docs/guide/session-jobs.md | 12 ++ .../SessionList.directory-action.test.tsx | 70 ++++++++---- web/src/components/SessionList.tsx | 81 ++++++++++---- .../settings/SettingsPrimitives.tsx | 10 +- .../hooks/usePinInProgressSessions.test.ts | 28 +++++ web/src/hooks/usePinInProgressSessions.ts | 105 +++++++++++++----- web/src/lib/locales/en.ts | 5 +- web/src/lib/locales/zh-CN.ts | 5 +- web/src/routes/settings/display.tsx | 15 ++- web/src/routes/settings/index.test.tsx | 7 +- 10 files changed, 260 insertions(+), 78 deletions(-) create mode 100644 web/src/hooks/usePinInProgressSessions.test.ts diff --git a/docs/guide/session-jobs.md b/docs/guide/session-jobs.md index 47048b1951..5516ae5a6e 100644 --- a/docs/guide/session-jobs.md +++ b/docs/guide/session-jobs.md @@ -161,6 +161,18 @@ Needs a hub/CLI build that includes `job` (soup / feat — global npm releases m Primary running job is enriched onto `GET /api/sessions` as `attachedJob` and pushed on `session-updated` SSE patches. +## Sidebar pin (Settings → Display) + +Attached jobs ship with a **tri-state** "Pin in-progress sessions" control (not a yes/no): + +| Mode | Floats to In progress | +|------|------------------------| +| Off | Nothing | +| Long-running jobs (default) | Sessions with a running attached job (even when the agent is idle) | +| All activity | Jobs **plus** thinking / pending / in-agent background tasks | + +Unset preference defaults to **Long-running jobs** — that is the product stand for this capability. Legacy `true` maps to All activity; legacy `false` maps to Off. + ## Related - [Supported Agents](./agents.md) — flavors and resume diff --git a/web/src/components/SessionList.directory-action.test.tsx b/web/src/components/SessionList.directory-action.test.tsx index fb49c666fa..9aaa00ce79 100644 --- a/web/src/components/SessionList.directory-action.test.tsx +++ b/web/src/components/SessionList.directory-action.test.tsx @@ -13,7 +13,8 @@ const SEARCH_PLACEHOLDER = 'Search title/path/Agent/machine/ID…' afterEach(() => { cleanup() localStorage.removeItem('hapi-session-preview-limit') - localStorage.removeItem('hapi-pin-in-progress-sessions') + // Explicit off — unset now defaults to `jobs` (session-attached jobs stand). + localStorage.setItem('hapi-pin-in-progress-sessions', 'off') }) function makeSession(overrides: Partial & { id: string }): SessionSummary { @@ -392,7 +393,7 @@ describe('SessionList collapse behavior', () => { } it('keeps a selected running path collapsed across live session-list refreshes', async () => { - localStorage.setItem('hapi-pin-in-progress-sessions', 'true') + localStorage.setItem('hapi-pin-in-progress-sessions', 'all') const baseSessions = [ makeSession({ id: 'session-running', @@ -430,6 +431,7 @@ describe('SessionList collapse behavior', () => { }) it('leaves active sessions in directory groups when pin-in-progress is off', () => { + localStorage.setItem('hapi-pin-in-progress-sessions', 'off') const sessions = [ makeSession({ id: 'session-running', @@ -453,8 +455,8 @@ describe('SessionList collapse behavior', () => { expect(screen.getByTitle('/work/hapi').nextElementSibling?.getAttribute('data-open')).toBe('true') }) - it('pins active sessions into In progress when the preference is on', () => { - localStorage.setItem('hapi-pin-in-progress-sessions', 'true') + it('pins active sessions into In progress when mode is all activity', () => { + localStorage.setItem('hapi-pin-in-progress-sessions', 'all') const sessions = [ makeSession({ id: 'session-running', @@ -478,7 +480,7 @@ describe('SessionList collapse behavior', () => { }) it('keeps project-pinned active sessions in their project group when the preference is on', () => { - localStorage.setItem('hapi-pin-in-progress-sessions', 'true') + localStorage.setItem('hapi-pin-in-progress-sessions', 'all') const sessions = [ makeSession({ id: 'session-pinned-running', @@ -502,6 +504,7 @@ describe('SessionList collapse behavior', () => { }) it('keeps In progress above project-pin groups; project pin stays first inside its group', () => { + // Legacy true → all (tri-state); floater under In progress; project folders stay below. localStorage.setItem('hapi-pin-in-progress-sessions', 'true') const sessions = [ makeSession({ @@ -553,6 +556,41 @@ describe('SessionList collapse behavior', () => { expect(screen.getByRole('button', { name: /Unpinned floater/ })).toBeInTheDocument() }) + it('pins idle sessions with a running attachedJob when mode is jobs (default)', () => { + localStorage.setItem('hapi-pin-in-progress-sessions', 'jobs') + const sessions = [ + makeSession({ + id: 'session-beets', + active: false, + updatedAt: 100, + metadata: { path: '/music', name: 'Music drain', flavor: 'claude' }, + attachedJob: { + key: 'beets', + label: 'beets import', + status: 'running', + remaining: 12, + heartbeatAt: 1, + startedAt: 1, + updatedAt: 1, + }, + }), + makeSession({ + id: 'session-thinking', + active: true, + thinking: true, + updatedAt: 90, + metadata: { path: '/work/hapi', name: 'Thinking agent', flavor: 'codex' }, + }), + ] + render(renderSessionList(sessions, null)) + + expect(screen.getByTitle('In progress')).toBeInTheDocument() + expect(screen.getByRole('button', { name: /Music drain/ })).toBeInTheDocument() + // Thinking agent is not a long-running job — stays in directory under jobs mode. + expect(screen.getByTitle('/work/hapi')).toBeInTheDocument() + expect(screen.getByRole('button', { name: /Thinking agent/ })).toBeInTheDocument() + }) + it('does not label quiet active sessions as Idle', () => { const sessions = [ makeSession({ @@ -569,8 +607,8 @@ describe('SessionList collapse behavior', () => { expect(screen.queryByTitle('Idle')).toBeNull() }) - it('keeps quiet active sessions in the Active section when pin-in-progress is on', () => { - localStorage.setItem('hapi-pin-in-progress-sessions', 'true') + it('keeps quiet active sessions in directory groups when pin-in-progress is on', () => { + localStorage.setItem('hapi-pin-in-progress-sessions', 'all') const sessions = [ makeSession({ id: 'session-running', @@ -598,17 +636,11 @@ describe('SessionList collapse behavior', () => { expect(screen.getByTitle('In progress')).toBeInTheDocument() expect(screen.getByText(/Running \(1\)/)).toBeInTheDocument() expect(screen.getByText(/pending \(1\)/)).toBeInTheDocument() - // Quiet active sessions float into their own Active section (finished - // executing, still connected) instead of falling into directory groups. - expect(screen.getByTitle('Active sessions')).toBeInTheDocument() - expect(screen.getByText(/Active \(1\)/)).toBeInTheDocument() - expect(screen.getByRole('button', { name: /Quiet task/ })).toBeInTheDocument() - // The directory header survives as an action-only header (copy-path / - // new-session-in-directory) even though every row floated. + expect(screen.queryByText(/Idle \(/)).toBeNull() + // Quiet active stays under its project directory, not an Idle pin bucket. expect(screen.getByTitle('/work/hapi')).toBeInTheDocument() - expect(screen.getByTitle('/work/hapi').nextElementSibling).toBeNull() - expect(screen.getByTitle('/work/other')).toBeInTheDocument() - expect(screen.getByTitle('/work/other').nextElementSibling).toBeNull() + expect(screen.getByRole('button', { name: /Quiet task/ })).toBeInTheDocument() + expect(getProjectPanel().getAttribute('data-open')).toBe('true') }) it('keeps new-session-in-directory actions for projects whose rows all floated', () => { @@ -743,7 +775,7 @@ describe('SessionList collapse behavior', () => { }) it('keeps the running section open while searching even when collapsed', () => { - localStorage.setItem('hapi-pin-in-progress-sessions', 'true') + localStorage.setItem('hapi-pin-in-progress-sessions', 'all') const sessions = [ makeSession({ id: 'session-running', @@ -781,7 +813,7 @@ describe('SessionList collapse behavior', () => { }) it('toggles the running section with the keyboard', () => { - localStorage.setItem('hapi-pin-in-progress-sessions', 'true') + localStorage.setItem('hapi-pin-in-progress-sessions', 'all') const sessions = [ makeSession({ id: 'session-running', diff --git a/web/src/components/SessionList.tsx b/web/src/components/SessionList.tsx index d09f346e94..4c8a4b4bb6 100644 --- a/web/src/components/SessionList.tsx +++ b/web/src/components/SessionList.tsx @@ -33,7 +33,10 @@ import { useTranslation } from '@/lib/use-translation' import { DEFAULT_SESSION_PREVIEW_LIMIT, useSessionPreviewLimit } from '@/hooks/useSessionPreviewLimit' import { useSessionListStatusMode } from '@/hooks/useSessionListStatusMode' import { useShowActiveSessionsOnly } from '@/hooks/useShowActiveSessionsOnly' -import { usePinInProgressSessions } from '@/hooks/usePinInProgressSessions' +import { + usePinInProgressSessions, + type PinInProgressMode +} from '@/hooks/usePinInProgressSessions' import { classifySessionAttention, sessionIsUnread } from '@/lib/sessionAttention' import { getSessionLastSeenAt, @@ -88,39 +91,56 @@ const RUNNING_BUCKETS = [ type RunningBucketKey = (typeof RUNNING_BUCKETS)[number]['key'] +function hasRunningAttachedJob(session: SessionSummary): boolean { + return session.attachedJob?.status === 'running' +} + +function hasAgentInProgressActivity(session: SessionSummary): boolean { + if (!session.active) { + return false + } + return session.thinking + || (session.backgroundTaskCount ?? 0) > 0 + || (session.pendingRequestsCount ?? 0) > 0 +} + export function emptyRunningBuckets(): Record { return { working: [], pending: [], active: [], idle: [] } } /** - * Split the connected sessions into the in-progress / active sub-buckets the - * pinned sections render. Pure so the bucketing rules stay testable. + * Split sessions into the in-progress / active sub-buckets the pinned sections + * render. Pure so the bucketing rules stay testable. Quiet connected never + * floats (#1404); keepalive-idle zombies (#1820) also stay out unless they have + * agent activity or a running attached job. */ export function bucketRunningSessions( sessions: SessionSummary[], - pinInProgressSessions: boolean, + pinInProgressMode: PinInProgressMode, compare: (a: SessionSummary, b: SessionSummary) => number = (a, b) => b.updatedAt - a.updatedAt ): Record { const buckets = emptyRunningBuckets() - if (!pinInProgressSessions) { + if (pinInProgressMode === 'off') { return buckets } for (const session of sessions) { if (session.globalPinned || session.pinned) { continue } - if (!session.active) { + if (!isPinnedInProgressSession(session, pinInProgressMode)) { continue } - if (session.thinking || (session.backgroundTaskCount ?? 0) > 0) { + const agentWorking = session.thinking || (session.backgroundTaskCount ?? 0) > 0 + const agentPending = session.active + && (session.pendingRequestsCount ?? 0) > 0 + && !agentWorking + if (agentWorking || hasRunningAttachedJob(session)) { buckets.working.push(session) - } else if ((session.pendingRequestsCount ?? 0) > 0) { + } else if (agentPending) { buckets.pending.push(session) } else if (session.metadata?.lifecycleState === SESSION_LIFECYCLE_IDLE) { - // Keepalive-only: socket up, no agent progress for hours. buckets.idle.push(session) } else { - // Quiet but connected: finished executing, operator will continue. buckets.active.push(session) } } @@ -131,13 +151,20 @@ export function bucketRunningSessions( } /** - * Sessions that warrant the optional pinned top sections. - * Any connected session floats — a session that just finished executing stays - * visible at the top (Active tier) because the operator usually continues the - * conversation; only disconnected sessions fall into directory groups. + * Sessions that float into the pinned In progress section. + * Mode is a degree: off → jobs (outliving attachedJob) → all (jobs + agent activity). */ -function isPinnedInProgressSession(session: SessionSummary): boolean { - return session.active +export function isPinnedInProgressSession( + session: SessionSummary, + mode: PinInProgressMode +): boolean { + if (mode === 'off') { + return false + } + if (mode === 'jobs') { + return hasRunningAttachedJob(session) + } + return hasRunningAttachedJob(session) || hasAgentInProgressActivity(session) } export type SessionTimeRange = { @@ -1238,7 +1265,7 @@ export function SessionList(props: { const lastSeenVersion = useSessionLastSeenVersion() // Transient unread lens — not a Settings preference. Cleared on reload; rows drop as they're seen. const [showUnreadOnly, setShowUnreadOnly] = useState(false) - const { pinInProgressSessions } = usePinInProgressSessions() + const { pinInProgressMode } = usePinInProgressSessions() const { machineFilter, setMachineFilter } = useSessionListMachineFilter() const showDetailedStatus = sessionListStatusMode === 'detailed' const [searchQuery, setSearchQuery] = useState('') @@ -1386,8 +1413,8 @@ export function SessionList(props: { } return b.updatedAt - a.updatedAt } - return bucketRunningSessions(machineFilteredSessions, pinInProgressSessions, byRelevanceOrRecent) - }, [machineFilteredSessions, pinInProgressSessions, searchScoreIndex, hasTextQuery]) + return bucketRunningSessions(machineFilteredSessions, pinInProgressMode, byRelevanceOrRecent) + }, [machineFilteredSessions, pinInProgressMode, searchScoreIndex, hasTextQuery]) const runningSessionTotal = runningSessions.working.length + runningSessions.pending.length const activeSessionTotal = runningSessions.active.length + runningSessions.idle.length @@ -1396,7 +1423,15 @@ export function SessionList(props: { const grouped = groupSessionsByDirectory( machineFilteredSessions.filter((session) => { if (session.globalPinned) return false - if (pinInProgressSessions && !session.pinned && isPinnedInProgressSession(session)) return false + // Project-pinned stay in the project group; only unpinned + // "in progress" sessions float to the In progress section. + if ( + pinInProgressMode !== 'off' + && !session.pinned + && isPinnedInProgressSession(session, pinInProgressMode) + ) { + return false + } return true }) ) @@ -1405,7 +1440,7 @@ export function SessionList(props: { } return grouped }, - [machineFilteredSessions, pinInProgressSessions, searchScoreIndex, hasTextQuery] + [machineFilteredSessions, pinInProgressMode, searchScoreIndex, hasTextQuery] ) // Directory groups whose rows all floated to the pinned sections still // render an action-only header so copy-path / new-session-in-directory @@ -1419,12 +1454,12 @@ export function SessionList(props: { [machineFilteredSessions] ) const actionOnlyGroups = useMemo(() => { - if (!pinInProgressSessions) { + if (pinInProgressMode === 'off') { return [] } const visibleKeys = new Set(groups.map((group) => group.key)) return allDirectoryGroups.filter((group) => !visibleKeys.has(group.key)) - }, [groups, allDirectoryGroups, pinInProgressSessions]) + }, [groups, allDirectoryGroups, pinInProgressMode]) const [collapseOverrides, setCollapseOverrides] = useState>( () => new Map() ) diff --git a/web/src/components/settings/SettingsPrimitives.tsx b/web/src/components/settings/SettingsPrimitives.tsx index 4bc32f91c4..e60655720d 100644 --- a/web/src/components/settings/SettingsPrimitives.tsx +++ b/web/src/components/settings/SettingsPrimitives.tsx @@ -82,9 +82,15 @@ export function SettingsChoiceGroup(props: { value: T options: ReadonlyArray<{ value: T; label: string; description?: string }> onChange: (value: T) => void - columns?: 2 | 4 | 5 + columns?: 2 | 3 | 4 | 5 }) { - const columns = props.columns === 5 ? 'grid-cols-5' : props.columns === 4 ? 'grid-cols-2 sm:grid-cols-4' : 'grid-cols-2' + const columns = props.columns === 5 + ? 'grid-cols-5' + : props.columns === 4 + ? 'grid-cols-2 sm:grid-cols-4' + : props.columns === 3 + ? 'grid-cols-3' + : 'grid-cols-2' return (
diff --git a/web/src/hooks/usePinInProgressSessions.test.ts b/web/src/hooks/usePinInProgressSessions.test.ts new file mode 100644 index 0000000000..d008ee0a43 --- /dev/null +++ b/web/src/hooks/usePinInProgressSessions.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' +import { + DEFAULT_PIN_IN_PROGRESS_MODE, + parsePinInProgressMode +} from './usePinInProgressSessions' + +describe('parsePinInProgressMode', () => { + it('defaults unset to jobs (capability stand)', () => { + expect(parsePinInProgressMode(null)).toBe('jobs') + expect(parsePinInProgressMode('')).toBe('jobs') + expect(DEFAULT_PIN_IN_PROGRESS_MODE).toBe('jobs') + }) + + it('migrates legacy boolean strings', () => { + expect(parsePinInProgressMode('true')).toBe('all') + expect(parsePinInProgressMode('false')).toBe('off') + }) + + it('accepts explicit tri-state values', () => { + expect(parsePinInProgressMode('off')).toBe('off') + expect(parsePinInProgressMode('jobs')).toBe('jobs') + expect(parsePinInProgressMode('all')).toBe('all') + }) + + it('falls back to jobs on garbage', () => { + expect(parsePinInProgressMode('maybe')).toBe('jobs') + }) +}) diff --git a/web/src/hooks/usePinInProgressSessions.ts b/web/src/hooks/usePinInProgressSessions.ts index 8f3bfc225a..ec1886dc7b 100644 --- a/web/src/hooks/usePinInProgressSessions.ts +++ b/web/src/hooks/usePinInProgressSessions.ts @@ -1,10 +1,24 @@ import { useCallback, useEffect, useState } from 'react' -export const DEFAULT_PIN_IN_PROGRESS_SESSIONS = false +/** + * Sidebar "In progress" pin policy (ships with session-attached jobs #1404). + * + * Degree of float, not a yes/no: + * - off — everything stays in project directories + * - jobs — only sessions with a running attachedJob (outliving work) + * - all — jobs + agent working/pending (legacy maximalist pin) + * + * Unset / never configured defaults to `jobs` — the product stand for this capability. + */ -function getPinInProgressSessionsStorageKey(): string { - return 'hapi-pin-in-progress-sessions' -} +export type PinInProgressMode = 'off' | 'jobs' | 'all' + +export const PIN_IN_PROGRESS_MODES: readonly PinInProgressMode[] = ['off', 'jobs', 'all'] as const + +/** New default when the preference has never been set. */ +export const DEFAULT_PIN_IN_PROGRESS_MODE: PinInProgressMode = 'jobs' + +export const PIN_IN_PROGRESS_STORAGE_KEY = 'hapi-pin-in-progress-sessions' function isBrowser(): boolean { return typeof window !== 'undefined' && typeof document !== 'undefined' @@ -32,33 +46,60 @@ function safeSetItem(key: string, value: string): void { } } -function safeRemoveItem(key: string): void { - if (!isBrowser()) { - return +/** + * Parse stored value. + * - absent / null → `jobs` (capability default) + * - legacy `true` → `all` + * - legacy `false` → `off` + * - `off` | `jobs` | `all` → as written + */ +export function parsePinInProgressMode(raw: string | null): PinInProgressMode { + if (raw === null || raw === '') { + return DEFAULT_PIN_IN_PROGRESS_MODE } - try { - localStorage.removeItem(key) - } catch { - // Ignore storage errors + if (raw === 'true') { + return 'all' + } + if (raw === 'false') { + return 'off' } + if (raw === 'off' || raw === 'jobs' || raw === 'all') { + return raw + } + return DEFAULT_PIN_IN_PROGRESS_MODE } -function parsePinInProgressSessions(raw: string | null): boolean { - if (raw === 'true') { - return true - } - return DEFAULT_PIN_IN_PROGRESS_SESSIONS +export function getInitialPinInProgressMode(): PinInProgressMode { + return parsePinInProgressMode(safeGetItem(PIN_IN_PROGRESS_STORAGE_KEY)) } +/** @deprecated Use getInitialPinInProgressMode — boolean form treated `all` as true. */ export function getInitialPinInProgressSessions(): boolean { - return parsePinInProgressSessions(safeGetItem(getPinInProgressSessionsStorageKey())) + return getInitialPinInProgressMode() !== 'off' +} + +export function getPinInProgressModeOptions(): ReadonlyArray<{ + value: PinInProgressMode + labelKey: string +}> { + return [ + { value: 'off', labelKey: 'settings.display.pinInProgressMode.off' }, + { value: 'jobs', labelKey: 'settings.display.pinInProgressMode.jobs' }, + { value: 'all', labelKey: 'settings.display.pinInProgressMode.all' }, + ] } export function usePinInProgressSessions(): { + pinInProgressMode: PinInProgressMode + setPinInProgressMode: (value: PinInProgressMode) => void + /** True when mode is not off (any pin bucket may show). */ pinInProgressSessions: boolean + /** @deprecated Prefer setPinInProgressMode. `true`→all, `false`→off. */ setPinInProgressSessions: (value: boolean) => void } { - const [pinInProgressSessions, setPinInProgressSessionsState] = useState(getInitialPinInProgressSessions) + const [pinInProgressMode, setPinInProgressModeState] = useState( + getInitialPinInProgressMode + ) useEffect(() => { if (!isBrowser()) { @@ -66,25 +107,31 @@ export function usePinInProgressSessions(): { } const onStorage = (event: StorageEvent) => { - if (event.key !== getPinInProgressSessionsStorageKey()) { + if (event.key !== PIN_IN_PROGRESS_STORAGE_KEY) { return } - setPinInProgressSessionsState(parsePinInProgressSessions(event.newValue)) + setPinInProgressModeState(parsePinInProgressMode(event.newValue)) } window.addEventListener('storage', onStorage) return () => window.removeEventListener('storage', onStorage) }, []) - const setPinInProgressSessions = useCallback((value: boolean) => { - setPinInProgressSessionsState(value) - - if (value === DEFAULT_PIN_IN_PROGRESS_SESSIONS) { - safeRemoveItem(getPinInProgressSessionsStorageKey()) - } else { - safeSetItem(getPinInProgressSessionsStorageKey(), String(value)) - } + const setPinInProgressMode = useCallback((value: PinInProgressMode) => { + setPinInProgressModeState(value) + // Always persist so an explicit Off is distinct from never-set→jobs default + // after the user has opened Settings and chosen. + safeSetItem(PIN_IN_PROGRESS_STORAGE_KEY, value) }, []) - return { pinInProgressSessions, setPinInProgressSessions } + const setPinInProgressSessions = useCallback((value: boolean) => { + setPinInProgressMode(value ? 'all' : 'off') + }, [setPinInProgressMode]) + + return { + pinInProgressMode, + setPinInProgressMode, + pinInProgressSessions: pinInProgressMode !== 'off', + setPinInProgressSessions + } } diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index cfdc81fb6c..282a95c807 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -876,7 +876,10 @@ export default { 'settings.display.activeSessionsOnly': 'Active sessions only', 'settings.display.activeSessionsOnly.desc': 'Hide inactive sessions in the sidebar. The session you have open stays visible.', 'settings.display.pinInProgressSessions': 'Pin in-progress sessions', - 'settings.display.pinInProgressSessions.desc': 'Move unpinned connected sessions into sections above project folders: running and pending work first, then quiet active sessions (finished executing, still connected). Global pins remain above them. Off keeps everything in directory groups.', + 'settings.display.pinInProgressSessions.desc': 'How loudly sessions float to the top In progress section above project folders. Default is long-running jobs (outliving batch work with a session job meter). Global pins remain above it; quiet active agents stay in project folders unless you choose All activity. Off keeps everything in directory groups.', + 'settings.display.pinInProgressMode.off': 'Off', + 'settings.display.pinInProgressMode.jobs': 'Long-running jobs', + 'settings.display.pinInProgressMode.all': 'All activity', 'settings.display.appBadge': 'Taskbar unread badge', 'settings.display.appBadge.desc': 'Show the number of sessions with new activity on an installed Edge or Chrome PWA. The host browser controls its appearance; this has no effect in a normal browser tab.', 'settings.display.sessionListStatus': 'Session list status hints', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index 2c3bf40054..fe0a5d8b36 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -874,7 +874,10 @@ export default { 'settings.display.activeSessionsOnly': '仅显示活跃会话', 'settings.display.activeSessionsOnly.desc': '在侧边栏隐藏非活跃会话；当前打开的会话仍会保留显示。', 'settings.display.pinInProgressSessions': '置顶进行中会话', - 'settings.display.pinInProgressSessions.desc': '将未置顶的已连接会话移到项目文件夹上方的分区：先运行中和待处理，再是安静的活跃会话（已执行完但仍在连接）。全局置顶仍在其上。关闭后全部保留在目录分组中。', + 'settings.display.pinInProgressSessions.desc': '侧边栏顶部「进行中」分区的置顶程度（位于项目目录分组之上）。默认为长时间任务（带会话 job 进度的后台批处理）。全局置顶仍在该分区之上；安静的活跃智能体仍留在项目目录中，除非选择「全部活动」。关闭后全部保留在目录分组中。', + 'settings.display.pinInProgressMode.off': '关闭', + 'settings.display.pinInProgressMode.jobs': '长时间任务', + 'settings.display.pinInProgressMode.all': '全部活动', 'settings.display.appBadge': '任务栏未读角标', 'settings.display.appBadge.desc': '在已安装的 Edge 或 Chrome PWA 图标上显示有新活动的会话数。角标样式由宿主浏览器控制；普通浏览器标签页不受影响。', 'settings.display.sessionListStatus': '会话列表状态提示', diff --git a/web/src/routes/settings/display.tsx b/web/src/routes/settings/display.tsx index 92f23f50d1..db83c7a86d 100644 --- a/web/src/routes/settings/display.tsx +++ b/web/src/routes/settings/display.tsx @@ -139,7 +139,7 @@ export default function SettingsDisplayPage() { const { terminalFontSize, setTerminalFontSize } = useTerminalFontSize() const { sessionListStatusMode, setSessionListStatusMode } = useSessionListStatusMode() const { showActiveSessionsOnly, setShowActiveSessionsOnly } = useShowActiveSessionsOnly() - const { pinInProgressSessions, setPinInProgressSessions } = usePinInProgressSessions() + const { pinInProgressMode, setPinInProgressMode } = usePinInProgressSessions() const { appBadgeEnabled, setAppBadgeEnabled } = useAppBadgePreference() const { openExternalLinksInNewTab, setOpenExternalLinksInNewTab } = useOpenExternalLinksInNewTab() const { preferences: sessionHeaderMetadata, setPreference: setSessionHeaderMetadata } = useSessionHeaderMetadata() @@ -178,7 +178,18 @@ export default function SettingsDisplayPage() { - + ({ })) vi.mock('@/hooks/usePinInProgressSessions', () => ({ - usePinInProgressSessions: () => ({ pinInProgressSessions: false, setPinInProgressSessions: vi.fn() }), + usePinInProgressSessions: () => ({ + pinInProgressMode: 'off' as const, + setPinInProgressMode: vi.fn(), + pinInProgressSessions: false, + setPinInProgressSessions: vi.fn(), + }), })) vi.mock('@/hooks/useAppBadgePreference', () => ({ From e96985c1a336349a5c23dcf59b412071ddbcbdb9 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:37:05 +0000 Subject: [PATCH 11/94] fix(jobs): cold-review pass 3 Blocker + Majors (#1404) CI: bump migration tip expects to SCHEMA 22; stub list attachedJobs mocks; align Codex/OpenCode/ACP fixtures with job steer + session_job tool list. Merge: write jobsAccepted/Transferred redirects after metadata merge so a name/summary copy cannot clobber the post-merge job owner map. MCP: session_job is own-session only (drop sessionId from schema) so auto-approve cannot silently write peer meters. Prefer hapi job run in steer. Co-authored-by: Cursor --- cli/src/codex/utils/appServerConfig.test.ts | 20 ++++++------- .../permission/BasePermissionHandler.ts | 5 ++-- .../modules/common/sessionJobInstruction.ts | 11 +++---- .../modules/sessionJob/sessionJobMcp.test.ts | 10 ++++++- cli/src/modules/sessionJob/sessionJobMcp.ts | 30 +++++++++---------- cli/src/opencode/utils/systemPrompt.test.ts | 2 +- hub/src/sync/sessionCache-merge-jobs.test.ts | 29 ++++++++++++++++++ hub/src/sync/sessionCache.ts | 17 +++++++---- hub/src/web/routes/sessions.test.ts | 5 +++- web/src/components/SessionList.tsx | 9 ++++-- .../hooks/usePinInProgressSessions.test.ts | 22 +++++++++++++- web/src/hooks/usePinInProgressSessions.ts | 10 ++++++- 12 files changed, 124 insertions(+), 46 deletions(-) diff --git a/cli/src/codex/utils/appServerConfig.test.ts b/cli/src/codex/utils/appServerConfig.test.ts index d45e677f92..71b2d03575 100644 --- a/cli/src/codex/utils/appServerConfig.test.ts +++ b/cli/src/codex/utils/appServerConfig.test.ts @@ -7,7 +7,7 @@ import { codexCollaborationSpawnAgentInstructions, supportsReasoningSummary } from './appServerConfig'; -import { codexSystemPrompt } from './systemPrompt'; +import { getCodexSystemPrompt } from './systemPrompt'; describe('appServerConfig', () => { const mcpServers = { hapi: { command: 'node', args: ['mcp'] } }; @@ -56,7 +56,7 @@ describe('appServerConfig', () => { command: 'node', args: ['mcp'] }, - developer_instructions: codexSystemPrompt + developer_instructions: getCodexSystemPrompt() }); }); @@ -98,7 +98,7 @@ describe('appServerConfig', () => { } } }, - developer_instructions: codexSystemPrompt + developer_instructions: getCodexSystemPrompt() }); }); @@ -201,7 +201,7 @@ describe('appServerConfig', () => { command: 'node', args: ['mcp'] }, - developer_instructions: `${codexSystemPrompt}\n\nOnly respond in Chinese.` + developer_instructions: `${getCodexSystemPrompt()}\n\nOnly respond in Chinese.` }); }); @@ -217,7 +217,7 @@ describe('appServerConfig', () => { command: 'node', args: ['mcp'] }, - developer_instructions: codexSystemPrompt, + developer_instructions: getCodexSystemPrompt(), model_reasoning_effort: 'ultra' }); }); @@ -383,7 +383,7 @@ describe('appServerConfig', () => { settings: { model: 'o3', reasoning_effort: 'high', - developer_instructions: withCollaborationInstructions(codexSystemPrompt) + developer_instructions: withCollaborationInstructions(getCodexSystemPrompt()) } }); expect(params.model).toBeUndefined(); @@ -409,7 +409,7 @@ describe('appServerConfig', () => { settings: { model: 'gpt-5.3-codex-spark', reasoning_effort: 'high', - developer_instructions: withCollaborationInstructions(codexSystemPrompt) + developer_instructions: withCollaborationInstructions(getCodexSystemPrompt()) } }); }); @@ -573,7 +573,7 @@ describe('appServerConfig', () => { mode: 'default', settings: { model: 'o3', - developer_instructions: withCollaborationInstructions(codexSystemPrompt) + developer_instructions: withCollaborationInstructions(getCodexSystemPrompt()) } }); }); @@ -593,7 +593,7 @@ describe('appServerConfig', () => { mode: 'default', settings: { model: 'o3', - developer_instructions: withCollaborationInstructions(codexSystemPrompt) + developer_instructions: withCollaborationInstructions(getCodexSystemPrompt()) } }); }); @@ -612,7 +612,7 @@ describe('appServerConfig', () => { mode: 'default', settings: { model: 'gpt-5', - developer_instructions: withCollaborationInstructions(codexSystemPrompt) + developer_instructions: withCollaborationInstructions(getCodexSystemPrompt()) } }); expect(params.model).toBeUndefined(); diff --git a/cli/src/modules/common/permission/BasePermissionHandler.ts b/cli/src/modules/common/permission/BasePermissionHandler.ts index 27924f0388..62bb54e75e 100644 --- a/cli/src/modules/common/permission/BasePermissionHandler.ts +++ b/cli/src/modules/common/permission/BasePermissionHandler.ts @@ -38,7 +38,8 @@ const AUTO_APPROVE_EXACT_TOOL_NAMES = new Set([ 'mcp__hapi__list_peers', // ACP permission requests often surface MCP tool title, not the snake_case name. 'list peer sessions', - // Own-session progress meter (tiann/hapi#1404) — hub REST only, not peer inject. + // Own-session progress meter (tiann/hapi#1404) — MCP schema has no sessionId; + // tool always targets this chat. Cross-session writes use CLI hapi job (not auto). 'session_job', 'hapi_session_job', 'happy__session_job', @@ -49,7 +50,7 @@ const AUTO_APPROVE_EXACT_TOOL_NAMES = new Set([ // resume+inject into another session or read peer histories, so permission // modes must still gate them. Treat both as write-like in read-only so ACP // titles such as "Ping Peer Session" / "Inspect Peer Session" also require -// approval. list_peers / session_job are auto-approved above. +// approval. list_peers / own-session session_job are auto-approved above. const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'save_memory']; const SENSITIVE_TOOL_NAME_HINTS = [ 'ping_peer', diff --git a/cli/src/modules/common/sessionJobInstruction.ts b/cli/src/modules/common/sessionJobInstruction.ts index c400615455..aaec5f96d7 100644 --- a/cli/src/modules/common/sessionJobInstruction.ts +++ b/cli/src/modules/common/sessionJobInstruction.ts @@ -14,11 +14,12 @@ export const SESSION_JOB_INSTRUCTION = [ '(batch imports, long scripts, external daemons), attach a session job so', 'the session list shows progress while you are idle — same class of HAPI', 'tooling as ping_peer / inspect_peer.', - 'Prefer MCP tool session_job (aliases: mcp__hapi__session_job, hapi_session_job,', - 'functions.hapi__session_job) with action=set before the process starts, then', - 'action=update every ~10m (idle agents cannot heartbeat).', - 'For a supervised shell child: hapi job run "$HAPI_SESSION_ID" ', - '--label -- … (auto-heartbeats + completed/failed on exit).', + 'Prefer supervised CLI for process-shaped work (idle agents cannot heartbeat):', + 'hapi job run "$HAPI_SESSION_ID" --label -- …', + '(auto-heartbeats + completed/failed on exit).', + 'Manual path: MCP tool session_job (aliases: mcp__hapi__session_job,', + 'hapi_session_job, functions.hapi__session_job) action=set, then action=update', + 'every ~10m from a self-heartbeating wrapper — never set-once and walk away.', 'Prefer honest remaining or done+total; omit counts when unknown', '(UI shows "running" + elapsed). Never invent a fake percent.', 'Full contract: hapi job --help.' diff --git a/cli/src/modules/sessionJob/sessionJobMcp.test.ts b/cli/src/modules/sessionJob/sessionJobMcp.test.ts index 87ac653dc6..b9a4fe95de 100644 --- a/cli/src/modules/sessionJob/sessionJobMcp.test.ts +++ b/cli/src/modules/sessionJob/sessionJobMcp.test.ts @@ -44,13 +44,21 @@ describe('sessionJobMcp', () => { expect(SESSION_JOB_TOOL_DESCRIPTION).toContain('hapi job run') }) - it('set requires label and defaults session to caller id', async () => { + it('set requires label and always targets the caller session id', async () => { + const { setSessionJob } = await import('./sessionJob') const result = await handleSessionJobTool( { action: 'set', jobKey: 'beets', label: 'beets import', remaining: 12 }, 'sid-1' ) expect(result.isError).toBe(false) expect(result.text).toContain('set beets') + expect(setSessionJob).toHaveBeenCalledWith( + expect.objectContaining({ sessionIdPrefix: 'sid-1' }) + ) + }) + + it('description claims own-session only', () => { + expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/Own-session only/i) }) it('rejects update with empty patch', async () => { diff --git a/cli/src/modules/sessionJob/sessionJobMcp.ts b/cli/src/modules/sessionJob/sessionJobMcp.ts index 519f064a9b..7f689f89bd 100644 --- a/cli/src/modules/sessionJob/sessionJobMcp.ts +++ b/cli/src/modules/sessionJob/sessionJobMcp.ts @@ -20,26 +20,23 @@ export const SESSION_JOB_TOOL_NAME = 'session_job' * Write for selection, not for humans browsing a README. */ export const SESSION_JOB_TOOL_DESCRIPTION = [ - 'Attach or update a hub-persisted progress meter on a HAPI session for work that', + 'Attach or update a hub-persisted progress meter on THIS HAPI session for work that', 'OUTLIVES this agent turn (nohup / batch import / rclone / compile / long drain /', - 'external daemon). The session list shows the meter while the agent is idle', - '(active:false). Call action=set BEFORE starting that process (or immediately when', - 'you start it). Heartbeat with action=update at least every ~10 minutes while it', - 'runs — an idle agent cannot. Prefer honest remaining or done+total; omit counts', - 'when unknown (UI shows "running" + elapsed). Never invent a percent or ETA.', - 'Finish with action=update status=completed|failed or action=clear.', - 'Default sessionId is this chat ($HAPI_SESSION_ID). Not for in-agent todos,', - 'thinking progress, or short tool calls. For a supervised shell child that', - 'auto-heartbeats, prefer CLI: hapi job run "$HAPI_SESSION_ID" --label … -- .', + 'external daemon). Own-session only (auto-approved) — not for injecting meters onto', + 'peer sessions (use CLI hapi job for that). The session list shows the meter while', + 'the agent is idle (active:false). Prefer CLI for process-shaped work:', + 'hapi job run "$HAPI_SESSION_ID" --label … -- (auto-heartbeats).', + 'Manual: action=set BEFORE starting the process, then action=update at least every', + '~10 minutes from a self-heartbeating wrapper — an idle agent cannot. Prefer honest', + 'remaining or done+total; omit counts when unknown (UI shows "running" + elapsed).', + 'Never invent a percent or ETA. Finish with action=update status=completed|failed', + 'or action=clear. Not for in-agent todos, thinking progress, or short tool calls.', ].join(' ') export const sessionJobInputSchema: z.ZodTypeAny = z.object({ action: z.enum(['set', 'update', 'clear', 'list']).describe( 'set=register/upsert running job; update=heartbeat/progress/status; clear=remove; list=show jobs' ), - sessionId: z.string().trim().min(1).optional().describe( - 'Target session id or prefix. Omit to use this chat ($HAPI_SESSION_ID).' - ), jobKey: z.string().trim().min(1).max(128).optional().describe( 'Stable job key (alnum . _ -). Required for set/update/clear.' ), @@ -61,7 +58,6 @@ export const sessionJobInputSchema: z.ZodTypeAny = z.object({ export type SessionJobToolArgs = { action: 'set' | 'update' | 'clear' | 'list' - sessionId?: string jobKey?: string label?: string status?: 'running' | 'completed' | 'failed' @@ -88,10 +84,12 @@ export async function handleSessionJobTool( args: SessionJobToolArgs, defaultSessionId: string ): Promise<{ text: string; isError: boolean }> { - const sessionIdPrefix = (args.sessionId?.trim() || defaultSessionId || process.env.HAPI_SESSION_ID || '').trim() + // Own-session only — sessionId is not in the schema so auto-approve cannot + // become a silent cross-session write (cold-review pass 3 Major). + const sessionIdPrefix = (defaultSessionId || process.env.HAPI_SESSION_ID || '').trim() if (!sessionIdPrefix) { return { - text: 'sessionId required (or set HAPI_SESSION_ID / call from a HAPI-wrapped session)', + text: 'own session id required (set HAPI_SESSION_ID / call from a HAPI-wrapped session)', isError: true } } diff --git a/cli/src/opencode/utils/systemPrompt.test.ts b/cli/src/opencode/utils/systemPrompt.test.ts index 483c901075..0adc0654f7 100644 --- a/cli/src/opencode/utils/systemPrompt.test.ts +++ b/cli/src/opencode/utils/systemPrompt.test.ts @@ -26,7 +26,7 @@ describe('OpenCode local HAPI instructions', () => { const instructions = await readFile(instructionsPath, 'utf8') expect(instructions).toContain('$name') expect(instructions).toContain('skill_lookup') - expect(instructions).toContain('hapi job set') + expect(instructions).toContain('hapi job run') expect(instructions).toContain(TITLE_INSTRUCTION.trim()) }) }) diff --git a/hub/src/sync/sessionCache-merge-jobs.test.ts b/hub/src/sync/sessionCache-merge-jobs.test.ts index 9171a12fbc..5b814cb418 100644 --- a/hub/src/sync/sessionCache-merge-jobs.test.ts +++ b/hub/src/sync/sessionCache-merge-jobs.test.ts @@ -73,6 +73,35 @@ describe('mergeSessions job redirect through SessionCache (#1404)', () => { expect(cache.resolveAttachedJobSessionId(newSession.id, 'default')).toBe(newSession.id) }) + it('keeps jobsAcceptedFromSessionIds when metadata merge also copies name from old', async () => { + const { store, cache } = setup() + const oldSession = cache.getOrCreateSession( + 'agent-jobs-named-old-' + Math.random().toString(36).slice(2, 8), + { path: '/tmp/project', host: 'localhost', flavor: 'codex', name: 'Lidarr drain' }, + null, + 'default' + ) + const newSession = cache.getOrCreateSession( + 'agent-jobs-named-new-' + Math.random().toString(36).slice(2, 8), + { path: '/tmp/project', host: 'localhost', flavor: 'codex' }, + null, + 'default' + ) + + store.sessionJobs.upsert(oldSession.id, 'beets', { + label: 'beets import', + status: 'running', + remaining: 4 + }) + + await cache.mergeSessions(oldSession.id, newSession.id, 'default') + + const refreshed = cache.refreshSession(newSession.id) + expect(refreshed?.metadata?.name).toBe('Lidarr drain') + expect(refreshed?.metadata?.jobsAcceptedFromSessionIds).toContain(oldSession.id) + expect(cache.resolveAttachedJobSessionId(oldSession.id, 'default')).toBe(newSession.id) + }) + it('keeps jobsTransferredToSessionId on a kept-alive source after mergeSessionHistory', async () => { const { store, cache } = setup() const { oldSession, newSession } = makeSessions(cache) diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 14bfae8d9b..3f23e8c3c5 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -1316,12 +1316,6 @@ export class SessionCache { const movedScratchlist = this.store.scratchlist.transfer(oldSessionId, newSessionId) const movedJobs = this.store.sessionJobs.transfer(oldSessionId, newSessionId) if (movedJobs.moved > 0 || movedJobs.collided > 0) { - // Agents keep addressing $HAPI_SESSION_ID from the pre-merge row. - // Record redirects so job REST routes can follow the live job owner. - this.recordJobsAcceptedFromSession(newSessionId, oldSessionId, namespace) - if (!options.deleteOldSession) { - this.recordJobsTransferredToSession(oldSessionId, newSessionId, namespace) - } this.emitAttachedJobChanged( newSessionId, this.store.sessionJobs.getPrimaryRunning(newSessionId) @@ -1397,6 +1391,17 @@ export class SessionCache { } } + // Job-owner redirects AFTER metadata merge. Writing them before the + // merge clobbers jobsAcceptedFromSessionIds when mergeSessionMetadata + // rebuilds from the stale pre-merge newStored.metadata snapshot + // (cold-review pass 3 Major — agents heartbeating $HAPI_SESSION_ID 404). + if (movedJobs.moved > 0 || movedJobs.collided > 0) { + this.recordJobsAcceptedFromSession(newSessionId, oldSessionId, namespace) + if (!options.deleteOldSession) { + this.recordJobsTransferredToSession(oldSessionId, newSessionId, namespace) + } + } + if (newStored.model === null && oldStored.model !== null) { const updated = this.store.sessions.setSessionModel(newSessionId, oldStored.model, namespace, { touchUpdatedAt: false diff --git a/hub/src/web/routes/sessions.test.ts b/hub/src/web/routes/sessions.test.ts index f82520f771..c798a6de21 100644 --- a/hub/src/web/routes/sessions.test.ts +++ b/hub/src/web/routes/sessions.test.ts @@ -180,7 +180,8 @@ function createApp(session: Session, opts?: { implementCodexPlan: opts?.implementCodexPlan, rewindConversation: opts?.rewindConversation ?? (async () => ({ type: 'success' })), suggestSessionTitle: opts?.suggestSessionTitle ?? (async () => 'Generated title'), - updateSessionSummary: opts?.updateSessionSummary ?? (async () => {}) + updateSessionSummary: opts?.updateSessionSummary ?? (async () => {}), + getPrimaryAttachedJobsBySessionIds: opts?.getPrimaryAttachedJobsBySessionIds ?? (() => new Map()) } as Partial const app = new Hono() @@ -1701,6 +1702,7 @@ describe('sessions routes', () => { return new Map(ids.map((id) => [id, 0])) }, getNextScheduledAtBySessionIds: (_ids: string[]) => new Map(), + getPrimaryAttachedJobsBySessionIds: () => new Map(), resolveSessionAccess: () => ({ ok: false, reason: 'not-found' as const }) } as unknown as Partial @@ -1733,6 +1735,7 @@ describe('sessions routes', () => { getSessionsByNamespace: () => sessions, getFutureScheduledMessageCounts: (ids: string[]) => new Map(ids.map((id) => [id, 0])), getNextScheduledAtBySessionIds: (_ids: string[]) => new Map(), + getPrimaryAttachedJobsBySessionIds: () => new Map(), resolveSessionAccess: () => ({ ok: false, reason: 'not-found' as const }) } as unknown as Partial diff --git a/web/src/components/SessionList.tsx b/web/src/components/SessionList.tsx index 4c8a4b4bb6..b3921c54c0 100644 --- a/web/src/components/SessionList.tsx +++ b/web/src/components/SessionList.tsx @@ -329,9 +329,14 @@ export function prepareSidebarSessions(sessions: SessionSummary[], selectedSessi // "Active sessions only" view: hide inactive sessions, but never hide the one the // operator currently has open — otherwise toggling the filter would yank the -// selected session out from under them. +// selected session out from under them. Idle sessions with a running attached +// job stay visible too — that is the headline use case for session jobs. export function filterActiveSessionsOnly(sessions: SessionSummary[], selectedSessionId?: string | null): SessionSummary[] { - return sessions.filter(session => session.active || session.id === selectedSessionId) + return sessions.filter(session => + session.active + || session.id === selectedSessionId + || hasRunningAttachedJob(session) + ) } // Transient unread lens: hide sessions the operator has already seen. diff --git a/web/src/hooks/usePinInProgressSessions.test.ts b/web/src/hooks/usePinInProgressSessions.test.ts index d008ee0a43..e6acff9ca4 100644 --- a/web/src/hooks/usePinInProgressSessions.test.ts +++ b/web/src/hooks/usePinInProgressSessions.test.ts @@ -1,6 +1,8 @@ -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it } from 'vitest' import { DEFAULT_PIN_IN_PROGRESS_MODE, + PIN_IN_PROGRESS_STORAGE_KEY, + getInitialPinInProgressMode, parsePinInProgressMode } from './usePinInProgressSessions' @@ -26,3 +28,21 @@ describe('parsePinInProgressMode', () => { expect(parsePinInProgressMode('maybe')).toBe('jobs') }) }) + +describe('getInitialPinInProgressMode', () => { + afterEach(() => { + localStorage.removeItem(PIN_IN_PROGRESS_STORAGE_KEY) + }) + + it('persists jobs default so later Off is distinct from unset', () => { + localStorage.removeItem(PIN_IN_PROGRESS_STORAGE_KEY) + expect(getInitialPinInProgressMode()).toBe('jobs') + expect(localStorage.getItem(PIN_IN_PROGRESS_STORAGE_KEY)).toBe('jobs') + }) + + it('rewrites legacy false to persisted off', () => { + localStorage.setItem(PIN_IN_PROGRESS_STORAGE_KEY, 'false') + expect(getInitialPinInProgressMode()).toBe('off') + expect(localStorage.getItem(PIN_IN_PROGRESS_STORAGE_KEY)).toBe('off') + }) +}) diff --git a/web/src/hooks/usePinInProgressSessions.ts b/web/src/hooks/usePinInProgressSessions.ts index ec1886dc7b..1c6c1dc356 100644 --- a/web/src/hooks/usePinInProgressSessions.ts +++ b/web/src/hooks/usePinInProgressSessions.ts @@ -70,7 +70,15 @@ export function parsePinInProgressMode(raw: string | null): PinInProgressMode { } export function getInitialPinInProgressMode(): PinInProgressMode { - return parsePinInProgressMode(safeGetItem(PIN_IN_PROGRESS_STORAGE_KEY)) + const raw = safeGetItem(PIN_IN_PROGRESS_STORAGE_KEY) + const mode = parsePinInProgressMode(raw) + // Persist so explicit Off is distinguishable from never-set→jobs. + // Legacy true/false also rewrite to all/off (upstream removed the key on false, + // so those users already look like unset — product stand maps them to jobs). + if (raw === null || raw === '' || raw === 'true' || raw === 'false') { + safeSetItem(PIN_IN_PROGRESS_STORAGE_KEY, mode) + } + return mode } /** @deprecated Use getInitialPinInProgressMode — boolean form treated `all` as true. */ From a12b8cac21c7cc6c4467e94c5fc5b71652ec6b28 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:52:17 +0000 Subject: [PATCH 12/94] =?UTF-8?q?fix(jobs):=20close=20pass-3=20minors=20?= =?UTF-8?q?=E2=80=94=20stable=20primary,=20own-client=20heartbeats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve JWT/session once for job run heartbeats; order primary job by started_at ASC; cover job redirect 404 + invalid key/body; idle jobs get a Jobs bucket (not Running); treat attachedJob.startedAt as render-relevant. Co-authored-by: Cursor --- .../modules/sessionJob/runSessionJob.test.ts | 30 +++++-- cli/src/modules/sessionJob/runSessionJob.ts | 40 +++++---- cli/src/modules/sessionJob/sessionJob.ts | 33 ++++++-- hub/src/store/migration-v25.test.ts | 11 +++ hub/src/store/sessionJobs.ts | 7 +- hub/src/web/routes/sessions-jobs.test.ts | 82 +++++++++++++++++++ .../SessionList.directory-action.test.tsx | 1 + web/src/components/SessionList.tsx | 11 ++- web/src/hooks/useSSE.test.ts | 17 ++++ web/src/hooks/useSSE.ts | 1 + web/src/lib/locales/en.ts | 1 + web/src/lib/locales/zh-CN.ts | 1 + 12 files changed, 197 insertions(+), 38 deletions(-) diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts index 66e444a09d..c1b3fc4240 100644 --- a/cli/src/modules/sessionJob/runSessionJob.test.ts +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -2,14 +2,18 @@ import { EventEmitter } from 'node:events' import { describe, expect, it, vi } from 'vitest' import { runSessionJob } from './runSessionJob' -function fakeChild(exitCode: number) { +function fakeChild(exitCode: number, deferExit = false) { const child = new EventEmitter() as EventEmitter & { pid: number killed: boolean + exit: () => void } child.pid = 4242 child.killed = false - queueMicrotask(() => child.emit('exit', exitCode, null)) + child.exit = () => child.emit('exit', exitCode, null) + if (!deferExit) { + queueMicrotask(() => child.exit()) + } return child } @@ -50,7 +54,8 @@ describe('runSessionJob', () => { } const timers: Array<() => void> = [] - const exitCode = await runSessionJob({ + const child = fakeChild(0, true) + const running = runSessionJob({ sessionIdPrefix: 'aaaa', jobKey: 'drain', label: 'drain', @@ -59,7 +64,7 @@ describe('runSessionJob', () => { accessToken: 'token', apiUrl: 'http://127.0.0.1:3006', http: http as never, - spawnImpl: (() => fakeChild(0)) as never, + spawnImpl: (() => child) as never, setIntervalImpl: ((fn: () => void) => { timers.push(fn) return 1 as unknown as NodeJS.Timeout @@ -67,11 +72,24 @@ describe('runSessionJob', () => { clearIntervalImpl: (() => undefined) as never }) + await vi.waitFor(() => expect(http.put).toHaveBeenCalled()) + expect(http.post).toHaveBeenCalledTimes(1) + expect(http.get).toHaveBeenCalledTimes(1) + + // Heartbeat ticks reuse resolved client (no extra auth). + expect(timers.length).toBe(1) + timers[0]!() + await vi.waitFor(() => expect(http.patch).toHaveBeenCalled()) + expect(http.post).toHaveBeenCalledTimes(1) + expect(http.get).toHaveBeenCalledTimes(1) + + child.exit() + const exitCode = await running expect(exitCode).toBe(0) - expect(http.put).toHaveBeenCalled() - expect(http.patch).toHaveBeenCalled() const lastPatch = http.patch.mock.calls.at(-1)?.[1] as { status?: string } expect(lastPatch.status).toBe('completed') + expect(http.post).toHaveBeenCalledTimes(1) + expect(http.get).toHaveBeenCalledTimes(1) }) it('marks failed on non-zero exit', async () => { diff --git a/cli/src/modules/sessionJob/runSessionJob.ts b/cli/src/modules/sessionJob/runSessionJob.ts index f654b1fcd8..232cdd8163 100644 --- a/cli/src/modules/sessionJob/runSessionJob.ts +++ b/cli/src/modules/sessionJob/runSessionJob.ts @@ -7,9 +7,11 @@ import { spawn, type ChildProcess } from 'node:child_process' import type { AttachedJobUpsert } from '@hapi/protocol' import { SessionJobError, + resolveSessionJobClient, setSessionJob, updateSessionJob, - type SessionJobClientOptions + type SessionJobClientOptions, + type SessionJobResolvedClient } from './sessionJob' export type RunSessionJobOptions = SessionJobClientOptions & { @@ -45,13 +47,18 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise { void updateSessionJob({ - sessionIdPrefix: options.sessionIdPrefix, + ...clientOpts, jobKey: options.jobKey, body: { detail: options.detail, status: 'running' - }, - apiUrl: options.apiUrl, - accessToken: options.accessToken, - http: options.http + } }).catch(() => { // Best-effort — exit path still marks terminal status. }) @@ -101,12 +105,9 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise( - options: SessionJobClientOptions, - fn: (ctx: { apiUrl: string; jwt: string; sessionId: string; http: AxiosInstance }) => Promise -): Promise { +/** One JWT + session id for the life of a supervised job. */ +export async function resolveSessionJobClient( + options: SessionJobClientOptions +): Promise { + if (options.resolved) { + return options.resolved + } const http = options.http ?? axios const apiUrl = resolveApiUrl(options.apiUrl) const accessToken = resolveAccessToken(options.accessToken) const jwt = await exchangeJwt(apiUrl, accessToken, http) const sessionId = await resolveSessionId(apiUrl, jwt, http, options.sessionIdPrefix) - return fn({ apiUrl, jwt, sessionId, http }) + return { apiUrl, jwt, sessionId } +} + +async function withClient( + options: SessionJobClientOptions, + fn: (ctx: SessionJobResolvedClient & { http: AxiosInstance }) => Promise +): Promise { + const http = options.http ?? axios + const resolved = await resolveSessionJobClient(options) + return fn({ ...resolved, http }) } export async function listSessionJobs( diff --git a/hub/src/store/migration-v25.test.ts b/hub/src/store/migration-v25.test.ts index 43b58ac498..a7afec2e3f 100644 --- a/hub/src/store/migration-v25.test.ts +++ b/hub/src/store/migration-v25.test.ts @@ -87,6 +87,17 @@ describe('Store V26→V27 migration: session_jobs table', () => { expect(primary?.key).toBe('beets') expect(primary?.remaining).toBe(100) + // Stable primary: earliest started_at wins even after a newer job heartbeats. + store.sessionJobs.upsert(session.id, 'newer', { + label: 'sidecar', + status: 'running', + remaining: 1, + startedAt: (primary!.startedAt) + 60_000 + }) + store.sessionJobs.patch(session.id, 'newer', { remaining: 0 }) + expect(store.sessionJobs.getPrimaryRunning(session.id)?.key).toBe('beets') + expect(store.sessionJobs.delete(session.id, 'newer')).toBe(true) + const patched = store.sessionJobs.patch(session.id, 'beets', { remaining: 80 }) expect(patched?.remaining).toBe(80) diff --git a/hub/src/store/sessionJobs.ts b/hub/src/store/sessionJobs.ts index fb933bef19..644fc9415a 100644 --- a/hub/src/store/sessionJobs.ts +++ b/hub/src/store/sessionJobs.ts @@ -83,13 +83,13 @@ export function getSessionJob( return row ? toStored(row) : null } -/** Newest `running` job for a session, or null. */ +/** Earliest-started `running` job for a session, or null (stable list chrome). */ export function getPrimaryRunningJob(db: Database, sessionId: string): StoredSessionJob | null { const row = db.prepare( `SELECT ${JOB_COLUMNS} FROM session_jobs WHERE session_id = ? AND status = 'running' - ORDER BY updated_at DESC, job_key ASC + ORDER BY started_at ASC, job_key ASC LIMIT 1` ).get(sessionId) as DbJobRow | undefined return row ? toStored(row) : null @@ -111,10 +111,11 @@ export function getPrimaryRunningJobsBySessionIds( `SELECT ${JOB_COLUMNS} FROM session_jobs WHERE status = 'running' AND session_id IN (${placeholders}) - ORDER BY updated_at DESC, job_key ASC` + ORDER BY started_at ASC, job_key ASC` ).all(...sessionIds) as DbJobRow[] for (const row of rows) { + // First row per session wins — earliest started_at (stable primary). if (result.has(row.session_id)) continue result.set(row.session_id, toAttachedJob(toStored(row))) } diff --git a/hub/src/web/routes/sessions-jobs.test.ts b/hub/src/web/routes/sessions-jobs.test.ts index 87a0161b5d..8622a34de1 100644 --- a/hub/src/web/routes/sessions-jobs.test.ts +++ b/hub/src/web/routes/sessions-jobs.test.ts @@ -127,4 +127,86 @@ describe('session-attached jobs routes (tiann/hapi#1404)', () => { expect(listed.jobs).toEqual([]) expect(listed.primary).toBeNull() }) + + it('follows jobsAccepted redirect when the pre-merge session id 404s', async () => { + const owner = createSession({ id: '22222222-2222-2222-2222-222222222222' }) + const deletedId = '11111111-1111-1111-1111-111111111111' + const jobs = new Map() + jobs.set('beets', { + key: 'beets', + label: 'beets import', + status: 'running', + remaining: 3, + heartbeatAt: 1, + startedAt: 1, + updatedAt: 1 + }) + + const engine = { + resolveSessionAccess: (id: string) => { + if (id === owner.id) { + return { ok: true as const, sessionId: owner.id, session: owner } + } + return { ok: false as const, reason: 'not-found' as const } + }, + resolveAttachedJobSessionId: (id: string) => (id === deletedId ? owner.id : id), + listSessionJobs: (sid: string) => (sid === owner.id ? [...jobs.values()] : []), + getPrimaryAttachedJob: (sid: string) => (sid === owner.id ? jobs.get('beets')! : null), + upsertSessionJob: () => ({ outcome: 'session-not-found' as const }), + patchSessionJob: () => null, + deleteSessionJob: () => false + } as unknown as SyncEngine + + const app = new Hono() + app.use('*', async (c, next) => { + c.set('namespace', 'default') + await next() + }) + app.route('/api', createSessionsRoutes(() => engine)) + + const res = await app.request(`http://localhost/api/sessions/${deletedId}/jobs`) + expect(res.status).toBe(200) + const body = await res.json() as { primary: AttachedJob | null } + expect(body.primary?.key).toBe('beets') + }) + + it('rejects invalid jobKey and invalid upsert body with 400', async () => { + const session = createSession() + const engine = { + resolveSessionAccess: () => ({ ok: true as const, sessionId: session.id, session }), + resolveAttachedJobSessionId: (id: string) => id, + listSessionJobs: () => [], + getPrimaryAttachedJob: () => null, + upsertSessionJob: () => ({ outcome: 'session-not-found' as const }), + patchSessionJob: () => null, + deleteSessionJob: () => false + } as unknown as SyncEngine + + const app = new Hono() + app.use('*', async (c, next) => { + c.set('namespace', 'default') + await next() + }) + app.route('/api', createSessionsRoutes(() => engine)) + + const badKey = await app.request( + `http://localhost/api/sessions/${session.id}/jobs/bad key!`, + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ label: 'x' }) + } + ) + expect(badKey.status).toBe(400) + + const badBody = await app.request( + `http://localhost/api/sessions/${session.id}/jobs/ok-key`, + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ remaining: 1 }) + } + ) + expect(badBody.status).toBe(400) + }) }) diff --git a/web/src/components/SessionList.directory-action.test.tsx b/web/src/components/SessionList.directory-action.test.tsx index 9aaa00ce79..3b5cb89be1 100644 --- a/web/src/components/SessionList.directory-action.test.tsx +++ b/web/src/components/SessionList.directory-action.test.tsx @@ -585,6 +585,7 @@ describe('SessionList collapse behavior', () => { render(renderSessionList(sessions, null)) expect(screen.getByTitle('In progress')).toBeInTheDocument() + expect(screen.getByText(/Jobs \(1\)/)).toBeInTheDocument() expect(screen.getByRole('button', { name: /Music drain/ })).toBeInTheDocument() // Thinking agent is not a long-running job — stays in directory under jobs mode. expect(screen.getByTitle('/work/hapi')).toBeInTheDocument() diff --git a/web/src/components/SessionList.tsx b/web/src/components/SessionList.tsx index b3921c54c0..c62dae2bea 100644 --- a/web/src/components/SessionList.tsx +++ b/web/src/components/SessionList.tsx @@ -80,6 +80,7 @@ type SessionGroup = { } const RUNNING_BUCKETS = [ + { key: 'jobs', labelKey: 'session.item.attachedJob', colorClass: 'text-[var(--app-badge-success-text)]', pulse: true }, { key: 'working', labelKey: 'session.item.running', colorClass: 'text-[var(--app-badge-success-text)]', pulse: true }, { key: 'pending', labelKey: 'session.item.pending', colorClass: 'text-[var(--app-badge-warning-text)]', pulse: true }, { key: 'active', labelKey: 'session.item.active', colorClass: 'text-[var(--app-hint)]', pulse: false }, @@ -105,7 +106,7 @@ function hasAgentInProgressActivity(session: SessionSummary): boolean { } export function emptyRunningBuckets(): Record { - return { working: [], pending: [], active: [], idle: [] } + return { jobs: [], working: [], pending: [], active: [], idle: [] } } /** @@ -134,8 +135,11 @@ export function bucketRunningSessions( const agentPending = session.active && (session.pendingRequestsCount ?? 0) > 0 && !agentWorking - if (agentWorking || hasRunningAttachedJob(session)) { + if (agentWorking) { buckets.working.push(session) + } else if (hasRunningAttachedJob(session)) { + // Idle outliving work — not "Running" agent activity. + buckets.jobs.push(session) } else if (agentPending) { buckets.pending.push(session) } else if (session.metadata?.lifecycleState === SESSION_LIFECYCLE_IDLE) { @@ -1420,7 +1424,8 @@ export function SessionList(props: { } return bucketRunningSessions(machineFilteredSessions, pinInProgressMode, byRelevanceOrRecent) }, [machineFilteredSessions, pinInProgressMode, searchScoreIndex, hasTextQuery]) - const runningSessionTotal = runningSessions.working.length + const runningSessionTotal = runningSessions.jobs.length + + runningSessions.working.length + runningSessions.pending.length const activeSessionTotal = runningSessions.active.length + runningSessions.idle.length const groups = useMemo( diff --git a/web/src/hooks/useSSE.test.ts b/web/src/hooks/useSSE.test.ts index 7917422acb..3a6592e631 100644 --- a/web/src/hooks/useSSE.test.ts +++ b/web/src/hooks/useSSE.test.ts @@ -326,6 +326,23 @@ describe('isRenderIrrelevantPatch', () => { expect(isRenderIrrelevantPatch(current, next)).toBe(false) }) + + it('reports attachedJob.startedAt changes as relevant', () => { + const job = { + key: 'beets', + label: 'beets', + status: 'running' as const, + heartbeatAt: 100, + startedAt: 100, + updatedAt: 100 + } + const current = makeSummary({ attachedJob: job }) + const next = makeSummary({ + attachedJob: { ...job, startedAt: 50 }, + activeAt: 11_000 + }) + expect(isRenderIrrelevantPatch(current, next)).toBe(false) + }) }) describe('isRenderIrrelevantSessionPatch', () => { diff --git a/web/src/hooks/useSSE.ts b/web/src/hooks/useSSE.ts index 39236f212c..926705726d 100644 --- a/web/src/hooks/useSSE.ts +++ b/web/src/hooks/useSSE.ts @@ -168,6 +168,7 @@ export function isRenderIrrelevantPatch(current: SessionSummary, next: SessionSu && current.attachedJob?.unit === next.attachedJob?.unit && current.attachedJob?.detail === next.attachedJob?.detail && current.attachedJob?.heartbeatAt === next.attachedJob?.heartbeatAt + && current.attachedJob?.startedAt === next.attachedJob?.startedAt && (current.attachedJob == null) === (next.attachedJob == null) && current.model === next.model && current.modelReasoningEffort === next.modelReasoningEffort diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index 282a95c807..a59e01c4be 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -189,6 +189,7 @@ export default { 'session.item.running': 'Running', 'session.item.active': 'Active', 'session.item.idle': 'Idle (keepalive only)', + 'session.item.attachedJob': 'Jobs', 'session.item.permission': 'Permission required', 'session.item.needsInput': 'Needs input', 'session.item.background': 'Background tasks running', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index fe0a5d8b36..64f6df56f8 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -189,6 +189,7 @@ export default { 'session.item.running': '运行中', 'session.item.active': '活跃', 'session.item.idle': '空闲（仅心跳）', + 'session.item.attachedJob': '长时间任务', 'session.item.permission': '需要权限', 'session.item.needsInput': '需要输入', 'session.item.background': '后台任务运行中', From 81013c0ebc15beedea5ca9a3d955e77d210b75f2 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:03:31 +0000 Subject: [PATCH 13/94] fix(jobs): refresh JWT for long job-run heartbeats (#1404) Cache sessionId once but re-exchange JWT before hub's 4h expiry and on 401 retry so days-long supervised jobs keep heartbeating and can still mark completed/failed. Log the first heartbeat failure to stderr. Co-authored-by: Cursor --- .../modules/sessionJob/runSessionJob.test.ts | 86 +++++++++ cli/src/modules/sessionJob/runSessionJob.ts | 21 ++- cli/src/modules/sessionJob/sessionJob.ts | 176 ++++++++++++------ 3 files changed, 224 insertions(+), 59 deletions(-) diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts index c1b3fc4240..5ff45b2658 100644 --- a/cli/src/modules/sessionJob/runSessionJob.test.ts +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -144,4 +144,90 @@ describe('runSessionJob', () => { const lastPatch = http.patch.mock.calls.at(-1)?.[1] as { status?: string } expect(lastPatch.status).toBe('failed') }) + + it('re-exchanges JWT on heartbeat 401 and still marks completed (hub 4h expiry)', async () => { + let jwtIssue = 0 + let patchCalls = 0 + const http = { + post: vi.fn(async () => { + jwtIssue += 1 + return { status: 200, data: { token: `jwt-${jwtIssue}` } } + }), + get: vi.fn(async () => ({ + status: 200, + data: { sessions: [{ id: 'aaaaaaaa-1111-1111-1111-111111111111' }] } + })), + put: vi.fn(async () => ({ + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: 'running', + heartbeatAt: 1, + startedAt: 1, + updatedAt: 1 + } + } + })), + patch: vi.fn(async (_url: string, body: { status?: string }, cfg?: { headers?: Record }) => { + patchCalls += 1 + const auth = cfg?.headers?.Authorization ?? '' + // First heartbeat still carries jwt-1 after hub expiry → 401. + if (patchCalls === 1 && auth.includes('jwt-1')) { + return { status: 401, data: { error: 'expired' } } + } + return { + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: body.status ?? 'running', + heartbeatAt: 2, + startedAt: 1, + updatedAt: 2 + } + } + } + }) + } + + const timers: Array<() => void> = [] + const child = fakeChild(0, true) + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined) + const running = runSessionJob({ + sessionIdPrefix: 'aaaa', + jobKey: 'drain', + label: 'drain', + command: ['true'], + heartbeatMs: 10, + accessToken: 'token', + apiUrl: 'http://127.0.0.1:3006', + http: http as never, + spawnImpl: (() => child) as never, + setIntervalImpl: ((fn: () => void) => { + timers.push(fn) + return 1 as unknown as NodeJS.Timeout + }) as never, + clearIntervalImpl: (() => undefined) as never + }) + + await vi.waitFor(() => expect(http.put).toHaveBeenCalled()) + expect(http.post).toHaveBeenCalledTimes(1) + expect(http.get).toHaveBeenCalledTimes(1) + + timers[0]!() + await vi.waitFor(() => expect(http.post).toHaveBeenCalledTimes(2)) + await vi.waitFor(() => expect(http.patch.mock.calls.length).toBeGreaterThanOrEqual(2)) + // Session list not re-fetched — only JWT refresh. + expect(http.get).toHaveBeenCalledTimes(1) + + child.exit() + const exitCode = await running + expect(exitCode).toBe(0) + const lastPatch = http.patch.mock.calls.at(-1)?.[1] as { status?: string } + expect(lastPatch.status).toBe('completed') + errSpy.mockRestore() + }) }) diff --git a/cli/src/modules/sessionJob/runSessionJob.ts b/cli/src/modules/sessionJob/runSessionJob.ts index 232cdd8163..1345399f56 100644 --- a/cli/src/modules/sessionJob/runSessionJob.ts +++ b/cli/src/modules/sessionJob/runSessionJob.ts @@ -47,11 +47,14 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise { void updateSessionJob({ ...clientOpts, @@ -79,8 +83,14 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise { - // Best-effort — exit path still marks terminal status. + }).catch((error: unknown) => { + // Best-effort — exit path still marks terminal status. Log once so + // a broken supervisor is visible (stuck chip with dead PID is worse). + if (!loggedHeartbeatFailure) { + loggedHeartbeatFailure = true + const message = error instanceof Error ? error.message : String(error) + console.error(`[hapi job run] heartbeat failed (will keep trying): ${message}`) + } }) }, heartbeatMs) // Don't keep the event loop alive solely for heartbeats if child already exited. @@ -134,8 +144,9 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise { @@ -196,43 +205,89 @@ export async function resolveSessionJobClient( const accessToken = resolveAccessToken(options.accessToken) const jwt = await exchangeJwt(apiUrl, accessToken, http) const sessionId = await resolveSessionId(apiUrl, jwt, http, options.sessionIdPrefix) - return { apiUrl, jwt, sessionId } + return { apiUrl, jwt, sessionId, jwtIssuedAtMs: Date.now() } +} + +async function refreshSessionJobJwt( + resolved: SessionJobResolvedClient, + options: SessionJobClientOptions +): Promise { + const http = options.http ?? axios + const accessToken = resolveAccessToken(options.accessToken) + resolved.jwt = await exchangeJwt(resolved.apiUrl, accessToken, http) + resolved.jwtIssuedAtMs = Date.now() +} + +async function ensureFreshJwt( + resolved: SessionJobResolvedClient, + options: SessionJobClientOptions, + force = false +): Promise { + const age = Date.now() - resolved.jwtIssuedAtMs + if (force || age >= SESSION_JOB_JWT_REFRESH_AFTER_MS) { + await refreshSessionJobJwt(resolved, options) + } } -async function withClient( +type AuthedResponse = { status: number; data?: unknown } + +/** + * Run an authed request; on 401 re-exchange JWT (keep cached sessionId) and retry once. + */ +async function withAuthedRequest( options: SessionJobClientOptions, - fn: (ctx: SessionJobResolvedClient & { http: AxiosInstance }) => Promise + request: (ctx: { apiUrl: string; jwt: string; sessionId: string; http: AxiosInstance }) => Promise, + handle: (response: AuthedResponse, sessionId: string) => T ): Promise { const http = options.http ?? axios const resolved = await resolveSessionJobClient(options) - return fn({ ...resolved, http }) + await ensureFreshJwt(resolved, options) + + const run = async () => request({ + apiUrl: resolved.apiUrl, + jwt: resolved.jwt, + sessionId: resolved.sessionId, + http + }) + + let response = await run() + if (response.status === 401) { + await ensureFreshJwt(resolved, options, true) + response = await run() + } + return handle(response, resolved.sessionId) } export async function listSessionJobs( options: SessionJobClientOptions ): Promise<{ sessionId: string; jobs: AttachedJob[]; primary: AttachedJob | null }> { - return withClient(options, async ({ apiUrl, jwt, sessionId, http }) => { - const response = await http.get(`${apiUrl}/api/sessions/${sessionId}/jobs`, { + return withAuthedRequest( + options, + ({ apiUrl, jwt, sessionId, http }) => http.get(`${apiUrl}/api/sessions/${sessionId}/jobs`, { headers: authHeaders(jwt), timeout: 15_000, validateStatus: () => true - }) - if (response.status < 200 || response.status >= 300) { - throw new SessionJobError('request_failed', `list jobs failed: HTTP ${response.status}`) - } - return { - sessionId, - jobs: Array.isArray(response.data?.jobs) ? response.data.jobs : [], - primary: response.data?.primary ?? null + }), + (response, sessionId) => { + if (response.status < 200 || response.status >= 300) { + throw new SessionJobError('request_failed', `list jobs failed: HTTP ${response.status}`) + } + const data = response.data as { jobs?: AttachedJob[]; primary?: AttachedJob | null } | undefined + return { + sessionId, + jobs: Array.isArray(data?.jobs) ? data.jobs : [], + primary: data?.primary ?? null + } } - }) + ) } export async function setSessionJob( options: SessionJobClientOptions & { jobKey: string; body: AttachedJobUpsert } ): Promise<{ sessionId: string; job: AttachedJob }> { - return withClient(options, async ({ apiUrl, jwt, sessionId, http }) => { - const response = await http.put( + return withAuthedRequest( + options, + ({ apiUrl, jwt, sessionId, http }) => http.put( `${apiUrl}/api/sessions/${sessionId}/jobs/${encodeURIComponent(options.jobKey)}`, options.body, { @@ -240,25 +295,29 @@ export async function setSessionJob( timeout: 15_000, validateStatus: () => true } - ) - if (response.status === 404) { - throw new SessionJobError('not_found', 'session or job not found') - } - if (response.status < 200 || response.status >= 300 || !response.data?.job) { - const detail = typeof response.data?.error === 'string' - ? response.data.error - : `HTTP ${response.status}` - throw new SessionJobError('request_failed', `set job failed: ${detail}`) + ), + (response, sessionId) => { + if (response.status === 404) { + throw new SessionJobError('not_found', 'session or job not found') + } + const data = response.data as { job?: AttachedJob; error?: string } | undefined + if (response.status < 200 || response.status >= 300 || !data?.job) { + const detail = typeof data?.error === 'string' + ? data.error + : `HTTP ${response.status}` + throw new SessionJobError('request_failed', `set job failed: ${detail}`) + } + return { sessionId, job: data.job } } - return { sessionId, job: response.data.job as AttachedJob } - }) + ) } export async function updateSessionJob( options: SessionJobClientOptions & { jobKey: string; body: AttachedJobPatch } ): Promise<{ sessionId: string; job: AttachedJob }> { - return withClient(options, async ({ apiUrl, jwt, sessionId, http }) => { - const response = await http.patch( + return withAuthedRequest( + options, + ({ apiUrl, jwt, sessionId, http }) => http.patch( `${apiUrl}/api/sessions/${sessionId}/jobs/${encodeURIComponent(options.jobKey)}`, options.body, { @@ -266,40 +325,49 @@ export async function updateSessionJob( timeout: 15_000, validateStatus: () => true } - ) - if (response.status === 404) { - throw new SessionJobError('not_found', 'job not found') - } - if (response.status < 200 || response.status >= 300 || !response.data?.job) { - const detail = typeof response.data?.error === 'string' - ? response.data.error - : `HTTP ${response.status}` - throw new SessionJobError('request_failed', `update job failed: ${detail}`) + ), + (response, sessionId) => { + if (response.status === 404) { + throw new SessionJobError('not_found', 'job not found') + } + const data = response.data as { job?: AttachedJob; error?: string } | undefined + if (response.status < 200 || response.status >= 300 || !data?.job) { + const detail = typeof data?.error === 'string' + ? data.error + : `HTTP ${response.status}` + if (response.status === 401) { + throw new SessionJobError('auth_failed', `update job failed: ${detail}`) + } + throw new SessionJobError('request_failed', `update job failed: ${detail}`) + } + return { sessionId, job: data.job } } - return { sessionId, job: response.data.job as AttachedJob } - }) + ) } export async function clearSessionJob( options: SessionJobClientOptions & { jobKey: string } ): Promise<{ sessionId: string }> { - return withClient(options, async ({ apiUrl, jwt, sessionId, http }) => { - const response = await http.delete( + return withAuthedRequest( + options, + ({ apiUrl, jwt, sessionId, http }) => http.delete( `${apiUrl}/api/sessions/${sessionId}/jobs/${encodeURIComponent(options.jobKey)}`, { headers: authHeaders(jwt), timeout: 15_000, validateStatus: () => true } - ) - if (response.status === 404) { - throw new SessionJobError('not_found', 'job not found') - } - if (response.status < 200 || response.status >= 300) { - throw new SessionJobError('request_failed', `clear job failed: HTTP ${response.status}`) + ), + (response, sessionId) => { + if (response.status === 404) { + throw new SessionJobError('not_found', 'job not found') + } + if (response.status < 200 || response.status >= 300) { + throw new SessionJobError('request_failed', `clear job failed: HTTP ${response.status}`) + } + return { sessionId } } - return { sessionId } - }) + ) } export function exitCodeForSessionJobError(error: SessionJobError): number { From 515219cee0ee177c5f1d2b039038981f770019ef Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:09:34 +0000 Subject: [PATCH 14/94] test(jobs): cover proactive JWT refresh; map 401 to auth_failed Co-authored-by: Cursor --- .../modules/sessionJob/runSessionJob.test.ts | 81 +++++++++++++++++++ cli/src/modules/sessionJob/sessionJob.ts | 31 ++++--- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts index 5ff45b2658..15d5dbee2f 100644 --- a/cli/src/modules/sessionJob/runSessionJob.test.ts +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -1,5 +1,6 @@ import { EventEmitter } from 'node:events' import { describe, expect, it, vi } from 'vitest' +import { SESSION_JOB_JWT_REFRESH_AFTER_MS, resolveSessionJobClient, updateSessionJob } from './sessionJob' import { runSessionJob } from './runSessionJob' function fakeChild(exitCode: number, deferExit = false) { @@ -230,4 +231,84 @@ describe('runSessionJob', () => { expect(lastPatch.status).toBe('completed') errSpy.mockRestore() }) + + it('proactively re-exchanges JWT after 3h without re-listing sessions', async () => { + let jwtIssue = 0 + const http = { + post: vi.fn(async () => { + jwtIssue += 1 + return { status: 200, data: { token: `jwt-${jwtIssue}` } } + }), + get: vi.fn(async () => ({ + status: 200, + data: { sessions: [{ id: 'aaaaaaaa-1111-1111-1111-111111111111' }] } + })), + patch: vi.fn(async (_url: string, _body: unknown, cfg?: { headers?: Record }) => ({ + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: 'running', + heartbeatAt: 2, + startedAt: 1, + updatedAt: 2 + }, + _auth: cfg?.headers?.Authorization + } + })) + } + + const resolved = await resolveSessionJobClient({ + sessionIdPrefix: 'aaaa', + accessToken: 'token', + apiUrl: 'http://127.0.0.1:3006', + http: http as never + }) + expect(http.post).toHaveBeenCalledTimes(1) + expect(http.get).toHaveBeenCalledTimes(1) + + // Inside the window: no second exchange. + await updateSessionJob({ + sessionIdPrefix: 'aaaa', + jobKey: 'drain', + body: { remaining: 9 }, + resolved, + accessToken: 'token', + apiUrl: 'http://127.0.0.1:3006', + http: http as never + }) + expect(http.post).toHaveBeenCalledTimes(1) + + // Past proactive refresh threshold: exchange once, keep session id. + resolved.jwtIssuedAtMs = Date.now() - SESSION_JOB_JWT_REFRESH_AFTER_MS - 1 + await updateSessionJob({ + sessionIdPrefix: 'aaaa', + jobKey: 'drain', + body: { remaining: 8 }, + resolved, + accessToken: 'token', + apiUrl: 'http://127.0.0.1:3006', + http: http as never + }) + expect(http.post).toHaveBeenCalledTimes(2) + expect(http.get).toHaveBeenCalledTimes(1) + expect(resolved.jwt).toBe('jwt-2') + const auth = (http.patch.mock.calls.at(-1)?.[2] as { headers?: Record } | undefined) + ?.headers?.Authorization + expect(auth).toContain('jwt-2') + + // Next tick inside the new window: no exchange storm. + await updateSessionJob({ + sessionIdPrefix: 'aaaa', + jobKey: 'drain', + body: { remaining: 7 }, + resolved, + accessToken: 'token', + apiUrl: 'http://127.0.0.1:3006', + http: http as never + }) + expect(http.post).toHaveBeenCalledTimes(2) + expect(http.get).toHaveBeenCalledTimes(1) + }) }) diff --git a/cli/src/modules/sessionJob/sessionJob.ts b/cli/src/modules/sessionJob/sessionJob.ts index bedda3bcf7..54cce77646 100644 --- a/cli/src/modules/sessionJob/sessionJob.ts +++ b/cli/src/modules/sessionJob/sessionJob.ts @@ -231,6 +231,20 @@ async function ensureFreshJwt( type AuthedResponse = { status: number; data?: unknown } +function httpStatusError( + action: string, + response: AuthedResponse, + errorDetail?: string +): SessionJobError { + const detail = typeof errorDetail === 'string' && errorDetail.length > 0 + ? errorDetail + : `HTTP ${response.status}` + if (response.status === 401) { + return new SessionJobError('auth_failed', `${action} failed: ${detail}`) + } + return new SessionJobError('request_failed', `${action} failed: ${detail}`) +} + /** * Run an authed request; on 401 re-exchange JWT (keep cached sessionId) and retry once. */ @@ -270,7 +284,7 @@ export async function listSessionJobs( }), (response, sessionId) => { if (response.status < 200 || response.status >= 300) { - throw new SessionJobError('request_failed', `list jobs failed: HTTP ${response.status}`) + throw httpStatusError('list jobs', response) } const data = response.data as { jobs?: AttachedJob[]; primary?: AttachedJob | null } | undefined return { @@ -302,10 +316,7 @@ export async function setSessionJob( } const data = response.data as { job?: AttachedJob; error?: string } | undefined if (response.status < 200 || response.status >= 300 || !data?.job) { - const detail = typeof data?.error === 'string' - ? data.error - : `HTTP ${response.status}` - throw new SessionJobError('request_failed', `set job failed: ${detail}`) + throw httpStatusError('set job', response, data?.error) } return { sessionId, job: data.job } } @@ -332,13 +343,7 @@ export async function updateSessionJob( } const data = response.data as { job?: AttachedJob; error?: string } | undefined if (response.status < 200 || response.status >= 300 || !data?.job) { - const detail = typeof data?.error === 'string' - ? data.error - : `HTTP ${response.status}` - if (response.status === 401) { - throw new SessionJobError('auth_failed', `update job failed: ${detail}`) - } - throw new SessionJobError('request_failed', `update job failed: ${detail}`) + throw httpStatusError('update job', response, data?.error) } return { sessionId, job: data.job } } @@ -363,7 +368,7 @@ export async function clearSessionJob( throw new SessionJobError('not_found', 'job not found') } if (response.status < 200 || response.status >= 300) { - throw new SessionJobError('request_failed', `clear job failed: HTTP ${response.status}`) + throw httpStatusError('clear job', response) } return { sessionId } } From 49d9e176e4cecf6a661b99bf7cf3ea1477e8f7ef Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:46:03 +0000 Subject: [PATCH 15/94] fix(jobs): stamp fresh startedAt on job run; prefer supervisor Re-running the same job key was sticky-reusing a prior row's startedAt, so elapsed lied. Drain in-flight heartbeats before terminal status. Align AGENTS/guide: CLI supervisor first, MCP as manual path. Co-authored-by: Cursor --- .../modules/sessionJob/runSessionJob.test.ts | 4 ++++ cli/src/modules/sessionJob/runSessionJob.ts | 11 ++++++++- docs/guide/session-jobs.md | 24 +++++++++---------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts index 15d5dbee2f..9b70ebc6a2 100644 --- a/cli/src/modules/sessionJob/runSessionJob.test.ts +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -74,6 +74,10 @@ describe('runSessionJob', () => { }) await vi.waitFor(() => expect(http.put).toHaveBeenCalled()) + const putBody = http.put.mock.calls[0]?.[1] as { startedAt?: number; status?: string } + expect(putBody.status).toBe('running') + expect(typeof putBody.startedAt).toBe('number') + expect(putBody.startedAt).toBeGreaterThan(0) expect(http.post).toHaveBeenCalledTimes(1) expect(http.get).toHaveBeenCalledTimes(1) diff --git a/cli/src/modules/sessionJob/runSessionJob.ts b/cli/src/modules/sessionJob/runSessionJob.ts index 1345399f56..ceb7202b34 100644 --- a/cli/src/modules/sessionJob/runSessionJob.ts +++ b/cli/src/modules/sessionJob/runSessionJob.ts @@ -40,6 +40,9 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise = Promise.resolve() const heartbeat = setIntervalFn(() => { - void updateSessionJob({ + inflightHeartbeat = updateSessionJob({ ...clientOpts, jobKey: options.jobKey, body: { @@ -113,6 +117,7 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise((resolve) => { child.on('error', async (error) => { clearIntervalFn(heartbeat) + await inflightHeartbeat.catch(() => undefined) try { await updateSessionJob({ ...clientOpts, @@ -137,6 +142,10 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise undefined) + const terminalStatus = exitCode === 0 ? 'completed' : 'failed' try { await updateSessionJob({ diff --git a/docs/guide/session-jobs.md b/docs/guide/session-jobs.md index 5516ae5a6e..7dbbdf28b9 100644 --- a/docs/guide/session-jobs.md +++ b/docs/guide/session-jobs.md @@ -37,18 +37,7 @@ If the operator would reopen the chat only to ask "how's it doing?", it belongs Treat this like `ping_peer` / `inspect_peer`: it is first-class HAPI tooling, not a docs footnote. -### MCP (preferred for agents) - -Tool name: `session_job` (Claude: `mcp__hapi__session_job`; Codex: `functions.hapi__session_job`; OpenCode/ACP: `hapi_session_job`). - -```json -{ "action": "set", "jobKey": "beets", "label": "beets import", - "remaining": 150, "done": 1637, "total": 1787, "unit": "units" } -``` - -Then `action=update` every ~10 minutes; finish with `status=completed|failed` or `action=clear`. Omit `sessionId` to target this chat. - -### CLI supervisor (preferred for shell children) +### CLI supervisor (preferred for process-shaped work) ```bash hapi job run "$HAPI_SESSION_ID" beets \ @@ -60,6 +49,17 @@ hapi job run "$HAPI_SESSION_ID" beets \ `hapi job run` registers the job, heartbeats on a timer while the child runs, then marks `completed`/`failed` from the exit code. An idle agent **cannot** heartbeat - set-once + manual update decays to amber. +### MCP (manual path when the child is not CLI-supervised) + +Tool name: `session_job` (Claude: `mcp__hapi__session_job`; Codex: `functions.hapi__session_job`; OpenCode/ACP: `hapi_session_job`). + +```json +{ "action": "set", "jobKey": "beets", "label": "beets import", + "remaining": 150, "done": 1637, "total": 1787, "unit": "units" } +``` + +Then `action=update` every ~10 minutes; finish with `status=completed|failed` or `action=clear`. Omit `sessionId` to target this chat. + ### CLI manual path ```bash From 23d8ec0974f5e052ba2e7ce66f11cf404a3c5b73 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:47:16 +0000 Subject: [PATCH 16/94] fix(jobs): typecheck-safe assertion for run startedAt Co-authored-by: Cursor --- cli/src/modules/sessionJob/runSessionJob.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts index 9b70ebc6a2..4f78dceb3c 100644 --- a/cli/src/modules/sessionJob/runSessionJob.test.ts +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -74,10 +74,10 @@ describe('runSessionJob', () => { }) await vi.waitFor(() => expect(http.put).toHaveBeenCalled()) - const putBody = http.put.mock.calls[0]?.[1] as { startedAt?: number; status?: string } - expect(putBody.status).toBe('running') - expect(typeof putBody.startedAt).toBe('number') - expect(putBody.startedAt).toBeGreaterThan(0) + expect(http.put.mock.calls[0]?.[1]).toEqual(expect.objectContaining({ + status: 'running', + startedAt: expect.any(Number) + })) expect(http.post).toHaveBeenCalledTimes(1) expect(http.get).toHaveBeenCalledTimes(1) From b5e669be2c09395f2afe93d722a63e8b13f09f9b Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:48:10 +0000 Subject: [PATCH 17/94] fix(jobs): type put mock args so startedAt assertion typechecks Co-authored-by: Cursor --- cli/src/modules/sessionJob/runSessionJob.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts index 4f78dceb3c..a1bcc6acd0 100644 --- a/cli/src/modules/sessionJob/runSessionJob.test.ts +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -26,15 +26,15 @@ describe('runSessionJob', () => { status: 200, data: { sessions: [{ id: 'aaaaaaaa-1111-1111-1111-111111111111' }] } })), - put: vi.fn(async () => ({ + put: vi.fn(async (_url: string, body: { status?: string; startedAt?: number }) => ({ status: 200, data: { job: { key: 'drain', label: 'drain', - status: 'running', + status: body.status ?? 'running', heartbeatAt: 1, - startedAt: 1, + startedAt: body.startedAt ?? 1, updatedAt: 1 } } From e7415d97ce5db15e6298d556f5125282f54387d6 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:49:32 +0000 Subject: [PATCH 18/94] =?UTF-8?q?fix(jobs):=20address=20Codex=20Majors=20?= =?UTF-8?q?=E2=80=94=20redirect=20ancestry,=20heartbeat=20status,=20ACP=20?= =?UTF-8?q?allowlist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inherit jobsAcceptedFromSessionIds across A→B→C merges; heartbeat PATCH omits status so late ticks cannot resurrect running; drop session_job from global name-only auto-approve (bridge/--allowedTools remain the trust path). Co-authored-by: Cursor --- .../permission/BasePermissionHandler.test.ts | 5 ++- .../permission/BasePermissionHandler.ts | 14 +++---- .../modules/sessionJob/runSessionJob.test.ts | 5 ++- cli/src/modules/sessionJob/runSessionJob.ts | 7 ++-- hub/src/sync/sessionCache-merge-jobs.test.ts | 39 +++++++++++++++++++ hub/src/sync/sessionCache.ts | 19 ++++++++- 6 files changed, 71 insertions(+), 18 deletions(-) diff --git a/cli/src/modules/common/permission/BasePermissionHandler.test.ts b/cli/src/modules/common/permission/BasePermissionHandler.test.ts index 3e8ea63108..ebd73f8b37 100644 --- a/cli/src/modules/common/permission/BasePermissionHandler.test.ts +++ b/cli/src/modules/common/permission/BasePermissionHandler.test.ts @@ -108,7 +108,8 @@ describe('resolveToolAutoApprovalDecision session_job', () => { 'hapi_session_job', 'mcp__hapi__session_job', 'Session-Attached Job' - ])('auto-approves own-session job meter %s', (toolName) => { - expect(resolveToolAutoApprovalDecision('default', toolName, 'call-1')).toBe('approved') + ])('does not name-only auto-approve spoofable job tool %s', (toolName) => { + // Bridge / --allowedTools own the approve path; global title allowlist must not. + expect(resolveToolAutoApprovalDecision('default', toolName, 'call-1')).toBeNull() }) }) diff --git a/cli/src/modules/common/permission/BasePermissionHandler.ts b/cli/src/modules/common/permission/BasePermissionHandler.ts index 62bb54e75e..639b782feb 100644 --- a/cli/src/modules/common/permission/BasePermissionHandler.ts +++ b/cli/src/modules/common/permission/BasePermissionHandler.ts @@ -37,20 +37,16 @@ const AUTO_APPROVE_EXACT_TOOL_NAMES = new Set([ 'happy__list_peers', 'mcp__hapi__list_peers', // ACP permission requests often surface MCP tool title, not the snake_case name. - 'list peer sessions', - // Own-session progress meter (tiann/hapi#1404) — MCP schema has no sessionId; - // tool always targets this chat. Cross-session writes use CLI hapi job (not auto). - 'session_job', - 'hapi_session_job', - 'happy__session_job', - 'mcp__hapi__session_job', - 'session-attached job' + 'list peer sessions' ]); +// session_job is intentionally NOT in this name-only allowlist: ACP titles are +// spoofable (tool-name derivation prefers backend title). Auto-approve only via +// trusted HAPI bridge config (Codex tools map / Claude --allowedTools). // ping_peer / inspect_peer intentionally omitted from always-approve: they can // resume+inject into another session or read peer histories, so permission // modes must still gate them. Treat both as write-like in read-only so ACP // titles such as "Ping Peer Session" / "Inspect Peer Session" also require -// approval. list_peers / own-session session_job are auto-approved above. +// approval. list_peers stays auto-approved above. const AUTO_APPROVE_TOOL_ID_HINTS = ['change_title', 'save_memory']; const SENSITIVE_TOOL_NAME_HINTS = [ 'ping_peer', diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts index a1bcc6acd0..fe52cdda90 100644 --- a/cli/src/modules/sessionJob/runSessionJob.test.ts +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -81,10 +81,13 @@ describe('runSessionJob', () => { expect(http.post).toHaveBeenCalledTimes(1) expect(http.get).toHaveBeenCalledTimes(1) - // Heartbeat ticks reuse resolved client (no extra auth). + // Heartbeat ticks reuse resolved client (no extra auth) and must not + // send status:running (late heartbeat must not resurrect after exit). expect(timers.length).toBe(1) timers[0]!() await vi.waitFor(() => expect(http.patch).toHaveBeenCalled()) + const heartbeatBody = http.patch.mock.calls[0]?.[1] as { status?: string } + expect(heartbeatBody.status).toBeUndefined() expect(http.post).toHaveBeenCalledTimes(1) expect(http.get).toHaveBeenCalledTimes(1) diff --git a/cli/src/modules/sessionJob/runSessionJob.ts b/cli/src/modules/sessionJob/runSessionJob.ts index ceb7202b34..238ef301c0 100644 --- a/cli/src/modules/sessionJob/runSessionJob.ts +++ b/cli/src/modules/sessionJob/runSessionJob.ts @@ -80,13 +80,12 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise = Promise.resolve() const heartbeat = setIntervalFn(() => { + // Never PATCH status:running on the heartbeat — a late in-flight + // request must not resurrect running after the terminal write. inflightHeartbeat = updateSessionJob({ ...clientOpts, jobKey: options.jobKey, - body: { - detail: options.detail, - status: 'running' - } + body: options.detail !== undefined ? { detail: options.detail } : {} }).catch((error: unknown) => { // Best-effort — exit path still marks terminal status. Log once so // a broken supervisor is visible (stuck chip with dead PID is worse). diff --git a/hub/src/sync/sessionCache-merge-jobs.test.ts b/hub/src/sync/sessionCache-merge-jobs.test.ts index 5b814cb418..83b7c2c1c6 100644 --- a/hub/src/sync/sessionCache-merge-jobs.test.ts +++ b/hub/src/sync/sessionCache-merge-jobs.test.ts @@ -73,6 +73,45 @@ describe('mergeSessions job redirect through SessionCache (#1404)', () => { expect(cache.resolveAttachedJobSessionId(newSession.id, 'default')).toBe(newSession.id) }) + it('preserves A→B→C jobsAccepted ancestry so deleted A still resolves on C', async () => { + const { store, cache } = setup() + const a = cache.getOrCreateSession( + 'agent-jobs-a-' + Math.random().toString(36).slice(2, 8), + { path: '/tmp/project', host: 'localhost', flavor: 'codex' }, + null, + 'default' + ) + const b = cache.getOrCreateSession( + 'agent-jobs-b-' + Math.random().toString(36).slice(2, 8), + { path: '/tmp/project', host: 'localhost', flavor: 'codex' }, + null, + 'default' + ) + const c = cache.getOrCreateSession( + 'agent-jobs-c-' + Math.random().toString(36).slice(2, 8), + { path: '/tmp/project', host: 'localhost', flavor: 'codex' }, + null, + 'default' + ) + + store.sessionJobs.upsert(a.id, 'beets', { + label: 'beets import', + status: 'running', + remaining: 9 + }) + await cache.mergeSessions(a.id, b.id, 'default') + expect(cache.resolveAttachedJobSessionId(a.id, 'default')).toBe(b.id) + + await cache.mergeSessions(b.id, c.id, 'default') + const refreshed = cache.refreshSession(c.id) + expect(refreshed?.metadata?.jobsAcceptedFromSessionIds).toEqual( + expect.arrayContaining([a.id, b.id]) + ) + expect(cache.resolveAttachedJobSessionId(a.id, 'default')).toBe(c.id) + expect(cache.resolveAttachedJobSessionId(b.id, 'default')).toBe(c.id) + expect(store.sessionJobs.getPrimaryRunning(c.id)?.key).toBe('beets') + }) + it('keeps jobsAcceptedFromSessionIds when metadata merge also copies name from old', async () => { const { store, cache } = setup() const oldSession = cache.getOrCreateSession( diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 3f23e8c3c5..ed653fe3f9 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -1554,8 +1554,23 @@ export class SessionCache { const prev = Array.isArray(meta.jobsAcceptedFromSessionIds) ? meta.jobsAcceptedFromSessionIds.filter((id): id is string => typeof id === 'string') : [] - if (prev.includes(fromSessionId)) return - meta.jobsAcceptedFromSessionIds = [...prev, fromSessionId] + // Preserve A→B→C ancestry: when B already accepted jobs from A and + // now merges into C, clients still holding A's HAPI_SESSION_ID must + // resolve through C after B is deleted. + const inheritedRaw = this.store.sessions + .getSessionByNamespace(fromSessionId, namespace) + ?.metadata?.jobsAcceptedFromSessionIds + const inherited = Array.isArray(inheritedRaw) + ? inheritedRaw.filter((id): id is string => typeof id === 'string') + : [] + const next = [...new Set([...prev, ...inherited, fromSessionId])] + if ( + next.length === prev.length + && next.every((id) => prev.includes(id)) + ) { + return + } + meta.jobsAcceptedFromSessionIds = next const result = this.store.sessions.updateSessionMetadata( toSessionId, meta, From eb6ea149eff96df7b73e64bf7e19010229e5c73a Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:49:49 +0000 Subject: [PATCH 19/94] fix(jobs): type from-session metadata when inheriting job redirects Co-authored-by: Cursor --- hub/src/sync/sessionCache.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index ed653fe3f9..76bc164a38 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -1557,9 +1557,10 @@ export class SessionCache { // Preserve A→B→C ancestry: when B already accepted jobs from A and // now merges into C, clients still holding A's HAPI_SESSION_ID must // resolve through C after B is deleted. - const inheritedRaw = this.store.sessions + const fromMeta = this.store.sessions .getSessionByNamespace(fromSessionId, namespace) - ?.metadata?.jobsAcceptedFromSessionIds + ?.metadata as Record | null | undefined + const inheritedRaw = fromMeta?.jobsAcceptedFromSessionIds const inherited = Array.isArray(inheritedRaw) ? inheritedRaw.filter((id): id is string => typeof id === 'string') : [] From 430017aa3163c9740c41baf99fcfdfc9b03b7629 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:58:42 +0000 Subject: [PATCH 20/94] fix(jobs): version attachedJob SSE; keep live job on merge collision Gate list cache on versioned attachedJob patches so dual EventSources cannot resurrect a cleared meter. On transfer key collision, prefer a running source over a terminal target (heartbeats no longer rewrite status). Co-authored-by: Cursor --- hub/src/store/migration-v23.test.ts | 24 +++++++++++++++++++ hub/src/store/sessionJobs.ts | 23 ++++++++++++++++-- hub/src/sync/sessionCache.ts | 6 ++++- shared/src/schemas.sessionPatch.test.ts | 13 +++++++--- shared/src/schemas.ts | 13 +++++++--- shared/src/sessionSummary.test.ts | 2 ++ shared/src/sessionSummary.ts | 3 +++ .../SessionAttentionIndicator.test.tsx | 1 + .../SessionList.directory-action.test.tsx | 1 + .../SessionList.machine-filter.test.tsx | 1 + web/src/components/SessionList.test.ts | 1 + web/src/hooks/useSSE.test.ts | 1 + web/src/hooks/useSSE.ts | 17 ++++++++++--- web/src/lib/sessionAttention.test.ts | 1 + web/src/lib/sessionReference.test.ts | 1 + 15 files changed, 96 insertions(+), 12 deletions(-) diff --git a/hub/src/store/migration-v23.test.ts b/hub/src/store/migration-v23.test.ts index 2705a3c738..7506a15175 100644 --- a/hub/src/store/migration-v23.test.ts +++ b/hub/src/store/migration-v23.test.ts @@ -46,4 +46,28 @@ describe('schema migration v22 to v27', () => { expect(version.user_version).toBe(27) migrated.close() }) + + it('on key collision keeps a running source over a terminal target', () => { + const store = new Store(':memory:') + const from = store.sessions.getOrCreateSession('from', { path: '/a' }, null, 'default') + const to = store.sessions.getOrCreateSession('to', { path: '/b' }, null, 'default') + store.sessionJobs.upsert(to.id, 'beets', { + label: 'stale', + status: 'completed', + remaining: 0 + }, 1_000) + store.sessionJobs.upsert(from.id, 'beets', { + label: 'live', + status: 'running', + remaining: 3 + }, 2_000) + const result = store.sessionJobs.transfer(from.id, to.id) + expect(result.collided).toBe(1) + expect(result.moved).toBe(1) + const primary = store.sessionJobs.getPrimaryRunning(to.id) + expect(primary?.label).toBe('live') + expect(primary?.status).toBe('running') + expect(store.sessionJobs.list(from.id)).toHaveLength(0) + store.close() + }) }) diff --git a/hub/src/store/sessionJobs.ts b/hub/src/store/sessionJobs.ts index 644fc9415a..098444189f 100644 --- a/hub/src/store/sessionJobs.ts +++ b/hub/src/store/sessionJobs.ts @@ -249,6 +249,9 @@ export function transferSessionJobs( fromSessionId: string, toSessionId: string ): { moved: number; collided: number } { + if (fromSessionId === toSessionId) { + return { moved: 0, collided: 0 } + } const rows = listSessionJobs(db, fromSessionId) let moved = 0 let collided = 0 @@ -256,8 +259,24 @@ export function transferSessionJobs( for (const job of rows) { const existing = getSessionJob(db, toSessionId, job.key) if (existing) { - db.prepare('DELETE FROM session_jobs WHERE session_id = ? AND job_key = ?') - .run(fromSessionId, job.key) + // Prefer a live source over a terminal target (or newer stamp). + // Redirected heartbeats omit status, so discarding a running source + // cannot be repaired by a later heartbeat. + const sourceWins = + (job.status === 'running' && existing.status !== 'running') + || (job.status === existing.status && job.updatedAt > existing.updatedAt) + if (sourceWins) { + db.prepare('DELETE FROM session_jobs WHERE session_id = ? AND job_key = ?') + .run(toSessionId, job.key) + db.prepare( + `UPDATE session_jobs SET session_id = ? + WHERE session_id = ? AND job_key = ?` + ).run(toSessionId, fromSessionId, job.key) + moved += 1 + } else { + db.prepare('DELETE FROM session_jobs WHERE session_id = ? AND job_key = ?') + .run(fromSessionId, job.key) + } collided += 1 continue } diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 76bc164a38..9a846e9a8e 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -764,11 +764,15 @@ export class SessionCache { const namespace = cached?.namespace ?? this.store.sessions.getSession(sessionId)?.namespace if (!namespace) return + // Clear uses wall clock so it outranks any in-flight heartbeat stamp. + const version = attachedJob?.updatedAt ?? Date.now() this.publisher.emit({ type: 'session-updated', sessionId, namespace, - data: { attachedJob } satisfies SessionPatch + data: { + attachedJob: { version, value: attachedJob } + } satisfies SessionPatch }) } diff --git a/shared/src/schemas.sessionPatch.test.ts b/shared/src/schemas.sessionPatch.test.ts index 6515ea5c66..4801a53038 100644 --- a/shared/src/schemas.sessionPatch.test.ts +++ b/shared/src/schemas.sessionPatch.test.ts @@ -102,7 +102,7 @@ describe('SessionPatchSchema structured patches (closes #884 follow-up)', () => expect(SessionPatchSchema.safeParse(fullSession).success).toBe(false); }); - it('accepts attachedJob payload or null (tiann/hapi#1404)', () => { + it('accepts versioned attachedJob payload or null (tiann/hapi#1404)', () => { const job = AttachedJobSchema.parse({ key: 'beets', label: 'beets import', @@ -114,8 +114,15 @@ describe('SessionPatchSchema structured patches (closes #884 follow-up)', () => startedAt: 1_000, updatedAt: 2_000 }) - expect(SessionPatchSchema.safeParse({ attachedJob: job }).success).toBe(true) - expect(SessionPatchSchema.safeParse({ attachedJob: null }).success).toBe(true) + expect(SessionPatchSchema.safeParse({ + attachedJob: { version: job.updatedAt, value: job } + }).success).toBe(true) + expect(SessionPatchSchema.safeParse({ + attachedJob: { version: 3_000, value: null } + }).success).toBe(true) + // Bare job / bare null are rejected — dual-SSE needs a watermark. + expect(SessionPatchSchema.safeParse({ attachedJob: job }).success).toBe(false) + expect(SessionPatchSchema.safeParse({ attachedJob: null }).success).toBe(false) }); it('rejects fake percent-only attached jobs without counters', () => { diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index 52d32adb9e..ea03d7e2d0 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -453,6 +453,14 @@ export const AttachedJobPatchSchema = z.object({ export type AttachedJobPatch = z.infer +// Dual SSE (global + per-session) has no shared delivery order. Version = +// job.updatedAt for a live primary, or Date.now() when cleared to null, so a +// lagged running heartbeat cannot resurrect a finished meter. +const VersionedAttachedJobPatchSchema = z.object({ + version: z.number(), + value: AttachedJobSchema.nullable() +}) + export const SessionPatchSchema = z.object({ active: z.boolean().optional(), thinking: z.boolean().optional(), @@ -488,9 +496,8 @@ export const SessionPatchSchema = z.object({ scratchlistUpdatedAt: z.number().optional(), // tiann/hapi#1404 — session-attached long-running jobs. Unlike // scratchlist (watermark → refetch), the list row needs the progress - // payload inline, so patches carry the primary running job (or null - // when cleared / none remain). - attachedJob: AttachedJobSchema.nullable().optional() + // payload inline. Versioned like todos so dual-SSE reorder is safe. + attachedJob: VersionedAttachedJobPatchSchema.optional() }).strict() export type SessionPatch = z.infer diff --git a/shared/src/sessionSummary.test.ts b/shared/src/sessionSummary.test.ts index c2a7f0b2b4..eb6f5b7ead 100644 --- a/shared/src/sessionSummary.test.ts +++ b/shared/src/sessionSummary.test.ts @@ -343,10 +343,12 @@ describe('summary derivation helpers', () => { } const summary = toSessionSummary(makeSession(), { attachedJob: job }) expect(summary.attachedJob).toEqual(job) + expect(summary.attachedJobUpdatedAt).toBe(job.updatedAt) }) it('defaults attachedJob to null', () => { expect(toSessionSummary(makeSession()).attachedJob).toBeNull() + expect(toSessionSummary(makeSession()).attachedJobUpdatedAt).toBe(0) }) it('toSessionSummaryMetadata returns null for null metadata', () => { diff --git a/shared/src/sessionSummary.ts b/shared/src/sessionSummary.ts index 6ded7a20d1..1684664abd 100644 --- a/shared/src/sessionSummary.ts +++ b/shared/src/sessionSummary.ts @@ -77,6 +77,8 @@ export type SessionSummary = { * Independent of agent `active` / thinking — work that outlives the agent. */ attachedJob: AttachedJob | null + /** Watermark for versioned `attachedJob` SSE patches (dual EventSource race). */ + attachedJobUpdatedAt: number model: string | null modelReasoningEffort?: string | null effort: string | null @@ -233,6 +235,7 @@ export function toSessionSummary( futureScheduledMessageCount: 0, nextScheduledAt: null, attachedJob: extras?.attachedJob ?? null, + attachedJobUpdatedAt: extras?.attachedJob?.updatedAt ?? 0, model: session.model, modelReasoningEffort: session.modelReasoningEffort, effort: session.effort diff --git a/web/src/components/SessionAttentionIndicator.test.tsx b/web/src/components/SessionAttentionIndicator.test.tsx index bbf540153c..eaa0654355 100644 --- a/web/src/components/SessionAttentionIndicator.test.tsx +++ b/web/src/components/SessionAttentionIndicator.test.tsx @@ -30,6 +30,7 @@ function makeSummary(overrides: Partial & { id: string }): Sessi futureScheduledMessageCount: 0, nextScheduledAt: null, attachedJob: null, + attachedJobUpdatedAt: 0, model: null, effort: null, ...overrides diff --git a/web/src/components/SessionList.directory-action.test.tsx b/web/src/components/SessionList.directory-action.test.tsx index 3b5cb89be1..278374d6f6 100644 --- a/web/src/components/SessionList.directory-action.test.tsx +++ b/web/src/components/SessionList.directory-action.test.tsx @@ -35,6 +35,7 @@ function makeSession(overrides: Partial & { id: string }): Sessi futureScheduledMessageCount: 0, nextScheduledAt: null, attachedJob: null, + attachedJobUpdatedAt: 0, model: null, effort: null, ...overrides diff --git a/web/src/components/SessionList.machine-filter.test.tsx b/web/src/components/SessionList.machine-filter.test.tsx index 4fc1477d44..587ede6166 100644 --- a/web/src/components/SessionList.machine-filter.test.tsx +++ b/web/src/components/SessionList.machine-filter.test.tsx @@ -30,6 +30,7 @@ function makeSession(overrides: Partial & { id: string }): Sessi futureScheduledMessageCount: 0, nextScheduledAt: null, attachedJob: null, + attachedJobUpdatedAt: 0, model: null, effort: null, ...overrides diff --git a/web/src/components/SessionList.test.ts b/web/src/components/SessionList.test.ts index 1c37f67fd1..f3689fc119 100644 --- a/web/src/components/SessionList.test.ts +++ b/web/src/components/SessionList.test.ts @@ -43,6 +43,7 @@ function makeSession(overrides: Partial & { id: string }): Sessi futureScheduledMessageCount: 0, nextScheduledAt: null, attachedJob: null, + attachedJobUpdatedAt: 0, model: null, effort: null, ...overrides diff --git a/web/src/hooks/useSSE.test.ts b/web/src/hooks/useSSE.test.ts index 3a6592e631..f849241eb2 100644 --- a/web/src/hooks/useSSE.test.ts +++ b/web/src/hooks/useSSE.test.ts @@ -180,6 +180,7 @@ function makeSummary(overrides: Partial = {}): SessionSummary { futureScheduledMessageCount: 0, nextScheduledAt: null, attachedJob: null, + attachedJobUpdatedAt: 0, model: null, effort: null, ...overrides diff --git a/web/src/hooks/useSSE.ts b/web/src/hooks/useSSE.ts index 926705726d..443b6d1196 100644 --- a/web/src/hooks/useSSE.ts +++ b/web/src/hooks/useSSE.ts @@ -170,6 +170,7 @@ export function isRenderIrrelevantPatch(current: SessionSummary, next: SessionSu && current.attachedJob?.heartbeatAt === next.attachedJob?.heartbeatAt && current.attachedJob?.startedAt === next.attachedJob?.startedAt && (current.attachedJob == null) === (next.attachedJob == null) + && (current.attachedJobUpdatedAt ?? 0) === (next.attachedJobUpdatedAt ?? 0) && current.model === next.model && current.modelReasoningEffort === next.modelReasoningEffort && current.effort === next.effort @@ -499,6 +500,7 @@ export function useSSE(options: { const summary = { ...toSessionSummary(session), attachedJob: existing?.attachedJob ?? null, + attachedJobUpdatedAt: existing?.attachedJobUpdatedAt ?? 0, futureScheduledMessageCount: existing?.futureScheduledMessageCount ?? 0, nextScheduledAt: existing?.nextScheduledAt ?? null } @@ -544,9 +546,8 @@ export function useSSE(options: { backgroundTaskCount: Object.prototype.hasOwnProperty.call(patch, 'backgroundTaskCount') ? patch.backgroundTaskCount ?? 0 : current.backgroundTaskCount, - attachedJob: Object.prototype.hasOwnProperty.call(patch, 'attachedJob') - ? patch.attachedJob ?? null - : current.attachedJob ?? null, + attachedJob: current.attachedJob ?? null, + attachedJobUpdatedAt: current.attachedJobUpdatedAt ?? 0, model: Object.prototype.hasOwnProperty.call(patch, 'model') ? patch.model ?? null : current.model, modelReasoningEffort: Object.prototype.hasOwnProperty.call(patch, 'modelReasoningEffort') ? patch.modelReasoningEffort ?? null @@ -572,6 +573,16 @@ export function useSSE(options: { nextSummary.metadata = toSessionSummaryMetadata(patch.metadata.value) nextSummary.metadataVersion = patch.metadata.version } + if ( + patch.attachedJob !== undefined + && isNewerVersionedPatch( + patch.attachedJob.version, + current.attachedJobUpdatedAt ?? 0 + ) + ) { + nextSummary.attachedJob = patch.attachedJob.value + nextSummary.attachedJobUpdatedAt = patch.attachedJob.version + } patched = true // The keep-alive patch repeats every field every ~10s per active diff --git a/web/src/lib/sessionAttention.test.ts b/web/src/lib/sessionAttention.test.ts index a503ac98a9..690475641b 100644 --- a/web/src/lib/sessionAttention.test.ts +++ b/web/src/lib/sessionAttention.test.ts @@ -23,6 +23,7 @@ function makeSummary(overrides: Partial & { id: string }): Sessi futureScheduledMessageCount: 0, nextScheduledAt: null, attachedJob: null, + attachedJobUpdatedAt: 0, model: null, effort: null, ...overrides diff --git a/web/src/lib/sessionReference.test.ts b/web/src/lib/sessionReference.test.ts index 407f296646..58bdd3ab57 100644 --- a/web/src/lib/sessionReference.test.ts +++ b/web/src/lib/sessionReference.test.ts @@ -29,6 +29,7 @@ function makeSession(overrides: Partial & { id: string }): Sessi futureScheduledMessageCount: 0, nextScheduledAt: null, attachedJob: null, + attachedJobUpdatedAt: 0, model: null, effort: null, ...overrides, From 7ee1bd549a552b3eddff05b55b77d1b3f7b447cd Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:08:23 +0000 Subject: [PATCH 21/94] fix(jobs): monotonic attachedJob emit watermark (not primary.updatedAt) Primary switches can go backwards in updatedAt and strand the list cache. Emit versions are Math.max(Date.now(), prev+1); REST list stamps a fresh wall-clock watermark so null jobs do not reset the client gate to 0. Co-authored-by: Cursor --- hub/src/sync/sessionCache.ts | 9 +++++++-- hub/src/web/routes/sessions.ts | 6 +++++- shared/src/schemas.ts | 6 +++--- shared/src/sessionSummary.ts | 14 +++++++++++--- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 9a846e9a8e..2421864264 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -756,6 +756,10 @@ export class SessionCache { * tiann/hapi#1404 — emit primary attached job (or null) so session-list * caches update inline without a dedicated refetch. */ + /** Monotonic emit watermark per session — never follows primary.updatedAt + * (primary switches can go backwards and would strand the web cache). */ + private attachedJobEmitVersion = new Map() + emitAttachedJobChanged( sessionId: string, attachedJob: import('@hapi/protocol').AttachedJob | null @@ -764,8 +768,9 @@ export class SessionCache { const namespace = cached?.namespace ?? this.store.sessions.getSession(sessionId)?.namespace if (!namespace) return - // Clear uses wall clock so it outranks any in-flight heartbeat stamp. - const version = attachedJob?.updatedAt ?? Date.now() + const prev = this.attachedJobEmitVersion.get(sessionId) ?? 0 + const version = Math.max(Date.now(), prev + 1) + this.attachedJobEmitVersion.set(sessionId, version) this.publisher.emit({ type: 'session-updated', sessionId, diff --git a/hub/src/web/routes/sessions.ts b/hub/src/web/routes/sessions.ts index 74ff6508fe..14608f0579 100644 --- a/hub/src/web/routes/sessions.ts +++ b/hub/src/web/routes/sessions.ts @@ -121,9 +121,13 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho const scheduledCounts = engine.getFutureScheduledMessageCounts(sessionRecords.map((session) => session.id)) const nextScheduledAt = engine.getNextScheduledAtBySessionIds(sessionRecords.map((session) => session.id)) const attachedJobs = engine.getPrimaryAttachedJobsBySessionIds(sessionRecords.map((session) => session.id)) + // Fresh wall-clock watermark so a REST refetch never resets the client + // SSE gate to 0 (which would let a lagged running patch resurrect). + const listJobWatermark = Date.now() const sessions = sessionRecords.map((session) => { const summary = toSessionSummary(session, { - attachedJob: attachedJobs.get(session.id) ?? null + attachedJob: attachedJobs.get(session.id) ?? null, + attachedJobUpdatedAt: listJobWatermark }) return { ...summary, diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index ea03d7e2d0..5371df4137 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -453,9 +453,9 @@ export const AttachedJobPatchSchema = z.object({ export type AttachedJobPatch = z.infer -// Dual SSE (global + per-session) has no shared delivery order. Version = -// job.updatedAt for a live primary, or Date.now() when cleared to null, so a -// lagged running heartbeat cannot resurrect a finished meter. +// Dual SSE (global + per-session) has no shared delivery order. Version is a +// monotonic per-session emit watermark (not primary.updatedAt — primary +// switches can go backwards). Lagged heartbeats cannot resurrect a clear. const VersionedAttachedJobPatchSchema = z.object({ version: z.number(), value: AttachedJobSchema.nullable() diff --git a/shared/src/sessionSummary.ts b/shared/src/sessionSummary.ts index 1684664abd..efa9d4d5ad 100644 --- a/shared/src/sessionSummary.ts +++ b/shared/src/sessionSummary.ts @@ -212,8 +212,14 @@ export function toSessionSummaryMetadata(metadata: Metadata | null | undefined): export function toSessionSummary( session: Session, - extras?: { attachedJob?: AttachedJob | null } + extras?: { + attachedJob?: AttachedJob | null + /** Explicit SSE/list watermark; required when attachedJob is null so + * a REST refetch does not reset the client gate to 0. */ + attachedJobUpdatedAt?: number + } ): SessionSummary { + const attachedJob = extras?.attachedJob ?? null return { hasConversationContent: session.hasConversationContent ?? false, id: session.id, @@ -234,8 +240,10 @@ export function toSessionSummary( backgroundTaskCount: session.backgroundTaskCount ?? 0, futureScheduledMessageCount: 0, nextScheduledAt: null, - attachedJob: extras?.attachedJob ?? null, - attachedJobUpdatedAt: extras?.attachedJob?.updatedAt ?? 0, + attachedJob, + attachedJobUpdatedAt: extras?.attachedJobUpdatedAt + ?? attachedJob?.updatedAt + ?? 0, model: session.model, modelReasoningEffort: session.modelReasoningEffort, effort: session.effort From 6a58daec40b75f4679074f1aacbe6b0fa1738de2 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:18:59 +0000 Subject: [PATCH 22/94] fix(jobs): one attachedJob watermark allocator for REST and SSE GET /sessions and emitAttachedJobChanged share allocateAttachedJobVersion so equal-ms terminal patches are not rejected after a list refetch. Also preserve real Unix signal exit codes in hapi job run. Co-authored-by: Cursor --- cli/src/modules/sessionJob/runSessionJob.ts | 4 +++- hub/src/sync/sessionCache.ts | 15 ++++++++++----- hub/src/sync/syncEngine.ts | 5 +++++ hub/src/web/routes/sessions.ts | 6 ++---- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/cli/src/modules/sessionJob/runSessionJob.ts b/cli/src/modules/sessionJob/runSessionJob.ts index 238ef301c0..512c607e68 100644 --- a/cli/src/modules/sessionJob/runSessionJob.ts +++ b/cli/src/modules/sessionJob/runSessionJob.ts @@ -4,6 +4,7 @@ */ import { spawn, type ChildProcess } from 'node:child_process' +import { constants as osConstants } from 'node:os' import type { AttachedJobUpsert } from '@hapi/protocol' import { SessionJobError, @@ -131,7 +132,8 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise { clearIntervalFn(heartbeat) if (signal) { - resolve(128 + (signal === 'SIGINT' ? 2 : signal === 'SIGTERM' ? 15 : 1)) + const signalNumber = osConstants.signals[signal] ?? 1 + resolve(128 + signalNumber) return } resolve(code ?? 1) diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 2421864264..784dc2b59a 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -756,10 +756,17 @@ export class SessionCache { * tiann/hapi#1404 — emit primary attached job (or null) so session-list * caches update inline without a dedicated refetch. */ - /** Monotonic emit watermark per session — never follows primary.updatedAt - * (primary switches can go backwards and would strand the web cache). */ + /** Monotonic watermark per session — shared by REST list snapshots and SSE + * emits so equal-ms terminal patches are not rejected after a refetch. */ private attachedJobEmitVersion = new Map() + allocateAttachedJobVersion(sessionId: string): number { + const prev = this.attachedJobEmitVersion.get(sessionId) ?? 0 + const version = Math.max(Date.now(), prev + 1) + this.attachedJobEmitVersion.set(sessionId, version) + return version + } + emitAttachedJobChanged( sessionId: string, attachedJob: import('@hapi/protocol').AttachedJob | null @@ -768,9 +775,7 @@ export class SessionCache { const namespace = cached?.namespace ?? this.store.sessions.getSession(sessionId)?.namespace if (!namespace) return - const prev = this.attachedJobEmitVersion.get(sessionId) ?? 0 - const version = Math.max(Date.now(), prev + 1) - this.attachedJobEmitVersion.set(sessionId, version) + const version = this.allocateAttachedJobVersion(sessionId) this.publisher.emit({ type: 'session-updated', sessionId, diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 871ac38cd1..40149fde7a 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -817,6 +817,11 @@ export class SyncEngine { return this.store.sessionJobs.getPrimaryRunningBySessionIds(sessionIds) } + /** Shared REST/SSE watermark allocator for attachedJob patches. */ + allocateAttachedJobVersion(sessionId: string): number { + return this.sessionCache.allocateAttachedJobVersion(sessionId) + } + upsertSessionJob( sessionId: string, jobKey: string, diff --git a/hub/src/web/routes/sessions.ts b/hub/src/web/routes/sessions.ts index 14608f0579..d5d81beb68 100644 --- a/hub/src/web/routes/sessions.ts +++ b/hub/src/web/routes/sessions.ts @@ -121,13 +121,11 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho const scheduledCounts = engine.getFutureScheduledMessageCounts(sessionRecords.map((session) => session.id)) const nextScheduledAt = engine.getNextScheduledAtBySessionIds(sessionRecords.map((session) => session.id)) const attachedJobs = engine.getPrimaryAttachedJobsBySessionIds(sessionRecords.map((session) => session.id)) - // Fresh wall-clock watermark so a REST refetch never resets the client - // SSE gate to 0 (which would let a lagged running patch resurrect). - const listJobWatermark = Date.now() const sessions = sessionRecords.map((session) => { const summary = toSessionSummary(session, { attachedJob: attachedJobs.get(session.id) ?? null, - attachedJobUpdatedAt: listJobWatermark + // Same allocator as SSE emits — equal-ms terminal patches stay applyable. + attachedJobUpdatedAt: engine.allocateAttachedJobVersion(session.id) }) return { ...summary, From c6452c8e97c7d10f47596f95f3d8859a746e3c76 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:19:30 +0000 Subject: [PATCH 23/94] fix(jobs): stub allocateAttachedJobVersion in sessions route mocks Co-authored-by: Cursor --- hub/src/web/routes/sessions-jobs.test.ts | 7 +++++++ hub/src/web/routes/sessions.test.ts | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/hub/src/web/routes/sessions-jobs.test.ts b/hub/src/web/routes/sessions-jobs.test.ts index 8622a34de1..e8408b3797 100644 --- a/hub/src/web/routes/sessions-jobs.test.ts +++ b/hub/src/web/routes/sessions-jobs.test.ts @@ -48,6 +48,13 @@ describe('session-attached jobs routes (tiann/hapi#1404)', () => { return map }, getPrimaryAttachedJob: () => [...jobs.values()].find((j) => j.status === 'running') ?? null, + allocateAttachedJobVersion: (() => { + let n = 0 + return () => { + n += 1 + return Date.now() + n + } + })(), listSessionJobs: () => [...jobs.values()], upsertSessionJob: (_sid: string, key: string, body: AttachedJobUpsert) => { const now = Date.now() diff --git a/hub/src/web/routes/sessions.test.ts b/hub/src/web/routes/sessions.test.ts index c798a6de21..7eaae8164f 100644 --- a/hub/src/web/routes/sessions.test.ts +++ b/hub/src/web/routes/sessions.test.ts @@ -181,7 +181,8 @@ function createApp(session: Session, opts?: { rewindConversation: opts?.rewindConversation ?? (async () => ({ type: 'success' })), suggestSessionTitle: opts?.suggestSessionTitle ?? (async () => 'Generated title'), updateSessionSummary: opts?.updateSessionSummary ?? (async () => {}), - getPrimaryAttachedJobsBySessionIds: opts?.getPrimaryAttachedJobsBySessionIds ?? (() => new Map()) + getPrimaryAttachedJobsBySessionIds: opts?.getPrimaryAttachedJobsBySessionIds ?? (() => new Map()), + allocateAttachedJobVersion: opts?.allocateAttachedJobVersion ?? (() => Date.now()) } as Partial const app = new Hono() @@ -1703,6 +1704,7 @@ describe('sessions routes', () => { }, getNextScheduledAtBySessionIds: (_ids: string[]) => new Map(), getPrimaryAttachedJobsBySessionIds: () => new Map(), + allocateAttachedJobVersion: () => Date.now(), resolveSessionAccess: () => ({ ok: false, reason: 'not-found' as const }) } as unknown as Partial @@ -1736,6 +1738,7 @@ describe('sessions routes', () => { getFutureScheduledMessageCounts: (ids: string[]) => new Map(ids.map((id) => [id, 0])), getNextScheduledAtBySessionIds: (_ids: string[]) => new Map(), getPrimaryAttachedJobsBySessionIds: () => new Map(), + allocateAttachedJobVersion: () => Date.now(), resolveSessionAccess: () => ({ ok: false, reason: 'not-found' as const }) } as unknown as Partial From e02de03b43dbe4e8d5ddfdd1f272b683c2e85651 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:29:34 +0000 Subject: [PATCH 24/94] fix(jobs): always record merge redirects even with zero jobs transferred Agents may attach the first outliving job after merge while still holding the pre-merge HAPI_SESSION_ID; redirects must exist before that first set. Co-authored-by: Cursor --- hub/src/sync/sessionCache-merge-jobs.test.ts | 22 ++++++++++++++++++++ hub/src/sync/sessionCache.ts | 10 ++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/hub/src/sync/sessionCache-merge-jobs.test.ts b/hub/src/sync/sessionCache-merge-jobs.test.ts index 83b7c2c1c6..5f9dbddf0f 100644 --- a/hub/src/sync/sessionCache-merge-jobs.test.ts +++ b/hub/src/sync/sessionCache-merge-jobs.test.ts @@ -73,6 +73,28 @@ describe('mergeSessions job redirect through SessionCache (#1404)', () => { expect(cache.resolveAttachedJobSessionId(newSession.id, 'default')).toBe(newSession.id) }) + it('records job redirects even when the source has no jobs yet', async () => { + const { store, cache } = setup() + const { oldSession, newSession } = makeSessions(cache) + + await cache.mergeSessions(oldSession.id, newSession.id, 'default') + + expect(store.sessions.getSession(oldSession.id)).toBeNull() + const refreshed = cache.refreshSession(newSession.id) + expect(refreshed?.metadata?.jobsAcceptedFromSessionIds).toContain(oldSession.id) + expect(cache.resolveAttachedJobSessionId(oldSession.id, 'default')).toBe(newSession.id) + + // First job attach after merge still lands on the canonical session + // when the agent keeps the pre-merge HAPI_SESSION_ID. + const upserted = store.sessionJobs.upsert( + cache.resolveAttachedJobSessionId(oldSession.id, 'default')!, + 'late', + { label: 'late attach', status: 'running', remaining: 1 } + ) + expect(upserted.outcome).toBe('upserted') + expect(store.sessionJobs.getPrimaryRunning(newSession.id)?.key).toBe('late') + }) + it('preserves A→B→C jobsAccepted ancestry so deleted A still resolves on C', async () => { const { store, cache } = setup() const a = cache.getOrCreateSession( diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 784dc2b59a..9613ab1c9d 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -1409,11 +1409,11 @@ export class SessionCache { // merge clobbers jobsAcceptedFromSessionIds when mergeSessionMetadata // rebuilds from the stale pre-merge newStored.metadata snapshot // (cold-review pass 3 Major — agents heartbeating $HAPI_SESSION_ID 404). - if (movedJobs.moved > 0 || movedJobs.collided > 0) { - this.recordJobsAcceptedFromSession(newSessionId, oldSessionId, namespace) - if (!options.deleteOldSession) { - this.recordJobsTransferredToSession(oldSessionId, newSessionId, namespace) - } + // Always record redirects even when the source had zero jobs yet — the + // first post-merge set/update still uses retained $HAPI_SESSION_ID. + this.recordJobsAcceptedFromSession(newSessionId, oldSessionId, namespace) + if (!options.deleteOldSession) { + this.recordJobsTransferredToSession(oldSessionId, newSessionId, namespace) } if (newStored.model === null && oldStored.model !== null) { From 664a5e9fffd81b94e2cc6943e8b5c91b6d81cb66 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:37:14 +0000 Subject: [PATCH 25/94] fix(jobs): install source redirect before merge awaits; allow empty update Point jobsTransferredToSession immediately after transfer so mid-merge heartbeats follow the new owner during scratchlist I/O. Empty CLI/MCP update bodies are heartbeat-only (hub stamps heartbeatAt). Co-authored-by: Cursor --- cli/src/commands/job.ts | 4 +--- cli/src/modules/sessionJob/sessionJobMcp.ts | 7 +------ hub/src/sync/sessionCache.ts | 17 ++++++++--------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/cli/src/commands/job.ts b/cli/src/commands/job.ts index b91d189254..b3753a86da 100644 --- a/cli/src/commands/job.ts +++ b/cli/src/commands/job.ts @@ -358,9 +358,7 @@ export async function handleJobCommand(args: string[]): Promise { ...(parsed.unit !== undefined ? { unit: parsed.unit } : {}), ...(parsed.detail !== undefined ? { detail: parsed.detail } : {}) } - if (Object.keys(body).length === 0) { - throw new SessionJobError('bad_args', 'update requires at least one field') - } + // Empty body is a heartbeat-only update; hub stamps heartbeatAt. const result = await updateSessionJob({ sessionIdPrefix: parsed.sessionIdPrefix, jobKey: parsed.jobKey, diff --git a/cli/src/modules/sessionJob/sessionJobMcp.ts b/cli/src/modules/sessionJob/sessionJobMcp.ts index 7f689f89bd..e5119663b9 100644 --- a/cli/src/modules/sessionJob/sessionJobMcp.ts +++ b/cli/src/modules/sessionJob/sessionJobMcp.ts @@ -148,12 +148,7 @@ export async function handleSessionJobTool( ...(args.unit !== undefined ? { unit: args.unit } : {}), ...(args.detail !== undefined ? { detail: args.detail } : {}) } - if (Object.keys(body).length === 0) { - return { - text: 'update requires at least one of label/status/done/total/remaining/unit/detail', - isError: true - } - } + // Empty body is a heartbeat-only update; hub stamps heartbeatAt. const result = await updateSessionJob({ sessionIdPrefix, jobKey, body }) return { text: `updated ${formatJobLine(result.job)} on ${result.sessionId}`, diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index 9613ab1c9d..ba3cefcd24 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -1329,6 +1329,11 @@ export class SessionCache { // promise that scratchlist survives reloads. const movedScratchlist = this.store.scratchlist.transfer(oldSessionId, newSessionId) const movedJobs = this.store.sessionJobs.transfer(oldSessionId, newSessionId) + // Install the source→target redirect BEFORE any await below. Merge can + // spend time on scratchlist attachment I/O while the old session row + // still exists; without this pointer, retained $HAPI_SESSION_ID hits + // the emptied source and terminal PATCHes 404. + this.recordJobsTransferredToSession(oldSessionId, newSessionId, namespace) if (movedJobs.moved > 0 || movedJobs.collided > 0) { this.emitAttachedJobChanged( newSessionId, @@ -1405,16 +1410,10 @@ export class SessionCache { } } - // Job-owner redirects AFTER metadata merge. Writing them before the - // merge clobbers jobsAcceptedFromSessionIds when mergeSessionMetadata - // rebuilds from the stale pre-merge newStored.metadata snapshot - // (cold-review pass 3 Major — agents heartbeating $HAPI_SESSION_ID 404). - // Always record redirects even when the source had zero jobs yet — the - // first post-merge set/update still uses retained $HAPI_SESSION_ID. + // Acceptor list AFTER metadata merge (writing before clobbers when + // mergeSessionMetadata rebuilds from the stale pre-merge snapshot). + // Source transfer pointer was installed immediately after job transfer. this.recordJobsAcceptedFromSession(newSessionId, oldSessionId, namespace) - if (!options.deleteOldSession) { - this.recordJobsTransferredToSession(oldSessionId, newSessionId, namespace) - } if (newStored.model === null && oldStored.model !== null) { const updated = this.store.sessions.setSessionModel(newSessionId, oldStored.model, namespace, { From a73c99f43e32ea458e6114a55cb1a250c750ab1f Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:48:13 +0000 Subject: [PATCH 26/94] fix(jobs): expect empty MCP update as heartbeat Co-authored-by: Cursor --- cli/src/modules/sessionJob/sessionJobMcp.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cli/src/modules/sessionJob/sessionJobMcp.test.ts b/cli/src/modules/sessionJob/sessionJobMcp.test.ts index b9a4fe95de..3fca228405 100644 --- a/cli/src/modules/sessionJob/sessionJobMcp.test.ts +++ b/cli/src/modules/sessionJob/sessionJobMcp.test.ts @@ -61,12 +61,20 @@ describe('sessionJobMcp', () => { expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/Own-session only/i) }) - it('rejects update with empty patch', async () => { + it('treats empty update as a heartbeat-only patch', async () => { + const { updateSessionJob } = await import('./sessionJob') const result = await handleSessionJobTool( { action: 'update', jobKey: 'beets' }, 'sid-1' ) - expect(result.isError).toBe(true) - expect(result.text).toMatch(/at least one/i) + expect(result.isError).toBe(false) + expect(result.text).toContain('updated') + expect(updateSessionJob).toHaveBeenCalledWith( + expect.objectContaining({ + sessionIdPrefix: 'sid-1', + jobKey: 'beets', + body: {} + }) + ) }) }) From a0275e77524b697075cfa0ba734c1c911c869e68 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:51:23 +0000 Subject: [PATCH 27/94] fix(jobs): treat merge redirect metadata as hub-owned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLI update-metadata must not forge or erase jobsAcceptedFromSessionIds / jobsTransferredToSessionId — same strip/restore as supersede/clear links. Co-authored-by: Cursor --- .../handlers/cli/sessionHandlers.test.ts | 30 +++++++++++++------ .../socket/handlers/cli/sessionHandlers.ts | 9 +++++- shared/src/schemas.ts | 1 + 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/hub/src/socket/handlers/cli/sessionHandlers.test.ts b/hub/src/socket/handlers/cli/sessionHandlers.test.ts index cec001db17..c034bd9953 100644 --- a/hub/src/socket/handlers/cli/sessionHandlers.test.ts +++ b/hub/src/socket/handlers/cli/sessionHandlers.test.ts @@ -438,9 +438,14 @@ describe('cli session handlers', () => { expect(uuids).toEqual(['msg-1', 'msg-2']) }) - it.each(['supersededBySessionId', 'opencodeClearOperation'] as const)( + it.each([ + ['supersededBySessionId', 'foreign-session'], + ['opencodeClearOperation', { replacementSessionId: 'foreign-session', state: 'reserved', updatedAt: Date.now() }], + ['jobsAcceptedFromSessionIds', ['foreign-session']], + ['jobsTransferredToSessionId', 'foreign-session'], + ] as const)( 'ignores a forged hub-owned %s addition from CLI metadata', - (field) => { + (field, forged) => { const store = new Store(':memory:') const session = store.sessions.getOrCreateSession('forged-clear-link', { path: '/tmp/project' }, null, 'default') const socket = new FakeSocket() @@ -454,20 +459,21 @@ describe('cli session handlers', () => { expectedVersion: session.metadataVersion, metadata: { path: '/tmp/project', - [field]: field === 'supersededBySessionId' - ? 'foreign-session' - : { replacementSessionId: 'foreign-session', state: 'reserved', updatedAt: Date.now() } + [field]: forged } }, () => {}) expect(store.sessions.getSessionByNamespace(session.id, 'default')?.metadata).not.toHaveProperty(field) } ) - it('preserves existing hub-owned clear metadata across CLI metadata updates', () => { + it('preserves existing hub-owned clear and job-redirect metadata across CLI metadata updates', () => { const store = new Store(':memory:') const operation = { replacementSessionId: 'owned-target', state: 'completed', updatedAt: Date.now() } const session = store.sessions.getOrCreateSession('preserve-clear-link', { - supersededBySessionId: 'owned-target', opencodeClearOperation: operation + supersededBySessionId: 'owned-target', + opencodeClearOperation: operation, + jobsAcceptedFromSessionIds: ['old-session'], + jobsTransferredToSessionId: 'merge-target', }, null, 'default') const socket = new FakeSocket() registerSessionHandlers(socket as unknown as CliSocketWithData, { @@ -481,11 +487,17 @@ describe('cli session handlers', () => { metadata: { lifecycleState: 'archived', supersededBySessionId: 'forged-target', - opencodeClearOperation: { replacementSessionId: 'forged-target', state: 'reserved', updatedAt: 0 } + opencodeClearOperation: { replacementSessionId: 'forged-target', state: 'reserved', updatedAt: 0 }, + jobsAcceptedFromSessionIds: ['forged-session'], + jobsTransferredToSessionId: 'forged-target', } }, () => {}) expect(store.sessions.getSessionByNamespace(session.id, 'default')?.metadata).toMatchObject({ - supersededBySessionId: 'owned-target', opencodeClearOperation: operation, lifecycleState: 'archived' + supersededBySessionId: 'owned-target', + opencodeClearOperation: operation, + jobsAcceptedFromSessionIds: ['old-session'], + jobsTransferredToSessionId: 'merge-target', + lifecycleState: 'archived', }) }) }) diff --git a/hub/src/socket/handlers/cli/sessionHandlers.ts b/hub/src/socket/handlers/cli/sessionHandlers.ts index dd1c24a463..89e90f2dfc 100644 --- a/hub/src/socket/handlers/cli/sessionHandlers.ts +++ b/hub/src/socket/handlers/cli/sessionHandlers.ts @@ -68,7 +68,14 @@ const updateStateSchema = z.object({ agentState: z.unknown().nullable() }) -const HUB_OWNED_METADATA_KEYS = ['supersededBySessionId', 'opencodeClearOperation'] as const +// Hub-only merge/clear links. CLI update-metadata must not forge or erase them +// (same strip/restore as supersededBySessionId — see sessionHandlers.test.ts). +const HUB_OWNED_METADATA_KEYS = [ + 'supersededBySessionId', + 'opencodeClearOperation', + 'jobsAcceptedFromSessionIds', + 'jobsTransferredToSessionId', +] as const function preserveHubOwnedMetadata(incoming: unknown, current: unknown): unknown { if (!incoming || typeof incoming !== 'object' || Array.isArray(incoming)) return incoming diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index 5371df4137..71edcd1fef 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -139,6 +139,7 @@ export const MetadataSchema = z.object({ // pre-merge $HAPI_SESSION_ID), and a kept-alive source points at the // post-merge owner. Must be declared here — SessionCache.refreshSession // parses via MetadataSchema and strips unknown keys (tiann/hapi#1404). + // Hub-owned: CLI update-metadata cannot forge/erase (HUB_OWNED_METADATA_KEYS). jobsAcceptedFromSessionIds: z.array(z.string()).optional(), jobsTransferredToSessionId: z.string().optional(), // Durable in-progress state for runner-backed OpenCode /clear. From 4281cc2e6e793c4f7e5f517b391d119efb17355e Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 01:03:36 +0000 Subject: [PATCH 28/94] fix(jobs): reconcile REST list against attachedJob watermark Slow /api/sessions can finish after SSE clear/progress and resurrect a stale attachedJob. Also prefer newer terminal results on merge key collisions. Co-authored-by: Cursor --- hub/src/store/migration-v23.test.ts | 25 +++++ hub/src/store/sessionJobs.ts | 14 ++- .../queries/reconcileAttachedJobs.test.ts | 101 ++++++++++++++++++ .../hooks/queries/reconcileAttachedJobs.ts | 34 ++++++ web/src/hooks/queries/useSessions.ts | 10 +- 5 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 web/src/hooks/queries/reconcileAttachedJobs.test.ts create mode 100644 web/src/hooks/queries/reconcileAttachedJobs.ts diff --git a/hub/src/store/migration-v23.test.ts b/hub/src/store/migration-v23.test.ts index 7506a15175..8e22b13744 100644 --- a/hub/src/store/migration-v23.test.ts +++ b/hub/src/store/migration-v23.test.ts @@ -70,4 +70,29 @@ describe('schema migration v22 to v27', () => { expect(store.sessionJobs.list(from.id)).toHaveLength(0) store.close() }) + + it('on key collision prefers a newer terminal source over an older terminal target', () => { + const store = new Store(':memory:') + const from = store.sessions.getOrCreateSession('from-term', { path: '/a' }, null, 'default') + const to = store.sessions.getOrCreateSession('to-term', { path: '/b' }, null, 'default') + store.sessionJobs.upsert(to.id, 'beets', { + label: 'old-complete', + status: 'completed', + remaining: 0 + }, 1_000) + store.sessionJobs.upsert(from.id, 'beets', { + label: 'new-fail', + status: 'failed', + remaining: 0 + }, 2_000) + const result = store.sessionJobs.transfer(from.id, to.id) + expect(result.collided).toBe(1) + expect(result.moved).toBe(1) + const kept = store.sessionJobs.list(to.id) + expect(kept).toHaveLength(1) + expect(kept[0]?.label).toBe('new-fail') + expect(kept[0]?.status).toBe('failed') + expect(store.sessionJobs.list(from.id)).toHaveLength(0) + store.close() + }) }) diff --git a/hub/src/store/sessionJobs.ts b/hub/src/store/sessionJobs.ts index 098444189f..88f32d2219 100644 --- a/hub/src/store/sessionJobs.ts +++ b/hub/src/store/sessionJobs.ts @@ -259,12 +259,16 @@ export function transferSessionJobs( for (const job of rows) { const existing = getSessionJob(db, toSessionId, job.key) if (existing) { - // Prefer a live source over a terminal target (or newer stamp). - // Redirected heartbeats omit status, so discarding a running source - // cannot be repaired by a later heartbeat. + // Prefer a live source over a terminal target. When both are + // running or both terminal (incl. completed vs failed), prefer + // the newer updatedAt — otherwise a later terminal result loses + // to an older one with a different status. Redirected heartbeats + // omit status, so discarding a running source cannot be repaired. + const sourceRunning = job.status === 'running' + const targetRunning = existing.status === 'running' const sourceWins = - (job.status === 'running' && existing.status !== 'running') - || (job.status === existing.status && job.updatedAt > existing.updatedAt) + (sourceRunning && !targetRunning) + || (sourceRunning === targetRunning && job.updatedAt > existing.updatedAt) if (sourceWins) { db.prepare('DELETE FROM session_jobs WHERE session_id = ? AND job_key = ?') .run(toSessionId, job.key) diff --git a/web/src/hooks/queries/reconcileAttachedJobs.test.ts b/web/src/hooks/queries/reconcileAttachedJobs.test.ts new file mode 100644 index 0000000000..66c0a18b9d --- /dev/null +++ b/web/src/hooks/queries/reconcileAttachedJobs.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest' +import type { SessionSummary, SessionsResponse } from '@/types/api' +import { reconcileAttachedJobsFromCache } from './reconcileAttachedJobs' + +function makeSummary(overrides: Partial & { id: string }): SessionSummary { + return { + active: false, + thinking: false, + activeAt: 0, + updatedAt: 0, + metadata: null, + metadataVersion: 0, + agentStateVersion: 0, + todosUpdatedAt: 0, + todoProgress: null, + pendingRequestsCount: 0, + pendingRequestKinds: [], + pendingRequests: [], + backgroundTaskCount: 0, + futureScheduledMessageCount: 0, + nextScheduledAt: null, + attachedJob: null, + attachedJobUpdatedAt: 0, + model: null, + effort: null, + ...overrides, + } +} + +describe('reconcileAttachedJobsFromCache', () => { + it('keeps a newer cached clear over a stale REST snapshot that still has the job', () => { + const id = 'sess-1' + const cached: SessionsResponse = { + sessions: [ + makeSummary({ + id, + attachedJob: null, + attachedJobUpdatedAt: 20, + }), + ], + } + const fetched: SessionsResponse = { + sessions: [ + makeSummary({ + id, + attachedJob: { + key: 'batch', + label: 'batch', + status: 'running', + startedAt: 1, + heartbeatAt: 1, + updatedAt: 10, + }, + attachedJobUpdatedAt: 10, + }), + ], + } + const next = reconcileAttachedJobsFromCache(fetched, cached) + expect(next.sessions[0]?.attachedJob).toBeNull() + expect(next.sessions[0]?.attachedJobUpdatedAt).toBe(20) + }) + + it('accepts a fresher REST snapshot', () => { + const id = 'sess-1' + const cached: SessionsResponse = { + sessions: [ + makeSummary({ + id, + attachedJob: { + key: 'batch', + label: 'batch', + status: 'running', + startedAt: 1, + heartbeatAt: 1, + updatedAt: 10, + }, + attachedJobUpdatedAt: 10, + }), + ], + } + const fetched: SessionsResponse = { + sessions: [ + makeSummary({ + id, + attachedJob: null, + attachedJobUpdatedAt: 30, + }), + ], + } + const next = reconcileAttachedJobsFromCache(fetched, cached) + expect(next.sessions[0]?.attachedJob).toBeNull() + expect(next.sessions[0]?.attachedJobUpdatedAt).toBe(30) + }) + + it('passes through when there is no cache', () => { + const fetched: SessionsResponse = { + sessions: [makeSummary({ id: 'a', attachedJobUpdatedAt: 1 })], + } + expect(reconcileAttachedJobsFromCache(fetched, undefined)).toBe(fetched) + }) +}) diff --git a/web/src/hooks/queries/reconcileAttachedJobs.ts b/web/src/hooks/queries/reconcileAttachedJobs.ts new file mode 100644 index 0000000000..372a311cbb --- /dev/null +++ b/web/src/hooks/queries/reconcileAttachedJobs.ts @@ -0,0 +1,34 @@ +import type { SessionSummary, SessionsResponse } from '@/types/api' + +/** + * Keep a fresher attachedJob from an in-flight SSE cache when a slower + * /api/sessions response would otherwise clobber it (clear/progress race). + */ +export function reconcileAttachedJobsFromCache( + fetched: SessionsResponse, + cached: SessionsResponse | undefined +): SessionsResponse { + if (!cached?.sessions?.length) { + return fetched + } + const cachedById = new Map(cached.sessions.map((session) => [session.id, session])) + return { + ...fetched, + sessions: fetched.sessions.map((session) => { + const previous = cachedById.get(session.id) + if (!previous) { + return session + } + const previousAt = previous.attachedJobUpdatedAt ?? 0 + const fetchedAt = session.attachedJobUpdatedAt ?? 0 + if (previousAt <= fetchedAt) { + return session + } + return { + ...session, + attachedJob: previous.attachedJob, + attachedJobUpdatedAt: previous.attachedJobUpdatedAt, + } satisfies SessionSummary + }), + } +} diff --git a/web/src/hooks/queries/useSessions.ts b/web/src/hooks/queries/useSessions.ts index 2786b7edda..4bcda81bb4 100644 --- a/web/src/hooks/queries/useSessions.ts +++ b/web/src/hooks/queries/useSessions.ts @@ -1,7 +1,8 @@ -import { useQuery } from '@tanstack/react-query' +import { useQuery, useQueryClient } from '@tanstack/react-query' import type { ApiClient } from '@/api/client' -import type { SessionSummary } from '@/types/api' +import type { SessionSummary, SessionsResponse } from '@/types/api' import { queryKeys } from '@/lib/query-keys' +import { reconcileAttachedJobsFromCache } from './reconcileAttachedJobs' export type UseSessionsOptions = { enabled?: boolean @@ -13,13 +14,16 @@ export function useSessions(api: ApiClient | null, options: UseSessionsOptions = error: string | null refetch: () => Promise } { + const queryClient = useQueryClient() const query = useQuery({ queryKey: queryKeys.sessions, queryFn: async () => { if (!api) { throw new Error('API unavailable') } - return await api.getSessions() + const fetched = await api.getSessions() + const cached = queryClient.getQueryData(queryKeys.sessions) + return reconcileAttachedJobsFromCache(fetched, cached) }, enabled: Boolean(api) && (options.enabled ?? true), }) From c74ed0f6a11abfe9e171ddba5f611970513e65d5 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 01:18:12 +0000 Subject: [PATCH 29/94] fix(jobs): retry terminal status write; reject startedAt on update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supervisor exit is the only running→terminal transition — retry transient hub failures. --started-at / startedAt only apply to set, not update/run. Co-authored-by: Cursor --- cli/src/commands/job.test.ts | 9 +++ cli/src/commands/job.ts | 8 +++ .../modules/sessionJob/runSessionJob.test.ts | 64 +++++++++++++++++++ cli/src/modules/sessionJob/runSessionJob.ts | 55 ++++++++++++---- .../modules/sessionJob/sessionJobMcp.test.ts | 9 +++ cli/src/modules/sessionJob/sessionJobMcp.ts | 4 ++ 6 files changed, 136 insertions(+), 13 deletions(-) diff --git a/cli/src/commands/job.test.ts b/cli/src/commands/job.test.ts index e3785f1561..58822c5ff7 100644 --- a/cli/src/commands/job.test.ts +++ b/cli/src/commands/job.test.ts @@ -59,6 +59,15 @@ describe('parseJobArgs', () => { ]) expect(parsed.startedAt).toBe(1_785_304_595_000) }) + + it('rejects --started-at on update', () => { + expect(() => parseJobArgs([ + 'update', + 'sid', + 'beets', + '--started-at=1785304595000' + ])).toThrow(/--started-at is only valid with job set/) + }) }) describe('resolveSessionByPrefix', () => { diff --git a/cli/src/commands/job.ts b/cli/src/commands/job.ts index b3753a86da..358d17cd64 100644 --- a/cli/src/commands/job.ts +++ b/cli/src/commands/job.ts @@ -209,6 +209,10 @@ export function parseJobArgs(args: string[]): ParsedJobArgs { throw new SessionJobError('bad_args', `unexpected arg: ${arg}`) } + if (result.startedAt !== undefined && result.action !== undefined && result.action !== 'set') { + throw new SessionJobError('bad_args', '--started-at is only valid with job set') + } + return result } @@ -285,6 +289,10 @@ export async function handleJobCommand(args: string[]): Promise { throw new SessionJobError('bad_args', 'missing job key') } + if (parsed.startedAt !== undefined && parsed.action !== 'set') { + throw new SessionJobError('bad_args', '--started-at is only valid with job set') + } + if (parsed.action === 'clear') { const result = await clearSessionJob({ sessionIdPrefix: parsed.sessionIdPrefix, diff --git a/cli/src/modules/sessionJob/runSessionJob.test.ts b/cli/src/modules/sessionJob/runSessionJob.test.ts index fe52cdda90..6dfe3cd454 100644 --- a/cli/src/modules/sessionJob/runSessionJob.test.ts +++ b/cli/src/modules/sessionJob/runSessionJob.test.ts @@ -100,6 +100,70 @@ describe('runSessionJob', () => { expect(http.get).toHaveBeenCalledTimes(1) }) + it('retries terminal status write after transient failures', async () => { + let patchCalls = 0 + const http = { + post: vi.fn(async () => ({ status: 200, data: { token: 'jwt' } })), + get: vi.fn(async () => ({ + status: 200, + data: { sessions: [{ id: 'aaaaaaaa-1111-1111-1111-111111111111' }] } + })), + put: vi.fn(async () => ({ + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: 'running', + heartbeatAt: 1, + startedAt: 1, + updatedAt: 1 + } + } + })), + patch: vi.fn(async (_url: string, body: { status?: string }) => { + patchCalls += 1 + if (body.status === 'completed' && patchCalls < 3) { + throw new Error('transient hub 503') + } + return { + status: 200, + data: { + job: { + key: 'drain', + label: 'drain', + status: body.status ?? 'running', + heartbeatAt: 2, + startedAt: 1, + updatedAt: 2 + } + } + } + }) + } + + const sleeps: number[] = [] + const exitCode = await runSessionJob({ + sessionIdPrefix: 'aaaa', + jobKey: 'drain', + label: 'drain', + command: ['true'], + accessToken: 'token', + apiUrl: 'http://127.0.0.1:3006', + http: http as never, + spawnImpl: (() => fakeChild(0)) as never, + setIntervalImpl: ((() => 1) as never), + clearIntervalImpl: (() => undefined) as never, + sleepImpl: async (ms) => { sleeps.push(ms) } + }) + + expect(exitCode).toBe(0) + expect(patchCalls).toBe(3) + expect(sleeps).toEqual([1_000, 2_000]) + const lastPatch = http.patch.mock.calls.at(-1)?.[1] as { status?: string } + expect(lastPatch.status).toBe('completed') + }) + it('marks failed on non-zero exit', async () => { const http = { post: vi.fn(async () => ({ status: 200, data: { token: 'jwt' } })), diff --git a/cli/src/modules/sessionJob/runSessionJob.ts b/cli/src/modules/sessionJob/runSessionJob.ts index 512c607e68..6822a82448 100644 --- a/cli/src/modules/sessionJob/runSessionJob.ts +++ b/cli/src/modules/sessionJob/runSessionJob.ts @@ -29,9 +29,42 @@ export type RunSessionJobOptions = SessionJobClientOptions & { spawnImpl?: typeof spawn setIntervalImpl?: typeof setInterval clearIntervalImpl?: typeof clearInterval + /** Injected for tests (terminal-status retry backoff). */ + sleepImpl?: (ms: number) => Promise } const DEFAULT_HEARTBEAT_MS = 5 * 60 * 1000 +const TERMINAL_STATUS_ATTEMPTS = 3 + +async function markTerminalWithRetry(options: { + clientOpts: SessionJobClientOptions & { resolved: SessionJobResolvedClient } + jobKey: string + status: 'completed' | 'failed' + detail?: string + sleep: (ms: number) => Promise +}): Promise { + let lastError: unknown + for (let attempt = 0; attempt < TERMINAL_STATUS_ATTEMPTS; attempt += 1) { + try { + await updateSessionJob({ + ...options.clientOpts, + jobKey: options.jobKey, + body: { + status: options.status, + ...(options.detail !== undefined ? { detail: options.detail } : {}), + }, + }) + return + } catch (error) { + lastError = error + if (attempt === TERMINAL_STATUS_ATTEMPTS - 1) { + break + } + await options.sleep(1_000 * 2 ** attempt) + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)) +} export async function runSessionJob(options: RunSessionJobOptions): Promise { if (options.command.length === 0) { @@ -71,6 +104,8 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise new Promise((resolve) => setTimeout(resolve, ms))) const heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS const child: ChildProcess = spawnFn(options.command[0]!, options.command.slice(1), { @@ -114,19 +149,11 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise((resolve) => { child.on('error', async (error) => { clearIntervalFn(heartbeat) - await inflightHeartbeat.catch(() => undefined) - try { - await updateSessionJob({ - ...clientOpts, - jobKey: options.jobKey, - body: { status: 'failed', detail: error.message } - }) - } catch { - // ignore - } + spawnErrorDetail = error.message resolve(127) }) child.on('exit', (code, signal) => { @@ -149,10 +176,12 @@ export async function runSessionJob(options: RunSessionJobOptions): Promise { }) ) }) + + it('rejects startedAt on update', async () => { + const result = await handleSessionJobTool( + { action: 'update', jobKey: 'beets', startedAt: 1_785_304_595_000 }, + 'sid-1' + ) + expect(result.isError).toBe(true) + expect(result.text).toMatch(/startedAt is only valid with action=set/) + }) }) diff --git a/cli/src/modules/sessionJob/sessionJobMcp.ts b/cli/src/modules/sessionJob/sessionJobMcp.ts index e5119663b9..ddf8f3589b 100644 --- a/cli/src/modules/sessionJob/sessionJobMcp.ts +++ b/cli/src/modules/sessionJob/sessionJobMcp.ts @@ -112,6 +112,10 @@ export async function handleSessionJobTool( } const jobKey = args.jobKey.trim() + if (args.startedAt !== undefined && args.action !== 'set') { + return { text: 'startedAt is only valid with action=set', isError: true } + } + if (args.action === 'clear') { const result = await clearSessionJob({ sessionIdPrefix, jobKey }) return { text: `cleared ${jobKey} on ${result.sessionId}`, isError: false } From ab7f6e0a10c79d738b6cc72673812e97db693461 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 01:29:51 +0000 Subject: [PATCH 30/94] fix(jobs): keep both live jobs on same-key merge Dual-running collisions remap the source key and record jobKeyRedirects so pre-merge supervisors cannot terminal-mark the winner. Also raise the flaky claudeRemote first-result test timeout under CI load. Co-authored-by: Cursor --- .../handlers/cli/sessionHandlers.test.ts | 4 + .../socket/handlers/cli/sessionHandlers.ts | 1 + hub/src/store/migration-v23.test.ts | 51 ++++++++++ hub/src/store/sessionJobs.ts | 64 +++++++++++-- hub/src/store/sessionJobsStore.ts | 3 +- hub/src/sync/sessionCache-merge-jobs.test.ts | 67 ++++++++++++++ hub/src/sync/sessionCache.ts | 92 +++++++++++++++++++ hub/src/sync/syncEngine.ts | 15 +++ hub/src/web/routes/sessions-jobs.test.ts | 3 + hub/src/web/routes/sessions.ts | 57 +++++++++--- shared/src/schemas.clear.test.ts | 8 +- shared/src/schemas.ts | 4 + 12 files changed, 346 insertions(+), 23 deletions(-) diff --git a/hub/src/socket/handlers/cli/sessionHandlers.test.ts b/hub/src/socket/handlers/cli/sessionHandlers.test.ts index c034bd9953..4db9022f03 100644 --- a/hub/src/socket/handlers/cli/sessionHandlers.test.ts +++ b/hub/src/socket/handlers/cli/sessionHandlers.test.ts @@ -443,6 +443,7 @@ describe('cli session handlers', () => { ['opencodeClearOperation', { replacementSessionId: 'foreign-session', state: 'reserved', updatedAt: Date.now() }], ['jobsAcceptedFromSessionIds', ['foreign-session']], ['jobsTransferredToSessionId', 'foreign-session'], + ['jobKeyRedirects', { 'foreign-session/beets': 'beets.foreign' }], ] as const)( 'ignores a forged hub-owned %s addition from CLI metadata', (field, forged) => { @@ -474,6 +475,7 @@ describe('cli session handlers', () => { opencodeClearOperation: operation, jobsAcceptedFromSessionIds: ['old-session'], jobsTransferredToSessionId: 'merge-target', + jobKeyRedirects: { 'old-session/beets': 'beets.oldsess1' }, }, null, 'default') const socket = new FakeSocket() registerSessionHandlers(socket as unknown as CliSocketWithData, { @@ -490,6 +492,7 @@ describe('cli session handlers', () => { opencodeClearOperation: { replacementSessionId: 'forged-target', state: 'reserved', updatedAt: 0 }, jobsAcceptedFromSessionIds: ['forged-session'], jobsTransferredToSessionId: 'forged-target', + jobKeyRedirects: { 'forged/beets': 'beets.forged' }, } }, () => {}) expect(store.sessions.getSessionByNamespace(session.id, 'default')?.metadata).toMatchObject({ @@ -497,6 +500,7 @@ describe('cli session handlers', () => { opencodeClearOperation: operation, jobsAcceptedFromSessionIds: ['old-session'], jobsTransferredToSessionId: 'merge-target', + jobKeyRedirects: { 'old-session/beets': 'beets.oldsess1' }, lifecycleState: 'archived', }) }) diff --git a/hub/src/socket/handlers/cli/sessionHandlers.ts b/hub/src/socket/handlers/cli/sessionHandlers.ts index 89e90f2dfc..ac117bbf3b 100644 --- a/hub/src/socket/handlers/cli/sessionHandlers.ts +++ b/hub/src/socket/handlers/cli/sessionHandlers.ts @@ -75,6 +75,7 @@ const HUB_OWNED_METADATA_KEYS = [ 'opencodeClearOperation', 'jobsAcceptedFromSessionIds', 'jobsTransferredToSessionId', + 'jobKeyRedirects', ] as const function preserveHubOwnedMetadata(incoming: unknown, current: unknown): unknown { diff --git a/hub/src/store/migration-v23.test.ts b/hub/src/store/migration-v23.test.ts index 8e22b13744..473a397190 100644 --- a/hub/src/store/migration-v23.test.ts +++ b/hub/src/store/migration-v23.test.ts @@ -95,4 +95,55 @@ describe('schema migration v22 to v27', () => { expect(store.sessionJobs.list(from.id)).toHaveLength(0) store.close() }) + + it('on dual-running same-key collision keeps both under remapped source key', () => { + const store = new Store(':memory:') + const fromId = 'aaaaaaaa-1111-1111-1111-111111111111' + const toId = 'bbbbbbbb-2222-2222-2222-222222222222' + const from = store.sessions.getOrCreateSession( + 'tag-from-dual', + { path: '/a' }, + null, + 'default', + undefined, + undefined, + undefined, + fromId + ) + const to = store.sessions.getOrCreateSession( + 'tag-to-dual', + { path: '/b' }, + null, + 'default', + undefined, + undefined, + undefined, + toId + ) + expect(from.id).toBe(fromId) + expect(to.id).toBe(toId) + store.sessionJobs.upsert(to.id, 'beets', { + label: 'target-live', + status: 'running', + remaining: 9 + }, 1_000) + store.sessionJobs.upsert(from.id, 'beets', { + label: 'source-live', + status: 'running', + remaining: 3 + }, 2_000) + const result = store.sessionJobs.transfer(from.id, to.id) + expect(result.collided).toBe(1) + expect(result.moved).toBe(1) + expect(result.keyRedirects).toEqual([ + { fromKey: 'beets', toKey: 'beets.aaaaaaaa' } + ]) + const onTarget = store.sessionJobs.list(to.id) + expect(onTarget).toHaveLength(2) + expect(onTarget.map((j) => j.key).sort()).toEqual(['beets', 'beets.aaaaaaaa']) + expect(store.sessionJobs.get(to.id, 'beets')?.label).toBe('target-live') + expect(store.sessionJobs.get(to.id, 'beets.aaaaaaaa')?.label).toBe('source-live') + expect(store.sessionJobs.list(from.id)).toHaveLength(0) + store.close() + }) }) diff --git a/hub/src/store/sessionJobs.ts b/hub/src/store/sessionJobs.ts index 88f32d2219..a2952a4eef 100644 --- a/hub/src/store/sessionJobs.ts +++ b/hub/src/store/sessionJobs.ts @@ -240,6 +240,40 @@ export function deleteSessionJob(db: Database, sessionId: string, jobKey: string return result.changes > 0 } +export type SessionJobKeyRedirect = { + fromKey: string + toKey: string +} + +export type TransferSessionJobsResult = { + moved: number + collided: number + /** Source keys remapped on the target so two live supervisors stay isolated. */ + keyRedirects: SessionJobKeyRedirect[] +} + +const JOB_KEY_MAX = 128 + +/** Allocate `base.` (then `.N`) that fits JOB_KEY_MAX and is free on target. */ +export function allocateRemappedJobKey( + db: Database, + toSessionId: string, + fromSessionId: string, + fromKey: string +): string { + const short = fromSessionId.replace(/-/g, '').slice(0, 8) || 'src' + const suffix0 = `.${short}` + const base = fromKey.slice(0, Math.max(1, JOB_KEY_MAX - suffix0.length)) + let candidate = `${base}${suffix0}` + let n = 0 + while (getSessionJob(db, toSessionId, candidate)) { + n += 1 + const suffix = `.${short}.${n}` + candidate = `${fromKey.slice(0, Math.max(1, JOB_KEY_MAX - suffix.length))}${suffix}` + } + return candidate +} + /** * Re-point jobs during session merge (same contract as scratchlist transfer). * Call BEFORE deleteSession so CASCADE does not race the move. @@ -248,27 +282,39 @@ export function transferSessionJobs( db: Database, fromSessionId: string, toSessionId: string -): { moved: number; collided: number } { +): TransferSessionJobsResult { if (fromSessionId === toSessionId) { - return { moved: 0, collided: 0 } + return { moved: 0, collided: 0, keyRedirects: [] } } const rows = listSessionJobs(db, fromSessionId) let moved = 0 let collided = 0 + const keyRedirects: SessionJobKeyRedirect[] = [] for (const job of rows) { const existing = getSessionJob(db, toSessionId, job.key) if (existing) { - // Prefer a live source over a terminal target. When both are - // running or both terminal (incl. completed vs failed), prefer - // the newer updatedAt — otherwise a later terminal result loses - // to an older one with a different status. Redirected heartbeats - // omit status, so discarding a running source cannot be repaired. const sourceRunning = job.status === 'running' const targetRunning = existing.status === 'running' + // Two live supervisors still PATCH the pre-merge key via session + // redirect. Collapsing them would let the loser terminal-mark the + // winner — keep both under distinct keys and record a key remap. + if (sourceRunning && targetRunning) { + const toKey = allocateRemappedJobKey(db, toSessionId, fromSessionId, job.key) + db.prepare( + `UPDATE session_jobs SET session_id = ?, job_key = ? + WHERE session_id = ? AND job_key = ?` + ).run(toSessionId, toKey, fromSessionId, job.key) + keyRedirects.push({ fromKey: job.key, toKey }) + moved += 1 + collided += 1 + continue + } + // Prefer a live source over a terminal target. When both are + // terminal (incl. completed vs failed), prefer the newer updatedAt. const sourceWins = (sourceRunning && !targetRunning) - || (sourceRunning === targetRunning && job.updatedAt > existing.updatedAt) + || (!sourceRunning && !targetRunning && job.updatedAt > existing.updatedAt) if (sourceWins) { db.prepare('DELETE FROM session_jobs WHERE session_id = ? AND job_key = ?') .run(toSessionId, job.key) @@ -291,5 +337,5 @@ export function transferSessionJobs( moved += 1 } - return { moved, collided } + return { moved, collided, keyRedirects } } diff --git a/hub/src/store/sessionJobsStore.ts b/hub/src/store/sessionJobsStore.ts index 8394730e5b..9f6a75384c 100644 --- a/hub/src/store/sessionJobsStore.ts +++ b/hub/src/store/sessionJobsStore.ts @@ -12,6 +12,7 @@ import { toAttachedJob, transferSessionJobs, upsertSessionJob, + type TransferSessionJobsResult, type UpsertSessionJobResult } from './sessionJobs' @@ -61,7 +62,7 @@ export class SessionJobsStore { return deleteSessionJob(this.db, sessionId, jobKey) } - transfer(fromSessionId: string, toSessionId: string): { moved: number; collided: number } { + transfer(fromSessionId: string, toSessionId: string): TransferSessionJobsResult { return transferSessionJobs(this.db, fromSessionId, toSessionId) } } diff --git a/hub/src/sync/sessionCache-merge-jobs.test.ts b/hub/src/sync/sessionCache-merge-jobs.test.ts index 5f9dbddf0f..f3a44adfec 100644 --- a/hub/src/sync/sessionCache-merge-jobs.test.ts +++ b/hub/src/sync/sessionCache-merge-jobs.test.ts @@ -188,4 +188,71 @@ describe('mergeSessions job redirect through SessionCache (#1404)', () => { expect(cache.resolveAttachedJobSessionId(oldSession.id, 'default')).toBe(newSession.id) }) + + it('remaps dual-running same-key jobs and routes PATCH via jobKeyRedirects', async () => { + const { store, cache } = setup() + const oldId = 'aaaaaaaa-1111-1111-1111-111111111111' + const newId = 'bbbbbbbb-2222-2222-2222-222222222222' + const oldSession = cache.getOrCreateSession( + 'tag-dual-old', + { path: '/a', host: 'local', flavor: 'codex' }, + null, + 'default', + undefined, + undefined, + undefined, + oldId + ) + const newSession = cache.getOrCreateSession( + 'tag-dual-new', + { path: '/b', host: 'local', flavor: 'codex' }, + null, + 'default', + undefined, + undefined, + undefined, + newId + ) + expect(oldSession.id).toBe(oldId) + expect(newSession.id).toBe(newId) + + store.sessionJobs.upsert(newSession.id, 'beets', { + label: 'target-live', + status: 'running', + remaining: 9 + }, 1_000) + store.sessionJobs.upsert(oldSession.id, 'beets', { + label: 'source-live', + status: 'running', + remaining: 3 + }, 2_000) + + await cache.mergeSessionHistory(oldSession.id, newSession.id, 'default', { + mergeAgentState: false + }) + + const onTarget = store.sessionJobs.list(newSession.id) + expect(onTarget).toHaveLength(2) + const refreshed = cache.refreshSession(newSession.id) + expect(refreshed?.metadata?.jobKeyRedirects).toEqual({ + [`${oldId}/beets`]: 'beets.aaaaaaaa' + }) + expect( + cache.resolveAttachedJobKey(oldId, newId, 'beets', 'default') + ).toBe('beets.aaaaaaaa') + expect( + cache.resolveAttachedJobKey(newId, newId, 'beets', 'default') + ).toBe('beets') + + // Terminal update via pre-merge session id + original key touches only the remapped row. + const patched = store.sessionJobs.patch( + newId, + cache.resolveAttachedJobKey(oldId, newId, 'beets', 'default'), + { status: 'completed' }, + 3_000 + ) + expect(patched?.key).toBe('beets.aaaaaaaa') + expect(patched?.status).toBe('completed') + expect(store.sessionJobs.get(newId, 'beets')?.status).toBe('running') + }) }) diff --git a/hub/src/sync/sessionCache.ts b/hub/src/sync/sessionCache.ts index ba3cefcd24..f9b2d48b70 100644 --- a/hub/src/sync/sessionCache.ts +++ b/hub/src/sync/sessionCache.ts @@ -1334,6 +1334,12 @@ export class SessionCache { // still exists; without this pointer, retained $HAPI_SESSION_ID hits // the emptied source and terminal PATCHes 404. this.recordJobsTransferredToSession(oldSessionId, newSessionId, namespace) + this.recordJobKeyRedirects( + newSessionId, + oldSessionId, + movedJobs.keyRedirects, + namespace + ) if (movedJobs.moved > 0 || movedJobs.collided > 0) { this.emitAttachedJobChanged( newSessionId, @@ -1629,6 +1635,71 @@ export class SessionCache { } } + /** + * Persist key remaps from dual-running same-key merges, and inherit any + * redirects the source already held (A→B→C). + */ + private recordJobKeyRedirects( + toSessionId: string, + fromSessionId: string, + redirects: Array<{ fromKey: string; toKey: string }>, + namespace: string + ): void { + for (let attempt = 0; attempt < 2; attempt += 1) { + const latest = this.store.sessions.getSessionByNamespace(toSessionId, namespace) + if (!latest) return + const meta = (latest.metadata && typeof latest.metadata === 'object' + ? { ...(latest.metadata as Record) } + : {}) as Record + const prevRaw = meta.jobKeyRedirects + const next: Record = {} + if (prevRaw && typeof prevRaw === 'object' && !Array.isArray(prevRaw)) { + for (const [k, v] of Object.entries(prevRaw as Record)) { + if (typeof v === 'string' && v.trim()) next[k] = v + } + } + const fromMeta = this.store.sessions + .getSessionByNamespace(fromSessionId, namespace) + ?.metadata as Record | null | undefined + const inheritedRaw = fromMeta?.jobKeyRedirects + if (inheritedRaw && typeof inheritedRaw === 'object' && !Array.isArray(inheritedRaw)) { + for (const [k, v] of Object.entries(inheritedRaw as Record)) { + if (typeof v === 'string' && v.trim()) next[k] = v + } + } + for (const { fromKey, toKey } of redirects) { + next[`${fromSessionId}/${fromKey}`] = toKey + } + const prevKeys = Object.keys( + prevRaw && typeof prevRaw === 'object' && !Array.isArray(prevRaw) + ? (prevRaw as Record) + : {} + ) + const nextKeys = Object.keys(next) + const unchanged = + prevKeys.length === nextKeys.length + && nextKeys.every((k) => (prevRaw as Record | undefined)?.[k] === next[k]) + if (unchanged) return + if (nextKeys.length === 0) { + delete meta.jobKeyRedirects + } else { + meta.jobKeyRedirects = next + } + const result = this.store.sessions.updateSessionMetadata( + toSessionId, + meta, + latest.metadataVersion, + namespace, + { touchUpdatedAt: false } + ) + if (result.result === 'success') { + this.refreshSession(toSessionId) + return + } + if (result.result !== 'version-mismatch') return + } + } + /** * Follow job-owner redirects after session merge/dedup so agents that still * hold the pre-merge `$HAPI_SESSION_ID` can heartbeat. @@ -1675,6 +1746,27 @@ export class SessionCache { return null } + /** + * Map a pre-merge job key onto the post-merge owner key when dual-running + * same-key merge remapped the source row. + */ + resolveAttachedJobKey( + requestedSessionId: string, + ownerSessionId: string, + jobKey: string, + namespace: string + ): string { + const access = this.resolveSessionAccess(ownerSessionId, namespace) + if (!access.ok) return jobKey + const meta = access.session.metadata as Record | null | undefined + const redirects = meta?.jobKeyRedirects + if (!redirects || typeof redirects !== 'object' || Array.isArray(redirects)) { + return jobKey + } + const mapped = (redirects as Record)[`${requestedSessionId}/${jobKey}`] + return typeof mapped === 'string' && mapped.trim() ? mapped : jobKey + } + private mergeSessionMetadata(oldMetadata: unknown | null, newMetadata: unknown | null): unknown | null { if (!oldMetadata || typeof oldMetadata !== 'object') { return newMetadata diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 40149fde7a..df2c480026 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -380,6 +380,21 @@ export class SyncEngine { return this.sessionCache.resolveAttachedJobSessionId(sessionId, namespace) } + /** Follow dual-running same-key remaps after merge (tiann/hapi#1404). */ + resolveAttachedJobKey( + requestedSessionId: string, + ownerSessionId: string, + jobKey: string, + namespace: string + ): string { + return this.sessionCache.resolveAttachedJobKey( + requestedSessionId, + ownerSessionId, + jobKey, + namespace + ) + } + getActiveSessions(): Session[] { return this.sessionCache.getActiveSessions() } diff --git a/hub/src/web/routes/sessions-jobs.test.ts b/hub/src/web/routes/sessions-jobs.test.ts index e8408b3797..a9d3cd4d56 100644 --- a/hub/src/web/routes/sessions-jobs.test.ts +++ b/hub/src/web/routes/sessions-jobs.test.ts @@ -36,6 +36,7 @@ describe('session-attached jobs routes (tiann/hapi#1404)', () => { const engine = { resolveSessionAccess: () => ({ ok: true as const, sessionId: session.id, session }), resolveAttachedJobSessionId: (id: string) => id, + resolveAttachedJobKey: (_requested: string, _owner: string, jobKey: string) => jobKey, getSessionsByNamespace: () => [session], getFutureScheduledMessageCounts: () => new Map(), getNextScheduledAtBySessionIds: () => new Map(), @@ -157,6 +158,7 @@ describe('session-attached jobs routes (tiann/hapi#1404)', () => { return { ok: false as const, reason: 'not-found' as const } }, resolveAttachedJobSessionId: (id: string) => (id === deletedId ? owner.id : id), + resolveAttachedJobKey: (_requested: string, _owner: string, jobKey: string) => jobKey, listSessionJobs: (sid: string) => (sid === owner.id ? [...jobs.values()] : []), getPrimaryAttachedJob: (sid: string) => (sid === owner.id ? jobs.get('beets')! : null), upsertSessionJob: () => ({ outcome: 'session-not-found' as const }), @@ -182,6 +184,7 @@ describe('session-attached jobs routes (tiann/hapi#1404)', () => { const engine = { resolveSessionAccess: () => ({ ok: true as const, sessionId: session.id, session }), resolveAttachedJobSessionId: (id: string) => id, + resolveAttachedJobKey: (_requested: string, _owner: string, jobKey: string) => jobKey, listSessionJobs: () => [], getPrimaryAttachedJob: () => null, upsertSessionJob: () => ({ outcome: 'session-not-found' as const }), diff --git a/hub/src/web/routes/sessions.ts b/hub/src/web/routes/sessions.ts index d5d81beb68..7de8342a0f 100644 --- a/hub/src/web/routes/sessions.ts +++ b/hub/src/web/routes/sessions.ts @@ -1308,17 +1308,21 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho function resolveJobOwnerSession( c: Context, engine: SyncEngine - ): { sessionId: string; session: Session } | Response { + ): { requestedSessionId: string; sessionId: string; session: Session } | Response { + const rawId = c.req.param('id') ?? '' const sessionResult = requireSessionFromParam(c, engine) if (sessionResult instanceof Response) { // Session may already be deleted after merge — still try acceptor redirect. - const rawId = c.req.param('id') ?? '' const namespace = c.get('namespace') const redirected = engine.resolveAttachedJobSessionId(rawId, namespace) if (redirected !== rawId) { const access = engine.resolveSessionAccess(redirected, namespace) if (access.ok) { - return { sessionId: access.sessionId, session: access.session } + return { + requestedSessionId: rawId, + sessionId: access.sessionId, + session: access.session, + } } } return sessionResult @@ -1326,13 +1330,39 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho const namespace = c.get('namespace') const ownerId = engine.resolveAttachedJobSessionId(sessionResult.sessionId, namespace) if (ownerId === sessionResult.sessionId) { - return sessionResult + return { + requestedSessionId: sessionResult.sessionId, + sessionId: sessionResult.sessionId, + session: sessionResult.session, + } } const access = engine.resolveSessionAccess(ownerId, namespace) if (!access.ok) { - return sessionResult + return { + requestedSessionId: sessionResult.sessionId, + sessionId: sessionResult.sessionId, + session: sessionResult.session, + } } - return { sessionId: access.sessionId, session: access.session } + return { + requestedSessionId: sessionResult.sessionId, + sessionId: access.sessionId, + session: access.session, + } + } + + function resolveJobKey( + c: Context, + engine: SyncEngine, + owner: { requestedSessionId: string; sessionId: string }, + jobKey: string + ): string { + return engine.resolveAttachedJobKey( + owner.requestedSessionId, + owner.sessionId, + jobKey, + c.get('namespace') + ) } app.get('/sessions/:id/jobs', (c) => { @@ -1359,10 +1389,11 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho if (sessionResult instanceof Response) { return sessionResult } - const jobKey = c.req.param('jobKey') - if (!jobKey || !JOB_KEY_RE.test(jobKey)) { + const rawJobKey = c.req.param('jobKey') + if (!rawJobKey || !JOB_KEY_RE.test(rawJobKey)) { return c.json({ error: 'Invalid jobKey (1-128 chars: alnum, . _ -)' }, 400) } + const jobKey = resolveJobKey(c, engine, sessionResult, rawJobKey) const body = await c.req.json().catch(() => null) const parsed = AttachedJobUpsertSchema.safeParse(body) if (!parsed.success) { @@ -1384,10 +1415,11 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho if (sessionResult instanceof Response) { return sessionResult } - const jobKey = c.req.param('jobKey') - if (!jobKey || !JOB_KEY_RE.test(jobKey)) { + const rawJobKey = c.req.param('jobKey') + if (!rawJobKey || !JOB_KEY_RE.test(rawJobKey)) { return c.json({ error: 'Invalid jobKey (1-128 chars: alnum, . _ -)' }, 400) } + const jobKey = resolveJobKey(c, engine, sessionResult, rawJobKey) const body = await c.req.json().catch(() => null) const parsed = AttachedJobPatchSchema.safeParse(body) if (!parsed.success) { @@ -1409,10 +1441,11 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho if (sessionResult instanceof Response) { return sessionResult } - const jobKey = c.req.param('jobKey') - if (!jobKey || !JOB_KEY_RE.test(jobKey)) { + const rawJobKey = c.req.param('jobKey') + if (!rawJobKey || !JOB_KEY_RE.test(rawJobKey)) { return c.json({ error: 'Invalid jobKey (1-128 chars: alnum, . _ -)' }, 400) } + const jobKey = resolveJobKey(c, engine, sessionResult, rawJobKey) const removed = engine.deleteSessionJob(sessionResult.sessionId, jobKey) if (!removed) { return c.json({ error: 'Job not found' }, 404) diff --git a/shared/src/schemas.clear.test.ts b/shared/src/schemas.clear.test.ts index b8197221c7..5d9eb46c13 100644 --- a/shared/src/schemas.clear.test.ts +++ b/shared/src/schemas.clear.test.ts @@ -15,10 +15,16 @@ describe('fresh-session clear schema contract', () => { path: '/tmp/project', host: 'host', jobsAcceptedFromSessionIds: ['old-session-id'], - jobsTransferredToSessionId: 'new-session-id' + jobsTransferredToSessionId: 'new-session-id', + jobKeyRedirects: { + 'old-session-id/beets': 'beets.oldsess1', + }, }) expect(parsed.jobsAcceptedFromSessionIds).toEqual(['old-session-id']) expect(parsed.jobsTransferredToSessionId).toBe('new-session-id') + expect(parsed.jobKeyRedirects).toEqual({ + 'old-session-id/beets': 'beets.oldsess1', + }) }) it('accepts cleared as an additive session-end reason', () => { diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index 71edcd1fef..bc2060cb38 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -142,6 +142,10 @@ export const MetadataSchema = z.object({ // Hub-owned: CLI update-metadata cannot forge/erase (HUB_OWNED_METADATA_KEYS). jobsAcceptedFromSessionIds: z.array(z.string()).optional(), jobsTransferredToSessionId: z.string().optional(), + // When merge remaps a live source job key (same-key dual-running), map + // `${fromSessionId}/${fromKey}` → toKey on the post-merge owner so + // pre-merge supervisors keep PATCHing the right row. + jobKeyRedirects: z.record(z.string(), z.string()).optional(), // Durable in-progress state for runner-backed OpenCode /clear. opencodeClearOperation: OpencodeClearOperationSchema.optional(), preferredPermissionMode: PermissionModeSchema.optional(), From 380ef3b8dfc1a235b01d9ede47b4636fd849ac73 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Sun, 9 Aug 2026 01:49:38 +0000 Subject: [PATCH 31/94] fix(jobs): hard-refuse MCP set; louder stale heartbeat chrome MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Process-shaped work must use Shell hapi job run. MCP set is refused with the run recipe (wardrobe freeze). Stale rows show "no heartbeat · age". Co-authored-by: Cursor --- .../common/sessionJobInstruction.test.ts | 3 +- .../modules/common/sessionJobInstruction.ts | 9 +- .../modules/sessionJob/sessionJobMcp.test.ts | 44 ++++------ cli/src/modules/sessionJob/sessionJobMcp.ts | 85 ++++++++++--------- docs/guide/session-jobs.md | 20 +++-- web/src/components/SessionRowSummary.tsx | 9 +- web/src/lib/attachedJob.test.ts | 27 ++++-- web/src/lib/attachedJob.ts | 22 +++-- 8 files changed, 125 insertions(+), 94 deletions(-) diff --git a/cli/src/modules/common/sessionJobInstruction.test.ts b/cli/src/modules/common/sessionJobInstruction.test.ts index 45ebbd72cd..12085a4d65 100644 --- a/cli/src/modules/common/sessionJobInstruction.test.ts +++ b/cli/src/modules/common/sessionJobInstruction.test.ts @@ -5,11 +5,12 @@ import { } from './sessionJobInstruction' describe('sessionJobInstruction', () => { - it('prefers MCP session_job + job run supervisor and forbids fake percent', () => { + it('requires job run and forbids MCP set / fake percent', () => { expect(SESSION_JOB_INSTRUCTION).toContain('session_job') expect(SESSION_JOB_INSTRUCTION).toContain('ping_peer') expect(SESSION_JOB_INSTRUCTION).toContain('hapi job run') expect(SESSION_JOB_INSTRUCTION).toContain('idle agents cannot') + expect(SESSION_JOB_INSTRUCTION).toMatch(/action=set \(refused\)/) expect(SESSION_JOB_INSTRUCTION).toContain('Never invent a fake percent') expect(SESSION_JOB_INSTRUCTION).toContain('HAPI_SESSION_ID') }) diff --git a/cli/src/modules/common/sessionJobInstruction.ts b/cli/src/modules/common/sessionJobInstruction.ts index aaec5f96d7..18515f2a2b 100644 --- a/cli/src/modules/common/sessionJobInstruction.ts +++ b/cli/src/modules/common/sessionJobInstruction.ts @@ -14,12 +14,11 @@ export const SESSION_JOB_INSTRUCTION = [ '(batch imports, long scripts, external daemons), attach a session job so', 'the session list shows progress while you are idle — same class of HAPI', 'tooling as ping_peer / inspect_peer.', - 'Prefer supervised CLI for process-shaped work (idle agents cannot heartbeat):', - 'hapi job run "$HAPI_SESSION_ID" --label -- …', + 'REQUIRED for process-shaped work (idle agents cannot heartbeat):', + 'Shell → hapi job run "$HAPI_SESSION_ID" --label -- …', '(auto-heartbeats + completed/failed on exit).', - 'Manual path: MCP tool session_job (aliases: mcp__hapi__session_job,', - 'hapi_session_job, functions.hapi__session_job) action=set, then action=update', - 'every ~10m from a self-heartbeating wrapper — never set-once and walk away.', + 'Do NOT use MCP session_job action=set (refused) or bare set+nohup — that freezes the bar.', + 'MCP session_job is only update / clear / list after job run created the meter.', 'Prefer honest remaining or done+total; omit counts when unknown', '(UI shows "running" + elapsed). Never invent a fake percent.', 'Full contract: hapi job --help.' diff --git a/cli/src/modules/sessionJob/sessionJobMcp.test.ts b/cli/src/modules/sessionJob/sessionJobMcp.test.ts index f168f51899..1bef390ba3 100644 --- a/cli/src/modules/sessionJob/sessionJobMcp.test.ts +++ b/cli/src/modules/sessionJob/sessionJobMcp.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it, vi } from 'vitest' -import { handleSessionJobTool, SESSION_JOB_TOOL_DESCRIPTION } from './sessionJobMcp' +import { + handleSessionJobTool, + SESSION_JOB_RUN_RECIPE, + SESSION_JOB_SET_REFUSED_TEXT, + SESSION_JOB_TOOL_DESCRIPTION +} from './sessionJobMcp' vi.mock('./sessionJob', () => ({ SessionJobError: class SessionJobError extends Error { @@ -9,18 +14,9 @@ vi.mock('./sessionJob', () => ({ this.code = code } }, - setSessionJob: vi.fn(async () => ({ - sessionId: 'sid-1', - job: { - key: 'beets', - label: 'beets import', - status: 'running', - remaining: 12, - heartbeatAt: 1, - startedAt: 1, - updatedAt: 1 - } - })), + setSessionJob: vi.fn(async () => { + throw new Error('setSessionJob must not be called from MCP') + }), updateSessionJob: vi.fn(async () => ({ sessionId: 'sid-1', job: { @@ -38,27 +34,23 @@ vi.mock('./sessionJob', () => ({ })) describe('sessionJobMcp', () => { - it('description steers outliving batch work and honest progress', () => { + it('description steers to job run and forbids MCP set', () => { expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/OUTLIVES/i) - expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/Never invent a percent/i) + expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/do NOT use action=set/i) expect(SESSION_JOB_TOOL_DESCRIPTION).toContain('hapi job run') + expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/Own-session only/i) }) - it('set requires label and always targets the caller session id', async () => { + it('hard-refuses action=set and never calls setSessionJob', async () => { const { setSessionJob } = await import('./sessionJob') const result = await handleSessionJobTool( { action: 'set', jobKey: 'beets', label: 'beets import', remaining: 12 }, 'sid-1' ) - expect(result.isError).toBe(false) - expect(result.text).toContain('set beets') - expect(setSessionJob).toHaveBeenCalledWith( - expect.objectContaining({ sessionIdPrefix: 'sid-1' }) - ) - }) - - it('description claims own-session only', () => { - expect(SESSION_JOB_TOOL_DESCRIPTION).toMatch(/Own-session only/i) + expect(result.isError).toBe(true) + expect(result.text).toBe(SESSION_JOB_SET_REFUSED_TEXT) + expect(result.text).toContain(SESSION_JOB_RUN_RECIPE) + expect(setSessionJob).not.toHaveBeenCalled() }) it('treats empty update as a heartbeat-only patch', async () => { @@ -84,6 +76,6 @@ describe('sessionJobMcp', () => { 'sid-1' ) expect(result.isError).toBe(true) - expect(result.text).toMatch(/startedAt is only valid with action=set/) + expect(result.text).toMatch(/startedAt is not valid over MCP/) }) }) diff --git a/cli/src/modules/sessionJob/sessionJobMcp.ts b/cli/src/modules/sessionJob/sessionJobMcp.ts index ddf8f3589b..63f6251468 100644 --- a/cli/src/modules/sessionJob/sessionJobMcp.ts +++ b/cli/src/modules/sessionJob/sessionJobMcp.ts @@ -1,50 +1,64 @@ /** * MCP surface for session-attached jobs (tiann/hapi#1404). * Same discovery class as ping_peer / inspect_peer — tool catalog, not docs-only. + * + * Hard contract: MCP cannot create a long-lived bar (action=set is refused). + * Create meters with Shell + `hapi job run` (babysitter). MCP is for + * update / clear / list after a supervisor or CLI wrapper owns heartbeats. */ import { z } from 'zod' -import type { AttachedJob, AttachedJobPatch, AttachedJobUpsert } from '@hapi/protocol' +import type { AttachedJob, AttachedJobPatch } from '@hapi/protocol' import { SessionJobError, clearSessionJob, listSessionJobs, - setSessionJob, updateSessionJob } from './sessionJob' export const SESSION_JOB_TOOL_NAME = 'session_job' +/** Exact Shell recipe agents should run instead of MCP set. */ +export const SESSION_JOB_RUN_RECIPE = + 'hapi job run "$HAPI_SESSION_ID" --label "