From d0a04da2d5a0e2a54d7337c20863d2f8ee41c00e Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:55:51 +0000 Subject: [PATCH 01/12] feat(overseer): opt-in OpenAI-compatible LLM summary fallback (#90) When AGENT_NOTIFY_SUMMARY is missing and HAPI_OVERSEER_LLM_FALLBACK is on, synthesize a Session Log-only event via chat-completions/responses using the full turn text. Failures fall through to the heuristic first-line backstop. Also restore skillLookupInstruction so the stacked tip typechecks. Co-authored-by: Cursor --- .../modules/common/skillLookupInstruction.ts | 11 ++ .../2026-07-24-overseer-summary-emission.md | 46 +++-- hub/README.md | 13 ++ hub/src/configuration.ts | 2 + .../overseerEventRecorder.fallback.test.ts | 29 +-- .../overseerEventRecorder.llmFallback.test.ts | 156 ++++++++++++++++ hub/src/sync/overseerEventRecorder.test.ts | 26 +-- hub/src/sync/overseerEventRecorder.ts | 78 ++++++-- hub/src/sync/overseerLlmFallback.test.ts | 149 +++++++++++++++ hub/src/sync/overseerLlmFallback.ts | 170 ++++++++++++++++++ .../sync/overseerLlmFallbackConfig.test.ts | 84 +++++++++ hub/src/sync/overseerLlmFallbackConfig.ts | 95 ++++++++++ hub/src/sync/syncEngine.ts | 17 +- 13 files changed, 815 insertions(+), 61 deletions(-) create mode 100644 cli/src/modules/common/skillLookupInstruction.ts create mode 100644 hub/src/sync/overseerEventRecorder.llmFallback.test.ts create mode 100644 hub/src/sync/overseerLlmFallback.test.ts create mode 100644 hub/src/sync/overseerLlmFallback.ts create mode 100644 hub/src/sync/overseerLlmFallbackConfig.test.ts create mode 100644 hub/src/sync/overseerLlmFallbackConfig.ts diff --git a/cli/src/modules/common/skillLookupInstruction.ts b/cli/src/modules/common/skillLookupInstruction.ts new file mode 100644 index 0000000000..aa04d33f8c --- /dev/null +++ b/cli/src/modules/common/skillLookupInstruction.ts @@ -0,0 +1,11 @@ +/** + * Discovery copy for agents that can host a durable system / instructions block + * (OpenCode, Grok). Do **not** prepend this to user turns β€” that path looks like + * prompt injection on Cursor ACP and similar remotes (tiann/hapi#1095). + * + * Cursor / Kimi / generic ACP rely on the `skill_lookup` MCP tool description + * (and Cursor's native `.cursor/mcp.json` overlay where session/new mcpServers + * are ignored) instead of a user-message prepend. + */ +export const SKILL_LOOKUP_INSTRUCTION = + 'When a user message starts with "$name", call HAPI\'s skill_lookup tool with "name" (without "$") before acting.' diff --git a/docs/plans/2026-07-24-overseer-summary-emission.md b/docs/plans/2026-07-24-overseer-summary-emission.md index dd6eea125f..c220083a29 100644 --- a/docs/plans/2026-07-24-overseer-summary-emission.md +++ b/docs/plans/2026-07-24-overseer-summary-emission.md @@ -1,6 +1,6 @@ # Overseer summary emission (Half B) β€” 2026-07-24 -Status: Pieces 1–2 live on soup; Piece 3 (non-Cursor systemPrompt + debug dates) in flight +Status: Pieces 1–3 live/in-flight; Option A LLM fallback implemented (default OFF, #90) Owner: feat/overseer-summary-emit (peer of πŸ”overseer prep) Scope: FORK-ONLY. Never upstream. The whole overseer feature is fork-private. @@ -129,7 +129,9 @@ ordinary, useful project config, never as surveillance: - kimi + generic ACP / pi: **tracked** β€” fork issue [#89](https://github.com/heavygee/hapi/issues/89); required for next overseer phase full-coverage. -- Better LLM / oneshot-agent fallback: designed below, **not implemented in v1**. +- Better LLM / oneshot-agent fallback: Option A implemented behind + `HAPI_OVERSEER_LLM_FALLBACK` (default off); see Β§ Better fallback / #90. + Option B oneshot agent remains out of scope. ## Better fallback (opt-in β€” tracked #90) @@ -142,28 +144,43 @@ and is a real cost tax - so it must be **opt-in**, clearly labeled, and rare ### Gate: rarity first, quality never second -Do **not** ship a better fallback until primary emission is good enough that +Do **not** enable a better fallback until primary emission is good enough that fallback is a thin residue - target **well under 5% of turns** (5% is already generous). Measure emit vs hub-synthesized ratio fleet-wide after Piece 3 is -live; only then enable LLM fallback. +live; only then flip the enable flag. When it *does* run, it must be **at least as useful as a primary self-report**: feed the **full last-turn assistant content** (no input-char truncation that would make the summary worse than the agent would have written). Rarity is the cost control; accuracy is non-negotiable on the rare path. -### Option A β€” raw OpenAI-compatible completions call +### Option A β€” raw OpenAI-compatible completions call (implemented) -Hub (or a tiny side worker) POSTs the full last assistant turn text to an -operator-configured base URL (`/v1/chat/completions` or `/v1/responses`) with a -fixed prompt: "emit exactly one AGENT_NOTIFY_SUMMARY JSON line." Local (Ollama / -vLLM / gateway) or remote (OpenAI) - same wire format. +Hub POSTs the full last assistant turn text to an operator-configured base URL +(`/v1/chat/completions` or `/v1/responses`) with a fixed prompt: "emit exactly +one AGENT_NOTIFY_SUMMARY JSON line." Local (Ollama / vLLM / gateway) or remote +(OpenAI) - same wire format. -- Pros: cheap to wire, no session surface, easy to bill/attribute as - `provenance: hub-llm-fallback`. -- Cons: large turns = large prompt tokens (accepted when rare); operator must - provision a key/URL; prefer Chat Completions for local-gateway compatibility, - Responses for OpenAI-native - support both behind one adapter. +**Enable (default OFF β€” never surprise usage):** + +```bash +export HAPI_OVERSEER_LLM_FALLBACK=1 +export HAPI_OVERSEER_LLM_BASE_URL=http://127.0.0.1:11434/v1 # include /v1 +export HAPI_OVERSEER_LLM_MODEL=llama3.3 +# optional: +export HAPI_OVERSEER_LLM_API_KEY=ollama # Bearer token; empty OK for local +export HAPI_OVERSEER_LLM_API=chat-completions # or: responses +export HAPI_OVERSEER_LLM_TIMEOUT_MS=30000 +``` + +Prefer `chat-completions` for local-gateway compatibility; use `responses` for +OpenAI-native. Failures / non-compliant model output fall through to the +heuristic first-line fallback. Events are marked +`provenance: hub-llm-fallback ...` with `payload.synthesis = "llm-fallback"`, +`attentionCandidate = 0` (Session Log only β€” not inbox / voice). + +**Cost warning:** every missed primary emit becomes a full-turn prompt. Enable +only after the rarity gate, or accept the bill deliberately. ### Option B β€” out-of-band oneshot agent @@ -176,6 +193,7 @@ in Session Log / inbox so the operator never wonders "wtf usage is this." multi-step retrieval if needed. - Cons: heavier; looks like a phantom session if not carefully labeled; higher cost variance; more moving parts. + **Out of scope for #90** β€” revisit only if Option A proves insufficient. ### Shared requirements (either option) diff --git a/hub/README.md b/hub/README.md index b224b5df78..1f45860b3c 100644 --- a/hub/README.md +++ b/hub/README.md @@ -42,6 +42,19 @@ See `src/configuration.ts` for all options. - `HAPI_RELAY_FORCE_TCP` - Force TCP relay mode (true/1). - `VAPID_SUBJECT` - Contact email/URL for Web Push. +### Optional (Overseer LLM fallback β€” fork, default OFF) + +Only when primary agents omit `AGENT_NOTIFY_SUMMARY`. Costs a full-turn LLM call +per miss β€” enable after miss rate is rare (~<5%). See +`docs/plans/2026-07-24-overseer-summary-emission.md`. + +- `HAPI_OVERSEER_LLM_FALLBACK` - `1`/`true` to enable (default: off). +- `HAPI_OVERSEER_LLM_BASE_URL` - OpenAI-compatible base including `/v1` (required when enabled). +- `HAPI_OVERSEER_LLM_MODEL` - Model id (required when enabled). +- `HAPI_OVERSEER_LLM_API_KEY` - Bearer token (optional for local gateways). +- `HAPI_OVERSEER_LLM_API` - `chat-completions` (default) or `responses`. +- `HAPI_OVERSEER_LLM_TIMEOUT_MS` - Request timeout (default: 30000). + ## Running Binary (single executable): diff --git a/hub/src/configuration.ts b/hub/src/configuration.ts index 20779a3a90..dd5f7a5c58 100644 --- a/hub/src/configuration.ts +++ b/hub/src/configuration.ts @@ -21,6 +21,8 @@ * - VAPID_SUBJECT: Contact email or URL for Web Push (defaults to mailto:admin@hapi.run) * - HAPI_HOME: Data directory (default: ~/.hapi) * - DB_PATH: SQLite database path (default: {HAPI_HOME}/hapi.db) + * - HAPI_OVERSEER_LLM_FALLBACK: Opt-in hub LLM summary fallback when AGENT_NOTIFY_SUMMARY is missing (default: off) + * - HAPI_OVERSEER_LLM_BASE_URL / _MODEL / _API_KEY / _API / _TIMEOUT_MS: OpenAI-compatible endpoint for that fallback */ import { existsSync, mkdirSync } from 'node:fs' diff --git a/hub/src/sync/overseerEventRecorder.fallback.test.ts b/hub/src/sync/overseerEventRecorder.fallback.test.ts index 67bb632733..f315c68b57 100644 --- a/hub/src/sync/overseerEventRecorder.fallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.fallback.test.ts @@ -34,12 +34,12 @@ function agentText(message: string) { } describe('OverseerEventRecorder turn fallback', () => { - it('synthesizes a session-log-only progress event when no summary line', () => { + it('synthesizes a session-log-only progress event when no summary line', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('cur', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), 'msg-fb', agentText('Refactored the parser and added tests.\n\nMore detail here.'), @@ -53,20 +53,21 @@ describe('OverseerEventRecorder turn fallback', () => { expect(event?.summary).toBe('Refactored the parser and added tests.') expect(event?.provenance).toContain('hub-synthesized') - const payload = JSON.parse(event!.payloadJson!) as { synthesized?: boolean } + const payload = JSON.parse(event!.payloadJson!) as { synthesized?: boolean; synthesis?: string } expect(payload.synthesized).toBe(true) + expect(payload.synthesis).toBe('heuristic') // Session log gets it; the attention inbox stays empty. expect(store.events.count()).toBe(1) expect(store.inbox.count()).toBe(0) }) - it('does not synthesize when a real AGENT_NOTIFY_SUMMARY is present', () => { + it('does not synthesize when a real AGENT_NOTIFY_SUMMARY is present', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('cur2', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), 'msg-real', agentText('All done.\nAGENT_NOTIFY_SUMMARY {"version":1,"status":"done","action":"Review PR","summary":"Shipped"}'), @@ -78,12 +79,12 @@ describe('OverseerEventRecorder turn fallback', () => { expect(store.events.count()).toBe(1) }) - it('does not synthesize when the summary line is malformed (validation_error wins)', () => { + it('does not synthesize when the summary line is malformed (validation_error wins)', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('cur3', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), 'msg-bad', agentText('Working.\nAGENT_NOTIFY_SUMMARY {not valid json'), @@ -94,25 +95,25 @@ describe('OverseerEventRecorder turn fallback', () => { expect(store.events.list({ eventType: 'progress' })).toHaveLength(0) }) - it('is idempotent for a redelivered message id', () => { + it('is idempotent for a redelivered message id', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('cur4', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') const snapshot = toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag) - recorder.onAgentMessage(snapshot, 'msg-dup', agentText('Progress update.'), Date.now()) - recorder.onAgentMessage(snapshot, 'msg-dup', agentText('Progress update.'), Date.now()) + await recorder.onAgentMessage(snapshot, 'msg-dup', agentText('Progress update.'), Date.now()) + await recorder.onAgentMessage(snapshot, 'msg-dup', agentText('Progress update.'), Date.now()) expect(store.events.list({ eventType: 'progress' })).toHaveLength(1) }) - it('caps a long first line with an ellipsis', () => { + it('caps a long first line with an ellipsis', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('cur5', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') const longLine = 'x'.repeat(500) - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), 'msg-long', agentText(longLine), @@ -123,13 +124,13 @@ describe('OverseerEventRecorder turn fallback', () => { expect(event?.summary?.endsWith('\u2026')).toBe(true) }) - it('ignores user messages', () => { + it('ignores user messages', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('cur6', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') const userContent = { role: 'user', content: { type: 'text', text: 'do the thing' } } - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), 'msg-user', userContent, diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts new file mode 100644 index 0000000000..688acf0011 --- /dev/null +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it, mock } from 'bun:test' +import type { NotifySummary } from '@hapi/protocol/messages' +import type { Session } from '@hapi/protocol/types' +import { Store } from '../store' +import type { OverseerLlmFallbackClient } from './overseerLlmFallback' +import { OverseerEventRecorder, toSessionSnapshot } from './overseerEventRecorder' + +function makeSession(id: string, flavor: string, overrides?: Partial): Session { + return { + id, + namespace: 'default', + seq: 0, + createdAt: Date.now(), + updatedAt: Date.now(), + active: true, + activeAt: Date.now(), + metadata: { flavor, path: '/tmp', host: 'local' }, + metadataVersion: 1, + agentState: null, + agentStateVersion: 1, + thinking: false, + thinkingAt: 0, + model: null, + modelReasoningEffort: null, + effort: null, + serviceTier: null, + ...overrides + } +} + +function agentText(message: string) { + return { + role: 'agent', + content: { type: 'codex', data: { type: 'message', message } } + } +} + +describe('OverseerEventRecorder LLM fallback', () => { + it('uses hub-llm-fallback provenance and keeps attn=0 when LLM succeeds', async () => { + const store = new Store(':memory:') + const synthesize = mock(async (plainText: string): Promise => { + expect(plainText).toContain('Refactored the parser') + expect(plainText).toContain('More detail here.') + return { + version: 1, + status: 'blocked', + action: 'Unblock CI', + summary: 'LLM distilled summary of the whole turn', + } + }) + const llmFallback: OverseerLlmFallbackClient = { synthesizeNotifySummary: synthesize } + const recorder = new OverseerEventRecorder(store.events, store.inbox, { llmFallback }) + const session = store.sessions.getOrCreateSession('llm1', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + + const event = await recorder.onAgentMessage( + toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), + 'msg-llm', + agentText('Refactored the parser and added tests.\n\nMore detail here.'), + Date.now() + ) + + expect(synthesize).toHaveBeenCalledTimes(1) + expect(event).not.toBeNull() + expect(event?.summary).toBe('LLM distilled summary of the whole turn') + expect(event?.eventType).toBe('blocked') + expect(event?.attentionCandidate).toBe(0) + expect(event?.operatorActionRequired).toBe(0) + expect(event?.provenance).toContain('hub-llm-fallback') + expect(store.inbox.count()).toBe(0) + + const payload = JSON.parse(event!.payloadJson!) as { + synthesized?: boolean + synthesis?: string + notify_summary?: NotifySummary + } + expect(payload.synthesized).toBe(true) + expect(payload.synthesis).toBe('llm-fallback') + expect(payload.notify_summary?.summary).toBe('LLM distilled summary of the whole turn') + }) + + it('falls through to heuristic when LLM returns null', async () => { + const store = new Store(':memory:') + const llmFallback: OverseerLlmFallbackClient = { + synthesizeNotifySummary: mock(async () => null), + } + const recorder = new OverseerEventRecorder(store.events, store.inbox, { llmFallback }) + const session = store.sessions.getOrCreateSession('llm2', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + + const event = await recorder.onAgentMessage( + toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), + 'msg-llm-fail', + agentText('First line wins.\nSecond line.'), + Date.now() + ) + + expect(event?.summary).toBe('First line wins.') + expect(event?.provenance).toContain('hub-synthesized') + expect(event?.eventType).toBe('progress') + }) + + it('does not call LLM when a real AGENT_NOTIFY_SUMMARY is present', async () => { + const store = new Store(':memory:') + const synthesize = mock(async () => ({ status: 'done', summary: 'should not run' })) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { synthesizeNotifySummary: synthesize }, + }) + const session = store.sessions.getOrCreateSession('llm3', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + + const event = await recorder.onAgentMessage( + toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), + 'msg-real', + agentText('All done.\nAGENT_NOTIFY_SUMMARY {"version":1,"status":"done","action":"Review PR","summary":"Shipped"}'), + Date.now() + ) + + expect(synthesize).toHaveBeenCalledTimes(0) + expect(event?.provenance).toBe('AGENT_NOTIFY_SUMMARY') + }) + + it('skips LLM when client is not configured (heuristic only)', async () => { + const store = new Store(':memory:') + const recorder = new OverseerEventRecorder(store.events, store.inbox) + const session = store.sessions.getOrCreateSession('llm4', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + + const event = await recorder.onAgentMessage( + toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), + 'msg-off', + agentText('Heuristic path.'), + Date.now() + ) + + expect(event?.provenance).toContain('hub-synthesized') + expect(event?.summary).toBe('Heuristic path.') + }) + + it('falls through to heuristic when LLM throws', async () => { + const store = new Store(':memory:') + const llmFallback: OverseerLlmFallbackClient = { + synthesizeNotifySummary: mock(async () => { + throw new Error('network down') + }), + } + const recorder = new OverseerEventRecorder(store.events, store.inbox, { llmFallback }) + const session = store.sessions.getOrCreateSession('llm5', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + + const event = await recorder.onAgentMessage( + toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), + 'msg-throw', + agentText('Recovered via heuristic.'), + Date.now() + ) + + expect(event?.summary).toBe('Recovered via heuristic.') + expect(event?.provenance).toContain('hub-synthesized') + }) +}) diff --git a/hub/src/sync/overseerEventRecorder.test.ts b/hub/src/sync/overseerEventRecorder.test.ts index 5a4ce9d0d7..0e674f4c97 100644 --- a/hub/src/sync/overseerEventRecorder.test.ts +++ b/hub/src/sync/overseerEventRecorder.test.ts @@ -28,7 +28,7 @@ function makeSession(id: string, flavor: string, overrides?: Partial): } describe('OverseerEventRecorder', () => { - it('records AGENT_NOTIFY_SUMMARY from codex assistant text', () => { + it('records AGENT_NOTIFY_SUMMARY from codex assistant text', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('test', { flavor: 'codex', path: '/tmp', host: 'local' }, null, 'default') @@ -44,7 +44,7 @@ describe('OverseerEventRecorder', () => { } } - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'codex'), session.tag), 'msg-1', content, @@ -72,7 +72,7 @@ describe('OverseerEventRecorder', () => { expect(item?.title).toBe('test') }) - it('captures done without action as captured-only', () => { + it('captures done without action as captured-only', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('test2', { flavor: 'claude', path: '/tmp', host: 'local' }, null, 'default') @@ -88,7 +88,7 @@ describe('OverseerEventRecorder', () => { } } - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'claude'), session.tag), 'msg-2', content, @@ -144,7 +144,7 @@ describe('OverseerEventRecorder', () => { expect(store.inbox.list()[0]?.title).toBe('perm') }) - it('denormalizes session display name and project into payload.session', () => { + it('denormalizes session display name and project into payload.session', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const stored = store.sessions.getOrCreateSession( @@ -168,7 +168,7 @@ describe('OverseerEventRecorder', () => { } } - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(live, stored.tag), 'msg-meta', content, @@ -185,7 +185,7 @@ describe('OverseerEventRecorder', () => { expect(payload.session.id).toBe(stored.id) }) - it('titles inbox items from payload.session.name after session delete', () => { + it('titles inbox items from payload.session.name after session delete', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const stored = store.sessions.getOrCreateSession( @@ -195,7 +195,7 @@ describe('OverseerEventRecorder', () => { 'default' ) - recorder.onAgentMessage( + await recorder.onAgentMessage( toSessionSnapshot(makeSession(stored.id, 'codex', { metadata: { flavor: 'codex', path: '/coding/hapi', name: 'meta HAPI triage', host: 'local' } }), stored.tag), @@ -223,7 +223,7 @@ describe('OverseerEventRecorder', () => { expect(itemAfter?.relatedSessionId).toBeNull() }) - it('scoops http(s) URLs into link_seen with artifact_refs kind:url', () => { + it('scoops http(s) URLs into link_seen with artifact_refs kind:url', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('links', { flavor: 'codex', path: '/tmp', host: 'local' }, null, 'default') @@ -243,7 +243,7 @@ describe('OverseerEventRecorder', () => { } } - const notify = recorder.onAgentMessage( + const notify = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'codex'), session.tag), 'msg-links', content, @@ -268,7 +268,7 @@ describe('OverseerEventRecorder', () => { expect(payload.session.id).toBe(session.id) }) - it('idempotently scoops the same URL from the same message once', () => { + it('idempotently scoops the same URL from the same message once', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('dedupe', { flavor: 'claude', path: '/tmp', host: 'local' }, null, 'default') @@ -283,8 +283,8 @@ describe('OverseerEventRecorder', () => { } } const snapshot = toSessionSnapshot(makeSession(session.id, 'claude'), session.tag) - recorder.onAgentMessage(snapshot, 'msg-dup', content, Date.now()) - recorder.onAgentMessage(snapshot, 'msg-dup', content, Date.now()) + await recorder.onAgentMessage(snapshot, 'msg-dup', content, Date.now()) + await recorder.onAgentMessage(snapshot, 'msg-dup', content, Date.now()) expect(store.events.list({ eventType: 'link_seen' })).toHaveLength(1) }) }) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index 2030a9d9ad..0eaa99a63b 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -24,9 +24,15 @@ import { import type { Session } from '@hapi/protocol/types' import type { EventStore, InsertSystemEventInput, StoredSystemEvent } from '../store' import type { InboxStore } from '../store/inboxStore' +import type { OverseerLlmFallbackClient } from './overseerLlmFallback' export type SessionSnapshot = OverseerSessionIdentity +export type OverseerEventRecorderOptions = { + /** Opt-in OpenAI-compatible synthesizer (issue #90). Default: unset / off. */ + llmFallback?: OverseerLlmFallbackClient | null +} + function asRecord(value: unknown): Record | null { return isObject(value) ? value as Record : null } @@ -155,11 +161,15 @@ function buildTags(notify: NotifySummary | null, flavor: string): string | null export class OverseerEventRecorder { private readonly lastAgentMessageAt = new Map() private readonly knownPermissionRequestIds = new Map>() + private readonly llmFallback: OverseerLlmFallbackClient | null constructor( private readonly events: EventStore, - private readonly inbox?: InboxStore - ) {} + private readonly inbox?: InboxStore, + options?: OverseerEventRecorderOptions + ) { + this.llmFallback = options?.llmFallback ?? null + } list(options: Parameters[0] = {}): StoredSystemEvent[] { return this.events.list(options) @@ -169,7 +179,7 @@ export class OverseerEventRecorder { return this.events.count() } - onAgentMessage(session: SessionSnapshot, messageId: string, content: unknown, ts: number): StoredSystemEvent | null { + async onAgentMessage(session: SessionSnapshot, messageId: string, content: unknown, ts: number): Promise { let primary: StoredSystemEvent | null = null if (isAgentMessageContent(content)) { @@ -236,13 +246,12 @@ export class OverseerEventRecorder { } // Deterministic backstop: an agent produced visible text but no - // AGENT_NOTIFY_SUMMARY (rule compliance can never be 100%). Synthesize - // a minimal, session-log-only capture so the overseer never has a - // fully blind agent turn. No LLM; attention stays 0 so the inbox is - // untouched. Marked hub-synthesized so it is never mistaken for a - // real self-report. + // AGENT_NOTIFY_SUMMARY (rule compliance can never be 100%). Prefer + // opt-in hub LLM synthesis when configured; otherwise (or on LLM + // failure) fall through to first-line heuristic. Attention stays 0 + // so Session Log only β€” never inbox / voice. if (!primary && plainText) { - primary = this.synthesizeTurnFallback(session, messageId, plainText, ts) + primary = await this.synthesizeTurnFallback(session, messageId, plainText, ts) } } @@ -346,20 +355,53 @@ export class OverseerEventRecorder { } /** - * Minimal per-turn fallback event when no AGENT_NOTIFY_SUMMARY was emitted. + * Per-turn fallback when no AGENT_NOTIFY_SUMMARY was emitted. + * + * 1. Opt-in LLM (issue #90): full turn text β†’ AGENT_NOTIFY_SUMMARY parse. + * Provenance `hub-llm-fallback`; attn forced to 0 (Session Log only). + * 2. Heuristic: first non-empty line, eventType `progress`, attn 0. * - * Summary is the first non-empty line of the assistant text (deterministic, - * no LLM). Status defaults to `progress` via the empty-status mapping, and - * attention stays 0, so these land in the Session Log only β€” never the - * attention inbox. This is the safety net under the Cursor rule overlay: - * even a dropped summary line yields a captured turn. + * LLM failures / empty / non-compliant output fall through to (2). */ - private synthesizeTurnFallback( + private async synthesizeTurnFallback( session: SessionSnapshot, messageId: string, plainText: string, ts: number - ): StoredSystemEvent | null { + ): Promise { + if (this.llmFallback) { + try { + const notify = await this.llmFallback.synthesizeNotifySummary(plainText) + if (notify) { + const eventType = mapNotifyStatusToEventType(notify.status) + return this.insertSystemEvent(session, { + ts, + sourceKind: 'system', + sourceRef: session.id, + eventType, + attentionCandidate: 0, + operatorActionRequired: 0, + summary: buildEventSummaryFromNotify(notify), + relatedSessionId: session.id, + provenance: 'hub-llm-fallback (no AGENT_NOTIFY_SUMMARY from primary agent)', + idempotencyKey: `session:${session.id}:message:${messageId}:turn_fallback`, + payloadFields: { + messageId, + synthesized: true, + synthesis: 'llm-fallback', + notify_summary: notify, + suggested_action: notify.action ?? null, + }, + notifyProject: notify.project ?? null, + severity: deriveSeverity(eventType), + tags: buildTags(notify, session.flavor), + }) + } + } catch { + // Fall through to heuristic β€” never let LLM errors blind a turn. + } + } + const summary = firstNonEmptyLine(plainText) if (!summary) return null @@ -375,7 +417,7 @@ export class OverseerEventRecorder { relatedSessionId: session.id, provenance: 'hub-synthesized from assistant text (no AGENT_NOTIFY_SUMMARY)', idempotencyKey: `session:${session.id}:message:${messageId}:turn_fallback`, - payloadFields: { messageId, synthesized: true }, + payloadFields: { messageId, synthesized: true, synthesis: 'heuristic' }, severity: deriveSeverity(eventType), tags: buildTags(null, session.flavor) }) diff --git a/hub/src/sync/overseerLlmFallback.test.ts b/hub/src/sync/overseerLlmFallback.test.ts new file mode 100644 index 0000000000..795db6b4a8 --- /dev/null +++ b/hub/src/sync/overseerLlmFallback.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it, mock } from 'bun:test' +import { + OVERSEER_LLM_FALLBACK_SYSTEM_PROMPT, + createOverseerLlmFallbackClient, + extractTextFromChatCompletionsBody, + extractTextFromResponsesBody, + parseNotifySummaryFromLlmText, + type OverseerLlmFetch, +} from './overseerLlmFallback' +import type { OverseerLlmFallbackEnabledConfig } from './overseerLlmFallbackConfig' + +const baseConfig: OverseerLlmFallbackEnabledConfig = { + enabled: true, + baseUrl: 'http://llm.test/v1', + apiKey: 'test-key', + model: 'test-model', + api: 'chat-completions', + timeoutMs: 5_000, +} + +describe('parseNotifySummaryFromLlmText', () => { + it('parses a bare AGENT_NOTIFY_SUMMARY line', () => { + const text = 'AGENT_NOTIFY_SUMMARY {"version":1,"status":"done","action":"Review PR","summary":"Shipped fix"}' + const notify = parseNotifySummaryFromLlmText(text) + expect(notify?.status).toBe('done') + expect(notify?.summary).toBe('Shipped fix') + expect(notify?.action).toBe('Review PR') + }) + + it('strips markdown fences before parse', () => { + const text = '```\nAGENT_NOTIFY_SUMMARY {"status":"blocked","summary":"Waiting on review"}\n```' + expect(parseNotifySummaryFromLlmText(text)?.summary).toBe('Waiting on review') + }) + + it('returns null for empty or non-compliant text', () => { + expect(parseNotifySummaryFromLlmText('')).toBeNull() + expect(parseNotifySummaryFromLlmText('just a paragraph')).toBeNull() + }) +}) + +describe('response body extractors', () => { + it('reads chat completions choices[0].message.content', () => { + expect(extractTextFromChatCompletionsBody({ + choices: [{ message: { content: 'AGENT_NOTIFY_SUMMARY {"status":"done","summary":"ok"}' } }], + })).toContain('AGENT_NOTIFY_SUMMARY') + }) + + it('reads responses output_text when present', () => { + expect(extractTextFromResponsesBody({ + output_text: 'AGENT_NOTIFY_SUMMARY {"status":"failed","summary":"boom"}', + })).toContain('failed') + }) + + it('aggregates responses output message content text parts', () => { + expect(extractTextFromResponsesBody({ + output: [{ + type: 'message', + content: [{ type: 'output_text', text: 'AGENT_NOTIFY_SUMMARY {"status":"stalled","summary":"idle"}' }], + }], + })).toContain('stalled') + }) +}) + +describe('createOverseerLlmFallbackClient', () => { + it('POSTs chat completions with full turn text and parses notify', async () => { + const fetchMock = mock(async (input: string, init?: RequestInit) => { + expect(input).toBe('http://llm.test/v1/chat/completions') + expect(init?.method).toBe('POST') + const headers = init?.headers as Record + expect(headers.Authorization).toBe('Bearer test-key') + const body = JSON.parse(String(init?.body)) as { + model: string + messages: Array<{ role: string; content: string }> + } + expect(body.model).toBe('test-model') + expect(body.messages[0]?.role).toBe('system') + expect(body.messages[0]?.content).toBe(OVERSEER_LLM_FALLBACK_SYSTEM_PROMPT) + expect(body.messages[1]?.content).toContain('FULL TURN BODY THAT IS LONG') + return new Response(JSON.stringify({ + choices: [{ + message: { + content: 'AGENT_NOTIFY_SUMMARY {"version":1,"status":"done","action":"Merge","summary":"Turn complete"}', + }, + }], + }), { status: 200, headers: { 'content-type': 'application/json' } }) + }) + + const client = createOverseerLlmFallbackClient(baseConfig, { + fetchImpl: fetchMock as unknown as OverseerLlmFetch, + }) + const notify = await client.synthesizeNotifySummary('FULL TURN BODY THAT IS LONG\nline two') + expect(notify?.summary).toBe('Turn complete') + expect(notify?.status).toBe('done') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('POSTs /responses when api=responses and uses store:false', async () => { + const fetchMock = mock(async (input: string, init?: RequestInit) => { + expect(input).toBe('http://llm.test/v1/responses') + const body = JSON.parse(String(init?.body)) as { + model: string + store: boolean + instructions: string + input: string + } + expect(body.store).toBe(false) + expect(body.instructions).toBe(OVERSEER_LLM_FALLBACK_SYSTEM_PROMPT) + expect(body.input).toContain('assistant turn text') + expect(body.model).toBe('test-model') + return new Response(JSON.stringify({ + output_text: 'AGENT_NOTIFY_SUMMARY {"status":"needs_review","summary":"Please look"}', + }), { status: 200 }) + }) + + const client = createOverseerLlmFallbackClient( + { ...baseConfig, api: 'responses' }, + { fetchImpl: fetchMock as unknown as OverseerLlmFetch }, + ) + const notify = await client.synthesizeNotifySummary('assistant turn text') + expect(notify?.status).toBe('needs_review') + }) + + it('returns null on HTTP error so caller can fall through', async () => { + const fetchMock = mock(async () => new Response('nope', { status: 500 })) + const client = createOverseerLlmFallbackClient(baseConfig, { + fetchImpl: fetchMock as unknown as OverseerLlmFetch, + }) + expect(await client.synthesizeNotifySummary('text')).toBeNull() + }) + + it('returns null when model output is not a notify line', async () => { + const fetchMock = mock(async () => new Response(JSON.stringify({ + choices: [{ message: { content: 'Sorry, I cannot help with that.' } }], + }), { status: 200 })) + const client = createOverseerLlmFallbackClient(baseConfig, { + fetchImpl: fetchMock as unknown as OverseerLlmFetch, + }) + expect(await client.synthesizeNotifySummary('text')).toBeNull() + }) + + it('returns null for empty turn text without calling fetch', async () => { + const fetchMock = mock(async () => new Response('{}', { status: 200 })) + const client = createOverseerLlmFallbackClient(baseConfig, { + fetchImpl: fetchMock as unknown as OverseerLlmFetch, + }) + expect(await client.synthesizeNotifySummary(' \n ')).toBeNull() + expect(fetchMock).toHaveBeenCalledTimes(0) + }) +}) diff --git a/hub/src/sync/overseerLlmFallback.ts b/hub/src/sync/overseerLlmFallback.ts new file mode 100644 index 0000000000..833e065b05 --- /dev/null +++ b/hub/src/sync/overseerLlmFallback.ts @@ -0,0 +1,170 @@ +import { extractNotifySummary, type NotifySummary } from '@hapi/protocol/messages' +import type { OverseerLlmFallbackEnabledConfig } from './overseerLlmFallbackConfig' + +/** + * Fixed system prompt for Option A hub LLM fallback. + * Asks for exactly one AGENT_NOTIFY_SUMMARY line β€” same contract as primary agents. + */ +export const OVERSEER_LLM_FALLBACK_SYSTEM_PROMPT = [ + 'You summarize an AI coding agent turn for session tracking.', + 'Reply with exactly one line and nothing else (no markdown fences, no prose):', + 'AGENT_NOTIFY_SUMMARY {"version":1,"status":"done|blocked|needs_review|needs_decision|failed|stalled","action":"<=12 words","summary":"one-line triage"}', + 'Use status blocked if unsure. action must be concrete when status is done and follow-up remains.', +].join('\n') + +export type OverseerLlmFallbackClient = { + synthesizeNotifySummary(plainText: string): Promise +} + +export type OverseerLlmFetch = ( + input: string, + init?: RequestInit +) => Promise + +export type OverseerLlmFallbackClientOptions = { + fetchImpl?: OverseerLlmFetch +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** Strip common markdown fences so local models that wrap output still parse. */ +function stripMarkdownFences(text: string): string { + const trimmed = text.trim() + const fenced = trimmed.match(/^```(?:\w+)?\s*\n?([\s\S]*?)\n?```$/u) + if (fenced?.[1]) return fenced[1].trim() + return trimmed +} + +/** + * Parse an AGENT_NOTIFY_SUMMARY from LLM output. + * Uses the same end-anchored extractor as primary agent turns. + */ +export function parseNotifySummaryFromLlmText(text: string): NotifySummary | null { + if (typeof text !== 'string' || text.trim().length === 0) return null + return extractNotifySummary(stripMarkdownFences(text)) +} + +export function extractTextFromChatCompletionsBody(body: unknown): string | null { + if (!isObject(body)) return null + const choices = body.choices + if (!Array.isArray(choices) || choices.length === 0) return null + const first = choices[0] + if (!isObject(first)) return null + const message = first.message + if (!isObject(message)) return null + const content = message.content + if (typeof content === 'string' && content.trim().length > 0) return content + if (Array.isArray(content)) { + const parts: string[] = [] + for (const part of content) { + if (typeof part === 'string') parts.push(part) + else if (isObject(part) && typeof part.text === 'string') parts.push(part.text) + } + const joined = parts.join('\n').trim() + return joined.length > 0 ? joined : null + } + return null +} + +export function extractTextFromResponsesBody(body: unknown): string | null { + if (!isObject(body)) return null + if (typeof body.output_text === 'string' && body.output_text.trim().length > 0) { + return body.output_text + } + const output = body.output + if (!Array.isArray(output)) return null + const parts: string[] = [] + for (const item of output) { + if (!isObject(item)) continue + if (item.type !== 'message') continue + const content = item.content + if (!Array.isArray(content)) continue + for (const part of content) { + if (!isObject(part)) continue + if ((part.type === 'output_text' || part.type === 'text') && typeof part.text === 'string') { + parts.push(part.text) + } + } + } + const joined = parts.join('\n').trim() + return joined.length > 0 ? joined : null +} + +function joinUrl(baseUrl: string, path: string): string { + return `${baseUrl.replace(/\/+$/, '')}/${path.replace(/^\/+/, '')}` +} + +export function createOverseerLlmFallbackClient( + config: OverseerLlmFallbackEnabledConfig, + options: OverseerLlmFallbackClientOptions = {} +): OverseerLlmFallbackClient { + const fetchImpl: OverseerLlmFetch = options.fetchImpl + ?? ((input, init) => fetch(input, init)) + + return { + async synthesizeNotifySummary(plainText: string): Promise { + const turn = plainText.trim() + if (!turn) return null + + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), config.timeoutMs) + + try { + const headers: Record = { + 'Content-Type': 'application/json', + } + if (config.apiKey) { + headers.Authorization = `Bearer ${config.apiKey}` + } + + let url: string + let body: Record + if (config.api === 'responses') { + url = joinUrl(config.baseUrl, 'responses') + body = { + model: config.model, + instructions: OVERSEER_LLM_FALLBACK_SYSTEM_PROMPT, + input: turn, + store: false, + } + } else { + url = joinUrl(config.baseUrl, 'chat/completions') + body = { + model: config.model, + messages: [ + { role: 'system', content: OVERSEER_LLM_FALLBACK_SYSTEM_PROMPT }, + { role: 'user', content: turn }, + ], + } + } + + const response = await fetchImpl(url, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: controller.signal, + }) + if (!response.ok) return null + + let parsed: unknown + try { + parsed = await response.json() + } catch { + return null + } + + const text = config.api === 'responses' + ? extractTextFromResponsesBody(parsed) + : extractTextFromChatCompletionsBody(parsed) + if (!text) return null + return parseNotifySummaryFromLlmText(text) + } catch { + return null + } finally { + clearTimeout(timer) + } + }, + } +} diff --git a/hub/src/sync/overseerLlmFallbackConfig.test.ts b/hub/src/sync/overseerLlmFallbackConfig.test.ts new file mode 100644 index 0000000000..40fc1ee191 --- /dev/null +++ b/hub/src/sync/overseerLlmFallbackConfig.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it } from 'bun:test' +import { loadOverseerLlmFallbackConfig } from './overseerLlmFallbackConfig' + +const ENV_KEYS = [ + 'HAPI_OVERSEER_LLM_FALLBACK', + 'HAPI_OVERSEER_LLM_BASE_URL', + 'HAPI_OVERSEER_LLM_API_KEY', + 'HAPI_OVERSEER_LLM_MODEL', + 'HAPI_OVERSEER_LLM_API', + 'HAPI_OVERSEER_LLM_TIMEOUT_MS', +] as const + +const saved: Partial> = {} + +function stashEnv(): void { + for (const key of ENV_KEYS) { + saved[key] = process.env[key] + delete process.env[key] + } +} + +function restoreEnv(): void { + for (const key of ENV_KEYS) { + const value = saved[key] + if (value === undefined) delete process.env[key] + else process.env[key] = value + } +} + +afterEach(() => { + restoreEnv() +}) + +describe('loadOverseerLlmFallbackConfig', () => { + it('defaults to disabled when env is unset', () => { + stashEnv() + const config = loadOverseerLlmFallbackConfig() + expect(config.enabled).toBe(false) + if (config.enabled) throw new Error('expected disabled') + expect(config.reasonDisabled).toBe('flag_off') + }) + + it('stays disabled when flag is on but base URL or model missing', () => { + stashEnv() + process.env.HAPI_OVERSEER_LLM_FALLBACK = '1' + process.env.HAPI_OVERSEER_LLM_MODEL = 'llama3.3' + const config = loadOverseerLlmFallbackConfig() + expect(config.enabled).toBe(false) + if (config.enabled) throw new Error('expected disabled') + expect(config.reasonDisabled).toBe('incomplete_config') + }) + + it('enables with chat-completions defaults when flag + url + model set', () => { + stashEnv() + process.env.HAPI_OVERSEER_LLM_FALLBACK = 'true' + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'http://127.0.0.1:11434/v1' + process.env.HAPI_OVERSEER_LLM_MODEL = 'llama3.3' + const config = loadOverseerLlmFallbackConfig() + expect(config.enabled).toBe(true) + if (!config.enabled) throw new Error('expected enabled') + expect(config.baseUrl).toBe('http://127.0.0.1:11434/v1') + expect(config.model).toBe('llama3.3') + expect(config.api).toBe('chat-completions') + expect(config.apiKey).toBe('') + expect(config.timeoutMs).toBe(30_000) + }) + + it('accepts responses api mode and custom timeout/key', () => { + stashEnv() + process.env.HAPI_OVERSEER_LLM_FALLBACK = '1' + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'https://api.openai.com/v1/' + process.env.HAPI_OVERSEER_LLM_MODEL = 'gpt-4.1-mini' + process.env.HAPI_OVERSEER_LLM_API = 'responses' + process.env.HAPI_OVERSEER_LLM_API_KEY = 'sk-test' + process.env.HAPI_OVERSEER_LLM_TIMEOUT_MS = '12000' + const config = loadOverseerLlmFallbackConfig() + expect(config.enabled).toBe(true) + if (!config.enabled) throw new Error('expected enabled') + expect(config.baseUrl).toBe('https://api.openai.com/v1') + expect(config.api).toBe('responses') + expect(config.apiKey).toBe('sk-test') + expect(config.timeoutMs).toBe(12_000) + }) +}) diff --git a/hub/src/sync/overseerLlmFallbackConfig.ts b/hub/src/sync/overseerLlmFallbackConfig.ts new file mode 100644 index 0000000000..fc9e8ff5de --- /dev/null +++ b/hub/src/sync/overseerLlmFallbackConfig.ts @@ -0,0 +1,95 @@ +/** + * Opt-in hub LLM fallback for missing AGENT_NOTIFY_SUMMARY (fork issue #90). + * + * Default OFF. Enable only after primary emission miss rate is rare (~<5%). + * Env-only for v1 β€” never surprise usage. + * + * HAPI_OVERSEER_LLM_FALLBACK=1 + * HAPI_OVERSEER_LLM_BASE_URL=http://127.0.0.1:11434/v1 + * HAPI_OVERSEER_LLM_MODEL=llama3.3 + * HAPI_OVERSEER_LLM_API_KEY= # optional for local gateways + * HAPI_OVERSEER_LLM_API=chat-completions|responses # default chat-completions + * HAPI_OVERSEER_LLM_TIMEOUT_MS=30000 + */ + +export type OverseerLlmApiMode = 'chat-completions' | 'responses' + +export type OverseerLlmFallbackEnabledConfig = { + enabled: true + baseUrl: string + apiKey: string + model: string + api: OverseerLlmApiMode + timeoutMs: number +} + +export type OverseerLlmFallbackDisabledConfig = { + enabled: false + reasonDisabled: 'flag_off' | 'incomplete_config' | 'invalid_api' | 'invalid_timeout' +} + +export type OverseerLlmFallbackConfig = + | OverseerLlmFallbackEnabledConfig + | OverseerLlmFallbackDisabledConfig + +const DEFAULT_TIMEOUT_MS = 30_000 + +function envTruthy(value: string | undefined): boolean { + if (!value) return false + const normalized = value.trim().toLowerCase() + return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on' +} + +function normalizeBaseUrl(raw: string): string { + return raw.trim().replace(/\/+$/, '') +} + +function parseApiMode(raw: string | undefined): OverseerLlmApiMode | null { + if (!raw || raw.trim() === '') return 'chat-completions' + const normalized = raw.trim().toLowerCase() + if (normalized === 'chat-completions' || normalized === 'chat_completions' || normalized === 'chat') { + return 'chat-completions' + } + if (normalized === 'responses' || normalized === 'response') { + return 'responses' + } + return null +} + +export function loadOverseerLlmFallbackConfig( + env: NodeJS.ProcessEnv = process.env +): OverseerLlmFallbackConfig { + if (!envTruthy(env.HAPI_OVERSEER_LLM_FALLBACK)) { + return { enabled: false, reasonDisabled: 'flag_off' } + } + + const baseUrlRaw = env.HAPI_OVERSEER_LLM_BASE_URL?.trim() ?? '' + const model = env.HAPI_OVERSEER_LLM_MODEL?.trim() ?? '' + if (!baseUrlRaw || !model) { + return { enabled: false, reasonDisabled: 'incomplete_config' } + } + + const api = parseApiMode(env.HAPI_OVERSEER_LLM_API) + if (!api) { + return { enabled: false, reasonDisabled: 'invalid_api' } + } + + let timeoutMs = DEFAULT_TIMEOUT_MS + const timeoutRaw = env.HAPI_OVERSEER_LLM_TIMEOUT_MS?.trim() + if (timeoutRaw) { + const parsed = Number.parseInt(timeoutRaw, 10) + if (!Number.isFinite(parsed) || parsed <= 0) { + return { enabled: false, reasonDisabled: 'invalid_timeout' } + } + timeoutMs = parsed + } + + return { + enabled: true, + baseUrl: normalizeBaseUrl(baseUrlRaw), + apiKey: env.HAPI_OVERSEER_LLM_API_KEY?.trim() ?? '', + model, + api, + timeoutMs, + } +} diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index f4bde7e656..978d34bbed 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -43,6 +43,8 @@ import { } from './rpcGateway' import { SessionCache } from './sessionCache' import { OverseerEventRecorder, toSessionSnapshot } from './overseerEventRecorder' +import { createOverseerLlmFallbackClient } from './overseerLlmFallback' +import { loadOverseerLlmFallbackConfig } from './overseerLlmFallbackConfig' import { OverseerEntity } from './overseerEntity' import { extractAssistantPlainText } from '@hapi/protocol/messages' import type { InboxOperatorAction } from '@hapi/protocol' @@ -165,7 +167,16 @@ export class SyncEngine { (sessionId, updatedAt) => this.recordSessionActivity(sessionId, updatedAt) ) this.rpcGateway = new RpcGateway(io, rpcRegistry) - this.overseerEvents = new OverseerEventRecorder(store.events, store.inbox) + const llmFallbackConfig = loadOverseerLlmFallbackConfig() + const llmFallback = llmFallbackConfig.enabled + ? createOverseerLlmFallbackClient(llmFallbackConfig) + : null + if (llmFallbackConfig.enabled) { + console.log( + `[overseer] LLM summary fallback ENABLED (api=${llmFallbackConfig.api}, model=${llmFallbackConfig.model}, base=${llmFallbackConfig.baseUrl})` + ) + } + this.overseerEvents = new OverseerEventRecorder(store.events, store.inbox, { llmFallback }) this.overseer = new OverseerEntity({ events: store.events, inbox: store.inbox, @@ -329,7 +340,9 @@ export class SyncEngine { event.message.id, event.message.content, event.message.createdAt - ) + ).catch((error) => { + console.error('[overseer] onAgentMessage failed', error) + }) } } From ce7269b31ddb751f0f39e812e605c82913ed3b20 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:07:29 +0000 Subject: [PATCH 02/12] fix(overseer): serialize LLM fallback and flush on keepalive Address Codex review on #91: per-session insert order, remap stalled away from hidden stale, flush deferred fallback on session-alive thinking-clear, and skip duplicate session-end completed_fallback after a successful LLM row. Mock AppContext in About settings tests so CI is green. Co-authored-by: Cursor --- .../overseerEventRecorder.llmFallback.test.ts | 127 ++++++++++++++++++ hub/src/sync/overseerEventRecorder.test.ts | 4 +- hub/src/sync/overseerEventRecorder.ts | 98 +++++++++----- hub/src/sync/syncEngine.ts | 23 +++- web/src/routes/settings/index.test.tsx | 9 ++ 5 files changed, 220 insertions(+), 41 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts index 8f69a8fbe0..1308cd97f8 100644 --- a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -152,4 +152,131 @@ describe('OverseerEventRecorder LLM fallback', () => { expect(event).toBeNull() expect(store.events.count()).toBe(0) }) + + it('maps stalled LLM status to progress so Session Log All still shows it', async () => { + const store = new Store(':memory:') + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => ({ + status: 'stalled', + summary: 'Agent went quiet mid-turn', + })), + }, + }) + const session = store.sessions.getOrCreateSession('llm-stale', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + + const event = await recorder.onAgentMessage( + toSessionSnapshot(makeSession(session.id, 'cursor'), session.tag), + 'msg-stalled', + agentText('Still working on the rebase.'), + Date.now() + ) + + expect(event?.eventType).toBe('progress') + expect(event?.attentionCandidate).toBe(0) + const payload = JSON.parse(event!.payloadJson!) as { notify_summary?: NotifySummary } + expect(payload.notify_summary?.status).toBe('stalled') + }) + + it('flushes deferred LLM fallback when thinking clears', async () => { + const store = new Store(':memory:') + const synthesize = mock(async () => ({ status: 'done', summary: 'End of turn' })) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { synthesizeNotifySummary: synthesize }, + }) + const live = store.sessions.getOrCreateSession('llm-think', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + const snapshot = toSessionSnapshot(makeSession(live.id, 'cursor'), live.tag) + + expect(await recorder.onAgentMessage(snapshot, 'msg-mid', agentText('Partial flush.'), Date.now(), { thinking: true })).toBeNull() + expect(synthesize).toHaveBeenCalledTimes(0) + + const flushed = await recorder.flushPendingLlmFallback(snapshot) + expect(synthesize).toHaveBeenCalledTimes(1) + expect(flushed?.summary).toBe('End of turn') + expect(store.events.count()).toBe(1) + }) + + it('flushes pending fallback from onSessionUpdated when thinking is false', async () => { + const store = new Store(':memory:') + const synthesize = mock(async () => ({ status: 'blocked', summary: 'Need a decision' })) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { synthesizeNotifySummary: synthesize }, + }) + const live = makeSession('sess-alive', 'cursor', { thinking: true }) + const stored = store.sessions.getOrCreateSession('llm-alive', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + + await recorder.onAgentMessage(snapshot, 'msg-pending', agentText('No notify yet.'), Date.now(), { thinking: true }) + expect(store.events.count()).toBe(0) + + live.thinking = false + await recorder.onSessionUpdated(live, stored.tag) + + expect(synthesize).toHaveBeenCalledTimes(1) + expect(store.events.list({ eventType: 'blocked' })).toHaveLength(1) + }) + + it('does not insert session-end completed_fallback after a successful LLM flush', async () => { + const store = new Store(':memory:') + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => ({ + status: 'done', + summary: 'LLM caught the last turn', + })), + }, + }) + const live = makeSession('sess-end', 'cursor', { thinking: true }) + const stored = store.sessions.getOrCreateSession('llm-end', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + + await recorder.onAgentMessage(snapshot, 'msg-last', agentText('Finishing up.'), Date.now(), { thinking: true }) + + const event = await recorder.onSessionEnd( + live, + stored.tag, + Date.now(), + 'completed', + () => 'Finishing up.' + ) + + expect(event?.provenance).toContain('hub-llm-fallback') + expect(store.events.count()).toBe(1) + expect(store.events.list().some((row) => row.provenance?.includes('session-end'))).toBe(false) + }) + + it('serializes LLM fallbacks on one session so earlier turns keep lower ids', async () => { + const store = new Store(':memory:') + let releaseFirst: ((value: NotifySummary) => void) | undefined + const firstGate = new Promise((resolve) => { + releaseFirst = resolve + }) + let firstStarted!: () => void + const firstStartedP = new Promise((resolve) => { + firstStarted = resolve + }) + const synthesize = mock(async (plainText: string): Promise => { + if (plainText.includes('FIRST')) { + firstStarted() + return firstGate + } + return { status: 'done', summary: 'second turn' } + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { synthesizeNotifySummary: synthesize }, + }) + const stored = store.sessions.getOrCreateSession('llm-ord', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + const snapshot = toSessionSnapshot(makeSession(stored.id, 'cursor'), stored.tag) + + const first = recorder.onAgentMessage(snapshot, 'msg-a', agentText('FIRST turn body'), Date.now()) + await firstStartedP + const second = recorder.onAgentMessage(snapshot, 'msg-b', agentText('SECOND turn body'), Date.now() + 1) + releaseFirst!({ status: 'done', summary: 'first turn' }) + await Promise.all([first, second]) + + const rows = store.events.list().sort((a, b) => a.id - b.id) + expect(rows.map((row) => row.summary)).toEqual(['first turn', 'second turn']) + }) }) diff --git a/hub/src/sync/overseerEventRecorder.test.ts b/hub/src/sync/overseerEventRecorder.test.ts index 0e674f4c97..69449867be 100644 --- a/hub/src/sync/overseerEventRecorder.test.ts +++ b/hub/src/sync/overseerEventRecorder.test.ts @@ -118,7 +118,7 @@ describe('OverseerEventRecorder', () => { expect(store.inbox.count()).toBe(0) }) - it('synthesizes approval_requested from permission prompts', () => { + it('synthesizes approval_requested from permission prompts', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('perm', { flavor: 'claude', path: '/tmp', host: 'local' }, null, 'default') @@ -130,7 +130,7 @@ describe('OverseerEventRecorder', () => { } } - recorder.onSessionUpdated(live, session.tag) + await recorder.onSessionUpdated(live, session.tag) const events = store.events.list({ eventType: 'approval_requested' }) expect(events).toHaveLength(1) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index a74dd594d4..e891da7d75 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -165,6 +165,10 @@ export class OverseerEventRecorder { /** Latest no-notify assistant text awaiting end-of-turn LLM attempt. */ private readonly pendingLlmFallback = new Map() private readonly sessionThinking = new Map() + /** Per-session tail so concurrent LLM calls insert in arrival order. */ + private readonly sessionWork = new Map>() + /** Sessions that already got a hub-llm-fallback row for the latest missed turn. */ + private readonly llmFallbackSucceeded = new Set() constructor( private readonly events: EventStore, @@ -231,6 +235,7 @@ export class OverseerEventRecorder { const notify = extractNotifySummary(plainText) if (notify) { this.pendingLlmFallback.delete(session.id) + this.llmFallbackSucceeded.delete(session.id) primary = this.recordNotifySummary(session, messageId, notify, ts) } } @@ -264,7 +269,9 @@ export class OverseerEventRecorder { this.pendingLlmFallback.set(session.id, { messageId, plainText, ts }) } else { this.pendingLlmFallback.delete(session.id) - primary = await this.tryLlmFallback(session, messageId, plainText, ts) + primary = await this.enqueueSessionWork(session.id, () => + this.tryLlmFallback(session, messageId, plainText, ts) + ) } } } @@ -273,57 +280,66 @@ export class OverseerEventRecorder { return primary } - onSessionUpdated(session: Session, tag?: string | null): void { - const prevThinking = this.sessionThinking.get(session.id) ?? false + async onSessionUpdated(session: Session, tag?: string | null): Promise { this.sessionThinking.set(session.id, session.thinking) this.syncPermissionRequests(session, tag ?? null) - if (prevThinking && !session.thinking) { - void this.flushPendingLlmFallback(toSessionSnapshot(session, tag ?? null)).catch((error) => { - console.error('[overseer] flushPendingLlmFallback failed', error) - }) + // Flush whenever thinking is clear and a deferred turn is waiting β€” + // not only on a trueβ†’false edge. Keepalives often never sent the + // thinking=true update through this recorder. + if (!session.thinking) { + await this.flushPendingLlmFallback(toSessionSnapshot(session, tag ?? null)) } } - onSessionEnd( + async onSessionEnd( session: Session, tag: string | null, ts: number, reason: string | undefined, getLastAgentPlainText: () => string | null - ): StoredSystemEvent | null { + ): Promise { this.knownPermissionRequestIds.delete(session.id) this.sessionThinking.delete(session.id) - void this.flushPendingLlmFallback(toSessionSnapshot(session, tag)).catch((error) => { - console.error('[overseer] flushPendingLlmFallback on session-end failed', error) - }) + const snapshot = toSessionSnapshot(session, tag) - if (reason !== 'completed') { - return null - } + return this.enqueueSessionWork(session.id, async () => { + const llmEvent = await this.flushPendingLlmFallbackUnlocked(snapshot) + if (reason !== 'completed') { + return llmEvent + } - const lastText = getLastAgentPlainText() - if (lastText && extractNotifySummary(lastText)) { - return null - } + const lastText = getLastAgentPlainText() + if (lastText && extractNotifySummary(lastText)) { + return llmEvent + } + // Successful LLM row already captured this missed turn β€” do not + // also write "session ended without AGENT_NOTIFY_SUMMARY". + if (llmEvent || this.llmFallbackSucceeded.has(session.id)) { + return llmEvent + } - const snapshot = toSessionSnapshot(session, tag) - return this.insertSystemEvent(snapshot, { - ts, - sourceKind: 'system', - sourceRef: session.id, - eventType: 'completed', - attentionCandidate: 0, - summary: 'Session ended without AGENT_NOTIFY_SUMMARY; hub inferred completion', - relatedSessionId: session.id, - provenance: 'hub-inferred from session-end completed signal', - idempotencyKey: `session:${session.id}:session_end:${ts}:completed_fallback`, - payloadFields: { reason }, - severity: deriveSeverity('completed'), - tags: buildTags(null, snapshot.flavor) + return this.insertSystemEvent(snapshot, { + ts, + sourceKind: 'system', + sourceRef: session.id, + eventType: 'completed', + attentionCandidate: 0, + summary: 'Session ended without AGENT_NOTIFY_SUMMARY; hub inferred completion', + relatedSessionId: session.id, + provenance: 'hub-inferred from session-end completed signal', + idempotencyKey: `session:${session.id}:session_end:${ts}:completed_fallback`, + payloadFields: { reason }, + severity: deriveSeverity('completed'), + tags: buildTags(null, snapshot.flavor) + }) }) } async flushPendingLlmFallback(session: SessionSnapshot): Promise { + return this.enqueueSessionWork(session.id, () => this.flushPendingLlmFallbackUnlocked(session)) + } + + private async flushPendingLlmFallbackUnlocked(session: SessionSnapshot): Promise { const pending = this.pendingLlmFallback.get(session.id) if (!pending) return null this.pendingLlmFallback.delete(session.id) @@ -331,6 +347,13 @@ export class OverseerEventRecorder { return this.tryLlmFallback(session, pending.messageId, pending.plainText, pending.ts) } + private enqueueSessionWork(sessionId: string, work: () => Promise): Promise { + const previous = this.sessionWork.get(sessionId) ?? Promise.resolve() + const run = previous.then(work, work) + this.sessionWork.set(sessionId, run.then(() => undefined, () => undefined)) + return run + } + /** * Opt-in LLM synthesis only. Failures return null β€” never invent a * first-line heuristic Session Log row. @@ -345,8 +368,11 @@ export class OverseerEventRecorder { try { const notify = await this.llmFallback.synthesizeNotifySummary(plainText) if (!notify) return null - const eventType = mapNotifyStatusToEventType(notify.status) - return this.insertSystemEvent(session, { + // Session Log All hides `stale` (ambient silence). Keep LLM + // fallbacks visible as captured-only progress. + const mapped = mapNotifyStatusToEventType(notify.status) + const eventType = mapped === 'stale' ? 'progress' : mapped + const stored = this.insertSystemEvent(session, { ts, sourceKind: 'system', sourceRef: session.id, @@ -368,6 +394,8 @@ export class OverseerEventRecorder { severity: deriveSeverity(eventType), tags: buildTags(notify, session.flavor), }) + if (stored) this.llmFallbackSucceeded.add(session.id) + return stored } catch { return null } diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 7334aa4f6c..0fb726fce9 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -307,10 +307,12 @@ export class SyncEngine { this.sessionCache.refreshSession(event.sessionId) const after = this.sessionCache.getSession(event.sessionId) if (after) { - this.overseerEvents.onSessionUpdated( + void this.overseerEvents.onSessionUpdated( after, this.store.sessions.getSession(after.id)?.tag ?? null - ) + ).catch((error) => { + console.error('[overseer] onSessionUpdated failed', error) + }) } if (after?.metadata && !this.hasSameAgentSessionIds(before?.metadata ?? null, after.metadata)) { if (!this.canRunCursorDedup(after)) { @@ -402,6 +404,17 @@ export class SyncEngine { }): void { this.sessionCache.handleSessionAlive(payload) this.triggerDedupIfNeeded(payload.sid) + const session = this.getSession(payload.sid) + if (session) { + // thinking=trueβ†’false usually arrives on session-alive, not + // session-updated. Flush deferred LLM fallback on that path. + void this.overseerEvents.onSessionUpdated( + session, + this.store.sessions.getSession(session.id)?.tag ?? null + ).catch((error) => { + console.error('[overseer] onSessionUpdated from session-alive failed', error) + }) + } } handleSessionReady(payload: { sid: string; time: number }): void { @@ -422,13 +435,15 @@ export class SyncEngine { this.sessionCache.handleSessionEnd(payload) const session = this.getSession(payload.sid) if (session) { - this.overseerEvents.onSessionEnd( + void this.overseerEvents.onSessionEnd( session, this.store.sessions.getSession(session.id)?.tag ?? null, payload.time, payload.reason, () => this.getLastAgentPlainText(session.id) - ) + ).catch((error) => { + console.error('[overseer] onSessionEnd failed', error) + }) } this.eventPublisher.emit({ type: 'session-ended', diff --git a/web/src/routes/settings/index.test.tsx b/web/src/routes/settings/index.test.tsx index e1698b676a..8fabe3055c 100644 --- a/web/src/routes/settings/index.test.tsx +++ b/web/src/routes/settings/index.test.tsx @@ -23,6 +23,15 @@ vi.mock('@tanstack/react-router', () => ({ useNavigate: () => navigate, })) +vi.mock('@/lib/app-context', () => ({ + useAppContext: () => ({ + api: { + fetchSystemEvents: vi.fn(async () => ({ total: 0, events: [] })), + fetchInboxItems: vi.fn(async () => ({ total: 0, items: [] })), + }, + }), +})) + vi.mock('@hapi/protocol', () => ({ PROTOCOL_VERSION: 1 })) vi.mock('@/hooks/useTheme', () => ({ From ec56cb7bf1e6404f28eeca4afc34ca9f34fa55ce Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:19:32 +0000 Subject: [PATCH 03/12] fix(overseer): accumulate ACP segments; scoop links before LLM Codex P2: keep full thinking-turn text, clear fallback-success per attempt, persist URLs before await, and drop idle session queues. Co-authored-by: Cursor --- .../overseerEventRecorder.llmFallback.test.ts | 95 +++++++++++++++++++ hub/src/sync/overseerEventRecorder.ts | 39 +++++++- 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts index 1308cd97f8..708b24f9b5 100644 --- a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -279,4 +279,99 @@ describe('OverseerEventRecorder LLM fallback', () => { const rows = store.events.list().sort((a, b) => a.id - b.id) expect(rows.map((row) => row.summary)).toEqual(['first turn', 'second turn']) }) + + it('accumulates ACP thinking segments before fallback', async () => { + const store = new Store(':memory:') + const synthesize = mock(async (plainText: string): Promise => { + expect(plainText).toContain('First chunk') + expect(plainText).toContain('Second chunk') + return { status: 'done', summary: 'both chunks' } + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { synthesizeNotifySummary: synthesize }, + }) + const stored = store.sessions.getOrCreateSession('llm-acc', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + const snapshot = toSessionSnapshot(makeSession(stored.id, 'cursor'), stored.tag) + + await recorder.onAgentMessage(snapshot, 'msg-1', agentText('First chunk'), Date.now(), { thinking: true }) + await recorder.onAgentMessage(snapshot, 'msg-2', agentText('Second chunk'), Date.now() + 1, { thinking: true }) + const flushed = await recorder.flushPendingLlmFallback(snapshot) + + expect(synthesize).toHaveBeenCalledTimes(1) + expect(flushed?.summary).toBe('both chunks') + }) + + it('writes completed_fallback when a later missed turn LLM fails', async () => { + const store = new Store(':memory:') + const synthesize = mock(async (plainText: string): Promise => { + if (plainText.includes('first turn')) { + return { status: 'done', summary: 'caught first' } + } + return null + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { synthesizeNotifySummary: synthesize }, + }) + const live = makeSession('sess-later', 'cursor', { thinking: true }) + const stored = store.sessions.getOrCreateSession('llm-later', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + + await recorder.onAgentMessage(snapshot, 'msg-first', agentText('first turn body'), Date.now(), { thinking: true }) + live.thinking = false + await recorder.onSessionUpdated(live, stored.tag) + expect(store.events.list({ eventType: 'completed' })).toHaveLength(1) + + await recorder.onAgentMessage(snapshot, 'msg-second', agentText('second turn miss'), Date.now() + 1) + expect(synthesize).toHaveBeenCalledTimes(2) + + const event = await recorder.onSessionEnd( + live, + stored.tag, + Date.now() + 2, + 'completed', + () => 'second turn miss' + ) + + expect(event?.provenance).toContain('session-end') + expect(store.events.list().filter((row) => row.provenance?.includes('session-end'))).toHaveLength(1) + expect(store.events.list().filter((row) => row.provenance?.includes('hub-llm-fallback'))).toHaveLength(1) + }) + + it('persists scooped links before awaiting LLM', async () => { + const store = new Store(':memory:') + let release!: (value: NotifySummary) => void + const gate = new Promise((resolve) => { + release = resolve + }) + let started!: () => void + const startedP = new Promise((resolve) => { + started = resolve + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => { + started() + return gate + }), + }, + }) + const stored = store.sessions.getOrCreateSession('llm-scoop', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + const snapshot = toSessionSnapshot(makeSession(stored.id, 'cursor'), stored.tag) + + const pending = recorder.onAgentMessage( + snapshot, + 'msg-url', + agentText('See https://example.com/docs for details.'), + Date.now() + ) + await startedP + + expect(store.events.list({ eventType: 'link_seen' })).toHaveLength(1) + expect(store.events.list().filter((row) => row.provenance?.includes('hub-llm-fallback'))).toHaveLength(0) + + release({ status: 'done', summary: 'linked turn' }) + await pending + expect(store.events.list().filter((row) => row.provenance?.includes('hub-llm-fallback'))).toHaveLength(1) + }) }) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index fada729d3f..b5e001eb6f 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -266,11 +266,15 @@ export class OverseerEventRecorder { } } + // Scoop URLs before any LLM await so a slow/crashed fallback + // cannot drop already-persisted assistant links. + this.scoopLinksFromContent(session, messageId, content, ts) + // Opt-in LLM only (#90). No first-line heuristic. Defer while // thinking so ACP mid-turn flushes do not each hit the LLM. if (!primary && plainText && this.llmFallback) { if (opts.thinking) { - this.pendingLlmFallback.set(session.id, { messageId, plainText, ts }) + this.rememberPendingLlmFallback(session.id, messageId, plainText, ts) } else { this.pendingLlmFallback.delete(session.id) primary = await this.enqueueSessionWork(session.id, () => @@ -278,6 +282,7 @@ export class OverseerEventRecorder { ) } } + return primary } this.scoopLinksFromContent(session, messageId, content, ts) @@ -340,9 +345,32 @@ export class OverseerEventRecorder { } async flushPendingLlmFallback(session: SessionSnapshot): Promise { + if (!this.pendingLlmFallback.has(session.id)) return null return this.enqueueSessionWork(session.id, () => this.flushPendingLlmFallbackUnlocked(session)) } + private rememberPendingLlmFallback( + sessionId: string, + messageId: string, + plainText: string, + ts: number + ): void { + const prev = this.pendingLlmFallback.get(sessionId) + // New ACP text segment in the same thinking turn β€” keep the whole turn. + // Same messageId is a redelivery; replace rather than duplicate. + const combined = prev && prev.messageId !== messageId + ? `${prev.plainText}\n${plainText}` + : plainText + if (!prev || prev.messageId !== messageId) { + this.llmFallbackSucceeded.delete(sessionId) + } + this.pendingLlmFallback.set(sessionId, { + messageId, + plainText: combined, + ts: prev?.ts ?? ts + }) + } + private async flushPendingLlmFallbackUnlocked(session: SessionSnapshot): Promise { const pending = this.pendingLlmFallback.get(session.id) if (!pending) return null @@ -354,7 +382,13 @@ export class OverseerEventRecorder { private enqueueSessionWork(sessionId: string, work: () => Promise): Promise { const previous = this.sessionWork.get(sessionId) ?? Promise.resolve() const run = previous.then(work, work) - this.sessionWork.set(sessionId, run.then(() => undefined, () => undefined)) + const tail: Promise = run.then(() => undefined, () => undefined) + this.sessionWork.set(sessionId, tail) + void tail.then(() => { + if (this.sessionWork.get(sessionId) === tail) { + this.sessionWork.delete(sessionId) + } + }) return run } @@ -369,6 +403,7 @@ export class OverseerEventRecorder { ts: number ): Promise { if (!this.llmFallback) return null + this.llmFallbackSucceeded.delete(session.id) try { const notify = await this.llmFallback.synthesizeNotifySummary(plainText) if (!notify) return null From 964f59fa29806ed6e43f33c78b858895fef539c9 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:27:41 +0000 Subject: [PATCH 04/12] fix(overseer): snapshot pending LLM work; reject empty notify Detach the deferred turn before queueing so a later ACP segment cannot merge into flushed work. LLM parse now requires status+summary; failed synthesis stays silent (docs matched to that). Co-authored-by: Cursor --- .../2026-07-24-overseer-summary-emission.md | 7 ++-- .../overseerEventRecorder.llmFallback.test.ts | 38 +++++++++++++++++++ hub/src/sync/overseerEventRecorder.ts | 24 ++++++++---- hub/src/sync/overseerLlmFallback.test.ts | 4 ++ hub/src/sync/overseerLlmFallback.ts | 14 ++++++- 5 files changed, 76 insertions(+), 11 deletions(-) diff --git a/docs/plans/2026-07-24-overseer-summary-emission.md b/docs/plans/2026-07-24-overseer-summary-emission.md index 275148ac00..b452ea993f 100644 --- a/docs/plans/2026-07-24-overseer-summary-emission.md +++ b/docs/plans/2026-07-24-overseer-summary-emission.md @@ -185,8 +185,9 @@ export HAPI_OVERSEER_LLM_TIMEOUT_MS=30000 ``` Prefer `chat-completions` for local-gateway compatibility; use `responses` for -OpenAI-native. Failures / non-compliant model output fall through to the -heuristic first-line fallback. Events are marked +OpenAI-native. Failures / non-compliant model output produce **no** Session Log +row (no first-line heuristic). Session-end may still write `completed_fallback` +if the session completes without a later successful notify/LLM row. Events are marked `provenance: hub-llm-fallback ...` with `payload.synthesis = "llm-fallback"`, `attentionCandidate = 0` (Session Log only β€” not inbox / voice). @@ -217,7 +218,7 @@ in Session Log / inbox so the operator never wonders "wtf usage is this." primary turn lacked a contract - never pretend the primary agent said it. - **Kill-criterion:** if opt-in users report surprise usage, the toggle and provenance labels failed - fix UX before expanding defaults. If fallback - summaries are worse than the heuristic first-line, do not ship. + summaries are worse than a primary `AGENT_NOTIFY_SUMMARY` emit, do not ship. Prefer **Option A** as the first better-fallback ship: smaller blast radius, easier to reason about cost, no phantom sessions. diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts index 708b24f9b5..228f6839fa 100644 --- a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -374,4 +374,42 @@ describe('OverseerEventRecorder LLM fallback', () => { await pending expect(store.events.list().filter((row) => row.provenance?.includes('hub-llm-fallback'))).toHaveLength(1) }) + + it('does not append a new turn onto a flush already queued', async () => { + const store = new Store(':memory:') + let releaseFirst!: (value: NotifySummary) => void + const firstGate = new Promise((resolve) => { + releaseFirst = resolve + }) + let firstStarted!: () => void + const firstStartedP = new Promise((resolve) => { + firstStarted = resolve + }) + const synthesize = mock(async (plainText: string): Promise => { + if (plainText.includes('TURN A')) { + firstStarted() + return firstGate + } + return { status: 'done', summary: plainText.includes('TURN C') ? 'turn c' : 'turn b' } + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { synthesizeNotifySummary: synthesize }, + }) + const stored = store.sessions.getOrCreateSession('llm-detach', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + const snapshot = toSessionSnapshot(makeSession(stored.id, 'cursor'), stored.tag) + + const first = recorder.onAgentMessage(snapshot, 'msg-a', agentText('TURN A body'), Date.now()) + await firstStartedP + await recorder.onAgentMessage(snapshot, 'msg-b', agentText('TURN B body'), Date.now() + 1, { thinking: true }) + const flushedB = recorder.flushPendingLlmFallback(snapshot) + await recorder.onAgentMessage(snapshot, 'msg-c', agentText('TURN C body'), Date.now() + 2, { thinking: true }) + + releaseFirst({ status: 'done', summary: 'turn a' }) + await first + const eventB = await flushedB + expect(eventB?.summary).toBe('turn b') + + const eventC = await recorder.flushPendingLlmFallback(snapshot) + expect(eventC?.summary).toBe('turn c') + }) }) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index b5e001eb6f..c9de387ae4 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -310,9 +310,12 @@ export class OverseerEventRecorder { this.knownPermissionRequestIds.delete(session.id) this.sessionThinking.delete(session.id) const snapshot = toSessionSnapshot(session, tag) + const pending = this.takePendingLlmFallback(session.id) return this.enqueueSessionWork(session.id, async () => { - const llmEvent = await this.flushPendingLlmFallbackUnlocked(snapshot) + const llmEvent = pending + ? await this.runPendingLlmFallback(snapshot, pending) + : null if (reason !== 'completed') { return llmEvent } @@ -345,8 +348,15 @@ export class OverseerEventRecorder { } async flushPendingLlmFallback(session: SessionSnapshot): Promise { - if (!this.pendingLlmFallback.has(session.id)) return null - return this.enqueueSessionWork(session.id, () => this.flushPendingLlmFallbackUnlocked(session)) + const pending = this.takePendingLlmFallback(session.id) + if (!pending) return null + return this.enqueueSessionWork(session.id, () => this.runPendingLlmFallback(session, pending)) + } + + private takePendingLlmFallback(sessionId: string): PendingLlmFallback | undefined { + const pending = this.pendingLlmFallback.get(sessionId) + if (pending) this.pendingLlmFallback.delete(sessionId) + return pending } private rememberPendingLlmFallback( @@ -371,10 +381,10 @@ export class OverseerEventRecorder { }) } - private async flushPendingLlmFallbackUnlocked(session: SessionSnapshot): Promise { - const pending = this.pendingLlmFallback.get(session.id) - if (!pending) return null - this.pendingLlmFallback.delete(session.id) + private async runPendingLlmFallback( + session: SessionSnapshot, + pending: PendingLlmFallback + ): Promise { if (extractNotifySummary(pending.plainText)) return null return this.tryLlmFallback(session, pending.messageId, pending.plainText, pending.ts) } diff --git a/hub/src/sync/overseerLlmFallback.test.ts b/hub/src/sync/overseerLlmFallback.test.ts index 795db6b4a8..ed6fb52568 100644 --- a/hub/src/sync/overseerLlmFallback.test.ts +++ b/hub/src/sync/overseerLlmFallback.test.ts @@ -35,6 +35,10 @@ describe('parseNotifySummaryFromLlmText', () => { it('returns null for empty or non-compliant text', () => { expect(parseNotifySummaryFromLlmText('')).toBeNull() expect(parseNotifySummaryFromLlmText('just a paragraph')).toBeNull() + expect(parseNotifySummaryFromLlmText('AGENT_NOTIFY_SUMMARY {}')).toBeNull() + expect(parseNotifySummaryFromLlmText('AGENT_NOTIFY_SUMMARY {"status":"done"}')).toBeNull() + expect(parseNotifySummaryFromLlmText('AGENT_NOTIFY_SUMMARY {"summary":"no status"}')).toBeNull() + expect(parseNotifySummaryFromLlmText('AGENT_NOTIFY_SUMMARY {"status":"nope","summary":"bad status"}')).toBeNull() }) }) diff --git a/hub/src/sync/overseerLlmFallback.ts b/hub/src/sync/overseerLlmFallback.ts index 833e065b05..c4dd1f6cf6 100644 --- a/hub/src/sync/overseerLlmFallback.ts +++ b/hub/src/sync/overseerLlmFallback.ts @@ -1,3 +1,4 @@ +import { NOTIFY_SUMMARY_STATUSES } from '@hapi/protocol' import { extractNotifySummary, type NotifySummary } from '@hapi/protocol/messages' import type { OverseerLlmFallbackEnabledConfig } from './overseerLlmFallbackConfig' @@ -41,9 +42,20 @@ function stripMarkdownFences(text: string): string { * Parse an AGENT_NOTIFY_SUMMARY from LLM output. * Uses the same end-anchored extractor as primary agent turns. */ +const LLM_NOTIFY_STATUSES = new Set(NOTIFY_SUMMARY_STATUSES) + +function isCompliantLlmNotify(notify: NotifySummary): boolean { + const summary = notify.summary?.trim() + if (!summary) return false + if (!notify.status || !LLM_NOTIFY_STATUSES.has(notify.status)) return false + return true +} + export function parseNotifySummaryFromLlmText(text: string): NotifySummary | null { if (typeof text !== 'string' || text.trim().length === 0) return null - return extractNotifySummary(stripMarkdownFences(text)) + const parsed = extractNotifySummary(stripMarkdownFences(text)) + if (!parsed || !isCompliantLlmNotify(parsed)) return null + return parsed } export function extractTextFromChatCompletionsBody(body: unknown): string | null { From 86d3c2a78b650bb6ac6983edf9370f4c39db9ff1 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:39:14 +0000 Subject: [PATCH 05/12] fix(overseer): flush on expire; await end before dedup; live Session Log Queue primary inserts with LLM work, clear success on new agent activity, reject dirty timeout env, and invalidate Session Log over SSE. Co-authored-by: Cursor --- .../overseerEventRecorder.llmFallback.test.ts | 105 +++++++++++++++ hub/src/sync/overseerEventRecorder.ts | 123 +++++++++++------- .../sync/overseerLlmFallbackConfig.test.ts | 18 +++ hub/src/sync/overseerLlmFallbackConfig.ts | 3 + hub/src/sync/syncEngine.ts | 57 +++++--- web/src/hooks/useSSE.ts | 4 + 6 files changed, 244 insertions(+), 66 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts index 228f6839fa..d3f11285c4 100644 --- a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -412,4 +412,109 @@ describe('OverseerEventRecorder LLM fallback', () => { const eventC = await recorder.flushPendingLlmFallback(snapshot) expect(eventC?.summary).toBe('turn c') }) + + it('queues a later AGENT_NOTIFY_SUMMARY behind an in-flight LLM insert', async () => { + const store = new Store(':memory:') + let releaseFirst!: (value: NotifySummary) => void + const firstGate = new Promise((resolve) => { + releaseFirst = resolve + }) + let firstStarted!: () => void + const firstStartedP = new Promise((resolve) => { + firstStarted = resolve + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => { + firstStarted() + return firstGate + }), + }, + }) + const stored = store.sessions.getOrCreateSession('llm-notify-ord', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + const snapshot = toSessionSnapshot(makeSession(stored.id, 'cursor'), stored.tag) + + const first = recorder.onAgentMessage(snapshot, 'msg-a', agentText('TURN A body'), Date.now()) + await firstStartedP + const second = recorder.onAgentMessage( + snapshot, + 'msg-b', + agentText('Done.\nAGENT_NOTIFY_SUMMARY {"version":1,"status":"done","action":"Review","summary":"real notify"}'), + Date.now() + 1 + ) + releaseFirst({ status: 'done', summary: 'llm first' }) + await Promise.all([first, second]) + + const rows = store.events.list().sort((a, b) => a.id - b.id) + expect(rows.map((row) => row.summary)).toEqual(['llm first', 'real notify']) + }) + + it('writes completed_fallback after a textless tool turn following LLM success', async () => { + const store = new Store(':memory:') + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => ({ + status: 'done', + summary: 'caught earlier turn', + })), + }, + }) + const live = makeSession('sess-tool', 'cursor') + const stored = store.sessions.getOrCreateSession('llm-tool', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + + await recorder.onAgentMessage(snapshot, 'msg-text', agentText('earlier turn body'), Date.now()) + await recorder.onAgentMessage(snapshot, 'msg-tool', { + role: 'agent', + content: { + type: 'codex', + data: { type: 'tool-call-result', output: { exit_code: 0 } }, + }, + }, Date.now() + 1) + + const event = await recorder.onSessionEnd( + live, + stored.tag, + Date.now() + 2, + 'completed', + () => 'earlier turn body' + ) + expect(event?.provenance).toContain('session-end') + }) + + it('publishes after a successful LLM insert', async () => { + const store = new Store(':memory:') + let published = 0 + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => ({ status: 'done', summary: 'ok' })), + }, + onAsyncSystemEvent: () => { + published += 1 + }, + }) + const stored = store.sessions.getOrCreateSession('llm-pub', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + await recorder.onAgentMessage( + toSessionSnapshot(makeSession(stored.id, 'cursor'), stored.tag), + 'msg-pub', + agentText('Turn body'), + Date.now() + ) + expect(published).toBe(1) + }) + + it('forgetSession drops deferred LLM state without flushing', async () => { + const store = new Store(':memory:') + const synthesize = mock(async () => ({ status: 'done', summary: 'should not run' })) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { synthesizeNotifySummary: synthesize }, + }) + const stored = store.sessions.getOrCreateSession('llm-forget', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + const snapshot = toSessionSnapshot(makeSession(stored.id, 'cursor'), stored.tag) + await recorder.onAgentMessage(snapshot, 'msg-p', agentText('pending'), Date.now(), { thinking: true }) + recorder.forgetSession(stored.id) + expect(await recorder.flushPendingLlmFallback(snapshot)).toBeNull() + expect(synthesize).toHaveBeenCalledTimes(0) + }) }) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index c9de387ae4..1881a19913 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -32,6 +32,8 @@ export type SessionSnapshot = OverseerSessionIdentity export type OverseerEventRecorderOptions = { /** Opt-in OpenAI-compatible synthesizer (issue #90). Default: unset / off. */ llmFallback?: OverseerLlmFallbackClient | null + /** Fired after an async Session Log insert so SSE clients can refetch. */ + onAsyncSystemEvent?: ((sessionId: string) => void) | null } export type OnAgentMessageOptions = { @@ -166,6 +168,7 @@ export class OverseerEventRecorder { private readonly lastAgentMessageAt = new Map() private readonly knownPermissionRequestIds = new Map>() private readonly llmFallback: OverseerLlmFallbackClient | null + private readonly onAsyncSystemEvent: ((sessionId: string) => void) | null /** Latest no-notify assistant text awaiting end-of-turn LLM attempt. */ private readonly pendingLlmFallback = new Map() private readonly sessionThinking = new Map() @@ -180,6 +183,7 @@ export class OverseerEventRecorder { options?: OverseerEventRecorderOptions ) { this.llmFallback = options?.llmFallback ?? null + this.onAsyncSystemEvent = options?.onAsyncSystemEvent ?? null } list(options: Parameters[0] = {}): StoredSystemEvent[] { @@ -201,46 +205,58 @@ export class OverseerEventRecorder { if (isAgentMessageContent(content)) { this.lastAgentMessageAt.set(session.id, ts) + this.llmFallbackSucceeded.delete(session.id) const agentBody = unwrapRoleWrappedRecordEnvelope(content) const agentContent = agentBody?.role === 'agent' ? agentBody.content : content + // Scoop URLs before any queued await so a slow/crashed fallback + // cannot drop already-persisted assistant links. + this.scoopLinksFromContent(session, messageId, content, ts) + const plainText = extractAssistantPlainText(agentContent) if (plainText) { if (detectEmptyHapiEventsSentinel(plainText)) { this.pendingLlmFallback.delete(session.id) - primary = this.insertSystemEvent(session, { - ts, - sourceKind: 'system', - eventType: 'validation_error', - attentionCandidate: 0, - summary: 'Malformed HAPI_EVENTS sentinel block (empty body)', - relatedSessionId: session.id, - provenance: 'hub-inferred from empty HAPI_EVENTS sentinel pair', - idempotencyKey: `session:${session.id}:message:${messageId}:validation_error:empty_hapi_events`, - payloadFields: { messageId, plainTextPreview: plainText.slice(0, 500) }, - severity: 1 - }) + primary = await this.enqueueSessionWork(session.id, () => + Promise.resolve(this.insertSystemEvent(session, { + ts, + sourceKind: 'system', + eventType: 'validation_error', + attentionCandidate: 0, + summary: 'Malformed HAPI_EVENTS sentinel block (empty body)', + relatedSessionId: session.id, + provenance: 'hub-inferred from empty HAPI_EVENTS sentinel pair', + idempotencyKey: `session:${session.id}:message:${messageId}:validation_error:empty_hapi_events`, + payloadFields: { messageId, plainTextPreview: plainText.slice(0, 500) }, + severity: 1 + })) + ) } else if (detectMalformedNotifySummaryLine(plainText)) { this.pendingLlmFallback.delete(session.id) - primary = this.insertSystemEvent(session, { - ts, - sourceKind: 'system', - eventType: 'validation_error', - attentionCandidate: 0, - summary: 'Malformed AGENT_NOTIFY_SUMMARY line on last turn', - relatedSessionId: session.id, - provenance: 'hub-inferred from malformed AGENT_NOTIFY_SUMMARY JSON', - idempotencyKey: `session:${session.id}:message:${messageId}:validation_error:malformed_notify`, - payloadFields: { messageId }, - severity: 1 - }) + primary = await this.enqueueSessionWork(session.id, () => + Promise.resolve(this.insertSystemEvent(session, { + ts, + sourceKind: 'system', + eventType: 'validation_error', + attentionCandidate: 0, + summary: 'Malformed AGENT_NOTIFY_SUMMARY line on last turn', + relatedSessionId: session.id, + provenance: 'hub-inferred from malformed AGENT_NOTIFY_SUMMARY JSON', + idempotencyKey: `session:${session.id}:message:${messageId}:validation_error:malformed_notify`, + payloadFields: { messageId }, + severity: 1 + })) + ) } else { const notify = extractNotifySummary(plainText) if (notify) { this.pendingLlmFallback.delete(session.id) - this.llmFallbackSucceeded.delete(session.id) - primary = this.recordNotifySummary(session, messageId, notify, ts) + primary = await this.enqueueSessionWork(session.id, () => { + const stored = this.recordNotifySummary(session, messageId, notify, ts) + if (stored) this.onAsyncSystemEvent?.(session.id) + return Promise.resolve(stored) + }) } } } @@ -248,28 +264,26 @@ export class OverseerEventRecorder { if (!primary) { const toolFailure = extractToolFailureSummary(agentContent) if (toolFailure) { - primary = this.insertSystemEvent(session, { - ts, - sourceKind: 'system', - sourceRef: session.id, - eventType: 'failed', - attentionCandidate: 1, - operatorActionRequired: 1, - summary: toolFailure, - relatedSessionId: session.id, - provenance: 'hub-inferred from tool-call-result exit code', - idempotencyKey: `session:${session.id}:message:${messageId}:tool_failed`, - payloadFields: { messageId }, - severity: deriveSeverity('failed'), - tags: buildTags(null, session.flavor) - }) + primary = await this.enqueueSessionWork(session.id, () => + Promise.resolve(this.insertSystemEvent(session, { + ts, + sourceKind: 'system', + sourceRef: session.id, + eventType: 'failed', + attentionCandidate: 1, + operatorActionRequired: 1, + summary: toolFailure, + relatedSessionId: session.id, + provenance: 'hub-inferred from tool-call-result exit code', + idempotencyKey: `session:${session.id}:message:${messageId}:tool_failed`, + payloadFields: { messageId }, + severity: deriveSeverity('failed'), + tags: buildTags(null, session.flavor) + })) + ) } } - // Scoop URLs before any LLM await so a slow/crashed fallback - // cannot drop already-persisted assistant links. - this.scoopLinksFromContent(session, messageId, content, ts) - // Opt-in LLM only (#90). No first-line heuristic. Defer while // thinking so ACP mid-turn flushes do not each hit the LLM. if (!primary && plainText && this.llmFallback) { @@ -330,7 +344,7 @@ export class OverseerEventRecorder { return llmEvent } - return this.insertSystemEvent(snapshot, { + const stored = this.insertSystemEvent(snapshot, { ts, sourceKind: 'system', sourceRef: session.id, @@ -344,6 +358,8 @@ export class OverseerEventRecorder { severity: deriveSeverity('completed'), tags: buildTags(null, snapshot.flavor) }) + if (stored) this.onAsyncSystemEvent?.(session.id) + return stored }) } @@ -443,7 +459,10 @@ export class OverseerEventRecorder { severity: deriveSeverity(eventType), tags: buildTags(notify, session.flavor), }) - if (stored) this.llmFallbackSucceeded.add(session.id) + if (stored) { + this.llmFallbackSucceeded.add(session.id) + this.onAsyncSystemEvent?.(session.id) + } return stored } catch { return null @@ -469,6 +488,16 @@ export class OverseerEventRecorder { this.lastAgentMessageAt.set(sessionId, ts) } + /** Drop deferred LLM state when a session is deleted (do not flush). */ + forgetSession(sessionId: string): void { + this.pendingLlmFallback.delete(sessionId) + this.sessionWork.delete(sessionId) + this.llmFallbackSucceeded.delete(sessionId) + this.lastAgentMessageAt.delete(sessionId) + this.knownPermissionRequestIds.delete(sessionId) + this.sessionThinking.delete(sessionId) + } + private scoopLinksFromContent( session: SessionSnapshot, messageId: string, diff --git a/hub/src/sync/overseerLlmFallbackConfig.test.ts b/hub/src/sync/overseerLlmFallbackConfig.test.ts index 40fc1ee191..1e99a995c6 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.test.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.test.ts @@ -81,4 +81,22 @@ describe('loadOverseerLlmFallbackConfig', () => { expect(config.apiKey).toBe('sk-test') expect(config.timeoutMs).toBe(12_000) }) + + it('rejects partially numeric timeout values', () => { + stashEnv() + process.env.HAPI_OVERSEER_LLM_FALLBACK = '1' + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'http://127.0.0.1:11434/v1' + process.env.HAPI_OVERSEER_LLM_MODEL = 'llama3.3' + process.env.HAPI_OVERSEER_LLM_TIMEOUT_MS = '30s' + const suffix = loadOverseerLlmFallbackConfig() + expect(suffix.enabled).toBe(false) + if (suffix.enabled) throw new Error('expected disabled') + expect(suffix.reasonDisabled).toBe('invalid_timeout') + + process.env.HAPI_OVERSEER_LLM_TIMEOUT_MS = '1e3' + const scientific = loadOverseerLlmFallbackConfig() + expect(scientific.enabled).toBe(false) + if (scientific.enabled) throw new Error('expected disabled') + expect(scientific.reasonDisabled).toBe('invalid_timeout') + }) }) diff --git a/hub/src/sync/overseerLlmFallbackConfig.ts b/hub/src/sync/overseerLlmFallbackConfig.ts index fc9e8ff5de..871f5e057b 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.ts @@ -77,6 +77,9 @@ export function loadOverseerLlmFallbackConfig( let timeoutMs = DEFAULT_TIMEOUT_MS const timeoutRaw = env.HAPI_OVERSEER_LLM_TIMEOUT_MS?.trim() if (timeoutRaw) { + if (!/^\d+$/.test(timeoutRaw)) { + return { enabled: false, reasonDisabled: 'invalid_timeout' } + } const parsed = Number.parseInt(timeoutRaw, 10) if (!Number.isFinite(parsed) || parsed <= 0) { return { enabled: false, reasonDisabled: 'invalid_timeout' } diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 0fb726fce9..f1c9743aa6 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -176,7 +176,12 @@ export class SyncEngine { `[overseer] LLM summary fallback ENABLED (api=${llmFallbackConfig.api}, model=${llmFallbackConfig.model}, base=${llmFallbackConfig.baseUrl})` ) } - this.overseerEvents = new OverseerEventRecorder(store.events, store.inbox, { llmFallback }) + this.overseerEvents = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback, + onAsyncSystemEvent: (sessionId) => { + this.eventPublisher.emit({ type: 'session-updated', sessionId }) + } + }) this.overseer = new OverseerEntity({ events: store.events, inbox: store.inbox, @@ -434,29 +439,32 @@ export class SyncEngine { this.sessionCache.handleSessionEnd(payload) const session = this.getSession(payload.sid) - if (session) { - void this.overseerEvents.onSessionEnd( - session, - this.store.sessions.getSession(session.id)?.tag ?? null, - payload.time, - payload.reason, - () => this.getLastAgentPlainText(session.id) - ).catch((error) => { - console.error('[overseer] onSessionEnd failed', error) - }) - } this.eventPublisher.emit({ type: 'session-ended', sessionId: payload.sid, reason: payload.reason }) - // Retry dedup now that this session is inactive β€” a prior dedup may have - // skipped it because it was still active at the time. Cursor ACP rows that - // never reached session-ready must not dedup-merge the original on failure. - if (shouldRetryDedup) { - this.triggerDedupIfNeeded(payload.sid) - } - this.sessionReadyIds.delete(payload.sid) + // Await recorder work before dedup so a queued LLM/completed_fallback + // insert still has a live relatedSessionId. + void (async () => { + try { + if (session) { + await this.overseerEvents.onSessionEnd( + session, + this.store.sessions.getSession(session.id)?.tag ?? null, + payload.time, + payload.reason, + () => this.getLastAgentPlainText(session.id) + ) + } + } catch (error) { + console.error('[overseer] onSessionEnd failed', error) + } + if (shouldRetryDedup) { + this.triggerDedupIfNeeded(payload.sid) + } + this.sessionReadyIds.delete(payload.sid) + })() } handleBackgroundTaskDelta(sessionId: string, delta: { started: number; completed: number }): void { @@ -473,6 +481,16 @@ export class SyncEngine { private expireInactive(): void { const expired = this.sessionCache.expireInactive() + for (const sessionId of expired) { + const session = this.sessionCache.getSession(sessionId) + if (!session) continue + void this.overseerEvents.onSessionUpdated( + session, + this.store.sessions.getSession(sessionId)?.tag ?? null + ).catch((error) => { + console.error('[overseer] onSessionUpdated from expireInactive failed', error) + }) + } // Sort by most recent first so dedup keeps the newest session when multiple // duplicates for the same agent thread expire in the same sweep. const sorted = expired @@ -776,6 +794,7 @@ export class SyncEngine { } async deleteSession(sessionId: string): Promise { + this.overseerEvents.forgetSession(sessionId) await this.sessionCache.deleteSession(sessionId) } diff --git a/web/src/hooks/useSSE.ts b/web/src/hooks/useSSE.ts index fe462d82af..6258dfad77 100644 --- a/web/src/hooks/useSSE.ts +++ b/web/src/hooks/useSSE.ts @@ -487,6 +487,10 @@ export function useSSE(options: { ingestIncomingMessages(event.sessionId, [event.message]) } + if (event.type === 'session-updated' || event.type === 'message-received' || event.type === 'session-ended') { + void queryClient.invalidateQueries({ queryKey: ['session-system-events', event.sessionId] }) + } + if (event.type === 'session-added' || event.type === 'session-updated' || event.type === 'session-removed') { if (event.type === 'session-removed') { removeSessionSummary(event.sessionId) From aee6587bee7c2bcaf83cf98af319cd1fb5c76f58 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:48:01 +0000 Subject: [PATCH 06/12] fix(overseer): await expire flush; key LLM success by message Dedup waits for expiry LLM work. Session-end skip only if the latest agent message is the one the LLM summarized. Queued inferred rows publish. Co-authored-by: Cursor --- .../overseerEventRecorder.llmFallback.test.ts | 45 +++++++++++++++++++ hub/src/sync/overseerEventRecorder.ts | 33 ++++++++++---- hub/src/sync/syncEngine.ts | 42 +++++++++-------- 3 files changed, 93 insertions(+), 27 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts index d3f11285c4..13515c8d74 100644 --- a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -483,6 +483,51 @@ describe('OverseerEventRecorder LLM fallback', () => { expect(event?.provenance).toContain('session-end') }) + it('does not let an in-flight earlier LLM cover a later tool-only turn', async () => { + const store = new Store(':memory:') + let releaseFirst!: (value: NotifySummary) => void + const firstGate = new Promise((resolve) => { + releaseFirst = resolve + }) + let firstStarted!: () => void + const firstStartedP = new Promise((resolve) => { + firstStarted = resolve + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => { + firstStarted() + return firstGate + }), + }, + }) + const live = makeSession('sess-inflight', 'cursor') + const stored = store.sessions.getOrCreateSession('llm-inflight', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + + const first = recorder.onAgentMessage(snapshot, 'msg-a', agentText('earlier turn'), Date.now()) + await firstStartedP + await recorder.onAgentMessage(snapshot, 'msg-tool', { + role: 'agent', + content: { + type: 'codex', + data: { type: 'tool-call-result', output: { exit_code: 0 } }, + }, + }, Date.now() + 1) + releaseFirst({ status: 'done', summary: 'caught earlier' }) + await first + + const event = await recorder.onSessionEnd( + live, + stored.tag, + Date.now() + 2, + 'completed', + () => 'earlier turn' + ) + expect(event?.provenance).toContain('session-end') + }) + it('publishes after a successful LLM insert', async () => { const store = new Store(':memory:') let published = 0 diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index 1881a19913..0277bea8ba 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -174,8 +174,9 @@ export class OverseerEventRecorder { private readonly sessionThinking = new Map() /** Per-session tail so concurrent LLM calls insert in arrival order. */ private readonly sessionWork = new Map>() - /** Sessions that already got a hub-llm-fallback row for the latest missed turn. */ - private readonly llmFallbackSucceeded = new Set() + /** Successful LLM fallback messageId per session (not a session-wide boolean). */ + private readonly llmFallbackSucceeded = new Map() + private readonly lastAgentMessageId = new Map() constructor( private readonly events: EventStore, @@ -205,7 +206,7 @@ export class OverseerEventRecorder { if (isAgentMessageContent(content)) { this.lastAgentMessageAt.set(session.id, ts) - this.llmFallbackSucceeded.delete(session.id) + this.lastAgentMessageId.set(session.id, messageId) const agentBody = unwrapRoleWrappedRecordEnvelope(content) const agentContent = agentBody?.role === 'agent' ? agentBody.content : content @@ -219,7 +220,7 @@ export class OverseerEventRecorder { if (detectEmptyHapiEventsSentinel(plainText)) { this.pendingLlmFallback.delete(session.id) primary = await this.enqueueSessionWork(session.id, () => - Promise.resolve(this.insertSystemEvent(session, { + Promise.resolve(this.insertInferredEvent(session, { ts, sourceKind: 'system', eventType: 'validation_error', @@ -235,7 +236,7 @@ export class OverseerEventRecorder { } else if (detectMalformedNotifySummaryLine(plainText)) { this.pendingLlmFallback.delete(session.id) primary = await this.enqueueSessionWork(session.id, () => - Promise.resolve(this.insertSystemEvent(session, { + Promise.resolve(this.insertInferredEvent(session, { ts, sourceKind: 'system', eventType: 'validation_error', @@ -265,7 +266,7 @@ export class OverseerEventRecorder { const toolFailure = extractToolFailureSummary(agentContent) if (toolFailure) { primary = await this.enqueueSessionWork(session.id, () => - Promise.resolve(this.insertSystemEvent(session, { + Promise.resolve(this.insertInferredEvent(session, { ts, sourceKind: 'system', sourceRef: session.id, @@ -340,7 +341,9 @@ export class OverseerEventRecorder { } // Successful LLM row already captured this missed turn β€” do not // also write "session ended without AGENT_NOTIFY_SUMMARY". - if (llmEvent || this.llmFallbackSucceeded.has(session.id)) { + const successForLatest = this.lastAgentMessageId.get(session.id) + && this.llmFallbackSucceeded.get(session.id) === this.lastAgentMessageId.get(session.id) + if (llmEvent || successForLatest) { return llmEvent } @@ -460,7 +463,7 @@ export class OverseerEventRecorder { tags: buildTags(notify, session.flavor), }) if (stored) { - this.llmFallbackSucceeded.add(session.id) + this.llmFallbackSucceeded.set(session.id, messageId) this.onAsyncSystemEvent?.(session.id) } return stored @@ -494,6 +497,7 @@ export class OverseerEventRecorder { this.sessionWork.delete(sessionId) this.llmFallbackSucceeded.delete(sessionId) this.lastAgentMessageAt.delete(sessionId) + this.lastAgentMessageId.delete(sessionId) this.knownPermissionRequestIds.delete(sessionId) this.sessionThinking.delete(sessionId) } @@ -612,6 +616,19 @@ export class OverseerEventRecorder { this.knownPermissionRequestIds.set(session.id, currentIds) } + private insertInferredEvent( + session: SessionSnapshot, + input: Omit & { + riskDetected?: 0 | 1 + payloadFields?: Record + notifyProject?: string | null + } + ): StoredSystemEvent | null { + const stored = this.insertSystemEvent(session, input) + if (stored) this.onAsyncSystemEvent?.(session.id) + return stored + } + private insertSystemEvent( session: SessionSnapshot, input: Omit & { diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index f1c9743aa6..c8d286b2a8 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -481,25 +481,29 @@ export class SyncEngine { private expireInactive(): void { const expired = this.sessionCache.expireInactive() - for (const sessionId of expired) { - const session = this.sessionCache.getSession(sessionId) - if (!session) continue - void this.overseerEvents.onSessionUpdated( - session, - this.store.sessions.getSession(sessionId)?.tag ?? null - ).catch((error) => { - console.error('[overseer] onSessionUpdated from expireInactive failed', error) - }) - } - // Sort by most recent first so dedup keeps the newest session when multiple - // duplicates for the same agent thread expire in the same sweep. - const sorted = expired - .map((id) => this.sessionCache.getSession(id)) - .filter((s): s is NonNullable => s != null) - .sort((a, b) => (b.activeAt - a.activeAt) || (b.updatedAt - a.updatedAt)) - for (const session of sorted) { - this.triggerDedupIfNeeded(session.id) - } + void (async () => { + for (const sessionId of expired) { + const session = this.sessionCache.getSession(sessionId) + if (!session) continue + try { + await this.overseerEvents.onSessionUpdated( + session, + this.store.sessions.getSession(sessionId)?.tag ?? null + ) + } catch (error) { + console.error('[overseer] onSessionUpdated from expireInactive failed', error) + } + } + // Sort by most recent first so dedup keeps the newest session when multiple + // duplicates for the same agent thread expire in the same sweep. + const sorted = expired + .map((id) => this.sessionCache.getSession(id)) + .filter((s): s is NonNullable => s != null) + .sort((a, b) => (b.activeAt - a.activeAt) || (b.updatedAt - a.updatedAt)) + for (const session of sorted) { + this.triggerDedupIfNeeded(session.id) + } + })() this.machineCache.expireInactive() this.overseerEvents.checkStaleSessions(this.sessionCache.getSessions()) // Piggybacked on the inactivity tick; not a logical part of expireInactive From 1bc50a71ffdafaa0eedfc79431e158011ca0cffe Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:56:39 +0000 Subject: [PATCH 07/12] fix(overseer): queue permissions; validate timeout max and base URL Permission rows join the per-session insert queue. Reject setTimeout overflow delays and non-absolute HTTP(S) fallback bases. Co-authored-by: Cursor --- .../overseerEventRecorder.llmFallback.test.ts | 39 +++++++++++++++++++ hub/src/sync/overseerEventRecorder.ts | 36 +++++++++-------- .../sync/overseerLlmFallbackConfig.test.ts | 23 +++++++++++ hub/src/sync/overseerLlmFallbackConfig.ts | 22 +++++++++-- 4 files changed, 100 insertions(+), 20 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts index 13515c8d74..e2913ec76e 100644 --- a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -562,4 +562,43 @@ describe('OverseerEventRecorder LLM fallback', () => { expect(await recorder.flushPendingLlmFallback(snapshot)).toBeNull() expect(synthesize).toHaveBeenCalledTimes(0) }) + + it('queues permission requests behind an in-flight LLM insert', async () => { + const store = new Store(':memory:') + let releaseFirst!: (value: NotifySummary) => void + const firstGate = new Promise((resolve) => { + releaseFirst = resolve + }) + let firstStarted!: () => void + const firstStartedP = new Promise((resolve) => { + firstStarted = resolve + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => { + firstStarted() + return firstGate + }), + }, + }) + const live = makeSession('sess-perm', 'cursor') + const stored = store.sessions.getOrCreateSession('llm-perm', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + + const first = recorder.onAgentMessage(snapshot, 'msg-a', agentText('TURN A body'), Date.now()) + await firstStartedP + live.agentState = { + requests: { + req1: { tool: 'Bash', arguments: { command: 'ls' } } + } + } + const perm = recorder.onSessionUpdated(live, stored.tag) + releaseFirst({ status: 'done', summary: 'llm first' }) + await Promise.all([first, perm]) + + const rows = store.events.list().sort((a, b) => a.id - b.id) + expect(rows.map((row) => row.eventType)).toEqual(['completed', 'approval_requested']) + expect(rows[0]?.summary).toBe('llm first') + }) }) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index 0277bea8ba..bde795962f 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -306,7 +306,7 @@ export class OverseerEventRecorder { async onSessionUpdated(session: Session, tag?: string | null): Promise { this.sessionThinking.set(session.id, session.thinking) - this.syncPermissionRequests(session, tag ?? null) + await this.syncPermissionRequests(session, tag ?? null) // Flush whenever thinking is clear and a deferred turn is waiting β€” // not only on a trueβ†’false edge. Keepalives often never sent the // thinking=true update through this recorder. @@ -580,7 +580,7 @@ export class OverseerEventRecorder { }) } - private syncPermissionRequests(session: Session, tag: string | null): void { + private async syncPermissionRequests(session: Session, tag: string | null): Promise { const requests = session.agentState?.requests ?? null if (!requests) { this.knownPermissionRequestIds.delete(session.id) @@ -596,21 +596,23 @@ export class OverseerEventRecorder { const request = asRecord(requests[requestId]) const toolName = typeof request?.tool === 'string' ? request.tool : 'tool' const summary = `Permission requested: ${toolName}` - this.insertSystemEvent(snapshot, { - ts: Date.now(), - sourceKind: 'system', - sourceRef: session.id, - eventType: 'approval_requested', - attentionCandidate: 1, - operatorActionRequired: 1, - summary, - relatedSessionId: session.id, - provenance: 'hub-inferred from permission prompt', - idempotencyKey: `session:${session.id}:permission:${requestId}`, - payloadFields: { requestId, request }, - severity: deriveSeverity('approval_requested'), - tags: buildTags(null, snapshot.flavor) - }) + await this.enqueueSessionWork(session.id, () => + Promise.resolve(this.insertInferredEvent(snapshot, { + ts: Date.now(), + sourceKind: 'system', + sourceRef: session.id, + eventType: 'approval_requested', + attentionCandidate: 1, + operatorActionRequired: 1, + summary, + relatedSessionId: session.id, + provenance: 'hub-inferred from permission prompt', + idempotencyKey: `session:${session.id}:permission:${requestId}`, + payloadFields: { requestId, request }, + severity: deriveSeverity('approval_requested'), + tags: buildTags(null, snapshot.flavor) + })) + ) } this.knownPermissionRequestIds.set(session.id, currentIds) diff --git a/hub/src/sync/overseerLlmFallbackConfig.test.ts b/hub/src/sync/overseerLlmFallbackConfig.test.ts index 1e99a995c6..20b273bae6 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.test.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.test.ts @@ -98,5 +98,28 @@ describe('loadOverseerLlmFallbackConfig', () => { expect(scientific.enabled).toBe(false) if (scientific.enabled) throw new Error('expected disabled') expect(scientific.reasonDisabled).toBe('invalid_timeout') + + process.env.HAPI_OVERSEER_LLM_TIMEOUT_MS = '2147483648' + const overflow = loadOverseerLlmFallbackConfig() + expect(overflow.enabled).toBe(false) + if (overflow.enabled) throw new Error('expected disabled') + expect(overflow.reasonDisabled).toBe('invalid_timeout') + }) + + it('rejects malformed fallback base URLs', () => { + stashEnv() + process.env.HAPI_OVERSEER_LLM_FALLBACK = '1' + process.env.HAPI_OVERSEER_LLM_MODEL = 'llama3.3' + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'localhost:11434/v1' + const missingScheme = loadOverseerLlmFallbackConfig() + expect(missingScheme.enabled).toBe(false) + if (missingScheme.enabled) throw new Error('expected disabled') + expect(missingScheme.reasonDisabled).toBe('invalid_base_url') + + process.env.HAPI_OVERSEER_LLM_BASE_URL = '/' + const slash = loadOverseerLlmFallbackConfig() + expect(slash.enabled).toBe(false) + if (slash.enabled) throw new Error('expected disabled') + expect(slash.reasonDisabled).toBe('invalid_base_url') }) }) diff --git a/hub/src/sync/overseerLlmFallbackConfig.ts b/hub/src/sync/overseerLlmFallbackConfig.ts index 871f5e057b..73144b83e2 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.ts @@ -25,7 +25,7 @@ export type OverseerLlmFallbackEnabledConfig = { export type OverseerLlmFallbackDisabledConfig = { enabled: false - reasonDisabled: 'flag_off' | 'incomplete_config' | 'invalid_api' | 'invalid_timeout' + reasonDisabled: 'flag_off' | 'incomplete_config' | 'invalid_api' | 'invalid_timeout' | 'invalid_base_url' } export type OverseerLlmFallbackConfig = @@ -33,6 +33,8 @@ export type OverseerLlmFallbackConfig = | OverseerLlmFallbackDisabledConfig const DEFAULT_TIMEOUT_MS = 30_000 +/** Node/Bun setTimeout clamps delays above 2^31-1 ms to 1 ms. */ +const MAX_TIMEOUT_MS = 2_147_483_647 function envTruthy(value: string | undefined): boolean { if (!value) return false @@ -44,6 +46,15 @@ function normalizeBaseUrl(raw: string): string { return raw.trim().replace(/\/+$/, '') } +function isAbsoluteHttpUrl(value: string): boolean { + try { + const parsed = new URL(value) + return parsed.protocol === 'http:' || parsed.protocol === 'https:' + } catch { + return false + } +} + function parseApiMode(raw: string | undefined): OverseerLlmApiMode | null { if (!raw || raw.trim() === '') return 'chat-completions' const normalized = raw.trim().toLowerCase() @@ -81,15 +92,20 @@ export function loadOverseerLlmFallbackConfig( return { enabled: false, reasonDisabled: 'invalid_timeout' } } const parsed = Number.parseInt(timeoutRaw, 10) - if (!Number.isFinite(parsed) || parsed <= 0) { + if (!Number.isFinite(parsed) || parsed <= 0 || parsed > MAX_TIMEOUT_MS) { return { enabled: false, reasonDisabled: 'invalid_timeout' } } timeoutMs = parsed } + const baseUrl = normalizeBaseUrl(baseUrlRaw) + if (!isAbsoluteHttpUrl(baseUrl)) { + return { enabled: false, reasonDisabled: 'invalid_base_url' } + } + return { enabled: true, - baseUrl: normalizeBaseUrl(baseUrlRaw), + baseUrl, apiKey: env.HAPI_OVERSEER_LLM_API_KEY?.trim() ?? '', model, api, From cb030a52c6d86775bf96d33ab5aede000a446384 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:06:39 +0000 Subject: [PATCH 08/12] fix(overseer): key LLM success by turn epoch; stamp permission ts Same-turn ACP tool/usage no longer duplicates completed_fallback. Permission rows keep the observe-time timestamp while queued behind LLM. Co-authored-by: Cursor --- .../overseerEventRecorder.llmFallback.test.ts | 97 +++++++++++++++++-- hub/src/sync/overseerEventRecorder.ts | 54 +++++++---- 2 files changed, 124 insertions(+), 27 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts index e2913ec76e..174e8f07c1 100644 --- a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -449,7 +449,43 @@ describe('OverseerEventRecorder LLM fallback', () => { expect(rows.map((row) => row.summary)).toEqual(['llm first', 'real notify']) }) - it('writes completed_fallback after a textless tool turn following LLM success', async () => { + it('does not write completed_fallback for same-turn ACP tool after LLM success', async () => { + const store = new Store(':memory:') + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => ({ + status: 'done', + summary: 'caught this turn', + })), + }, + }) + const live = makeSession('sess-same', 'cursor') + const stored = store.sessions.getOrCreateSession('llm-same', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + + await recorder.onAgentMessage(snapshot, 'msg-text', agentText('turn body'), Date.now()) + await recorder.onAgentMessage(snapshot, 'msg-tool', { + role: 'agent', + content: { + type: 'codex', + data: { type: 'tool-call-result', output: { exit_code: 0 } }, + }, + }, Date.now() + 1) + + const event = await recorder.onSessionEnd( + live, + stored.tag, + Date.now() + 2, + 'completed', + () => 'turn body' + ) + expect(event).toBeNull() + expect(store.events.list().filter((row) => row.provenance?.includes('hub-llm-fallback'))).toHaveLength(1) + expect(store.events.list().some((row) => row.provenance?.includes('session-end'))).toBe(false) + }) + + it('writes completed_fallback for a later tool-only turn after a user message', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox, { llmFallback: { @@ -465,25 +501,29 @@ describe('OverseerEventRecorder LLM fallback', () => { const snapshot = toSessionSnapshot(live, stored.tag) await recorder.onAgentMessage(snapshot, 'msg-text', agentText('earlier turn body'), Date.now()) + await recorder.onAgentMessage(snapshot, 'msg-user', { + role: 'user', + content: { type: 'text', text: 'do the next thing' }, + }, Date.now() + 1) await recorder.onAgentMessage(snapshot, 'msg-tool', { role: 'agent', content: { type: 'codex', data: { type: 'tool-call-result', output: { exit_code: 0 } }, }, - }, Date.now() + 1) + }, Date.now() + 2) const event = await recorder.onSessionEnd( live, stored.tag, - Date.now() + 2, + Date.now() + 3, 'completed', () => 'earlier turn body' ) expect(event?.provenance).toContain('session-end') }) - it('does not let an in-flight earlier LLM cover a later tool-only turn', async () => { + it('does not let an in-flight earlier LLM cover a later user+tool turn', async () => { const store = new Store(':memory:') let releaseFirst!: (value: NotifySummary) => void const firstGate = new Promise((resolve) => { @@ -508,20 +548,24 @@ describe('OverseerEventRecorder LLM fallback', () => { const first = recorder.onAgentMessage(snapshot, 'msg-a', agentText('earlier turn'), Date.now()) await firstStartedP + await recorder.onAgentMessage(snapshot, 'msg-user', { + role: 'user', + content: { type: 'text', text: 'continue' }, + }, Date.now() + 1) await recorder.onAgentMessage(snapshot, 'msg-tool', { role: 'agent', content: { type: 'codex', data: { type: 'tool-call-result', output: { exit_code: 0 } }, }, - }, Date.now() + 1) + }, Date.now() + 2) releaseFirst({ status: 'done', summary: 'caught earlier' }) await first const event = await recorder.onSessionEnd( live, stored.tag, - Date.now() + 2, + Date.now() + 3, 'completed', () => 'earlier turn' ) @@ -601,4 +645,45 @@ describe('OverseerEventRecorder LLM fallback', () => { expect(rows.map((row) => row.eventType)).toEqual(['completed', 'approval_requested']) expect(rows[0]?.summary).toBe('llm first') }) + + it('stamps permission requests at observe time, not queue-drain time', async () => { + const store = new Store(':memory:') + let releaseFirst!: (value: NotifySummary) => void + const firstGate = new Promise((resolve) => { + releaseFirst = resolve + }) + let firstStarted!: () => void + const firstStartedP = new Promise((resolve) => { + firstStarted = resolve + }) + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => { + firstStarted() + return firstGate + }), + }, + }) + const live = makeSession('sess-perm-ts', 'cursor') + const stored = store.sessions.getOrCreateSession('llm-perm-ts', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + + const first = recorder.onAgentMessage(snapshot, 'msg-a', agentText('TURN A body'), Date.now()) + await firstStartedP + live.agentState = { + requests: { + req1: { tool: 'Bash', arguments: { command: 'ls' } } + } + } + const marked = Date.now() + const perm = recorder.onSessionUpdated(live, stored.tag) + await Bun.sleep(40) + releaseFirst({ status: 'done', summary: 'llm first' }) + await Promise.all([first, perm]) + + const row = store.events.list({ eventType: 'approval_requested' })[0] + expect(row).toBeDefined() + expect(row!.ts).toBeLessThan(marked + 25) + }) }) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index bde795962f..9410855f3e 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -48,6 +48,7 @@ type PendingLlmFallback = { messageId: string plainText: string ts: number + epoch: number } function asRecord(value: unknown): Record | null { @@ -174,9 +175,9 @@ export class OverseerEventRecorder { private readonly sessionThinking = new Map() /** Per-session tail so concurrent LLM calls insert in arrival order. */ private readonly sessionWork = new Map>() - /** Successful LLM fallback messageId per session (not a session-wide boolean). */ - private readonly llmFallbackSucceeded = new Map() - private readonly lastAgentMessageId = new Map() + private readonly turnEpoch = new Map() + /** Epoch of the last successful LLM fallback for this session. */ + private readonly llmFallbackSucceededEpoch = new Map() constructor( private readonly events: EventStore, @@ -206,7 +207,6 @@ export class OverseerEventRecorder { if (isAgentMessageContent(content)) { this.lastAgentMessageAt.set(session.id, ts) - this.lastAgentMessageId.set(session.id, messageId) const agentBody = unwrapRoleWrappedRecordEnvelope(content) const agentContent = agentBody?.role === 'agent' ? agentBody.content : content @@ -292,20 +292,27 @@ export class OverseerEventRecorder { this.rememberPendingLlmFallback(session.id, messageId, plainText, ts) } else { this.pendingLlmFallback.delete(session.id) + this.bumpTurnEpoch(session.id) + const epoch = this.currentTurnEpoch(session.id) primary = await this.enqueueSessionWork(session.id, () => - this.tryLlmFallback(session, messageId, plainText, ts) + this.tryLlmFallback(session, messageId, plainText, ts, epoch) ) } } return primary } + this.bumpTurnEpoch(session.id) this.scoopLinksFromContent(session, messageId, content, ts) return primary } async onSessionUpdated(session: Session, tag?: string | null): Promise { + const wasThinking = this.sessionThinking.get(session.id) === true this.sessionThinking.set(session.id, session.thinking) + if (session.thinking && !wasThinking) { + this.bumpTurnEpoch(session.id) + } await this.syncPermissionRequests(session, tag ?? null) // Flush whenever thinking is clear and a deferred turn is waiting β€” // not only on a trueβ†’false edge. Keepalives often never sent the @@ -339,11 +346,9 @@ export class OverseerEventRecorder { if (lastText && extractNotifySummary(lastText)) { return llmEvent } - // Successful LLM row already captured this missed turn β€” do not - // also write "session ended without AGENT_NOTIFY_SUMMARY". - const successForLatest = this.lastAgentMessageId.get(session.id) - && this.llmFallbackSucceeded.get(session.id) === this.lastAgentMessageId.get(session.id) - if (llmEvent || successForLatest) { + // Successful LLM row already captured this turn (epoch), including + // same-turn ACP tool/usage messages after the text flush. + if (llmEvent || this.llmFallbackSucceededEpoch.get(session.id) === this.currentTurnEpoch(session.id)) { return llmEvent } @@ -390,13 +395,11 @@ export class OverseerEventRecorder { const combined = prev && prev.messageId !== messageId ? `${prev.plainText}\n${plainText}` : plainText - if (!prev || prev.messageId !== messageId) { - this.llmFallbackSucceeded.delete(sessionId) - } this.pendingLlmFallback.set(sessionId, { messageId, plainText: combined, - ts: prev?.ts ?? ts + ts: prev?.ts ?? ts, + epoch: prev?.epoch ?? this.currentTurnEpoch(sessionId) }) } @@ -405,7 +408,7 @@ export class OverseerEventRecorder { pending: PendingLlmFallback ): Promise { if (extractNotifySummary(pending.plainText)) return null - return this.tryLlmFallback(session, pending.messageId, pending.plainText, pending.ts) + return this.tryLlmFallback(session, pending.messageId, pending.plainText, pending.ts, pending.epoch) } private enqueueSessionWork(sessionId: string, work: () => Promise): Promise { @@ -429,10 +432,10 @@ export class OverseerEventRecorder { session: SessionSnapshot, messageId: string, plainText: string, - ts: number + ts: number, + epoch: number ): Promise { if (!this.llmFallback) return null - this.llmFallbackSucceeded.delete(session.id) try { const notify = await this.llmFallback.synthesizeNotifySummary(plainText) if (!notify) return null @@ -463,7 +466,7 @@ export class OverseerEventRecorder { tags: buildTags(notify, session.flavor), }) if (stored) { - this.llmFallbackSucceeded.set(session.id, messageId) + this.llmFallbackSucceededEpoch.set(session.id, epoch) this.onAsyncSystemEvent?.(session.id) } return stored @@ -495,13 +498,21 @@ export class OverseerEventRecorder { forgetSession(sessionId: string): void { this.pendingLlmFallback.delete(sessionId) this.sessionWork.delete(sessionId) - this.llmFallbackSucceeded.delete(sessionId) + this.llmFallbackSucceededEpoch.delete(sessionId) + this.turnEpoch.delete(sessionId) this.lastAgentMessageAt.delete(sessionId) - this.lastAgentMessageId.delete(sessionId) this.knownPermissionRequestIds.delete(sessionId) this.sessionThinking.delete(sessionId) } + private currentTurnEpoch(sessionId: string): number { + return this.turnEpoch.get(sessionId) ?? 0 + } + + private bumpTurnEpoch(sessionId: string): void { + this.turnEpoch.set(sessionId, this.currentTurnEpoch(sessionId) + 1) + } + private scoopLinksFromContent( session: SessionSnapshot, messageId: string, @@ -596,9 +607,10 @@ export class OverseerEventRecorder { const request = asRecord(requests[requestId]) const toolName = typeof request?.tool === 'string' ? request.tool : 'tool' const summary = `Permission requested: ${toolName}` + const ts = Date.now() await this.enqueueSessionWork(session.id, () => Promise.resolve(this.insertInferredEvent(snapshot, { - ts: Date.now(), + ts, sourceKind: 'system', sourceRef: session.id, eventType: 'approval_requested', From f7af139f9e7e0f701034323fff5dc2f2ca573cb4 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:15:25 +0000 Subject: [PATCH 09/12] fix(overseer): coalesce permissions; clear epochs; last-segment ts Reserve permission IDs before queueing, drop turn maps on session-end, stamp accumulated ACP text at the final segment, and reject base URLs with query/hash. User-message redelivery does not bump turn epoch. Co-authored-by: Cursor --- .../overseerEventRecorder.llmFallback.test.ts | 37 +++++++- hub/src/sync/overseerEventRecorder.ts | 87 +++++++++++-------- .../sync/overseerLlmFallbackConfig.test.ts | 12 +++ hub/src/sync/overseerLlmFallbackConfig.ts | 4 +- 4 files changed, 102 insertions(+), 38 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts index 174e8f07c1..bbf9a5bb44 100644 --- a/hub/src/sync/overseerEventRecorder.llmFallback.test.ts +++ b/hub/src/sync/overseerEventRecorder.llmFallback.test.ts @@ -293,12 +293,15 @@ describe('OverseerEventRecorder LLM fallback', () => { const stored = store.sessions.getOrCreateSession('llm-acc', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') const snapshot = toSessionSnapshot(makeSession(stored.id, 'cursor'), stored.tag) - await recorder.onAgentMessage(snapshot, 'msg-1', agentText('First chunk'), Date.now(), { thinking: true }) - await recorder.onAgentMessage(snapshot, 'msg-2', agentText('Second chunk'), Date.now() + 1, { thinking: true }) + const firstTs = Date.now() + const lastTs = firstTs + 120_000 + await recorder.onAgentMessage(snapshot, 'msg-1', agentText('First chunk'), firstTs, { thinking: true }) + await recorder.onAgentMessage(snapshot, 'msg-2', agentText('Second chunk'), lastTs, { thinking: true }) const flushed = await recorder.flushPendingLlmFallback(snapshot) expect(synthesize).toHaveBeenCalledTimes(1) expect(flushed?.summary).toBe('both chunks') + expect(flushed?.ts).toBe(lastTs) }) it('writes completed_fallback when a later missed turn LLM fails', async () => { @@ -686,4 +689,34 @@ describe('OverseerEventRecorder LLM fallback', () => { expect(row).toBeDefined() expect(row!.ts).toBeLessThan(marked + 25) }) + + it('does not bump turn epoch on redelivered user messages', async () => { + const store = new Store(':memory:') + const recorder = new OverseerEventRecorder(store.events, store.inbox, { + llmFallback: { + synthesizeNotifySummary: mock(async () => ({ status: 'done', summary: 'turn' })), + }, + }) + const live = makeSession('sess-redeliver', 'cursor', { thinking: true }) + const stored = store.sessions.getOrCreateSession('llm-redeliver', { flavor: 'cursor', path: '/tmp', host: 'local' }, null, 'default') + live.id = stored.id + const snapshot = toSessionSnapshot(live, stored.tag) + const user = { role: 'user', content: { type: 'text', text: 'go' } } + + await recorder.onAgentMessage(snapshot, 'user-1', user, Date.now()) + await recorder.onAgentMessage(snapshot, 'msg-a', agentText('working'), Date.now() + 1, { thinking: true }) + await recorder.flushPendingLlmFallback(snapshot) + await recorder.onAgentMessage(snapshot, 'user-1', user, Date.now() + 2) + + const event = await recorder.onSessionEnd( + live, + stored.tag, + Date.now() + 3, + 'completed', + () => 'working' + ) + expect(event).toBeNull() + expect(store.events.list().filter((row) => row.provenance?.includes('hub-llm-fallback'))).toHaveLength(1) + expect(store.events.list().some((row) => row.provenance?.includes('session-end'))).toBe(false) + }) }) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index 9410855f3e..bc339cba11 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -178,6 +178,7 @@ export class OverseerEventRecorder { private readonly turnEpoch = new Map() /** Epoch of the last successful LLM fallback for this session. */ private readonly llmFallbackSucceededEpoch = new Map() + private readonly seenUserMessageIds = new Map>() constructor( private readonly events: EventStore, @@ -302,7 +303,12 @@ export class OverseerEventRecorder { return primary } - this.bumpTurnEpoch(session.id) + const seen = this.seenUserMessageIds.get(session.id) ?? new Set() + if (!seen.has(messageId)) { + seen.add(messageId) + this.seenUserMessageIds.set(session.id, seen) + this.bumpTurnEpoch(session.id) + } this.scoopLinksFromContent(session, messageId, content, ts) return primary } @@ -335,39 +341,43 @@ export class OverseerEventRecorder { const pending = this.takePendingLlmFallback(session.id) return this.enqueueSessionWork(session.id, async () => { - const llmEvent = pending - ? await this.runPendingLlmFallback(snapshot, pending) - : null - if (reason !== 'completed') { - return llmEvent - } + try { + const llmEvent = pending + ? await this.runPendingLlmFallback(snapshot, pending) + : null + if (reason !== 'completed') { + return llmEvent + } - const lastText = getLastAgentPlainText() - if (lastText && extractNotifySummary(lastText)) { - return llmEvent - } - // Successful LLM row already captured this turn (epoch), including - // same-turn ACP tool/usage messages after the text flush. - if (llmEvent || this.llmFallbackSucceededEpoch.get(session.id) === this.currentTurnEpoch(session.id)) { - return llmEvent - } + const lastText = getLastAgentPlainText() + if (lastText && extractNotifySummary(lastText)) { + return llmEvent + } + // Successful LLM row already captured this turn (epoch), including + // same-turn ACP tool/usage messages after the text flush. + if (llmEvent || this.llmFallbackSucceededEpoch.get(session.id) === this.currentTurnEpoch(session.id)) { + return llmEvent + } - const stored = this.insertSystemEvent(snapshot, { - ts, - sourceKind: 'system', - sourceRef: session.id, - eventType: 'completed', - attentionCandidate: 0, - summary: 'Session ended without AGENT_NOTIFY_SUMMARY; hub inferred completion', - relatedSessionId: session.id, - provenance: 'hub-inferred from session-end completed signal', - idempotencyKey: `session:${session.id}:session_end:${ts}:completed_fallback`, - payloadFields: { reason }, - severity: deriveSeverity('completed'), - tags: buildTags(null, snapshot.flavor) - }) - if (stored) this.onAsyncSystemEvent?.(session.id) - return stored + const stored = this.insertSystemEvent(snapshot, { + ts, + sourceKind: 'system', + sourceRef: session.id, + eventType: 'completed', + attentionCandidate: 0, + summary: 'Session ended without AGENT_NOTIFY_SUMMARY; hub inferred completion', + relatedSessionId: session.id, + provenance: 'hub-inferred from session-end completed signal', + idempotencyKey: `session:${session.id}:session_end:${ts}:completed_fallback`, + payloadFields: { reason }, + severity: deriveSeverity('completed'), + tags: buildTags(null, snapshot.flavor) + }) + if (stored) this.onAsyncSystemEvent?.(session.id) + return stored + } finally { + this.clearTurnState(session.id) + } }) } @@ -398,7 +408,7 @@ export class OverseerEventRecorder { this.pendingLlmFallback.set(sessionId, { messageId, plainText: combined, - ts: prev?.ts ?? ts, + ts, epoch: prev?.epoch ?? this.currentTurnEpoch(sessionId) }) } @@ -498,13 +508,18 @@ export class OverseerEventRecorder { forgetSession(sessionId: string): void { this.pendingLlmFallback.delete(sessionId) this.sessionWork.delete(sessionId) - this.llmFallbackSucceededEpoch.delete(sessionId) - this.turnEpoch.delete(sessionId) + this.clearTurnState(sessionId) this.lastAgentMessageAt.delete(sessionId) this.knownPermissionRequestIds.delete(sessionId) this.sessionThinking.delete(sessionId) } + private clearTurnState(sessionId: string): void { + this.turnEpoch.delete(sessionId) + this.llmFallbackSucceededEpoch.delete(sessionId) + this.seenUserMessageIds.delete(sessionId) + } + private currentTurnEpoch(sessionId: string): number { return this.turnEpoch.get(sessionId) ?? 0 } @@ -604,6 +619,8 @@ export class OverseerEventRecorder { for (const requestId of currentIds) { if (known.has(requestId)) continue + known.add(requestId) + this.knownPermissionRequestIds.set(session.id, new Set(known)) const request = asRecord(requests[requestId]) const toolName = typeof request?.tool === 'string' ? request.tool : 'tool' const summary = `Permission requested: ${toolName}` diff --git a/hub/src/sync/overseerLlmFallbackConfig.test.ts b/hub/src/sync/overseerLlmFallbackConfig.test.ts index 20b273bae6..fdab8d11ca 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.test.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.test.ts @@ -121,5 +121,17 @@ describe('loadOverseerLlmFallbackConfig', () => { expect(slash.enabled).toBe(false) if (slash.enabled) throw new Error('expected disabled') expect(slash.reasonDisabled).toBe('invalid_base_url') + + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'https://host/v1?tenant=x' + const query = loadOverseerLlmFallbackConfig() + expect(query.enabled).toBe(false) + if (query.enabled) throw new Error('expected disabled') + expect(query.reasonDisabled).toBe('invalid_base_url') + + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'https://host/v1#frag' + const hash = loadOverseerLlmFallbackConfig() + expect(hash.enabled).toBe(false) + if (hash.enabled) throw new Error('expected disabled') + expect(hash.reasonDisabled).toBe('invalid_base_url') }) }) diff --git a/hub/src/sync/overseerLlmFallbackConfig.ts b/hub/src/sync/overseerLlmFallbackConfig.ts index 73144b83e2..51c334f326 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.ts @@ -49,7 +49,9 @@ function normalizeBaseUrl(raw: string): string { function isAbsoluteHttpUrl(value: string): boolean { try { const parsed = new URL(value) - return parsed.protocol === 'http:' || parsed.protocol === 'https:' + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return false + if (parsed.search !== '' || parsed.hash !== '') return false + return true } catch { return false } From 62af0596f9bb9141adad20ac34ac76f85991af39 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:24:38 +0000 Subject: [PATCH 10/12] fix(overseer): do not restore permissions after session end Mark sessions ended before queued permission work resumes, and forget recorder state only after deleteSession succeeds. Co-authored-by: Cursor --- hub/src/sync/overseerEventRecorder.ts | 8 +++++++- hub/src/sync/syncEngine.ts | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.ts b/hub/src/sync/overseerEventRecorder.ts index bc339cba11..07e0e97ee5 100644 --- a/hub/src/sync/overseerEventRecorder.ts +++ b/hub/src/sync/overseerEventRecorder.ts @@ -179,6 +179,7 @@ export class OverseerEventRecorder { /** Epoch of the last successful LLM fallback for this session. */ private readonly llmFallbackSucceededEpoch = new Map() private readonly seenUserMessageIds = new Map>() + private readonly sessionEnded = new Set() constructor( private readonly events: EventStore, @@ -335,6 +336,7 @@ export class OverseerEventRecorder { reason: string | undefined, getLastAgentPlainText: () => string | null ): Promise { + this.sessionEnded.add(session.id) this.knownPermissionRequestIds.delete(session.id) this.sessionThinking.delete(session.id) const snapshot = toSessionSnapshot(session, tag) @@ -512,6 +514,7 @@ export class OverseerEventRecorder { this.lastAgentMessageAt.delete(sessionId) this.knownPermissionRequestIds.delete(sessionId) this.sessionThinking.delete(sessionId) + this.sessionEnded.delete(sessionId) } private clearTurnState(sessionId: string): void { @@ -607,6 +610,7 @@ export class OverseerEventRecorder { } private async syncPermissionRequests(session: Session, tag: string | null): Promise { + if (this.sessionEnded.has(session.id)) return const requests = session.agentState?.requests ?? null if (!requests) { this.knownPermissionRequestIds.delete(session.id) @@ -644,7 +648,9 @@ export class OverseerEventRecorder { ) } - this.knownPermissionRequestIds.set(session.id, currentIds) + if (!this.sessionEnded.has(session.id)) { + this.knownPermissionRequestIds.set(session.id, currentIds) + } } private insertInferredEvent( diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index c8d286b2a8..35fc91b4b5 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -798,8 +798,8 @@ export class SyncEngine { } async deleteSession(sessionId: string): Promise { - this.overseerEvents.forgetSession(sessionId) await this.sessionCache.deleteSession(sessionId) + this.overseerEvents.forgetSession(sessionId) } async applySessionConfig( From 83ab6138c17f4e672e9e2f8572f2db3955f396f4 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:29:59 +0000 Subject: [PATCH 11/12] fix(overseer): await parent sentinel test; reject LLM URL userinfo Parent flavors landed a sync onAgentMessage assertion that broke hub typecheck on the PR merge commit. Also refuse credential-bearing fallback base URLs and redact userinfo if it ever reaches the startup log. Co-authored-by: Cursor --- hub/src/sync/overseerEventRecorder.test.ts | 4 ++-- hub/src/sync/overseerLlmFallbackConfig.test.ts | 16 +++++++++++++++- hub/src/sync/overseerLlmFallbackConfig.ts | 14 ++++++++++++++ hub/src/sync/syncEngine.ts | 4 ++-- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/hub/src/sync/overseerEventRecorder.test.ts b/hub/src/sync/overseerEventRecorder.test.ts index 045f56da55..c71fe66064 100644 --- a/hub/src/sync/overseerEventRecorder.test.ts +++ b/hub/src/sync/overseerEventRecorder.test.ts @@ -98,12 +98,12 @@ describe('OverseerEventRecorder', () => { expect(event?.attentionCandidate).toBe(0) }) - it('drops sentinel notify actions from suggested_action and inbox', () => { + it('drops sentinel notify actions from suggested_action and inbox', async () => { const store = new Store(':memory:') const recorder = new OverseerEventRecorder(store.events, store.inbox) const session = store.sessions.getOrCreateSession('test-sentinel', { flavor: 'claude', path: '/tmp', host: 'local' }, null, 'default') - const event = recorder.onAgentMessage( + const event = await recorder.onAgentMessage( toSessionSnapshot(makeSession(session.id, 'claude'), session.tag), 'msg-sentinel', { diff --git a/hub/src/sync/overseerLlmFallbackConfig.test.ts b/hub/src/sync/overseerLlmFallbackConfig.test.ts index fdab8d11ca..946e376832 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.test.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from 'bun:test' -import { loadOverseerLlmFallbackConfig } from './overseerLlmFallbackConfig' +import { loadOverseerLlmFallbackConfig, redactOverseerLlmBaseUrlForLog } from './overseerLlmFallbackConfig' const ENV_KEYS = [ 'HAPI_OVERSEER_LLM_FALLBACK', @@ -133,5 +133,19 @@ describe('loadOverseerLlmFallbackConfig', () => { expect(hash.enabled).toBe(false) if (hash.enabled) throw new Error('expected disabled') expect(hash.reasonDisabled).toBe('invalid_base_url') + + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'https://user:secret@gateway/v1' + const userinfo = loadOverseerLlmFallbackConfig() + expect(userinfo.enabled).toBe(false) + if (userinfo.enabled) throw new Error('expected disabled') + expect(userinfo.reasonDisabled).toBe('invalid_base_url') + }) + + it('redacts URL userinfo for startup logs', () => { + expect(redactOverseerLlmBaseUrlForLog('https://user:secret@gateway/v1')).toBe( + 'https://REDACTED:REDACTED@gateway/v1' + ) + expect(redactOverseerLlmBaseUrlForLog('https://gateway/v1')).toBe('https://gateway/v1') + expect(redactOverseerLlmBaseUrlForLog('not a url')).toBe('[invalid-url]') }) }) diff --git a/hub/src/sync/overseerLlmFallbackConfig.ts b/hub/src/sync/overseerLlmFallbackConfig.ts index 51c334f326..5f49cb99b0 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.ts @@ -46,11 +46,25 @@ function normalizeBaseUrl(raw: string): string { return raw.trim().replace(/\/+$/, '') } +export function redactOverseerLlmBaseUrlForLog(url: string): string { + try { + const parsed = new URL(url) + if (parsed.username !== '' || parsed.password !== '') { + parsed.username = parsed.username !== '' ? 'REDACTED' : '' + parsed.password = parsed.password !== '' ? 'REDACTED' : '' + } + return parsed.toString().replace(/\/$/, '') + } catch { + return '[invalid-url]' + } +} + function isAbsoluteHttpUrl(value: string): boolean { try { const parsed = new URL(value) if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return false if (parsed.search !== '' || parsed.hash !== '') return false + if (parsed.username !== '' || parsed.password !== '') return false return true } catch { return false diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 35fc91b4b5..646f4bd180 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -44,7 +44,7 @@ import { import { SessionCache } from './sessionCache' import { OverseerEventRecorder, toSessionSnapshot } from './overseerEventRecorder' import { createOverseerLlmFallbackClient } from './overseerLlmFallback' -import { loadOverseerLlmFallbackConfig } from './overseerLlmFallbackConfig' +import { loadOverseerLlmFallbackConfig, redactOverseerLlmBaseUrlForLog } from './overseerLlmFallbackConfig' import { OverseerEntity } from './overseerEntity' import { extractAssistantPlainText } from '@hapi/protocol/messages' import type { InboxOperatorAction } from '@hapi/protocol' @@ -173,7 +173,7 @@ export class SyncEngine { : null if (llmFallbackConfig.enabled) { console.log( - `[overseer] LLM summary fallback ENABLED (api=${llmFallbackConfig.api}, model=${llmFallbackConfig.model}, base=${llmFallbackConfig.baseUrl})` + `[overseer] LLM summary fallback ENABLED (api=${llmFallbackConfig.api}, model=${llmFallbackConfig.model}, base=${redactOverseerLlmBaseUrlForLog(llmFallbackConfig.baseUrl)})` ) } this.overseerEvents = new OverseerEventRecorder(store.events, store.inbox, { From 5c78aad9a5aa305eea0d0b2ff9cead995f4f7596 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:44:05 +0000 Subject: [PATCH 12/12] fix(overseer): reject empty URL delimiters; warn on bad fallback config Bare trailing ?/# pass URL() with empty search/hash, then joinUrl glues the chat path into query or fragment. Log reasonDisabled when the opt-in flag is on but config is invalid so operators see the typo. Co-authored-by: Cursor --- hub/src/sync/overseerLlmFallbackConfig.test.ts | 12 ++++++++++++ hub/src/sync/overseerLlmFallbackConfig.ts | 3 ++- hub/src/sync/syncEngine.ts | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hub/src/sync/overseerLlmFallbackConfig.test.ts b/hub/src/sync/overseerLlmFallbackConfig.test.ts index 946e376832..5b9b846915 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.test.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.test.ts @@ -139,6 +139,18 @@ describe('loadOverseerLlmFallbackConfig', () => { expect(userinfo.enabled).toBe(false) if (userinfo.enabled) throw new Error('expected disabled') expect(userinfo.reasonDisabled).toBe('invalid_base_url') + + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'https://host/v1?' + const emptyQuery = loadOverseerLlmFallbackConfig() + expect(emptyQuery.enabled).toBe(false) + if (emptyQuery.enabled) throw new Error('expected disabled') + expect(emptyQuery.reasonDisabled).toBe('invalid_base_url') + + process.env.HAPI_OVERSEER_LLM_BASE_URL = 'https://host/v1#' + const emptyHash = loadOverseerLlmFallbackConfig() + expect(emptyHash.enabled).toBe(false) + if (emptyHash.enabled) throw new Error('expected disabled') + expect(emptyHash.reasonDisabled).toBe('invalid_base_url') }) it('redacts URL userinfo for startup logs', () => { diff --git a/hub/src/sync/overseerLlmFallbackConfig.ts b/hub/src/sync/overseerLlmFallbackConfig.ts index 5f49cb99b0..dd11f32fdf 100644 --- a/hub/src/sync/overseerLlmFallbackConfig.ts +++ b/hub/src/sync/overseerLlmFallbackConfig.ts @@ -63,7 +63,8 @@ function isAbsoluteHttpUrl(value: string): boolean { try { const parsed = new URL(value) if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return false - if (parsed.search !== '' || parsed.hash !== '') return false + // URL() strips empty `?` / `#`; joinUrl would then glue the path into query/hash. + if (value.includes('?') || value.includes('#')) return false if (parsed.username !== '' || parsed.password !== '') return false return true } catch { diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index 646f4bd180..b9996e3520 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -175,6 +175,8 @@ export class SyncEngine { console.log( `[overseer] LLM summary fallback ENABLED (api=${llmFallbackConfig.api}, model=${llmFallbackConfig.model}, base=${redactOverseerLlmBaseUrlForLog(llmFallbackConfig.baseUrl)})` ) + } else if (llmFallbackConfig.reasonDisabled !== 'flag_off') { + console.warn(`[overseer] LLM summary fallback disabled (${llmFallbackConfig.reasonDisabled})`) } this.overseerEvents = new OverseerEventRecorder(store.events, store.inbox, { llmFallback,