From 49900f43e427849226ff05c29bbd925a342f0980 Mon Sep 17 00:00:00 2001 From: "jcool (Hermes agent)" Date: Thu, 24 Sep 2026 15:12:57 +0000 Subject: [PATCH] feat(voice): add respondToGather to collapse the conversational-turn round trip Every turn of a live agent-driven conversation is a race against Bandwidth's answer/gather callback retry window: the callback lands, and the agent has a bounded window to queue a BXML response via respondToCallback before the call gives up and hangs up. The current pattern for a conversational turn is two separate tool calls -- generateBXML, then respondToCallback -- with a full model reasoning step in between. Each of those is wall-clock time that narrows the window before the race is lost. Ran an internal PoC (Claude Code + this server, live Bandwidth Build account) attempting exactly this pattern for a scripted greeting (not even a full conversation, just turn zero). 8 of 9 live call attempts connected and hung up with no audio delivered -- the agent's tool-call round trip lost the race against the callback retry window every time but one. respondToGather(call_id, verbs, auto_gather) merges generateBXML + respondToCallback into a single tool call: generate the BXML from verb descriptions and queue it as the callback response in one round trip. Same first-write-wins semantics as respondToCallback. Does not by itself guarantee winning the race -- it removes one full model-reasoning gap from the loop, which is the highest-leverage, lowest-risk fix available without changing the callback timing model itself (a separate, bigger conversation -- see the accompanying spike proposal for making the server's callback loop respond fast enough for genuine multi-turn conversation without depending on the outer agent's per-turn latency at all). - src/tools/voice.py: new respond_to_gather_flow() (mirrors the generate_bxml_flow / respond_to_callback_flow pattern already in this file) plus the respondToGather tool registration. - src/profiles.py: added to the voice profile. - src/instructions.py: voice section now tells agents to prefer respondToGather for every turn after the first. - src/specs/AGENTS.md, README.md: tool inventory updated. - test/test_bxml.py: two new tests -- generates+queues in one call, and first-write-wins semantics match respondToCallback. Full suite: 119 passed (117 existing + 2 new), no regressions. --- README.md | 1 + src/instructions.py | 4 ++-- src/profiles.py | 1 + src/specs/AGENTS.md | 1 + src/tools/voice.py | 49 ++++++++++++++++++++++++++++++++++++++ test/test_bxml.py | 58 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6914d7e..a34cd53 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,7 @@ Kicks off a new Bandwidth Build account. Only one tool is exposed — SMS verifi - `updateCall` / `updateCallBxml` — redirect, hang up, or replace BXML - `generateBXML` — build valid BXML from a verb list - `respondToCallback` — queue a BXML response for an active callback (first-write-wins) +- `respondToGather` — generate BXML from verbs and queue it as the callback response in one call; prefer this over `generateBXML` + `respondToCallback` for conversational turns, since fewer round trips means less chance of losing the race against Bandwidth's callback retry window - `getCallbackEvents` — read recent voice / messaging callback events - `configureCallbacks` — point an application's webhook URLs at this server diff --git a/src/instructions.py b/src/instructions.py index 69abf40..3f5b230 100644 --- a/src/instructions.py +++ b/src/instructions.py @@ -62,7 +62,7 @@ - Conversation: `generateBXML(verbs=[{"type": "SpeakSentence", "text": "How can I help?", "voice": "julie"}])` (auto_gather=True is default, enables barge-in) 5. **Create the call**: `createCall(accountId, from, to, applicationId, answerUrl)` where answerUrl = BW_MCP_BASE_URL + `/callbacks/voice/answer`. 6. **Queue BXML immediately**: `respondToCallback(call_id, bxml)` — do this right after createCall. The BXML is delivered when the callee picks up. -7. **For conversations**: Poll `getCallbackEvents(event_type="voice.gather")` for caller speech, generate new BXML, deliver with `respondToCallback`. +7. **For conversations**: Poll `getCallbackEvents(event_type="voice.gather")` for caller speech, then respond in one call with `respondToGather(call_id, verbs)` — it generates the BXML and queues it in a single tool call, instead of a separate generateBXML + respondToCallback round trip. Every conversational turn races Bandwidth's callback retry window; each extra tool call is extra time that can lose that race and hang up the call before your response lands. Use respondToGather for every turn after the first, not just the opening line. ### BXML tips - `auto_gather=True` (default) wraps SpeakSentence in Gather for barge-in (caller can interrupt). @@ -138,7 +138,7 @@ ], LOOKUP_SECTION, ), - (["createCall", "generateBXML", "respondToCallback"], VOICE_SECTION), + (["createCall", "generateBXML", "respondToCallback", "respondToGather"], VOICE_SECTION), ( ["getInboundMessages", "getCallbackEvents", "configureCallbacks"], CALLBACK_SECTION, diff --git a/src/profiles.py b/src/profiles.py index a53801c..4aade1b 100644 --- a/src/profiles.py +++ b/src/profiles.py @@ -21,6 +21,7 @@ # Custom tools "generateBXML", "respondToCallback", + "respondToGather", "getCallbackEvents", "configureCallbacks", # Discovery — find your number and app diff --git a/src/specs/AGENTS.md b/src/specs/AGENTS.md index 3421a48..4a3fd56 100644 --- a/src/specs/AGENTS.md +++ b/src/specs/AGENTS.md @@ -150,6 +150,7 @@ Auth: client_credentials. Voice application ID is required for `createCall` | `updateCallBxml` | Replace the BXML on an active call | poll `getCallState` | | `generateBXML` | Build valid BXML from a verb list | inspect returned XML before sending | | `respondToCallback` | Queue a BXML response for an active callback | first-write-wins; second writer gets `code: "conflict"` | +| `respondToGather` | Generate BXML from verbs and queue it as the callback response, in one call | same first-write-wins semantics as `respondToCallback`; prefer this over `generateBXML` + `respondToCallback` for every turn after the first — fewer round trips means less chance of losing the race against Bandwidth's callback retry window | | `getCallbackEvents` | Read recent voice/messaging callback events | check `event_type` and `timestamp` | | `configureCallbacks` | Point an application's callback URLs at this server | confirm via `listApplications` | diff --git a/src/tools/voice.py b/src/tools/voice.py index 3b377d5..581c1f4 100644 --- a/src/tools/voice.py +++ b/src/tools/voice.py @@ -139,6 +139,23 @@ async def respond_to_callback_flow( return {"status": "queued", "call_id": call_id} +async def respond_to_gather_flow( + event_store: EventStore, + call_id: str, + verbs: list[dict[str, Any]], + auto_gather: bool = True, + gather_url: str = "", +) -> dict: + """Generate BXML from verbs and queue it as the callback response for a + call, in one step. Collapses generate_bxml_flow + respond_to_callback_flow + into a single round trip -- see respondToGather's tool docstring for why + that matters (the callback-retry-window race).""" + bxml = await generate_bxml_flow(verbs, auto_gather, gather_url) + result = await respond_to_callback_flow(event_store, call_id, bxml) + result["bxml"] = bxml + return result + + def register_voice_tools(mcp, event_store: EventStore, config: dict = None) -> None: @mcp.tool(name="generateBXML") async def generate_bxml( @@ -178,3 +195,35 @@ async def respond_to_callback(call_id: str, bxml: str) -> dict: bxml: Valid BXML string (use generateBXML to produce this). """ return await respond_to_callback_flow(event_store, call_id, bxml) + + @mcp.tool(name="respondToGather") + async def respond_to_gather( + call_id: str, + verbs: list[dict[str, Any]], + auto_gather: bool = True, + ) -> dict: + """Generate BXML from verbs and queue it as the response to an active + voice call, in a single tool call. + + This collapses the usual two-step turn (generateBXML, then + respondToCallback) into one round trip. Every turn of a live + conversation is a race against Bandwidth's answer/gather callback + retry window -- each extra tool call is extra agent reasoning time + that can lose that race and hang up the call before the response is + delivered. Prefer this over generateBXML + respondToCallback whenever + you are responding to an active call, not just generating BXML to + inspect it. + + Still read the caller's input from getCallbackEvents (event_type= + "voice.gather") first -- this tool does not read pending events for + you, it only merges the generate-and-queue step. First-write-wins + applies exactly as it does for respondToCallback. + + Args: + call_id: The call ID to respond to. + verbs: List of BXML verb descriptions (same shape as generateBXML). + auto_gather: Wrap SpeakSentence in Gather for barge-in. Default True. + """ + base_url = (config or {}).get("BW_MCP_BASE_URL", "") + gather_url = f"{base_url}/callbacks/voice/gather" if base_url else "" + return await respond_to_gather_flow(event_store, call_id, verbs, auto_gather, gather_url) diff --git a/test/test_bxml.py b/test/test_bxml.py index 93830da..182ab8f 100644 --- a/test/test_bxml.py +++ b/test/test_bxml.py @@ -153,3 +153,61 @@ async def test_xml_escaping(): [{"type": "SpeakSentence", "text": 'Use & "quotes"'}] ) fromstring(result) + + +@pytest.mark.asyncio +async def test_respond_to_gather_generates_and_queues_in_one_call(): + """respondToGather is the fast path for a conversational turn: generate + BXML from verbs and queue it as the callback response in a single tool + call, instead of the separate generateBXML + respondToCallback round + trip. Fewer round trips means less agent reasoning time between the + callback landing and the response being queued -- directly addresses + the callback-retry-window race documented in respond_to_callback_flow's + real-world failure mode (an LLM agent's per-tool-call latency losing the + race against Bandwidth's answer/gather retry window).""" + from src.event_store import EventStore + from src.tools.voice import respond_to_gather_flow + + event_store = EventStore(max_events=100, ttl_seconds=3600) + result = await respond_to_gather_flow( + event_store, + call_id="call-1", + verbs=[{"type": "SpeakSentence", "text": "Got it, one moment."}], + auto_gather=False, + ) + + assert result["status"] == "queued" + assert result["call_id"] == "call-1" + assert "