diff --git a/api/entrypoints/routers.py b/api/entrypoints/routers.py index b01db2d512d..2699ab130ee 100644 --- a/api/entrypoints/routers.py +++ b/api/entrypoints/routers.py @@ -182,6 +182,10 @@ from oss.src.dbs.postgres.sessions.streams.dbes import SessionStreamDBE # noqa: F401 from oss.src.dbs.postgres.sessions.streams.dao import SessionStreamsDAO from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.dbs.postgres.sessions.commands.dbes import SessionCommandDBE # noqa: F401 +from oss.src.dbs.postgres.sessions.commands.dao import SessionCommandsDAO +from oss.src.core.sessions.commands.service import SessionCommandsService +from oss.src.dbs.http.sessions.control_delivery_direct import DirectControlDelivery from oss.src.tasks.asyncio.sessions.orphan_sweep import orphan_sweep_loop from oss.src.dbs.redis.shared.engine import get_lock_engine @@ -838,6 +842,7 @@ async def lifespan(*args, **kwargs): interactions_service = SessionInteractionsService( interactions_dao=interactions_dao, watch_publisher=_sessions_watch_publisher, + records_service=records_service, ) triggers_service = TriggersService( @@ -1115,6 +1120,26 @@ async def _dispatch_detached_run(*, project_id, user_id, request) -> str: records_service=records_service, ) +# Durable session commands (Stop). The control-delivery adapter is chosen by one setting. +# `direct` posts the command to the runner's own /cancel over the hop that already carries hard +# kill; `long_poll` is not built yet, and naming it fails at boot rather than silently falling +# back to a transport the operator did not choose. +_control_adapter = (env.agenta.sessions.commands.adapter or "direct").strip().lower() +if _control_adapter != "direct": + raise RuntimeError( + f"AGENTA_SESSIONS_CONTROL_ADAPTER={_control_adapter!r} is not available in this build. " + "Only 'direct' is implemented; the long-poll adapter is a later change." + ) + +session_commands_dao = SessionCommandsDAO() +session_commands_service = SessionCommandsService( + commands_dao=session_commands_dao, + streams_service=session_streams_service, + interactions_service=interactions_service, + lock_engine=_lock_engine, + delivery=DirectControlDelivery(), +) + sessions = SessionsRouter( streams_service=session_streams_service, records_service=records_service, @@ -1125,6 +1150,7 @@ async def _dispatch_detached_run(*, project_id, user_id, request) -> str: mounts_service=mounts_service, turns_service=session_turns_service, sessions_service=sessions_service, + commands_service=session_commands_service, respond_task=_interactions_worker.respond_interaction, interactions_dispatcher=_interactions_dispatcher, ) @@ -1599,6 +1625,12 @@ async def _dispatch_detached_run(*, project_id, user_id, request) -> str: tags=["Sessions"], ) +# After `root`, so the literal /sessions/ routes always win a path match. +app.include_router( + router=sessions.control.router, + tags=["Sessions"], +) + @app.get("/health", operation_id="health_check", tags=["Status"]) async def health_check(): diff --git a/api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py new file mode 100644 index 00000000000..d150e2b11ab --- /dev/null +++ b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py @@ -0,0 +1,164 @@ +"""add session commands, and the two session_streams columns a Stop needs + +A user Stop reached the runner only through the absence of a Redis lock, discovered on the next +heartbeat up to 30 seconds later. Nothing recorded that a Stop had been asked for, so a Stop +against an unreachable runner was simply lost and no execution ever reached a terminal outcome +anyone could read. + +`session_commands` is that record. One row per durable request to change an execution. `state` +is where the COMMAND is (pending, claimed, applied, obsolete); `outcome` is what happened to the +EXECUTION (stopped, not_running, superseded_by_newer_turn, failed, lost). The two are separate +columns because they answer different questions and settle at different times. + +Two columns join `session_streams`: + + * `stopping_turn_id` names the execution an accepted Stop is waiting on, written in the same + transaction as the command insert and cleared at settlement. + * `turn_started_at` records when the row's current `turn_id` started. Nothing else could serve + the stale-Stop guard: `updated_at` is the heartbeat timestamp and moves every 30 seconds, + runner-minted turn ids are uuid4 and carry no time, the Redis lock value is a bare turn id + that a Lua compare reads whole, and the `session_turns` append is fire-and-forget so a + running turn may have no row at all. + +Both are nullable and backfill to NULL. A row written before this migration yields no +comparison, and the guard then does not fire — deliberately, because a guard that refused every +Stop it could not verify would break the common case to protect a rare one. + +Revision ID: oss000000022 +Revises: oss000000021 +Create Date: 2026-09-02 23:30:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision: str = "oss000000022" +down_revision: Union[str, None] = "oss000000021" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "session_commands", + sa.Column("id", sa.UUID(as_uuid=True), nullable=False), + sa.Column("project_id", sa.UUID(as_uuid=True), nullable=False), + sa.Column("session_id", sa.String(), nullable=False), + sa.Column("kind", sa.String(), nullable=False), + sa.Column("target_turn_id", sa.String(), nullable=True), + sa.Column("expected_turn_id", sa.String(), nullable=True), + sa.Column("state", sa.String(), nullable=False), + sa.Column("claimed_by", sa.String(), nullable=True), + sa.Column("claim_expires_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column( + "claim_count", + sa.Integer(), + server_default="0", + nullable=False, + ), + sa.Column("outcome", sa.String(), nullable=True), + sa.Column("idempotency_key", sa.String(), nullable=True), + sa.Column("settled_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("data", sa.JSON(), nullable=True), + sa.Column( + "flags", + postgresql.JSONB(none_as_null=True), + nullable=True, + ), + sa.Column( + "tags", + postgresql.JSONB(none_as_null=True), + nullable=True, + ), + sa.Column("meta", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.func.current_timestamp(), + nullable=True, + ), + sa.Column("updated_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("deleted_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("created_by_id", sa.UUID(as_uuid=True), nullable=True), + sa.Column("updated_by_id", sa.UUID(as_uuid=True), nullable=True), + sa.Column("deleted_by_id", sa.UUID(as_uuid=True), nullable=True), + sa.CheckConstraint("kind IN ('cancel')", name="ck_session_commands_kind"), + sa.CheckConstraint( + "state IN ('pending', 'claimed', 'applied', 'obsolete')", + name="ck_session_commands_state", + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["projects.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("project_id", "id"), + sa.UniqueConstraint( + "project_id", + "session_id", + "idempotency_key", + name="uq_session_commands_idempotency", + ), + ) + # One open command per target execution, enforced by the database because admission's + # read-then-insert races itself: two Stops in the same instant both find no open command. + op.create_index( + "uq_session_commands_open_target", + "session_commands", + ["project_id", "session_id", "kind", "target_turn_id"], + unique=True, + postgresql_where=sa.text( + "state IN ('pending', 'claimed') AND deleted_at IS NULL" + ), + ) + op.create_index( + "ix_session_commands_open", + "session_commands", + ["project_id", "session_id", "created_at"], + postgresql_where=sa.text( + "state IN ('pending', 'claimed') AND deleted_at IS NULL" + ), + ) + op.create_index( + "ix_session_commands_claims", + "session_commands", + ["claim_expires_at"], + postgresql_where=sa.text("state = 'claimed' AND deleted_at IS NULL"), + ) + op.create_index( + "ix_session_commands_project_session", + "session_commands", + ["project_id", "session_id", "created_at"], + ) + # The runner reports an outcome with the command id alone; it holds no project credential, + # so that read cannot use the primary key's leading column. + op.create_index( + "ix_session_commands_id", + "session_commands", + ["id"], + ) + + op.add_column( + "session_streams", + sa.Column("stopping_turn_id", sa.String(), nullable=True), + ) + op.add_column( + "session_streams", + sa.Column("turn_started_at", sa.TIMESTAMP(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("session_streams", "turn_started_at") + op.drop_column("session_streams", "stopping_turn_id") + op.drop_index("ix_session_commands_id", table_name="session_commands") + op.drop_index("ix_session_commands_project_session", table_name="session_commands") + op.drop_index("ix_session_commands_claims", table_name="session_commands") + op.drop_index("ix_session_commands_open", table_name="session_commands") + op.drop_index("uq_session_commands_open_target", table_name="session_commands") + op.drop_table("session_commands") diff --git a/api/oss/src/apis/fastapi/sessions/models.py b/api/oss/src/apis/fastapi/sessions/models.py index 01f47695ce5..3eed3666e15 100644 --- a/api/oss/src/apis/fastapi/sessions/models.py +++ b/api/oss/src/apis/fastapi/sessions/models.py @@ -346,3 +346,77 @@ class SessionRecordIngestRequest(BaseModel): # Both forward-fill only (tracing-DB rule) — absent on producers that predate this. turn_id: Optional[str] = None span_id: Optional[OTelSpanId] = None + + +# --------------------------------------------------------------------------- +# Session control: durable commands (Stop) +# --------------------------------------------------------------------------- + + +class SessionCancelRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + # Optional stale-request guard. When present, the API cancels only this execution and + # refuses the request if another one is running. When absent, it cancels whichever + # execution is active when the request is applied. A person never types this: the browser + # fills it from the session's own state, and a first-party client always sends it. + expected_execution_id: Optional[str] = None + + +class SessionCommandRef(BaseModel): + """The durable command an accepted request created. Identity and DELIVERY state only. + + A client must not read execution state from it. `state` says where the command is; the + session's own state says what the execution is doing. + """ + + id: UUID + state: Literal["pending", "claimed", "applied", "obsolete"] + + +class SessionExecutionRef(BaseModel): + """What the caller should render. `id` is null when the session was idle.""" + + id: Optional[str] = None + state: Literal["stopping", "idle"] + + +class SessionCancelResponse(BaseModel): + command: SessionCommandRef + execution: SessionExecutionRef + + +class SessionExecutionOutcome(BaseModel): + model_config = ConfigDict(extra="forbid") + + # The execution the runner acted on. Null when it held none. + id: Optional[str] = None + # stopped: cancelled as asked. not_running: no such execution on this runner. + # superseded_by_newer_turn: the held execution started after the command arrived. + # failed: the cancel itself failed. + state: Literal["stopped", "failed", "not_running", "superseded_by_newer_turn"] + # Short and human-readable, present only when `state` is "failed". + error: Optional[str] = Field(default=None, max_length=2000) + + +class SessionControlOutcomeRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + replica_id: str = Field(min_length=1, max_length=128) + # The command's terminal state. `applied` means the runner did the work; `obsolete` means + # there was nothing to do. + result: Literal["applied", "obsolete"] + execution: SessionExecutionOutcome + + +class SessionCommandSettlement(BaseModel): + id: UUID + state: Literal["applied", "obsolete"] + outcome: Literal[ + "stopped", "not_running", "superseded_by_newer_turn", "failed", "lost" + ] + settled_at: Optional[datetime] = None + + +class SessionControlOutcomeResponse(BaseModel): + command: SessionCommandSettlement diff --git a/api/oss/src/apis/fastapi/sessions/router.py b/api/oss/src/apis/fastapi/sessions/router.py index 3bf5eb22760..85587afe6d5 100644 --- a/api/oss/src/apis/fastapi/sessions/router.py +++ b/api/oss/src/apis/fastapi/sessions/router.py @@ -18,6 +18,7 @@ import re from functools import wraps +from secrets import compare_digest from uuid import UUID from fastapi import ( @@ -66,6 +67,13 @@ SessionStreamNotFound, ) from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.core.sessions.commands.service import SessionCommandsService +from oss.src.core.sessions.commands.types import ( + ExecutionExpectationFailed, + SessionCommandIdempotencyConflict, + SessionCommandNotClaimable, + SessionCommandNotFound, +) from oss.src.core.sessions.records.service import RecordsService from oss.src.core.sessions.records.dtos import SessionRecordEvent from oss.src.core.sessions.records.streaming import publish_record @@ -118,6 +126,13 @@ from oss.src.core.workflows.service import WorkflowsService from oss.src.apis.fastapi.sessions.models import ( + SessionCancelRequest, + SessionCancelResponse, + SessionCommandRef, + SessionCommandSettlement, + SessionControlOutcomeRequest, + SessionControlOutcomeResponse, + SessionExecutionRef, # streams SessionDetachRequest, SessionStreamQueryRequest, @@ -1845,6 +1860,214 @@ async def unarchive_session( ) +# --------------------------------------------------------------------------- +# Session control — durable commands (Stop) +# --------------------------------------------------------------------------- + + +def _handle_command_exceptions(): + """Map the commands plane's domain errors onto status codes. + + A separate decorator from `_handle_session_exceptions` so the two planes' error vocabularies + stay apart: a conflict here means "the execution you named is not the one running", which is + a different thing from the streams plane's "this session is already busy". + """ + + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + try: + return await func(*args, **kwargs) + except SessionIdInvalid as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=e.message, + ) from e + except ExecutionExpectationFailed as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "message": e.message, + "current_execution_id": e.current, + }, + ) from e + except SessionCommandIdempotencyConflict as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=e.message, + ) from e + except SessionCommandNotFound as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=e.message, + ) from e + except SessionCommandNotClaimable as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"message": e.message, "state": e.state}, + ) from e + + return wrapper + + return decorator + + +class SessionControlRouter: + """The Stop plane: one public route and one internal one. + + `POST /sessions/{session_id}/cancel` is the product's Stop. It is deliberately NOT behind + the runner concurrency limit: refusing to STOP work because a project is at its run limit + would be the exact wrong answer to a busy project. + + `POST /sessions/control/commands/{command_id}/outcome` is how the runner reports what + happened. It authenticates with the shared runner token rather than a project credential, + because the runner holds no project credential of its own for a command it was handed. The + command id resolves the project, so a caller still cannot reach across tenants: it can only + settle a command whose id it already knows and that it currently holds the claim on. + """ + + def __init__( + self, + *, + commands_service: SessionCommandsService, + ) -> None: + self._service = commands_service + self.router = APIRouter() + + self.router.add_api_route( + "/sessions/{session_id}/cancel", + self.cancel_session_execution, + methods=["POST"], + operation_id="cancel_session_execution", + tags=["Sessions"], + ) + self.router.add_api_route( + "/sessions/control/commands/{command_id}/outcome", + self.report_command_outcome, + methods=["POST"], + operation_id="report_session_command_outcome", + tags=["Sessions"], + include_in_schema=False, + ) + + @intercept_exceptions() + @_handle_command_exceptions() + async def cancel_session_execution( + self, + request: Request, + session_id: str, + payload: Optional[SessionCancelRequest] = None, + ) -> JSONResponse: + project_id = request.state.project_id + user_id = request.state.user_id + + has_permission = await check_action_access( + user_uid=str(user_id), + project_id=str(project_id), + permission=Permission.RUN_SESSIONS, + ) + if not has_permission: + raise FORBIDDEN_EXCEPTION + + if not env.agenta.sessions.durable_stop: + legacy = await self._service.request_cancel_legacy( + project_id=UUID(str(project_id)), + user_id=UUID(str(user_id)), + session_id=session_id, + expected_execution_id=( + payload.expected_execution_id if payload else None + ), + ) + return JSONResponse( + status_code=status.HTTP_200_OK, + content=legacy.model_dump(mode="json"), + ) + + idempotency_key = request.headers.get("Idempotency-Key") + if idempotency_key is not None: + idempotency_key = ( + idempotency_key.strip()[:_MAX_IDEMPOTENCY_KEY_CHARACTERS] or None + ) + + admission = await self._service.request_cancel( + project_id=UUID(str(project_id)), + user_id=UUID(str(user_id)) if user_id else None, + session_id=session_id, + expected_execution_id=payload.expected_execution_id if payload else None, + idempotency_key=idempotency_key, + ) + + body = SessionCancelResponse( + command=SessionCommandRef( + id=admission.command.id, + state=admission.command.state.value, + ), + execution=SessionExecutionRef( + id=admission.execution_id, + state="stopping" if admission.accepted else "idle", + ), + ) + # 202 and not 200 for the accepted case: the work is not done when the response + # returns. The caller learns the outcome from the session's own state. + return JSONResponse( + status_code=( + status.HTTP_202_ACCEPTED if admission.accepted else status.HTTP_200_OK + ), + content=body.model_dump(mode="json"), + ) + + @intercept_exceptions() + @_handle_command_exceptions() + async def report_command_outcome( + self, + request: Request, + command_id: UUID, + payload: SessionControlOutcomeRequest, + ) -> SessionControlOutcomeResponse: + _assert_runner_token(request) + + settled = await self._service.report_outcome( + command_id=command_id, + replica_id=payload.replica_id, + result=payload.result, + execution_id=payload.execution.id, + execution_state=payload.execution.state, + error=payload.execution.error, + ) + return SessionControlOutcomeResponse( + command=SessionCommandSettlement( + id=settled.id, + state=settled.state.value, + outcome=settled.outcome.value if settled.outcome else "failed", + settled_at=settled.settled_at, + ) + ) + + +def _assert_runner_token(request: Request) -> None: + """The runner proves it is the platform runtime with the shared secret both sides hold. + + Constant-time compare, so a wrong token leaks no length or prefix through timing. A missing + configured token fails closed: an unset secret must never mean "let everyone in". + """ + expected = env.runner.token + if not expected: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="runner token is not configured on this deployment", + ) + presented = request.headers.get("X-Agenta-Runner-Token") or "" + if not presented: + authorization = request.headers.get("Authorization") or "" + if authorization.lower().startswith("bearer "): + presented = authorization[7:].strip() + if not compare_digest(presented.encode("utf-8"), expected.encode("utf-8")): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Unauthorized", + ) + + # --------------------------------------------------------------------------- # Top-level composer # --------------------------------------------------------------------------- @@ -1861,6 +2084,11 @@ class SessionsRouter: sessions_router.mounts.router → prefix /sessions sessions_router.turns.router → prefix /sessions/turns sessions_router.root.router → no prefix (paths include /sessions/query, /sessions/, /sessions/archive, /sessions/unarchive) + sessions_router.control.router → no prefix (paths include /sessions/{session_id}/cancel and /sessions/control/…) + + `control` MUST be mounted AFTER `root`. `/sessions/{session_id}/cancel` is a two-segment + path and `/sessions/query` is one, so they cannot actually collide — but mounting the + literal routes first keeps that true for any two-segment literal added later. """ def __init__( @@ -1875,6 +2103,7 @@ def __init__( mounts_service: MountsService, turns_service: SessionTurnsService, sessions_service: SessionsService, + commands_service: SessionCommandsService, respond_task: Optional[Any] = None, interactions_dispatcher: Optional[Any] = None, ) -> None: @@ -1898,3 +2127,4 @@ def __init__( ) self.turns = SessionTurnsRouter(turns_service=turns_service) self.root = SessionsRootRouter(sessions_service=sessions_service) + self.control = SessionControlRouter(commands_service=commands_service) diff --git a/api/oss/src/core/sessions/commands/__init__.py b/api/oss/src/core/sessions/commands/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/oss/src/core/sessions/commands/dtos.py b/api/oss/src/core/sessions/commands/dtos.py new file mode 100644 index 00000000000..27b2f9c2ad2 --- /dev/null +++ b/api/oss/src/core/sessions/commands/dtos.py @@ -0,0 +1,123 @@ +"""Durable session commands — the data shapes. + +A command is one durable request to change an execution. Version one has one kind, `cancel`, +which the product calls Stop. + +Two ideas are kept apart on purpose, and the separation is the point of the whole record: + + * `state` says where the COMMAND is in its delivery (pending, claimed, applied, obsolete). + * `outcome` says what happened to the EXECUTION (stopped, not_running, ...). + +A client that draws a Stop button reads the execution; a client that retries safely reads the +command id. Merging them is what makes today's cancel ambiguous. +""" + +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional +from uuid import UUID + +from pydantic import BaseModel + +from oss.src.core.shared.dtos import Identifier, Lifecycle + + +class SessionCommandKind(str, Enum): + cancel = "cancel" + + +class SessionCommandState(str, Enum): + """Where the command is in its delivery. `applied` and `obsolete` are terminal.""" + + pending = "pending" # durable, not yet taken by a runner + claimed = "claimed" # a runner holds a lease on it + applied = "applied" # the runner did the work and reported + obsolete = "obsolete" # there was nothing to do, or nobody could ever do it + + +class SessionCommandOutcome(str, Enum): + """What happened to the targeted execution. Null while the command is open.""" + + stopped = "stopped" # cancelled as asked + not_running = "not_running" # no such execution anywhere + superseded_by_newer_turn = ( + "superseded_by_newer_turn" # a later turn holds the session + ) + failed = "failed" # the cancel itself failed + lost = "lost" # nobody ever reported; the sweep settled it + + +class SessionCommand(Identifier, Lifecycle): + project_id: UUID + session_id: str + kind: SessionCommandKind + + # The execution the API resolved at admission and pinned. Null when nothing ran. + target_turn_id: Optional[str] = None + # The execution the caller asserted was running, stored exactly as sent. Null when none. + expected_turn_id: Optional[str] = None + + # The command's own arguments. Empty for `cancel`; reserved for steer and queue. + data: Optional[Dict[str, Any]] = None + + state: SessionCommandState + claimed_by: Optional[str] = None + claim_expires_at: Optional[datetime] = None + claim_count: int = 0 + + outcome: Optional[SessionCommandOutcome] = None + idempotency_key: Optional[str] = None + settled_at: Optional[datetime] = None + + tags: Optional[Dict[str, Any]] = None + meta: Optional[Dict[str, Any]] = None + + +class SessionCommandCreate(BaseModel): + """One insert. `state`/`outcome`/`settled_at` are carried because admission can insert a + command that is ALREADY settled (nothing was running, or a newer turn took the session), + and that must be one write, not an insert followed by an update.""" + + project_id: UUID + session_id: str + kind: SessionCommandKind = SessionCommandKind.cancel + + target_turn_id: Optional[str] = None + expected_turn_id: Optional[str] = None + data: Optional[Dict[str, Any]] = None + + state: SessionCommandState = SessionCommandState.pending + outcome: Optional[SessionCommandOutcome] = None + settled_at: Optional[datetime] = None + + idempotency_key: Optional[str] = None + + # The instant the service stamped as the request's arrival. It is stored as `created_at` + # rather than left to the server default, so the value the stale-Stop guard COMPARED is the + # value the row CARRIES. A guard that compares one timestamp and stores another is not a + # guard the runner can repeat. + created_at: Optional[datetime] = None + + +class SessionCommandSettle(BaseModel): + """The terminal transition, guarded on the states the caller expects to find. + + A SET and not one state, because the outcome report races the claim that is taken on the + runner's behalf. Admission inserts the command `pending`, hands it to the runner, and only + then writes `claimed`; a runner that aborts fast reports its outcome while the row is still + `pending`. Guarding on `claimed` alone refused that report with a conflict and left the + command open until the sweep called it lost. Both states are legitimate at the moment of the + write, so the compare-and-set covers both. + + `replica_id` guards a settlement that follows a claim: only the replica that holds the + claim may write the outcome. A `pending` row has no claim to violate, so the guard admits a + null `claimed_by` as well. It is None altogether when the API itself settles a command + nobody ever took, which is the `not_held` case and the sweep's `lost` case. + """ + + project_id: UUID + command_id: UUID + state: SessionCommandState + outcome: SessionCommandOutcome + expected_states: List[SessionCommandState] = [SessionCommandState.claimed] + replica_id: Optional[str] = None diff --git a/api/oss/src/core/sessions/commands/interfaces.py b/api/oss/src/core/sessions/commands/interfaces.py new file mode 100644 index 00000000000..169e186c3a7 --- /dev/null +++ b/api/oss/src/core/sessions/commands/interfaces.py @@ -0,0 +1,178 @@ +"""The two ports of the session commands plane. + +`SessionCommandsDAOInterface` is storage. `ControlDeliveryPort` is transport: how the API +reaches whichever runner process holds a session. Durability, authorization, idempotency, the +state machine and terminal settlement live in the service and must not move into an adapter. +""" + +from abc import ABC, abstractmethod +from datetime import datetime +from typing import List, NamedTuple, Optional +from uuid import UUID + +from pydantic import BaseModel + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandKind, + SessionCommandSettle, +) + + +class SessionScope(BaseModel): + """One session a runner holds warm. The routing input of a claim.""" + + project_id: UUID + session_id: str + + +class CommandCreateResult(NamedTuple): + """The stored command and whether this call inserted it.""" + + command: SessionCommand + inserted: bool + + +class DeliveryReceipt(BaseModel): + """What the TRANSPORT learned, never what happened to the execution. + + * `accepted` — a runner took the command and will report through the outcome route. + * `unreachable` — the transport failed. The command is durable, so a later claim or the + settlement sweep recovers it. + * `not_held` — a reachable runner said it does not hold that session, which lets the + service settle at once instead of waiting for the deadline. + """ + + status: str # "accepted" | "unreachable" | "not_held" + detail: Optional[str] = None + # Which runner process took it, when the transport learned that. The service uses it as the + # claim owner, so the outcome route's guard reads the same way on every transport. + replica_id: Optional[str] = None + + +class ControlDeliveryPort(ABC): + """How the API reaches the runner that holds a session. Transport only.""" + + @abstractmethod + async def deliver(self, *, command: SessionCommand) -> DeliveryReceipt: + """Make `command` reachable by whoever holds its session, promptly. + + Best effort: a failure here never fails admission, because the command is already + durable. + """ + + @abstractmethod + async def acknowledge(self, *, command_id: UUID, replica_id: str) -> None: + """Record that a replica took the command, for adapters that keep their own delivery + bookkeeping. A no-op where the claim compare-and-set already IS the acknowledgement.""" + + +class SessionCommandsDAOInterface(ABC): + @abstractmethod + async def create_command( + self, + *, + user_id: Optional[UUID], + command: SessionCommandCreate, + stopping_turn_id: Optional[str] = None, + ) -> SessionCommand: + """Insert one command and, in the SAME transaction, stamp the session row's + `stopping_turn_id`. Idempotent on `(project_id, session_id, idempotency_key)`.""" + + @abstractmethod + async def create_command_with_status( + self, + *, + user_id: Optional[UUID], + command: SessionCommandCreate, + stopping_turn_id: Optional[str] = None, + ) -> CommandCreateResult: + """Create a command and report whether this call inserted it.""" + + @abstractmethod + async def fetch_by_idempotency_key( + self, + *, + project_id: UUID, + session_id: str, + idempotency_key: str, + ) -> Optional[SessionCommand]: + """The command previously created for this session-scoped retry key.""" + + @abstractmethod + async def fetch_open_command( + self, + *, + project_id: UUID, + session_id: str, + kind: SessionCommandKind, + target_turn_id: Optional[str], + ) -> Optional[SessionCommand]: + """The open (`pending` or `claimed`) command for this exact target, if one exists. + This is what collapses two Stops in a row onto one command.""" + + @abstractmethod + async def fetch_command( + self, + *, + command_id: UUID, + project_id: Optional[UUID] = None, + ) -> Optional[SessionCommand]: + """One command by id. `project_id` is optional because the runner reports an outcome + with the command id alone and holds no project credential.""" + + @abstractmethod + async def claim_commands( + self, + *, + sessions: List[SessionScope], + replica_id: str, + lease_seconds: int, + limit: int, + ) -> List[SessionCommand]: + """Take up to `limit` pending commands for these sessions. Compare-and-set, so two API + replicas serving two claims at once never hand out the same command twice.""" + + @abstractmethod + async def claim_for_delivery( + self, + *, + project_id: UUID, + command_id: UUID, + replica_id: str, + lease_seconds: int, + ) -> Optional[SessionCommand]: + """Move ONE command from `pending` to `claimed` for a runner that just accepted it over + a direct call. The long-poll adapter reaches the same transition through + `claim_commands`; both exist so the outcome route's guard reads the same either way.""" + + @abstractmethod + async def settle_command( + self, + *, + settle: SessionCommandSettle, + ) -> Optional[SessionCommand]: + """Terminal transition, guarded on `state='claimed' AND claimed_by=:replica_id`. + None means the claim had expired or somebody else settled it first.""" + + @abstractmethod + async def clear_stopping_turn( + self, + *, + project_id: UUID, + session_id: str, + turn_id: Optional[str] = None, + ) -> None: + """Clear `session_streams.stopping_turn_id`. With `turn_id`, only when it matches, so a + late settlement cannot clear a NEWER Stop's marker.""" + + @abstractmethod + async def expire_claims( + self, + *, + now: datetime, + max_deliveries: int, + ) -> List[SessionCommand]: + """Commands whose claim lease has passed. The settlement sweep reads this. Not called + in this slice; the execution watchdog owns settlement (see the slice document).""" diff --git a/api/oss/src/core/sessions/commands/service.py b/api/oss/src/core/sessions/commands/service.py new file mode 100644 index 00000000000..68e3012c389 --- /dev/null +++ b/api/oss/src/core/sessions/commands/service.py @@ -0,0 +1,623 @@ +"""Durable session commands — admission, delivery and settlement. + +Version one has one command kind, `cancel`, which the product calls Stop. + +WHAT STOP MEANS HERE. Stop ends the WORK, not the session. The sandbox stays warm, the native +harness session stays resumable, and the next message continues the same conversation. That is +why this service never force-deletes the Redis `alive` key: it leaves it to its own time to +live, exactly as the end of an ordinary turn does. Force-deleting `alive` is what makes today's +cancel read as a session teardown. + +THE ORDER OF ADMISSION. + + 1. Stamp the arrival time FIRST, before reading anything. + 2. Resolve the target execution once, from Redis `running`, falling back to `alive`. + 3. Apply the three late-Stop guards (below). + 4. Insert the command and stamp `session_streams.stopping_turn_id` in ONE transaction. + 5. Only then call the runner. Delivery failure never fails the request, because the command + is already durable. + +Redis is not written at admission. The stopping execution keeps `alive` and `running` while it +stops, which is what prevents a second message from starting underneath it. + +THE LATE-STOP GUARDS. A Stop that arrives after its turn ended must not kill the next turn. + + * The caller's `expected_execution_id`, when sent, must name the running execution. It does + not, the request is refused with a conflict and nothing is written. + * When no expectation was sent and the running execution started AFTER this request arrived, + the command is inserted already settled and targets nothing. + * The target is resolved once and pinned. A turn that starts later has a different id, so a + pinned command can never reach it. The runner repeats the comparison against its own memory, + which is exact. +""" + +from datetime import datetime, timezone +from typing import List, Optional, Tuple +from uuid import UUID + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandKind, + SessionCommandOutcome, + SessionCommandSettle, + SessionCommandState, +) +from oss.src.core.sessions.commands.interfaces import ( + CommandCreateResult, + ControlDeliveryPort, + SessionCommandsDAOInterface, +) +from oss.src.core.sessions.commands.types import ( + ExecutionExpectationFailed, + SessionCommandIdempotencyConflict, + SessionCommandNotClaimable, + SessionCommandNotFound, +) +from oss.src.core.sessions.interactions.service import SessionInteractionsService +from oss.src.core.sessions.streams.dtos import ( + SessionStreamCommandRequest, + SessionStreamCommandResponse, +) +from oss.src.core.sessions.streams.service import SessionStreamsService +from oss.src.core.sessions.streams.types import SessionIdInvalid, SessionTurnMismatch +from oss.src.dbs.redis.shared.engine import LockEngine +from oss.src.dbs.redis.sessions.contract import ( + HEARTBEAT_INTERVAL_SECONDS, + validate_session_id, +) +from oss.src.dbs.redis.sessions.locks import ( + get_alive_owner, + get_owner, + get_running_owner, + mark_turn_superseded, + release_running, +) +from oss.src.utils.env import env +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) + + +class CancelAdmission: + """What admission decided, in the shape the route answers with.""" + + def __init__( + self, + *, + command: SessionCommand, + execution_id: Optional[str], + accepted: bool, + ) -> None: + self.command = command + # What the caller should render: the execution being stopped, or nothing. + self.execution_id = execution_id + # True when an execution was running or parked and the command is on its way. The route + # answers 202 for it and 200 otherwise. + self.accepted = accepted + + +class SessionCommandsService: + def __init__( + self, + *, + commands_dao: SessionCommandsDAOInterface, + streams_service: SessionStreamsService, + interactions_service: SessionInteractionsService, + lock_engine: LockEngine, + delivery: ControlDeliveryPort, + ) -> None: + self._dao = commands_dao + self._streams = streams_service + self._interactions = interactions_service + self._lock = lock_engine + self._delivery = delivery + + # -- admission ---------------------------------------------------------- # + + async def request_cancel_legacy( + self, + *, + project_id: UUID, + user_id: UUID, + session_id: str, + expected_execution_id: Optional[str] = None, + ) -> SessionStreamCommandResponse: + """Use the heartbeat-carried Stop path kept for rollout rollback.""" + try: + return await self._streams.command( + project_id=project_id, + user_id=user_id, + request=SessionStreamCommandRequest( + session_id=session_id, + expected_execution_id=expected_execution_id, + ), + ) + except SessionTurnMismatch as error: + raise ExecutionExpectationFailed( + expected=error.expected_turn_id, + current=error.actual_turn_id, + ) from error + + async def request_cancel( + self, + *, + project_id: UUID, + user_id: Optional[UUID], + session_id: str, + expected_execution_id: Optional[str] = None, + idempotency_key: Optional[str] = None, + ) -> CancelAdmission: + if not validate_session_id(session_id): + raise SessionIdInvalid(session_id) + + # FIRST, before any read. The value compared below is the value stored as the row's + # `created_at`, so the runner can repeat the same comparison against its own memory. + received_at = datetime.now(timezone.utc) + + if idempotency_key is not None: + existing = await self._dao.fetch_by_idempotency_key( + project_id=project_id, + session_id=session_id, + idempotency_key=idempotency_key, + ) + if existing is not None: + if existing.expected_turn_id != expected_execution_id: + raise SessionCommandIdempotencyConflict( + idempotency_key=idempotency_key + ) + return self._admission_for_existing(existing) + + target_turn_id, turn_started_at = await self._resolve_target( + project_id=project_id, + session_id=session_id, + expected_turn_id=expected_execution_id, + ) + + if ( + expected_execution_id is not None + and target_turn_id != expected_execution_id + ): + # Compared against the TARGET, which is `running` with a fallback to `alive`, and + # never against `running` alone. An execution parked on an approval has released + # `running` and still holds `alive` under the same turn id, and it is exactly the + # execution the user is looking at when they press Stop on the approval card. The + # browser always sends the id it streamed, so comparing against `running` alone + # refused every named Stop on a parked approval while the same Stop without an + # expectation was accepted — the guard fired on the one case it exists to allow. + # + # Nothing is inserted and nothing is delivered. The caller was looking at a run + # that has already ended, and its next read tells it so. + raise ExecutionExpectationFailed( + expected=expected_execution_id, current=target_turn_id + ) + + if target_turn_id is None: + # No eligible execution is running. Record the intent so a retry with the same key + # gets the same answer, and settle it in the same write. + created = await self._insert( + project_id=project_id, + user_id=user_id, + session_id=session_id, + received_at=received_at, + target_turn_id=None, + expected_turn_id=expected_execution_id, + idempotency_key=idempotency_key, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.not_running, + ) + if not created.inserted: + return self._admission_for_existing(created.command) + command = created.command + return CancelAdmission(command=command, execution_id=None, accepted=False) + + if ( + expected_execution_id is None + and turn_started_at is not None + and turn_started_at > received_at + ): + # The execution now running began AFTER the user pressed Stop, so it is not the one + # they meant. Do not target it, do not touch Redis, and tell the caller there is + # nothing of theirs left to stop. + created = await self._insert( + project_id=project_id, + user_id=user_id, + session_id=session_id, + received_at=received_at, + target_turn_id=None, + expected_turn_id=None, + idempotency_key=idempotency_key, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.superseded_by_newer_turn, + ) + if not created.inserted: + return self._admission_for_existing(created.command) + command = created.command + return CancelAdmission(command=command, execution_id=None, accepted=False) + + # Two Stops in a row are one intent. Collapse onto the open command for the same target + # BEFORE inserting, so this holds even when the caller sends a different idempotency key. + open_command = await self._dao.fetch_open_command( + project_id=project_id, + session_id=session_id, + kind=SessionCommandKind.cancel, + target_turn_id=target_turn_id, + ) + if open_command is not None: + if open_command.state == SessionCommandState.pending: + # Nobody has taken it. The first delivery may have failed, so try again; the + # runner deduplicates by command id, so a duplicate arrival aborts nothing twice. + await self._deliver(open_command) + return CancelAdmission( + command=open_command, + execution_id=target_turn_id, + accepted=True, + ) + + created = await self._insert( + project_id=project_id, + user_id=user_id, + session_id=session_id, + received_at=received_at, + target_turn_id=target_turn_id, + expected_turn_id=expected_execution_id, + idempotency_key=idempotency_key, + state=SessionCommandState.pending, + outcome=None, + stopping_turn_id=target_turn_id, + ) + if not created.inserted: + return self._admission_for_existing(created.command) + command = created.command + # The row is committed. Everything from here is promptness, not correctness. + await self._deliver(command) + return CancelAdmission( + command=command, execution_id=target_turn_id, accepted=True + ) + + async def _resolve_target( + self, + *, + project_id: UUID, + session_id: str, + expected_turn_id: Optional[str], + ) -> Tuple[Optional[str], Optional[datetime]]: + """The execution to stop, and when it started. + + An unfenced Stop targets only `running`. A named Stop may fall back to `alive` so it can + still reach the parked approval the caller observed. + """ + turn_id = await get_running_owner( + self._lock, project_id=str(project_id), session_id=session_id + ) + if turn_id is None and expected_turn_id is not None: + turn_id = await get_alive_owner( + self._lock, project_id=str(project_id), session_id=session_id + ) + if turn_id is None: + return None, None + + stream = await self._streams.fetch_header( + project_id=project_id, session_id=session_id + ) + started_at = None + if stream is not None and stream.turn_id == turn_id: + # Only when the row agrees about WHICH turn is running. A start time read off a row + # that names a different turn would compare two unrelated things. + started_at = stream.turn_started_at + return turn_id, started_at + + async def _insert( + self, + *, + project_id: UUID, + user_id: Optional[UUID], + session_id: str, + received_at: datetime, + target_turn_id: Optional[str], + expected_turn_id: Optional[str], + idempotency_key: Optional[str], + state: SessionCommandState, + outcome: Optional[SessionCommandOutcome], + stopping_turn_id: Optional[str] = None, + ) -> CommandCreateResult: + return await self._dao.create_command_with_status( + user_id=user_id, + command=SessionCommandCreate( + project_id=project_id, + session_id=session_id, + kind=SessionCommandKind.cancel, + target_turn_id=target_turn_id, + expected_turn_id=expected_turn_id, + state=state, + outcome=outcome, + settled_at=received_at if outcome is not None else None, + idempotency_key=idempotency_key, + created_at=received_at, + ), + stopping_turn_id=stopping_turn_id, + ) + + @staticmethod + def _admission_for_existing(command: SessionCommand) -> CancelAdmission: + """Replay the command's original target without delivering it again.""" + return CancelAdmission( + command=command, + execution_id=command.target_turn_id, + accepted=command.target_turn_id is not None, + ) + + # -- delivery ----------------------------------------------------------- # + + async def _deliver(self, command: SessionCommand) -> None: + """Hand the command to the transport, then record what the transport learned. + + Never raises. The user's request has already succeeded by the time this runs. + """ + try: + receipt = await self._delivery.deliver(command=command) + except Exception as e: # noqa: BLE001 — transport failure is never a request failure + log.warning( + "control delivery raised for command=%s session=%s: %s", + command.id, + command.session_id, + e, + ) + return + + if receipt.status == "accepted": + # Take the claim on the runner's behalf, so the outcome route's guard reads the same + # way on every transport: only the holder of the claim writes the outcome. + await self._dao.claim_for_delivery( + project_id=command.project_id, + command_id=command.id, + replica_id=receipt.replica_id or "direct", + lease_seconds=env.agenta.sessions.commands.lease_seconds, + ) + return + + if receipt.status == "not_held": + await self._settle_not_held(command) + return + + log.warning( + "control delivery unreachable for command=%s session=%s: %s", + command.id, + command.session_id, + receipt.detail or "no detail", + ) + + async def _settle_not_held(self, command: SessionCommand) -> None: + """A reachable runner said it does not hold this session. Two different things look + alike here, and the user must not be told the wrong one. + + `running` is the discriminator, not the heartbeat. A `not_held` while SOME execution + holds `running` means a process is executing this session and it is not the one we + called. Settle that `lost`, so the user learns the Stop failed, and log it at error + level. + + With no `running` execution anywhere, nothing is executing and the work the user meant + to stop is over. That is the everyday case: the turn ended a moment before the Stop + arrived, the runner had already dropped it, and the answer is `not_running`. Judging it + on the heartbeat instead called every one of those a failed Stop, because a turn that + has just ended leaves `alive` set and a fresh beat behind it, exactly as a running one + does. + """ + outcome = SessionCommandOutcome.not_running + running_owner = await get_running_owner( + self._lock, + project_id=str(command.project_id), + session_id=command.session_id, + ) + if running_owner is not None and await self._session_is_beating( + project_id=command.project_id, session_id=command.session_id + ): + outcome = SessionCommandOutcome.lost + # Name the process that DOES hold the session, so the log says where the Stop + # should have gone rather than only that it did not arrive. + owner = await get_owner( + self._lock, + project_id=str(command.project_id), + session_id=command.session_id, + ) + log.error( + "control delivery: the runner answered not_held for session=%s while " + "execution %s holds `running` and the row is beating. A process is executing " + "that session and it is not the one we called, so this deployment has more " + "than one runner replica and the direct adapter cannot route to it. Settling " + "the command lost, so the user is told the Stop failed rather than that the " + "work had already finished. command=%s target_turn=%s owner_replica=%s", + command.session_id, + running_owner, + command.id, + command.target_turn_id, + owner or "unknown", + ) + await self.settle( + command_id=command.id, + project_id=command.project_id, + replica_id=None, + expected_states=[SessionCommandState.pending], + state=SessionCommandState.obsolete, + outcome=outcome, + execution_id=command.target_turn_id, + ) + + async def _session_is_beating(self, *, project_id: UUID, session_id: str) -> bool: + """Is a runner process keeping this session's row fresh right now?""" + stream = await self._streams.fetch_header( + project_id=project_id, session_id=session_id + ) + if stream is None or stream.updated_at is None: + return False + if not (stream.flags and stream.flags.is_alive): + return False + updated_at = stream.updated_at + if updated_at.tzinfo is None: + updated_at = updated_at.replace(tzinfo=timezone.utc) + age = (datetime.now(timezone.utc) - updated_at).total_seconds() + return age < HEARTBEAT_INTERVAL_SECONDS * 2 + + # -- settlement --------------------------------------------------------- # + + async def report_outcome( + self, + *, + command_id: UUID, + replica_id: str, + result: str, + execution_id: Optional[str], + execution_state: str, + error: Optional[str] = None, + ) -> SessionCommand: + """The runner reporting what happened to the execution. Both adapters land here, so + settlement has one path on every transport.""" + command = await self._dao.fetch_command(command_id=command_id) + if command is None: + raise SessionCommandNotFound(command_id=str(command_id)) + + outcome = _OUTCOME_BY_EXECUTION_STATE.get(execution_state) + if outcome is None: + outcome = SessionCommandOutcome.failed + state = ( + SessionCommandState.applied + if result == "applied" + else SessionCommandState.obsolete + ) + if error: + log.warning( + "session command %s reported a failed cancel for execution=%s: %s", + command_id, + execution_id, + error[:2000], + ) + + settled = await self.settle( + command_id=command_id, + project_id=command.project_id, + replica_id=replica_id, + # Both, and checked at the moment of the write. Admission inserts `pending`, + # delivers, and only then writes `claimed` on the runner's behalf, so a runner that + # aborts fast reports its outcome while the row is still `pending`. Guarding on + # `claimed` alone refused that report with a conflict and left a correctly stopped + # execution sitting `claimed` until the sweep called it lost — the user watching + # "stopping" for the whole sweep window, and a Stop that worked recorded as lost. + expected_states=[ + SessionCommandState.pending, + SessionCommandState.claimed, + ], + state=state, + outcome=outcome, + execution_id=execution_id or command.target_turn_id, + ) + if settled is None: + stored = await self._dao.fetch_command(command_id=command_id) + raise SessionCommandNotClaimable( + command_id=str(command_id), + state=stored.state.value if stored else "unknown", + ) + return settled + + async def settle( + self, + *, + command_id: UUID, + project_id: UUID, + replica_id: Optional[str], + expected_states: List[SessionCommandState], + state: SessionCommandState, + outcome: SessionCommandOutcome, + execution_id: Optional[str], + ) -> Optional[SessionCommand]: + """Settle the command and the execution together, guarded on the command's state. + + The guard is what makes this idempotent: a second report finds a terminal row, changes + nothing, and the side effects below do not run twice. + """ + settled = await self._dao.settle_command( + settle=SessionCommandSettle( + project_id=project_id, + command_id=command_id, + state=state, + outcome=outcome, + expected_states=expected_states, + replica_id=replica_id, + ) + ) + if settled is None: + return None + + session_id = settled.session_id + target = settled.target_turn_id + + await self._dao.clear_stopping_turn( + project_id=project_id, + session_id=session_id, + turn_id=target, + ) + + if outcome == SessionCommandOutcome.stopped and target: + # Order matters. Tombstone first, so a late beat from the stopped execution cannot + # re-arm the locks it is about to lose; that beat would otherwise find `alive` free + # and take it straight back under the same turn id. + await mark_turn_superseded( + self._lock, + project_id=str(project_id), + session_id=session_id, + turn_id=target, + ) + # Owner-checked, so it can only release its OWN execution's key. + await release_running( + self._lock, + project_id=str(project_id), + session_id=session_id, + turn_id=target, + ) + # `alive` is deliberately left to its own time to live, exactly as the end of a + # normal turn leaves it. Warm resume is the required outcome of Stop, so the session + # must end up in the state a finished turn leaves it in, not in a torn-down one. + + # Mirror the nest onto the row HERE, because nothing else will. The tombstone + # above refuses the stopped execution's own final `is_running=false` beat before it + # can reach the heartbeat's mirror write, and the read model the product polls + # (`query_streams`) reads Postgres and never Redis. Skipping this leaves the row + # saying `is_running: true` until the orphan sweep collapses it, so the tab that + # pressed Stop shows a "running somewhere else" strip over its own session. + await self._streams.mirror_liveness( + project_id=project_id, + session_id=session_id, + ) + + if outcome in ( + SessionCommandOutcome.stopped, + SessionCommandOutcome.not_running, + SessionCommandOutcome.lost, + ): + if target: + # An approval card whose execution was stopped is a card whose buttons do + # nothing. Scoped to this execution, so a newer turn's gates survive. + await self._interactions.cancel_session_pending( + project_id=project_id, + session_id=session_id, + only_turn_id=target, + command_id=command_id, + ) + await self._streams.publish_session_ended( + project_id=project_id, + session_id=session_id, + ) + return settled + + +# The runner names what happened to the EXECUTION; the command's `outcome` column stores it. +_OUTCOME_BY_EXECUTION_STATE = { + "stopped": SessionCommandOutcome.stopped, + "not_running": SessionCommandOutcome.not_running, + "superseded_by_newer_turn": SessionCommandOutcome.superseded_by_newer_turn, + "failed": SessionCommandOutcome.failed, +} + +__all__ = [ + "CancelAdmission", + "SessionCommandsService", +] diff --git a/api/oss/src/core/sessions/commands/types.py b/api/oss/src/core/sessions/commands/types.py new file mode 100644 index 00000000000..47092a44c01 --- /dev/null +++ b/api/oss/src/core/sessions/commands/types.py @@ -0,0 +1,51 @@ +"""Domain errors of the session commands plane. The router maps each to a status code.""" + +from typing import Optional + + +class SessionCommandError(Exception): + """Base of every commands-plane domain error.""" + + +class ExecutionExpectationFailed(SessionCommandError): + """`expected_execution_id` does not name the execution that is running. + + Carries the current execution id (or None) so the caller can refresh rather than guess. + """ + + def __init__(self, *, expected: str, current: Optional[str]) -> None: + self.expected = expected + self.current = current + self.message = ( + f"expected execution '{expected}' is not the running execution " + f"(current: {current or 'none'})" + ) + super().__init__(self.message) + + +class SessionCommandIdempotencyConflict(SessionCommandError): + """An idempotency key was reused for a different cancel request.""" + + def __init__(self, *, idempotency_key: str) -> None: + self.idempotency_key = idempotency_key + self.message = ( + f"idempotency key '{idempotency_key}' belongs to a different request" + ) + super().__init__(self.message) + + +class SessionCommandNotFound(SessionCommandError): + def __init__(self, *, command_id: str) -> None: + self.command_id = command_id + self.message = f"no session command with id '{command_id}'" + super().__init__(self.message) + + +class SessionCommandNotClaimable(SessionCommandError): + """A settle arrived for a command this replica does not hold, or that is already terminal.""" + + def __init__(self, *, command_id: str, state: str) -> None: + self.command_id = command_id + self.state = state + self.message = f"session command '{command_id}' is '{state}' and cannot be settled by this caller" + super().__init__(self.message) diff --git a/api/oss/src/core/sessions/interactions/interfaces.py b/api/oss/src/core/sessions/interactions/interfaces.py index 60336b51395..3eb9702c316 100644 --- a/api/oss/src/core/sessions/interactions/interfaces.py +++ b/api/oss/src/core/sessions/interactions/interfaces.py @@ -47,7 +47,7 @@ async def cancel_session_pending( except_turn_id: Optional[str] = None, except_tokens: Optional[List[str]] = None, only_turn_id: Optional[str] = None, - ) -> int: ... + ) -> List[SessionInteraction]: ... @abstractmethod async def query_interactions( diff --git a/api/oss/src/core/sessions/interactions/service.py b/api/oss/src/core/sessions/interactions/service.py index 02d685404f8..751c3aeac28 100644 --- a/api/oss/src/core/sessions/interactions/service.py +++ b/api/oss/src/core/sessions/interactions/service.py @@ -1,5 +1,5 @@ from typing import List, Optional -from uuid import UUID +from uuid import NAMESPACE_DNS, UUID, uuid5 from oss.src.core.sessions.interactions.dtos import ( SessionInteraction, @@ -11,12 +11,19 @@ SessionInteractionsDAOInterface, ) from oss.src.core.sessions.interactions.types import InteractionNotFound +from oss.src.core.sessions.records.dtos import SessionRecordEvent +from oss.src.core.sessions.records.service import RecordsService from oss.src.core.shared.dtos import Windowing from oss.src.dbs.redis.sessions.contract import ( WATCH_INTERACTION_PENDING, WATCH_INTERACTION_RESOLVED, ) from oss.src.core.sessions.watch.interfaces import SessionsWatchPublisherInterface +from oss.src.utils.logging import get_module_logger + + +_RECORD_NAMESPACE = uuid5(uuid5(NAMESPACE_DNS, "agenta"), "records") +log = get_module_logger(__name__) class SessionInteractionsService: @@ -25,9 +32,11 @@ def __init__( *, interactions_dao: SessionInteractionsDAOInterface, watch_publisher: Optional[SessionsWatchPublisherInterface] = None, + records_service: Optional[RecordsService] = None, ) -> None: self.interactions_dao = interactions_dao self._watch = watch_publisher + self._records = records_service async def _publish_interaction( self, *, project_id: UUID, session_id: str, status: str @@ -102,6 +111,7 @@ async def cancel_session_pending( except_turn_id: Optional[str] = None, except_tokens: Optional[List[str]] = None, only_turn_id: Optional[str] = None, + command_id: Optional[UUID] = None, ) -> int: cancelled = await self.interactions_dao.cancel_session_pending( project_id=project_id, @@ -110,13 +120,49 @@ async def cancel_session_pending( except_tokens=except_tokens, only_turn_id=only_turn_id, ) + if cancelled and command_id is not None and self._records is not None: + try: + await self._records.append_many( + events=[ + SessionRecordEvent( + project_id=project_id, + session_id=interaction.session_id, + record_id=uuid5( + _RECORD_NAMESPACE, + f"{interaction.session_id}:{interaction.token}:" + f"interaction_response:{interaction.turn_id or ''}", + ), + record_type="interaction_response", + record_source="agent", + attributes={ + "type": "interaction_response", + "id": interaction.token, + "kind": interaction.kind.value, + "payload": { + "outcome": "cancelled", + "turnId": interaction.turn_id, + "commandId": str(command_id), + }, + }, + turn_id=interaction.turn_id, + ) + for interaction in cancelled + ] + ) + except Exception: + log.warning( + "Failed to append cancellation records for session=%s command=%s", + session_id, + command_id, + exc_info=True, + ) if cancelled: await self._publish_interaction( project_id=project_id, session_id=session_id, status=WATCH_INTERACTION_RESOLVED, ) - return cancelled + return len(cancelled) async def query_interactions( self, diff --git a/api/oss/src/core/sessions/streams/dtos.py b/api/oss/src/core/sessions/streams/dtos.py index 2a611aa2937..ab462ebca7c 100644 --- a/api/oss/src/core/sessions/streams/dtos.py +++ b/api/oss/src/core/sessions/streams/dtos.py @@ -41,6 +41,11 @@ class SessionStream(Identifier, Header, Lifecycle): tags: Optional[Dict[str, Any]] = None meta: Optional[Dict[str, Any]] = None turn_id: Optional[str] = None + # When `turn_id` started. Stamped only when the id changes, so repeated heartbeats never + # move it. The stale-Stop guard compares a cancel request's arrival time against this. + turn_started_at: Optional[datetime] = None + # The execution an accepted Stop is waiting on. Null when nothing is stopping. + stopping_turn_id: Optional[str] = None # What this session runs. Filled once, from the first beat that knows — turn appends # are fire-and-forget, so a session whose only reference carrier was a dropped append # is unopenable forever. @@ -143,6 +148,16 @@ class SessionStreamCommandRequest(BaseModel): data: Optional[WorkflowServiceRequestData] = None force: bool = False detached: bool = False # fire-and-forget mode + expected_execution_id: Optional[str] = None + + @field_validator("expected_execution_id") + @classmethod + def _blank_expected_execution_id_means_absent( + cls, value: Optional[str] + ) -> Optional[str]: + if value is None: + return None + return value.strip() or None class SessionStreamCommandResponse(BaseModel): @@ -151,6 +166,9 @@ class SessionStreamCommandResponse(BaseModel): turn_id: Optional[str] = None watcher_id: Optional[str] = None detached: bool = False + # Cancel only: every turn this cancel tombstoned. Usually one. It is a list because + # `alive` and `running` can be held by different turns during a handover, and both die. + cancelled_turn_ids: List[str] = Field(default_factory=list) class SessionHeartbeatRequest(BaseModel): diff --git a/api/oss/src/core/sessions/streams/runner_client.py b/api/oss/src/core/sessions/streams/runner_client.py index 45e6689aca8..39c3f8dd1f1 100644 --- a/api/oss/src/core/sessions/streams/runner_client.py +++ b/api/oss/src/core/sessions/streams/runner_client.py @@ -17,6 +17,8 @@ the runner's own orphan sweep / idle-TTL eviction is the fallback net for a missed signal. """ +from typing import NamedTuple, Optional + import httpx from oss.src.utils.env import env @@ -60,3 +62,109 @@ async def kill_runner_sandbox(*, project_id: str, session_id: str) -> bool: except httpx.HTTPError as e: log.warning("kill: runner /kill call failed for session=%s: %s", session_id, e) return False + + +_CANCEL_TIMEOUT_SECONDS = 5.0 + + +class RunnerCancelResult: + """What the direct hop learned, as three named cases. + + * `accepted` — the runner holds the session and took the command. The outcome arrives + later on the outcome route, never in this response. + * `not_held` — the runner answered, and it does not hold that session. + * `unreachable` — no answer, a non-2xx that is not 404, or no runner configured at all. + """ + + accepted = "accepted" + not_held = "not_held" + unreachable = "unreachable" + + +class RunnerCancelResponse(NamedTuple): + """The acknowledgement, and WHICH runner process gave it. + + `replica_id` is what the API records as the claim holder, so the outcome route's guard + (`state='claimed' AND claimed_by=:replica_id`) matches the id the runner reports with. Take + it from the answer rather than assuming one: a claim written under a name the runner does + not use refuses the runner's own outcome report, which leaves the command open and the + session marked stopping forever. + """ + + status: str + replica_id: Optional[str] = None + + +async def cancel_runner_execution( + *, + command_id: str, + project_id: str, + session_id: str, + target_turn_id: Optional[str], + created_at: str, + timeout_seconds: float = _CANCEL_TIMEOUT_SECONDS, +) -> RunnerCancelResponse: + """POST the runner's `/cancel`. Returns the acknowledgement and the answering replica. + + Never raises. The command row is already committed when this runs, so a failure here costs + promptness, not the Stop: a later claim or the settlement sweep still reaches it. + + The body is camelCase because the runner's own HTTP surface is (see its `/kill`). + """ + base_url = env.runner.internal_url + token = env.runner.token + if not base_url or not token: + log.warning( + "cancel: no runner internal_url/token configured; command %s cannot be delivered", + command_id, + ) + return RunnerCancelResponse(RunnerCancelResult.unreachable) + + url = base_url.rstrip("/") + "/cancel" + try: + async with httpx.AsyncClient(timeout=timeout_seconds) as client: + response = await client.post( + url, + json={ + "commandId": command_id, + "projectId": project_id, + "sessionId": session_id, + "targetTurnId": target_turn_id, + "createdAt": created_at, + }, + headers={"Authorization": f"Bearer {token}"}, + ) + except httpx.HTTPError as e: + log.warning( + "cancel: runner /cancel call failed for session=%s command=%s: %s", + session_id, + command_id, + e, + ) + return RunnerCancelResponse(RunnerCancelResult.unreachable) + + if response.status_code == 404: + return RunnerCancelResponse(RunnerCancelResult.not_held) + if response.status_code >= 300: + log.warning( + "cancel: runner /cancel returned %s for session=%s command=%s", + response.status_code, + session_id, + command_id, + ) + return RunnerCancelResponse(RunnerCancelResult.unreachable) + + replica_id = None + try: + payload = response.json() + if isinstance(payload, dict): + replica_id = payload.get("replicaId") + except ValueError: + # A 2xx with no JSON body still means accepted; the claim then falls back to a + # placeholder and the runner's report is refused, so log it rather than hide it. + log.warning( + "cancel: runner /cancel answered %s with no JSON body for command=%s", + response.status_code, + command_id, + ) + return RunnerCancelResponse(RunnerCancelResult.accepted, replica_id) diff --git a/api/oss/src/core/sessions/streams/service.py b/api/oss/src/core/sessions/streams/service.py index 95b75f2153c..8c1d2de2549 100644 --- a/api/oss/src/core/sessions/streams/service.py +++ b/api/oss/src/core/sessions/streams/service.py @@ -67,6 +67,7 @@ SessionIdInvalid, SessionStreamAlreadyExists, SessionTurnInUse, + SessionTurnMismatch, ) from oss.src.core.sessions.streams.interfaces import SessionStreamsDAOInterface from oss.src.core.sessions.streams.runner_client import kill_runner_sandbox @@ -167,26 +168,74 @@ async def _supersede_turns( turn_id=turn_id, ) - async def _displace_turns(self, *, project_id: UUID, session_id: str) -> None: - """Tear alive+running off whichever turn holds them, tombstoning it first. + async def _displace_turns( + self, + *, + project_id: UUID, + session_id: str, + expected_turn_id: Optional[str] = None, + running_only: bool = False, + ) -> List[str]: + """Tombstone and release the selected turn owners. The order is the point. Clearing first leaves a window in which the turn being displaced heartbeats, finds `alive` free and nx-acquires it straight back - a cancelled session then reads as alive for a whole ALIVE_TTL. Tombstoning first makes - that beat refuse itself. The keys are still re-read after the clear, so a turn that - took them inside the window is tombstoned too. + that beat refuse itself. Broad displacement re-reads the keys after clearing them; + running-only cancellation uses owner-checked releases so it cannot touch another turn. """ + alive_owner = await get_alive_owner( + self._lock, + project_id=str(project_id), + session_id=session_id, + ) + running_owner = await get_running_owner( + self._lock, + project_id=str(project_id), + session_id=session_id, + ) + if running_only: + if running_owner is None: + return [] + await self._supersede_turns( + project_id=project_id, + session_id=session_id, + turn_ids=(running_owner,), + ) + await release_alive( + self._lock, + project_id=str(project_id), + session_id=session_id, + turn_id=running_owner, + ) + await release_running( + self._lock, + project_id=str(project_id), + session_id=session_id, + turn_id=running_owner, + ) + return [running_owner] + + if expected_turn_id is not None: + actual = next( + ( + owner + for owner in (running_owner, alive_owner) + if owner is not None and owner != expected_turn_id + ), + None, + ) + if actual is not None: + raise SessionTurnMismatch( + session_id, + expected_turn_id=expected_turn_id, + actual_turn_id=actual, + ) + await self._supersede_turns( project_id=project_id, session_id=session_id, - turn_ids=( - await get_alive_owner( - self._lock, project_id=str(project_id), session_id=session_id - ), - await get_running_owner( - self._lock, project_id=str(project_id), session_id=session_id - ), - ), + turn_ids=(alive_owner, running_owner, expected_turn_id), ) displaced_alive = await force_cancel_alive( self._lock, project_id=str(project_id), session_id=session_id @@ -199,6 +248,19 @@ async def _displace_turns(self, *, project_id: UUID, session_id: str) -> None: session_id=session_id, turn_ids=(displaced_alive, displaced_running), ) + return list( + dict.fromkeys( + turn_id + for turn_id in ( + alive_owner, + running_owner, + expected_turn_id, + displaced_alive, + displaced_running, + ) + if turn_id is not None + ) + ) async def _publish_lifecycle( self, *, project_id: UUID, session_id: str, state: str @@ -211,6 +273,19 @@ async def _publish_lifecycle( state=state, ) + async def publish_session_ended(self, *, project_id: UUID, session_id: str) -> None: + """Announce that a turn ended, on the channel every open browser already listens to. + + Public because the durable-command plane settles a Stop and has to publish the same + notification the ordinary end-of-turn path publishes. There is one `ended` event, not a + Stop-shaped one and a turn-shaped one; a client cannot be asked to tell them apart. + """ + await self._publish_lifecycle( + project_id=project_id, + session_id=session_id, + state=WATCH_LIFECYCLE_ENDED, + ) + async def _publish_changed(self, *, project_id: UUID, session_id: str) -> None: if self._watch is None: return @@ -287,21 +362,28 @@ async def command( ) elif mode == CommandMode.cancel: - await self._displace_turns(project_id=project_id, session_id=session_id) - await self._mark_stream_ended( + cancelled_turn_ids = await self._displace_turns( project_id=project_id, - user_id=user_id, session_id=session_id, + expected_turn_id=request.expected_execution_id, + running_only=request.expected_execution_id is None, ) - await self._publish_lifecycle( - project_id=project_id, - session_id=session_id, - state=WATCH_LIFECYCLE_ENDED, - ) + if cancelled_turn_ids: + await self._mark_stream_ended( + project_id=project_id, + user_id=user_id, + session_id=session_id, + ) + await self._publish_lifecycle( + project_id=project_id, + session_id=session_id, + state=WATCH_LIFECYCLE_ENDED, + ) return SessionStreamCommandResponse( mode=mode, session_id=session_id, detached=True, + cancelled_turn_ids=cancelled_turn_ids, ) else: # ATTACH @@ -1085,6 +1167,33 @@ async def _start_turn( await self._publish_changed(project_id=project_id, session_id=session_id) return turn_id + async def mirror_liveness( + self, + *, + project_id: UUID, + session_id: str, + user_id: Optional[UUID] = None, + ) -> None: + """Write the Redis nest onto the row, for a caller that changed the nest itself. + + Durable Stop settlement is that caller, and it is the one nest change no heartbeat can + mirror. Settlement tombstones the stopped execution BEFORE it releases `running`, so the + runner's own final `is_running=false` beat is refused by the tombstone check in + `heartbeat` above and returns before the mirror write at the end of that method. The + order cannot be swapped: a late beat that found `alive` free would take it straight back + under the dead turn's id. Without this method the row therefore keeps `is_running: true` + until the orphan sweep collapses it minutes later, and `query_streams` reads Postgres + alone, so the tab that pressed Stop sees its own session running somewhere else. + + Re-reads Redis rather than writing a literal `false`, so a newer turn that has already + taken `running` is reported, not erased. + """ + await self._mirror_flags( + project_id=project_id, + user_id=user_id, + session_id=session_id, + ) + async def _mirror_flags( self, *, diff --git a/api/oss/src/core/sessions/streams/types.py b/api/oss/src/core/sessions/streams/types.py index d55490c499a..2a4e58cd330 100644 --- a/api/oss/src/core/sessions/streams/types.py +++ b/api/oss/src/core/sessions/streams/types.py @@ -36,6 +36,24 @@ def __init__(self, session_id: str, liveness: dict): super().__init__(self.message) +class SessionTurnMismatch(SessionStreamError): + def __init__( + self, + session_id: str, + *, + expected_turn_id: str, + actual_turn_id: str | None, + ) -> None: + self.session_id = session_id + self.expected_turn_id = expected_turn_id + self.actual_turn_id = actual_turn_id + self.message = ( + f"expected execution '{expected_turn_id}' is not the running execution " + f"(current: {actual_turn_id or 'none'})" + ) + super().__init__(self.message) + + class ConcurrencyLimitExceeded(SessionStreamError): """Raised when the per-project concurrent-run limit is exceeded.""" diff --git a/api/oss/src/dbs/http/__init__.py b/api/oss/src/dbs/http/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/oss/src/dbs/http/sessions/__init__.py b/api/oss/src/dbs/http/sessions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/oss/src/dbs/http/sessions/control_delivery_direct.py b/api/oss/src/dbs/http/sessions/control_delivery_direct.py new file mode 100644 index 00000000000..dd470dcfb4d --- /dev/null +++ b/api/oss/src/dbs/http/sessions/control_delivery_direct.py @@ -0,0 +1,81 @@ +"""The direct-call control-delivery adapter. + +The API posts the command to the runner's own `/cancel`, over the same authenticated hop that +already carries hard kill. There is no held connection, no poll loop and no per-session Redis +channel: one runner process, one request. + +WHAT THIS ADAPTER IS NOT ALLOWED TO DO. Durability, authorization, idempotency, the state +machine and terminal settlement all live in `SessionCommandsService`. This file is transport. +Replacing it with a long-poll adapter must change no route, no data shape and no transition. + +THE ORDER IS NOT NEGOTIABLE. The command row is committed BEFORE `deliver` is called. Calling +first and recording afterwards would give back every failure the record exists to close: a crash +between the call and the insert leaves an aborted execution with no terminal outcome written +anywhere. + +WHERE IT FAILS, AND HOW THAT IS MADE LOUD. `env.runner.internal_url` is one service address. +Behind a load balancer with two runner replicas the call reaches the right process only by luck. +That failure is quiet at the transport level, because the wrong process honestly answers "I do +not hold that session" — the same answer a session that really ended gives. + +The detector is exact, and it is NOT in this file. A `not_held` for a session whose row says +alive with a heartbeat younger than one interval means some process is running that session and +it is not the one we just called; nothing else produces that. It needs the session row, so it +lives in `SessionCommandsService._settle_not_held`, next to the settlement it decides: the +command settles `lost` rather than `not_running`, so the user is told the Stop failed instead of +being told the work had already finished. + +There is deliberately no replica census here. An earlier version counted the replica ids that +had heartbeated recently and refused to deliver when it saw more than one. It refused after +every ordinary runner restart, because a runner mints a fresh id at boot when +`AGENTA_RUNNER_REPLICA_ID` is unset, so its own previous id was still inside the window. That +broke Stop for the whole window after every deploy, which is worse than the failure it guarded. +""" + +from typing import Optional +from uuid import UUID + +from oss.src.core.sessions.commands.dtos import SessionCommand +from oss.src.core.sessions.commands.interfaces import ( + ControlDeliveryPort, + DeliveryReceipt, +) +from oss.src.core.sessions.streams.runner_client import ( + RunnerCancelResult, + cancel_runner_execution, +) +from oss.src.utils.env import env +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) + + +class DirectControlDelivery(ControlDeliveryPort): + def __init__(self, *, timeout_seconds: Optional[float] = None) -> None: + self._timeout = ( + timeout_seconds + if timeout_seconds is not None + else env.agenta.sessions.commands.delivery_timeout_seconds + ) + + async def deliver(self, *, command: SessionCommand) -> DeliveryReceipt: + answer = await cancel_runner_execution( + command_id=str(command.id), + project_id=str(command.project_id), + session_id=command.session_id, + target_turn_id=command.target_turn_id, + created_at=command.created_at.isoformat() if command.created_at else "", + timeout_seconds=self._timeout, + ) + if answer.status == RunnerCancelResult.accepted: + # The answering replica's own id, so the claim the service writes matches the id + # the runner reports its outcome with. + return DeliveryReceipt(status="accepted", replica_id=answer.replica_id) + if answer.status == RunnerCancelResult.not_held: + return DeliveryReceipt(status="not_held") + return DeliveryReceipt(status="unreachable") + + async def acknowledge(self, *, command_id: UUID, replica_id: str) -> None: + """A no-op: the claim compare-and-set in the DAO IS the acknowledgement, and the direct + adapter keeps no delivery bookkeeping of its own.""" + return None diff --git a/api/oss/src/dbs/postgres/sessions/commands/__init__.py b/api/oss/src/dbs/postgres/sessions/commands/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/oss/src/dbs/postgres/sessions/commands/dao.py b/api/oss/src/dbs/postgres/sessions/commands/dao.py new file mode 100644 index 00000000000..70fa358a567 --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/commands/dao.py @@ -0,0 +1,397 @@ +"""Storage for durable session commands. + +Every state transition is one `UPDATE ... WHERE RETURNING *`, decided by +`scalar_one_or_none()`. That is what makes two API replicas unable to both win a claim or both +write a terminal outcome, and it is the same pattern +`SessionInteractionsDAO.transition_interaction` already uses. +""" + +from datetime import datetime, timedelta, timezone +from typing import List, Optional +from uuid import UUID + +from sqlalchemy import and_, func, or_, select, update as sa_update +from sqlalchemy.exc import IntegrityError + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandKind, + SessionCommandSettle, + SessionCommandState, +) +from oss.src.core.sessions.commands.interfaces import ( + CommandCreateResult, + SessionCommandsDAOInterface, + SessionScope, +) +from oss.src.dbs.postgres.sessions.commands.dbes import SessionCommandDBE +from oss.src.dbs.postgres.sessions.commands.mappings import ( + map_command_dbe_to_dto, + map_command_dto_to_dbe_create, +) +from oss.src.dbs.postgres.sessions.streams.dbes import SessionStreamDBE +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) + +_OPEN_STATES = (SessionCommandState.pending.value, SessionCommandState.claimed.value) + + +class SessionCommandsDAO(SessionCommandsDAOInterface): + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine + + async def create_command( + self, + *, + user_id: Optional[UUID], + command: SessionCommandCreate, + stopping_turn_id: Optional[str] = None, + ) -> SessionCommand: + result = await self.create_command_with_status( + user_id=user_id, + command=command, + stopping_turn_id=stopping_turn_id, + ) + return result.command + + async def create_command_with_status( + self, + *, + user_id: Optional[UUID], + command: SessionCommandCreate, + stopping_turn_id: Optional[str] = None, + ) -> CommandCreateResult: + """Insert the command and stamp the session row's `stopping_turn_id` together. + + One transaction, on purpose. A user whose Stop was recorded but whose session row never + learned it is waiting has a session that renders as plainly running while a command + exists to stop it, and nothing later reconciles the two. + + `session_streams` is written from here rather than through the streams DAO because + sharing one transaction is the whole requirement, and the streams DAO opens its own. + """ + dbe = map_command_dto_to_dbe_create(user_id=user_id, command=command) + + try: + async with self.engine.session() as session: + session.add(dbe) + if stopping_turn_id is not None: + await session.execute( + sa_update(SessionStreamDBE) + .where( + SessionStreamDBE.project_id == command.project_id, + SessionStreamDBE.session_id == command.session_id, + SessionStreamDBE.deleted_at.is_(None), + ) + .values(stopping_turn_id=stopping_turn_id) + ) + await session.commit() + await session.refresh(dbe) + return CommandCreateResult( + command=map_command_dbe_to_dto(dbe), inserted=True + ) + except IntegrityError: + # One of two unique constraints refused this insert, and both mean the same thing: + # a command for this intent already exists. Return it rather than a second command. + # + # uq_session_commands_idempotency — the caller retried with the same key. + # uq_session_commands_open_target — another request is already stopping this + # execution, which is what makes two Stops in + # the SAME INSTANT one command. Admission's own + # read cannot see a row that has not committed + # yet, so the database is the decider. + if command.idempotency_key is not None: + existing = await self.fetch_by_idempotency_key( + project_id=command.project_id, + session_id=command.session_id, + idempotency_key=command.idempotency_key, + ) + if existing is not None: + return CommandCreateResult(command=existing, inserted=False) + open_command = await self.fetch_open_command( + project_id=command.project_id, + session_id=command.session_id, + kind=command.kind, + target_turn_id=command.target_turn_id, + ) + if open_command is None: + raise + return CommandCreateResult(command=open_command, inserted=False) + + async def fetch_by_idempotency_key( + self, + *, + project_id: UUID, + session_id: str, + idempotency_key: str, + ) -> Optional[SessionCommand]: + async with self.engine.session() as session: + stmt = select(SessionCommandDBE).where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.session_id == session_id, + SessionCommandDBE.idempotency_key == idempotency_key, + ) + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def fetch_open_command( + self, + *, + project_id: UUID, + session_id: str, + kind: SessionCommandKind, + target_turn_id: Optional[str], + ) -> Optional[SessionCommand]: + async with self.engine.session() as session: + stmt = ( + select(SessionCommandDBE) + .where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.session_id == session_id, + SessionCommandDBE.kind == kind.value, + SessionCommandDBE.state.in_(_OPEN_STATES), + SessionCommandDBE.deleted_at.is_(None), + ( + SessionCommandDBE.target_turn_id.is_(None) + if target_turn_id is None + else SessionCommandDBE.target_turn_id == target_turn_id + ), + ) + .order_by(SessionCommandDBE.created_at.desc()) + .limit(1) + ) + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def fetch_command( + self, + *, + command_id: UUID, + project_id: Optional[UUID] = None, + ) -> Optional[SessionCommand]: + async with self.engine.session() as session: + stmt = select(SessionCommandDBE).where( + SessionCommandDBE.id == command_id, + ) + if project_id is not None: + stmt = stmt.where(SessionCommandDBE.project_id == project_id) + result = await session.execute(stmt) + dbe = result.scalars().first() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def claim_commands( + self, + *, + sessions: List[SessionScope], + replica_id: str, + lease_seconds: int, + limit: int, + ) -> List[SessionCommand]: + """Take pending commands for the sessions the caller declares it holds warm. + + The runner declaring what it holds is the routing input, not a replica id: a parked + session's Redis owner key expires, but the session is still in the runner's pool. + """ + if not sessions or limit <= 0: + return [] + + scope_filter = or_( + *[ + and_( + SessionCommandDBE.project_id == scope.project_id, + SessionCommandDBE.session_id == scope.session_id, + ) + for scope in sessions + ] + ) + + async with self.engine.session() as session: + selectable = ( + select(SessionCommandDBE.project_id, SessionCommandDBE.id) + .where( + SessionCommandDBE.state == SessionCommandState.pending.value, + SessionCommandDBE.deleted_at.is_(None), + scope_filter, + ) + .order_by(SessionCommandDBE.created_at) + .limit(limit) + # Two API replicas serving two claims at the same time must neither block on + # each other nor hand out the same command twice. + .with_for_update(skip_locked=True) + ) + rows = (await session.execute(selectable)).all() + if not rows: + await session.commit() + return [] + + keys = or_( + *[ + and_( + SessionCommandDBE.project_id == row[0], + SessionCommandDBE.id == row[1], + ) + for row in rows + ] + ) + now = datetime.now(timezone.utc) + stmt = ( + sa_update(SessionCommandDBE) + .where( + keys, + SessionCommandDBE.state == SessionCommandState.pending.value, + ) + .values( + state=SessionCommandState.claimed.value, + claimed_by=replica_id, + claim_expires_at=now + timedelta(seconds=lease_seconds), + claim_count=SessionCommandDBE.claim_count + 1, + updated_at=now, + ) + .returning(SessionCommandDBE) + ) + claimed = (await session.execute(stmt)).scalars().all() + await session.commit() + return [map_command_dbe_to_dto(dbe) for dbe in claimed] + + async def claim_for_delivery( + self, + *, + project_id: UUID, + command_id: UUID, + replica_id: str, + lease_seconds: int, + ) -> Optional[SessionCommand]: + """`pending` to `claimed` for one named command, after a runner accepted it directly. + + None means somebody else already took or settled it, which is not an error: the runner + that answered will still report, and the outcome route decides on the stored state. + """ + async with self.engine.session() as session: + now = datetime.now(timezone.utc) + stmt = ( + sa_update(SessionCommandDBE) + .where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.id == command_id, + SessionCommandDBE.state == SessionCommandState.pending.value, + ) + .values( + state=SessionCommandState.claimed.value, + claimed_by=replica_id, + claim_expires_at=now + timedelta(seconds=lease_seconds), + claim_count=SessionCommandDBE.claim_count + 1, + updated_at=now, + ) + .returning(SessionCommandDBE) + ) + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + await session.commit() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def settle_command( + self, + *, + settle: SessionCommandSettle, + ) -> Optional[SessionCommand]: + """Terminal transition. None means the command was in none of the states the caller + expected, so the caller reads the stored row and answers 409 instead of letting a runner + retry. + + One statement, so the guard is evaluated at the moment of the write. Reading the state + first and updating after would reopen the very race this exists to close: the claim can + commit between the read and the write. + """ + async with self.engine.session() as session: + now = datetime.now(timezone.utc) + stmt = sa_update(SessionCommandDBE).where( + SessionCommandDBE.project_id == settle.project_id, + SessionCommandDBE.id == settle.command_id, + SessionCommandDBE.state.in_( + [state.value for state in settle.expected_states] + ), + ) + if settle.replica_id is not None: + # Only the replica holding the claim may write the outcome. A row still + # `pending` holds no claim, and refusing it there is what turned a correct + # abort into a command the sweep later called lost. + stmt = stmt.where( + or_( + SessionCommandDBE.claimed_by.is_(None), + SessionCommandDBE.claimed_by == settle.replica_id, + ) + ) + stmt = stmt.values( + state=settle.state.value, + outcome=settle.outcome.value, + settled_at=now, + updated_at=now, + ).returning(SessionCommandDBE) + result = await session.execute(stmt) + dbe = result.scalar_one_or_none() + await session.commit() + return map_command_dbe_to_dto(dbe) if dbe is not None else None + + async def clear_stopping_turn( + self, + *, + project_id: UUID, + session_id: str, + turn_id: Optional[str] = None, + ) -> None: + async with self.engine.session() as session: + stmt = ( + sa_update(SessionStreamDBE) + .where( + SessionStreamDBE.project_id == project_id, + SessionStreamDBE.session_id == session_id, + ) + .values(stopping_turn_id=None) + ) + if turn_id is not None: + # Only clear OUR marker. A settlement that arrives after a second Stop was + # admitted must not tell the browser the newer Stop already finished. + stmt = stmt.where(SessionStreamDBE.stopping_turn_id == turn_id) + await session.execute(stmt) + await session.commit() + + async def expire_claims( + self, + *, + now: datetime, + max_deliveries: int, + ) -> List[SessionCommand]: + async with self.engine.session() as session: + stmt = ( + select(SessionCommandDBE) + .where( + SessionCommandDBE.state == SessionCommandState.claimed.value, + SessionCommandDBE.deleted_at.is_(None), + SessionCommandDBE.claim_expires_at < now, + SessionCommandDBE.claim_count < max_deliveries, + ) + .order_by(SessionCommandDBE.claim_expires_at) + .limit(200) + ) + result = await session.execute(stmt) + rows = result.scalars().all() + return [map_command_dbe_to_dto(dbe) for dbe in rows] + + async def count_open(self, *, project_id: UUID, session_id: str) -> int: + """Open commands for a session. Diagnostics and tests only.""" + async with self.engine.session() as session: + stmt = select(func.count()).where( + SessionCommandDBE.project_id == project_id, + SessionCommandDBE.session_id == session_id, + SessionCommandDBE.state.in_(_OPEN_STATES), + SessionCommandDBE.deleted_at.is_(None), + ) + result = await session.execute(stmt) + return int(result.scalar() or 0) diff --git a/api/oss/src/dbs/postgres/sessions/commands/dbas.py b/api/oss/src/dbs/postgres/sessions/commands/dbas.py new file mode 100644 index 00000000000..0163f162bb7 --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/commands/dbas.py @@ -0,0 +1,52 @@ +from sqlalchemy import Column, Integer, String, TIMESTAMP + +from oss.src.dbs.postgres.shared.dbas import ( + DataDBA, + FlagsDBA, + IdentifierDBA, + LifecycleDBA, + MetaDBA, + ProjectScopeDBA, + TagsDBA, +) + + +class SessionCommandDBA( + ProjectScopeDBA, + LifecycleDBA, + IdentifierDBA, + DataDBA, + FlagsDBA, + TagsDBA, + MetaDBA, +): + """One durable request to change an execution. + + The delivery columns (`state`, `claimed_by`, `claim_expires_at`, `claim_count`) are flat + rather than nested in `data` because a claim query filters and orders on them and a JSON + blob cannot be indexed for that. Their names carry the grouping. + + `state` and `outcome` are never merged. `state` says where the COMMAND is; `outcome` says + what happened to the EXECUTION. + """ + + __abstract__ = True + + # Bare correlator, not a foreign key — the same rule every other sessions table follows. + session_id = Column(String, nullable=False) + kind = Column(String, nullable=False) + + # The execution the API resolved ONCE at admission and pinned. A turn that starts later has + # a different id, so a pinned command can never reach it. Null when nothing was running. + target_turn_id = Column(String, nullable=True) + # What the caller asserted, stored as sent, so a 409 stays explainable after the fact. + expected_turn_id = Column(String, nullable=True) + + state = Column(String, nullable=False) + claimed_by = Column(String, nullable=True) + claim_expires_at = Column(TIMESTAMP(timezone=True), nullable=True) + claim_count = Column(Integer, nullable=False, default=0, server_default="0") + + outcome = Column(String, nullable=True) + idempotency_key = Column(String, nullable=True) + settled_at = Column(TIMESTAMP(timezone=True), nullable=True) diff --git a/api/oss/src/dbs/postgres/sessions/commands/dbes.py b/api/oss/src/dbs/postgres/sessions/commands/dbes.py new file mode 100644 index 00000000000..f4a755aba9a --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/commands/dbes.py @@ -0,0 +1,80 @@ +from sqlalchemy import ( + CheckConstraint, + ForeignKeyConstraint, + Index, + PrimaryKeyConstraint, + UniqueConstraint, + text, +) + +from oss.src.dbs.postgres.shared.base import Base +from oss.src.dbs.postgres.sessions.commands.dbas import SessionCommandDBA + + +class SessionCommandDBE(Base, SessionCommandDBA): + __tablename__ = "session_commands" + + __table_args__ = ( + ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + PrimaryKeyConstraint("project_id", "id"), + # The caller's retry identity. Postgres treats nulls as distinct in a unique index, so a + # command with no client key never collides with another. + UniqueConstraint( + "project_id", + "session_id", + "idempotency_key", + name="uq_session_commands_idempotency", + ), + CheckConstraint("kind IN ('cancel')", name="ck_session_commands_kind"), + CheckConstraint( + "state IN ('pending', 'claimed', 'applied', 'obsolete')", + name="ck_session_commands_state", + ), + # ONE open command per target execution. Two Stops are one intent, and admission's + # read-then-insert cannot enforce that on its own: two requests that arrive in the same + # instant both find no open command and both insert. The database decides instead, and + # the DAO turns the losing insert into a read of the winner. + # + # `target_turn_id` is NULL only on a command that is inserted already settled, which the + # predicate excludes, so the fact that Postgres treats NULLs as distinct costs nothing. + Index( + "uq_session_commands_open_target", + "project_id", + "session_id", + "kind", + "target_turn_id", + unique=True, + postgresql_where=text( + "state IN ('pending', 'claimed') AND deleted_at IS NULL" + ), + ), + # The claim query's index, and the open-command collapse read at admission. Partial on + # the open states because a settled command is never claimed again. + Index( + "ix_session_commands_open", + "project_id", + "session_id", + "created_at", + postgresql_where=text( + "state IN ('pending', 'claimed') AND deleted_at IS NULL" + ), + ), + # The settlement sweep's index: expired leases, nothing else. + Index( + "ix_session_commands_claims", + "claim_expires_at", + postgresql_where=text("state = 'claimed' AND deleted_at IS NULL"), + ), + Index( + "ix_session_commands_project_session", + "project_id", + "session_id", + "created_at", + ), + # The runner reports an outcome with the command id ALONE (it holds no project + # credential), so that read needs an index that does not lead with the project. + Index( + "ix_session_commands_id", + "id", + ), + ) diff --git a/api/oss/src/dbs/postgres/sessions/commands/mappings.py b/api/oss/src/dbs/postgres/sessions/commands/mappings.py new file mode 100644 index 00000000000..65a36df1eca --- /dev/null +++ b/api/oss/src/dbs/postgres/sessions/commands/mappings.py @@ -0,0 +1,72 @@ +from typing import Optional +from uuid import UUID + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandKind, + SessionCommandOutcome, + SessionCommandState, +) +from oss.src.dbs.postgres.sessions.commands.dbes import SessionCommandDBE + + +def map_command_dto_to_dbe_create( + *, + user_id: Optional[UUID], + command: SessionCommandCreate, +) -> SessionCommandDBE: + return SessionCommandDBE( + project_id=command.project_id, + # + created_by_id=user_id, + # Stamped, not defaulted: the stale-Stop guard compares this value, so the row must + # carry exactly the instant that was compared. + **({"created_at": command.created_at} if command.created_at else {}), + # + session_id=command.session_id, + kind=command.kind.value, + target_turn_id=command.target_turn_id, + expected_turn_id=command.expected_turn_id, + # + state=command.state.value, + claim_count=0, + outcome=command.outcome.value if command.outcome else None, + settled_at=command.settled_at, + idempotency_key=command.idempotency_key, + # + data=command.data, + ) + + +def map_command_dbe_to_dto(dbe: SessionCommandDBE) -> SessionCommand: + return SessionCommand( + id=dbe.id, + # + created_at=dbe.created_at, + updated_at=dbe.updated_at, + deleted_at=dbe.deleted_at, + created_by_id=dbe.created_by_id, + updated_by_id=dbe.updated_by_id, + deleted_by_id=dbe.deleted_by_id, + # + project_id=dbe.project_id, + session_id=dbe.session_id, + kind=SessionCommandKind(dbe.kind), + # + target_turn_id=dbe.target_turn_id, + expected_turn_id=dbe.expected_turn_id, + data=dbe.data, + # + state=SessionCommandState(dbe.state), + claimed_by=dbe.claimed_by, + claim_expires_at=dbe.claim_expires_at, + claim_count=dbe.claim_count or 0, + # + outcome=SessionCommandOutcome(dbe.outcome) if dbe.outcome else None, + idempotency_key=dbe.idempotency_key, + settled_at=dbe.settled_at, + # + tags=dbe.tags, + meta=dbe.meta, + ) diff --git a/api/oss/src/dbs/postgres/sessions/interactions/dao.py b/api/oss/src/dbs/postgres/sessions/interactions/dao.py index 97043a77b46..a3432af8bdb 100644 --- a/api/oss/src/dbs/postgres/sessions/interactions/dao.py +++ b/api/oss/src/dbs/postgres/sessions/interactions/dao.py @@ -142,12 +142,12 @@ async def cancel_session_pending( except_turn_id: Optional[str] = None, except_tokens: Optional[List[str]] = None, only_turn_id: Optional[str] = None, - ) -> int: + ) -> List[SessionInteraction]: """Cancel still-pending interactions for a session. With `except_turn_id`, spare the current turn's own gates (used at turn start to cancel prior turns' unanswered gates; without it, cancel all of them, e.g. on kill). `except_tokens` spares prior-turn gates the current turn answers in-band, so the resume can resolve them instead. With - `only_turn_id`, touch nothing but that one turn's gates. Returns the count cancelled.""" + `only_turn_id`, touch nothing but that one turn's gates. Returns the rows cancelled.""" async with self.engine.session() as session: stmt = ( sa_update(SessionInteractionDBE) @@ -160,6 +160,7 @@ async def cancel_session_pending( status="cancelled", updated_at=datetime.now(timezone.utc), ) + .returning(SessionInteractionDBE) ) if only_turn_id is not None: stmt = stmt.where(SessionInteractionDBE.turn_id == only_turn_id) @@ -168,8 +169,11 @@ async def cancel_session_pending( if except_tokens: stmt = stmt.where(SessionInteractionDBE.token.notin_(except_tokens)) result = await session.execute(stmt) + cancelled = [ + map_interaction_dbe_to_dto(dbe) for dbe in result.scalars().all() + ] await session.commit() - return result.rowcount or 0 + return cancelled async def query_interactions( self, diff --git a/api/oss/src/dbs/postgres/sessions/streams/dbes.py b/api/oss/src/dbs/postgres/sessions/streams/dbes.py index 7dd82b6cb7d..47b7afb8e4e 100644 --- a/api/oss/src/dbs/postgres/sessions/streams/dbes.py +++ b/api/oss/src/dbs/postgres/sessions/streams/dbes.py @@ -63,6 +63,24 @@ class SessionStreamDBE( # (resumable, still listed); `archived_at` marks a deliberately-hidden one (restorable). archived_at = Column(TIMESTAMP(timezone=True), nullable=True) + # The execution an accepted Stop is waiting on. Written in the same transaction as the + # command insert, cleared at settlement. Null means nothing is stopping. + # + # A column and not a bit inside `flags`, because `flags` is the Redis mirror and every + # heartbeat rewrites it whole (`streams/service.py`, the unconditional mirror write), so a + # value stored there would be erased on the next beat. `SessionStreamEdit` carries only + # flags/tags/meta/turn_id, so the heartbeat path cannot touch this column by accident. + stopping_turn_id = Column(String, nullable=True) + + # When the row's CURRENT `turn_id` started. It exists for the stale-Stop guard, which has to + # compare a Stop's arrival time with the running execution's start time, and there was + # nowhere to read that: `updated_at` is the heartbeat timestamp and moves every 30 seconds, + # runner-minted turn ids are uuid4 and carry no time, the Redis lock value is a bare turn id + # that a Lua compare reads whole, and the `session_turns` append is fire-and-forget so a + # running turn may have no row. Stamped only when the id actually changes, so the repeated + # heartbeats that restamp the same id never move it. + turn_started_at = Column(TIMESTAMP(timezone=True), nullable=True) + __table_args__ = ( ForeignKeyConstraint( ["project_id"], diff --git a/api/oss/src/dbs/postgres/sessions/streams/mappings.py b/api/oss/src/dbs/postgres/sessions/streams/mappings.py index 2442b3e433f..33d43c872d4 100644 --- a/api/oss/src/dbs/postgres/sessions/streams/mappings.py +++ b/api/oss/src/dbs/postgres/sessions/streams/mappings.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from typing import Any, Dict, Optional from uuid import UUID @@ -135,6 +136,10 @@ def map_stream_dto_to_dbe_create( tags=stream.tags, meta=stream.meta, turn_id=stream.turn_id, + # A create that already names a turn IS that turn's start. Without this, the first row a + # `_start_turn` writes carries no start time and the stale-Stop guard cannot fire on the + # very first turn of a session. + turn_started_at=datetime.now(timezone.utc) if stream.turn_id else None, references=references_to_json(stream.references), ) @@ -157,6 +162,8 @@ def map_stream_dbe_to_dto( name=stream_dbe.name, description=stream_dbe.description, turn_id=stream_dbe.turn_id, + turn_started_at=stream_dbe.turn_started_at, + stopping_turn_id=stream_dbe.stopping_turn_id, references=references_from_json(stream_dbe.references), archived_at=stream_dbe.archived_at, flags=SessionStreamFlags.model_validate(stream_dbe.flags) @@ -199,6 +206,11 @@ def map_stream_dto_to_dbe_edit( if stream.meta is not None: stream_dbe.meta = stream.meta if stream.turn_id is not None: + # Stamp the start time only when the id actually CHANGES. A heartbeat restamps the same + # id every 30 seconds, and a start time that moved with each beat would make every Stop + # look like it arrived before its own turn began. + if stream_dbe.turn_id != stream.turn_id: + stream_dbe.turn_started_at = datetime.now(timezone.utc) stream_dbe.turn_id = stream.turn_id diff --git a/api/oss/src/middlewares/auth.py b/api/oss/src/middlewares/auth.py index 302bd481790..f9f3cc930ab 100644 --- a/api/oss/src/middlewares/auth.py +++ b/api/oss/src/middlewares/auth.py @@ -71,6 +71,12 @@ "/api/tools/connections/callback", "/preview/tools/connections/callback", "/api/preview/tools/connections/callback", + # SESSIONS CONTROL — the runner reports a command's outcome with the shared runner token, + # not a project credential: it holds none for a command it was handed. The route checks the + # token itself and resolves the project from the command id, so this exemption widens no + # tenant boundary. + "/sessions/control/commands/", + "/api/sessions/control/commands/", # TRIGGERS — inbound provider events arrive from Composio with no auth token "/triggers/composio/events/", "/api/triggers/composio/events/", diff --git a/api/oss/src/utils/env.py b/api/oss/src/utils/env.py index f3358fb7dbd..acc9e8864ed 100644 --- a/api/oss/src/utils/env.py +++ b/api/oss/src/utils/env.py @@ -569,10 +569,61 @@ class SessionAttachmentsConfig(BaseModel): model_config = ConfigDict(extra="ignore") +class SessionsCommandsConfig(BaseModel): + """Durable session commands: how a Stop reaches the runner, and how long it may wait. + + `adapter` picks the control-delivery transport behind `ControlDeliveryPort`: + + * `direct` — the API posts the command to the runner's own `/cancel`, over the + authenticated hop that already carries hard kill. One runner process, no held + connection, no poll loop. This is the default. + * `long_poll` — the runner holds a claim request open and the API answers it. Correct for + two or more runner replicas and for a runner the API cannot reach inbound. Not built in + this slice; naming it here fails loudly rather than silently falling back. + + `direct` calls one service address, so with two runner replicas behind a load balancer the + call lands on the right process only by luck. Nothing here guards that, on purpose: the + detector is exact and lives in the service, where a `not_held` for a session that is alive + and beating is the wrong-replica failure and nothing else produces it. + """ + + adapter: str = os.getenv("AGENTA_SESSIONS_CONTROL_ADAPTER") or "direct" + + # How long a claimed command may go unreported before the settlement sweep acts. Three + # heartbeat intervals. + lease_seconds: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_COMMAND_LEASE_SECONDS") or 90 + ) + # Bounds a delivery loop where a runner accepts a command and never reports. + max_deliveries: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_COMMAND_MAX_DELIVERIES") or 3 + ) + sweep_seconds: int = ( + _parse_optional_positive_int_env("AGENTA_SESSIONS_COMMAND_SWEEP_SECONDS") or 10 + ) + # A command nobody ever claimed is a runner that is not there. + admission_timeout_seconds: int = ( + _parse_optional_positive_int_env( + "AGENTA_SESSIONS_COMMAND_ADMISSION_TIMEOUT_SECONDS" + ) + or 90 + ) + # How long the direct call waits for the runner to acknowledge. The runner answers before + # it cancels anything, so this covers a network hop, not a harness cancel. + delivery_timeout_seconds: float = float( + os.getenv("AGENTA_SESSIONS_COMMAND_DELIVERY_TIMEOUT_SECONDS") or 5.0 + ) + model_config = ConfigDict(extra="ignore") + + class SessionsConfig(BaseModel): """Agenta sessions sub-namespace.""" + durable_stop: bool = ( + os.getenv("AGENTA_SESSIONS_DURABLE_STOP") or "false" + ).lower() in _TRUTHY attachments: SessionAttachmentsConfig = SessionAttachmentsConfig() + commands: SessionsCommandsConfig = SessionsCommandsConfig() records: SessionsRecordsConfig = SessionsRecordsConfig() model_config = ConfigDict(extra="ignore") diff --git a/api/oss/tests/pytest/unit/middlewares/test_auth_public_endpoints.py b/api/oss/tests/pytest/unit/middlewares/test_auth_public_endpoints.py new file mode 100644 index 00000000000..5108924ead3 --- /dev/null +++ b/api/oss/tests/pytest/unit/middlewares/test_auth_public_endpoints.py @@ -0,0 +1,35 @@ +import pytest +from starlette.requests import Request + +from oss.src.middlewares.auth import _check_authentication_token +from oss.src.utils.exceptions import UnauthorizedException + + +def _request(path: str) -> Request: + return Request( + { + "type": "http", + "method": "POST", + "path": path, + "headers": [], + "query_string": b"", + "scheme": "http", + "server": ("testserver", 80), + "root_path": "", + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", ["", "/api"]) +async def test_session_named_control_still_requires_project_auth(prefix): + with pytest.raises(UnauthorizedException): + await _check_authentication_token(_request(f"{prefix}/sessions/control/cancel")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", ["", "/api"]) +async def test_runner_command_outcome_route_remains_auth_exempt(prefix): + await _check_authentication_token( + _request(f"{prefix}/sessions/control/commands/command-id/outcome") + ) diff --git a/api/oss/tests/pytest/unit/sessions/test_command_matrix_inputs_data.py b/api/oss/tests/pytest/unit/sessions/test_command_matrix_inputs_data.py index d260c85e830..40b2dcf1cf0 100644 --- a/api/oss/tests/pytest/unit/sessions/test_command_matrix_inputs_data.py +++ b/api/oss/tests/pytest/unit/sessions/test_command_matrix_inputs_data.py @@ -29,7 +29,13 @@ SessionStreamCommandRequest, ) from oss.src.core.sessions.streams.service import SessionStreamsService -from oss.src.core.sessions.streams.types import SessionTurnInUse +from oss.src.core.sessions.streams.types import SessionTurnInUse, SessionTurnMismatch +from oss.src.dbs.redis.sessions.locks import ( + acquire_alive, + get_alive_owner, + get_running_owner, + is_turn_superseded, +) from unit.sessions.test_project_scoped_locks import _FakeRedis @@ -158,6 +164,139 @@ async def test_no_inputs_no_force_is_cancel(lock_engine): assert result.mode == CommandMode.cancel +@pytest.mark.asyncio +async def test_unfenced_cancel_before_new_turn_admission_ignores_the_parked_owner( + lock_engine, +): + session_id = _session_id() + await acquire_alive( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + turn_id="turn-A", + ) + existing = SessionStream( + id=uuid4(), + project_id=_PROJECT, + session_id=session_id, + turn_id="turn-A", + ) + dao = _FakeStreamsDAO(existing) + svc = _service(lock_engine, dao=dao) + + # Turn B was submitted by the browser but has not reached `_start_turn` yet. + result = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest(session_id=session_id), + ) + + assert result.mode == CommandMode.cancel + assert result.cancelled_turn_ids == [] + assert dao.row == existing + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=session_id + ) + == "turn-A" + ) + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=session_id + ) + is None + ) + assert not await is_turn_superseded( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + turn_id="turn-A", + ) + + +@pytest.mark.asyncio +async def test_unfenced_cancel_targets_the_turn_once_it_is_running(lock_engine): + dao = _FakeStreamsDAO() + svc = _service(lock_engine, dao=dao) + session_id = _session_id() + started = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=session_id, + data=WorkflowServiceRequestData(inputs={"messages": ["hi"]}), + ), + ) + + result = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest(session_id=session_id), + ) + + assert started.turn_id is not None + assert result.cancelled_turn_ids == [started.turn_id] + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=session_id + ) + is None + ) + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=session_id + ) + is None + ) + assert await is_turn_superseded( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + turn_id=started.turn_id, + ) + + +@pytest.mark.asyncio +async def test_cancel_with_a_stale_execution_guard_touches_no_holder(lock_engine): + svc = _service(lock_engine) + session_id = _session_id() + started = await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=session_id, + data=WorkflowServiceRequestData(inputs={"messages": ["first"]}), + ), + ) + + with pytest.raises(SessionTurnMismatch): + await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=session_id, + expected_execution_id="another-turn", + ), + ) + + assert ( + await get_alive_owner( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + ) + == started.turn_id + ) + assert ( + await get_running_owner( + lock_engine, + project_id=str(_PROJECT), + session_id=session_id, + ) + == started.turn_id + ) + + @pytest.mark.asyncio async def test_no_inputs_and_force_is_attach(lock_engine): svc = _service(lock_engine) diff --git a/api/oss/tests/pytest/unit/sessions/test_interaction_cancel_records.py b/api/oss/tests/pytest/unit/sessions/test_interaction_cancel_records.py new file mode 100644 index 00000000000..743d734d4b5 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_interaction_cancel_records.py @@ -0,0 +1,205 @@ +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest +from fastapi import FastAPI, HTTPException, Request + +from oss.src.apis.fastapi.sessions.models import SessionInteractionRespondRequest +from oss.src.apis.fastapi.sessions.router import InteractionsRouter +from oss.src.core.sessions.interactions.dtos import ( + SessionInteraction, + SessionInteractionKind, + SessionInteractionStatus, +) +from oss.src.core.sessions.interactions.service import SessionInteractionsService + + +class _RecordingPublisher: + def __init__(self, journal): + self.journal = journal + self.calls = [] + + async def interaction(self, *, project_id, session_id, status): + self.journal.append("publish") + self.calls.append((project_id, session_id, status)) + + +class _RecordingRecordsService: + def __init__(self, journal): + self.journal = journal + self.events = [] + + async def append_many(self, *, events): + self.journal.append("records") + self.events.extend(events) + return [] + + +class _FailingRecordsService: + async def append_many(self, *, events): + raise RuntimeError("records unavailable") + + +def _interaction(*, project_id, token, turn_id="turn-1"): + return SessionInteraction( + id=uuid4(), + project_id=project_id, + session_id="sess-1", + turn_id=turn_id, + token=token, + kind=SessionInteractionKind.user_approval, + status=SessionInteractionStatus.cancelled, + ) + + +@pytest.mark.asyncio +async def test_stop_cancel_writes_one_record_per_cancelled_interaction_before_publish(): + project_id = uuid4() + command_id = uuid4() + cancelled = [ + _interaction(project_id=project_id, token="gate-1"), + _interaction(project_id=project_id, token="gate-2"), + ] + dao = AsyncMock() + dao.cancel_session_pending = AsyncMock(return_value=cancelled) + journal = [] + records = _RecordingRecordsService(journal) + publisher = _RecordingPublisher(journal) + service = SessionInteractionsService( + interactions_dao=dao, + records_service=records, + watch_publisher=publisher, + ) + + count = await service.cancel_session_pending( + project_id=project_id, + session_id="sess-1", + only_turn_id="turn-1", + command_id=command_id, + ) + + assert count == 2 + assert len(records.events) == 2 + assert len({event.record_id for event in records.events}) == 2 + for event, interaction in zip(records.events, cancelled): + assert event.record_type == "interaction_response" + assert event.record_source == "agent" + assert event.turn_id == "turn-1" + assert event.attributes == { + "type": "interaction_response", + "id": interaction.token, + "kind": "user_approval", + "payload": { + "outcome": "cancelled", + "turnId": "turn-1", + "commandId": str(command_id), + }, + } + assert journal == ["records", "publish"] + assert publisher.calls == [(str(project_id), "sess-1", "resolved")] + + +@pytest.mark.asyncio +async def test_stop_cancel_writes_no_record_when_nothing_was_pending(): + project_id = uuid4() + dao = AsyncMock() + dao.cancel_session_pending = AsyncMock(return_value=[]) + journal = [] + records = _RecordingRecordsService(journal) + publisher = _RecordingPublisher(journal) + service = SessionInteractionsService( + interactions_dao=dao, + records_service=records, + watch_publisher=publisher, + ) + + count = await service.cancel_session_pending( + project_id=project_id, + session_id="sess-1", + only_turn_id="turn-1", + command_id=uuid4(), + ) + + assert count == 0 + assert records.events == [] + assert publisher.calls == [] + assert journal == [] + + +@pytest.mark.asyncio +async def test_record_failure_does_not_block_interaction_resolution_publish(): + project_id = uuid4() + dao = AsyncMock() + dao.cancel_session_pending = AsyncMock( + return_value=[_interaction(project_id=project_id, token="gate-1")] + ) + journal = [] + publisher = _RecordingPublisher(journal) + service = SessionInteractionsService( + interactions_dao=dao, + records_service=_FailingRecordsService(), + watch_publisher=publisher, + ) + + count = await service.cancel_session_pending( + project_id=project_id, + session_id="sess-1", + only_turn_id="turn-1", + command_id=uuid4(), + ) + + assert count == 1 + assert journal == ["publish"] + assert publisher.calls == [(str(project_id), "sess-1", "resolved")] + + +@pytest.mark.asyncio +async def test_answer_after_stop_returns_the_terminal_interaction_409_contract(): + project_id = uuid4() + user_id = uuid4() + interaction_id = uuid4() + interactions_service = AsyncMock() + interactions_service.fetch_interaction.return_value = SessionInteraction( + id=interaction_id, + project_id=project_id, + session_id="sess-1", + turn_id="turn-1", + token="gate-1", + kind=SessionInteractionKind.user_approval, + status=SessionInteractionStatus.cancelled, + ) + respond_task = AsyncMock() + respond_task.kiq = AsyncMock() + router = InteractionsRouter( + interactions_service=interactions_service, + workflows_service=AsyncMock(), + respond_task=respond_task, + ) + request = Request( + { + "type": "http", + "method": "POST", + "path": f"/sessions/interactions/{interaction_id}/respond", + "headers": [], + "app": FastAPI(), + } + ) + request.state.project_id = project_id + request.state.user_id = user_id + + with patch( + "oss.src.apis.fastapi.sessions.router.check_action_access", + new_callable=AsyncMock, + return_value=True, + ): + with pytest.raises(HTTPException) as exc_info: + await router.respond_interaction( + request=request, + interaction_id=interaction_id, + body=SessionInteractionRespondRequest(answer={"approved": True}), + ) + + assert exc_info.value.status_code == 409 + assert exc_info.value.detail == "Interaction is no longer pending" + interactions_service.transition_interaction.assert_not_awaited() + respond_task.kiq.assert_not_awaited() diff --git a/api/oss/tests/pytest/unit/sessions/test_runner_client_kill.py b/api/oss/tests/pytest/unit/sessions/test_runner_client_kill.py index 1966fdbeff5..2650cee59c4 100644 --- a/api/oss/tests/pytest/unit/sessions/test_runner_client_kill.py +++ b/api/oss/tests/pytest/unit/sessions/test_runner_client_kill.py @@ -9,7 +9,11 @@ import httpx import pytest -from oss.src.core.sessions.streams.runner_client import kill_runner_sandbox +from oss.src.core.sessions.streams.runner_client import ( + RunnerCancelResult, + cancel_runner_execution, + kill_runner_sandbox, +) class _FakeRunnerEnv: @@ -120,3 +124,44 @@ async def post(self, *a, **kw): result = await kill_runner_sandbox(project_id="proj-1", session_id="sess-1") assert result is False + + +@pytest.mark.asyncio +async def test_cancel_accepts_non_object_json_without_crashing(): + class _FakeResponse: + status_code = 200 + + @staticmethod + def json(): + return ["accepted"] + + class _FakeClient: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def post(self, *args, **kwargs): + return _FakeResponse() + + with ( + patch("oss.src.core.sessions.streams.runner_client.env") as mock_env, + patch( + "oss.src.core.sessions.streams.runner_client.httpx.AsyncClient", + return_value=_FakeClient(), + ), + ): + mock_env.runner = _FakeRunnerEnv( + internal_url="http://runner:8765", token="shared-secret" + ) + result = await cancel_runner_execution( + command_id="command-1", + project_id="project-1", + session_id="session-1", + target_turn_id="turn-1", + created_at="2026-09-04T00:00:00Z", + ) + + assert result.status == RunnerCancelResult.accepted + assert result.replica_id is None diff --git a/api/oss/tests/pytest/unit/sessions/test_session_cancel_admission.py b/api/oss/tests/pytest/unit/sessions/test_session_cancel_admission.py new file mode 100644 index 00000000000..fb31281f6f6 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_session_cancel_admission.py @@ -0,0 +1,1149 @@ +"""What a Stop request decides before anything durable is written. + +Admission is where a Stop can go wrong in the two ways that matter to a user. It can miss the +run they meant, and it can kill a run they never meant. These pin the rules that stop both: + + * the arrival time is stamped BEFORE any read, and stored as the row's `created_at`, so the + value the guard compared is the value the runner can re-compare; + * a stale `expected_execution_id` is refused and writes nothing at all; + * an execution that started AFTER the request arrived is never targeted; + * only a named Stop can reach a parked session, which holds `alive` and not `running`; + * two Stops in a row collapse onto one command; + * Redis is not written at admission, so the stopping execution keeps its locks while it stops. +""" + +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional +from unittest.mock import patch +from uuid import UUID, uuid4 + +import pytest +import pytest_asyncio +import uuid_utils.compat as uuid + +from oss.src.core.sessions.commands.dtos import ( + SessionCommand, + SessionCommandCreate, + SessionCommandOutcome, + SessionCommandState, +) +from oss.src.core.sessions.commands.interfaces import ( + CommandCreateResult, + DeliveryReceipt, +) +from oss.src.core.sessions.commands.service import SessionCommandsService +from oss.src.core.sessions.commands.types import ( + ExecutionExpectationFailed, + SessionCommandIdempotencyConflict, +) +from oss.src.core.sessions.streams.dtos import ( + CommandMode, + SessionStream, + SessionStreamCommandResponse, + SessionStreamFlags, +) +from oss.src.core.sessions.streams.types import SessionTurnMismatch +from oss.src.dbs.redis.sessions.locks import ( + acquire_alive, + acquire_running, + get_alive_owner, + get_running_owner, + get_session_liveness, + is_turn_superseded, + release_running, +) + +from unit.sessions.test_project_scoped_locks import _FakeRedis + + +_PROJECT = uuid4() +_USER = uuid4() +_SESSION = "session_cancel_admission" + + +class _FakeCommandsDAO: + """Enough of the DAO to observe what admission wrote, and how many times.""" + + def __init__(self) -> None: + self.rows: List[SessionCommand] = [] + self.stopping_turn_ids: List[Optional[str]] = [] + self.claims: List[Dict] = [] + + async def create_command( + self, *, user_id, command: SessionCommandCreate, stopping_turn_id=None + ): + row = SessionCommand( + id=uuid.uuid7(), + project_id=command.project_id, + session_id=command.session_id, + kind=command.kind, + target_turn_id=command.target_turn_id, + expected_turn_id=command.expected_turn_id, + data=command.data, + state=command.state, + outcome=command.outcome, + settled_at=command.settled_at, + idempotency_key=command.idempotency_key, + created_at=command.created_at, + ) + self.rows.append(row) + self.stopping_turn_ids.append(stopping_turn_id) + return row + + async def create_command_with_status( + self, *, user_id, command: SessionCommandCreate, stopping_turn_id=None + ): + if command.idempotency_key is not None: + for row in self.rows: + if ( + row.project_id == command.project_id + and row.session_id == command.session_id + and row.idempotency_key == command.idempotency_key + ): + return CommandCreateResult(command=row, inserted=False) + row = await self.create_command( + user_id=user_id, + command=command, + stopping_turn_id=stopping_turn_id, + ) + return CommandCreateResult(command=row, inserted=True) + + async def fetch_by_idempotency_key( + self, *, project_id, session_id, idempotency_key + ): + for row in self.rows: + if ( + row.project_id == project_id + and row.session_id == session_id + and row.idempotency_key == idempotency_key + ): + return row + return None + + async def fetch_open_command(self, *, project_id, session_id, kind, target_turn_id): + for row in reversed(self.rows): + if ( + row.project_id == project_id + and row.session_id == session_id + and row.kind == kind + and row.target_turn_id == target_turn_id + and row.state + in (SessionCommandState.pending, SessionCommandState.claimed) + ): + return row + return None + + async def fetch_command(self, *, command_id, project_id=None): + for row in self.rows: + if row.id == command_id: + return row + return None + + async def claim_for_delivery( + self, *, project_id, command_id, replica_id, lease_seconds + ): + # A copy, never a mutation of the object the caller holds — the real DAO returns a + # fresh row from RETURNING *, so admission's own view of the command stays as it was. + self.claims.append({"command_id": command_id, "replica_id": replica_id}) + for index, row in enumerate(self.rows): + if row.id == command_id and row.state == SessionCommandState.pending: + claimed = row.model_copy( + update={ + "state": SessionCommandState.claimed, + "claimed_by": replica_id, + } + ) + self.rows[index] = claimed + return claimed + return None + + async def claim_commands(self, **_): + return [] + + async def settle_command(self, *, settle): + for index, row in enumerate(self.rows): + if row.id == settle.command_id and row.state in settle.expected_states: + # Mirrors the real guard: a `pending` row holds no claim, so a null + # `claimed_by` passes; a claimed row must be claimed by the reporter. + if ( + settle.replica_id is not None + and row.claimed_by is not None + and row.claimed_by != settle.replica_id + ): + return None + settled = row.model_copy( + update={ + "state": settle.state, + "outcome": settle.outcome, + "settled_at": datetime.now(timezone.utc), + } + ) + self.rows[index] = settled + return settled + return None + + async def clear_stopping_turn(self, *, project_id, session_id, turn_id=None): + self.stopping_turn_ids.append(None) + + async def expire_claims(self, *, now, max_deliveries): + return [] + + +class _FakeStreamsService: + """The reads admission makes, plus the row settlement writes. + + `mirrored` stands in for the `session_streams` row. It records the nest exactly as the real + `_mirror_flags` would read it — from Redis, at the moment settlement calls — so a test can + assert what the ROW says and not merely that a call happened. `query_streams`, which is what + the product's liveness polls read, serves that row and never looks at Redis. + """ + + def __init__( + self, stream: Optional[SessionStream] = None, lock_engine=None + ) -> None: + self.stream = stream + self.ended: List[str] = [] + self.lock_engine = lock_engine + self.mirrored: List[Dict[str, bool]] = [] + + async def fetch_header(self, *, project_id: UUID, session_id: str): + return self.stream + + async def command(self, *, project_id, user_id, request): + actual = self.stream.turn_id if self.stream is not None else None + if ( + request.expected_execution_id is not None + and actual != request.expected_execution_id + ): + raise SessionTurnMismatch( + request.session_id, + expected_turn_id=request.expected_execution_id, + actual_turn_id=actual, + ) + return SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id=request.session_id, + turn_id=actual, + detached=True, + ) + + async def publish_session_ended(self, *, project_id: UUID, session_id: str): + self.ended.append(session_id) + + async def mirror_liveness(self, *, project_id: UUID, session_id: str, user_id=None): + snap = await get_session_liveness( + self.lock_engine, project_id=str(project_id), session_id=session_id + ) + self.mirrored.append( + { + "is_alive": snap["alive"], + "is_running": snap["running"], + "is_attached": snap["attached"], + } + ) + + +class _FakeInteractionsService: + def __init__(self) -> None: + self.cancelled: List[Optional[str]] = [] + self.command_ids: List[Optional[UUID]] = [] + + async def cancel_session_pending( + self, + *, + project_id, + session_id, + only_turn_id=None, + command_id=None, + **_, + ): + self.cancelled.append(only_turn_id) + self.command_ids.append(command_id) + return 1 + + +class _RecordingDelivery: + def __init__(self, status: str = "accepted") -> None: + self.status = status + self.delivered: List[SessionCommand] = [] + + async def deliver(self, *, command: SessionCommand) -> DeliveryReceipt: + self.delivered.append(command) + return DeliveryReceipt(status=self.status, replica_id="runner-1") + + async def acknowledge(self, *, command_id, replica_id) -> None: + return None + + +def _stream( + turn_id: Optional[str], turn_started_at: Optional[datetime] +) -> SessionStream: + return SessionStream( + id=uuid4(), + project_id=_PROJECT, + session_id=_SESSION, + turn_id=turn_id, + turn_started_at=turn_started_at, + flags=SessionStreamFlags(is_alive=True, is_running=True), + updated_at=datetime.now(timezone.utc), + ) + + +@pytest_asyncio.fixture +async def lock_engine(): + from oss.src.dbs.redis.shared.engine import LockEngine + + eng = LockEngine() + with patch.object(eng, "_client", return_value=_FakeRedis()): + yield eng + + +def _service(lock_engine, *, dao=None, streams=None, interactions=None, delivery=None): + streams = streams or _FakeStreamsService() + # The fake mirrors from Redis, so it reads the same engine the service writes through. + if streams.lock_engine is None: + streams.lock_engine = lock_engine + return SessionCommandsService( + commands_dao=dao or _FakeCommandsDAO(), + streams_service=streams, + interactions_service=interactions or _FakeInteractionsService(), + lock_engine=lock_engine, + delivery=delivery or _RecordingDelivery(), + ) + + +async def _run_turn(lock_engine, turn_id: str) -> None: + await acquire_alive( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id=turn_id + ) + await acquire_running( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id=turn_id + ) + + +@pytest.mark.asyncio +async def test_stop_on_a_running_turn_is_accepted_and_pins_the_target(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + started = datetime.now(timezone.utc) - timedelta(seconds=30) + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService(_stream("turn-A", started)), + delivery=delivery, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is True + assert admission.execution_id == "turn-A" + assert admission.command.state == SessionCommandState.pending + assert admission.command.target_turn_id == "turn-A" + # The row and the session marker are written together. + assert dao.stopping_turn_ids == ["turn-A"] + assert len(delivery.delivered) == 1 + + +@pytest.mark.asyncio +async def test_admission_does_not_touch_redis(lock_engine): + await _run_turn(lock_engine, "turn-A") + svc = _service( + lock_engine, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + ) + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + + # The stopping execution keeps both locks WHILE it stops, which is what prevents a second + # message from starting underneath it. + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-A" + ) + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-A" + ) + + +@pytest.mark.asyncio +async def test_stop_when_nothing_runs_is_settled_at_once(lock_engine): + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service(lock_engine, dao=dao, delivery=delivery) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is False + assert admission.execution_id is None + assert admission.command.state == SessionCommandState.obsolete + assert admission.command.outcome == SessionCommandOutcome.not_running + assert delivery.delivered == [], "nothing to deliver to" + assert dao.stopping_turn_ids == [None], "no session is stopping" + + +@pytest.mark.asyncio +async def test_stale_expected_execution_id_is_refused_and_writes_nothing(lock_engine): + await _run_turn(lock_engine, "turn-B") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-B", datetime.now(timezone.utc) - timedelta(seconds=5)) + ), + delivery=delivery, + ) + + with pytest.raises(ExecutionExpectationFailed) as excinfo: + await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + ) + + assert excinfo.value.current == "turn-B" + assert dao.rows == [], "a refused Stop must insert nothing" + assert delivery.delivered == [] + + +@pytest.mark.asyncio +async def test_legacy_cancel_keeps_the_expected_execution_guard(lock_engine): + await _run_turn(lock_engine, "turn-B") + svc = _service( + lock_engine, + streams=_FakeStreamsService( + _stream("turn-B", datetime.now(timezone.utc) - timedelta(seconds=5)) + ), + ) + + with pytest.raises(ExecutionExpectationFailed) as excinfo: + await svc.request_cancel_legacy( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + ) + + assert excinfo.value.current == "turn-B" + assert ( + await get_running_owner( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + ) + == "turn-B" + ) + + +@pytest.mark.asyncio +async def test_a_turn_that_started_after_the_request_is_never_targeted(lock_engine): + # The race: the user presses Stop, turn one ends, turn two starts, and only then does the + # request get applied. Turn two must not hear about it. + await _run_turn(lock_engine, "turn-two") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-two", datetime.now(timezone.utc) + timedelta(seconds=5)) + ), + delivery=delivery, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is False + assert admission.execution_id is None + assert admission.command.target_turn_id is None + assert admission.command.outcome == SessionCommandOutcome.superseded_by_newer_turn + assert delivery.delivered == [], "the newer turn is never contacted" + # And its locks are untouched. + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-two" + ) + + +@pytest.mark.asyncio +async def test_the_guard_does_not_fire_when_the_start_time_is_unknown(lock_engine): + # A row written before `turn_started_at` existed yields no comparison. Failing this way + # round is deliberate: refusing every Stop we cannot verify would break the common case. + await _run_turn(lock_engine, "turn-A") + svc = _service(lock_engine, streams=_FakeStreamsService(_stream("turn-A", None))) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is True + assert admission.command.target_turn_id == "turn-A" + + +@pytest.mark.asyncio +async def test_the_stored_created_at_is_the_value_that_was_compared(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + before = datetime.now(timezone.utc) + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + ) + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + after = datetime.now(timezone.utc) + + stored = dao.rows[0].created_at + assert stored is not None + # Stamped by the service, not defaulted by the server: the runner repeats this comparison. + assert before <= stored <= after + + +@pytest.mark.asyncio +async def test_unfenced_stop_before_new_turn_admission_ignores_the_parked_owner( + lock_engine, +): + # Turn B was submitted by the browser but has not established `running` yet. + await acquire_alive( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-parked", + ) + delivery = _RecordingDelivery() + dao = _FakeCommandsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService(_stream("turn-parked", None)), + delivery=delivery, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is False + assert admission.execution_id is None + assert admission.command.outcome == SessionCommandOutcome.not_running + assert delivery.delivered == [] + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-parked" + ) + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + is None + ) + assert not await is_turn_superseded( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-parked", + ) + + +@pytest.mark.asyncio +async def test_a_named_stop_reaches_a_parked_approval(lock_engine): + """The Stop the browser actually sends, on the session state Stop exists to reach. + + A parked approval has released `running` and still holds `alive` under the same turn id. + The browser always sends `expected_execution_id`, because it knows the id it streamed. If + the expectation is compared against `running` alone it is None here, so the named Stop is + refused with a conflict while the identical Stop without an expectation is accepted — the + guard firing on the one case it exists to allow, and the gate left pending. + """ + await acquire_alive( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-parked", + ) + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + streams=_FakeStreamsService(_stream("turn-parked", None)), + delivery=delivery, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-parked", + ) + + assert admission.accepted is True + assert admission.execution_id == "turn-parked" + assert len(delivery.delivered) == 1 + + +@pytest.mark.asyncio +async def test_a_named_stop_on_a_parked_session_still_refuses_a_different_turn( + lock_engine, +): + """The guard must keep working on the fallback, not merely stop firing. + + A user looking at a turn that finished, on a session now parked under a NEWER turn, must + still be refused: the id they named is not the one that would be stopped. + """ + await acquire_alive( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-new", + ) + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService(_stream("turn-new", None)), + delivery=delivery, + ) + + with pytest.raises(ExecutionExpectationFailed) as excinfo: + await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-old", + ) + + assert excinfo.value.current == "turn-new" + assert dao.rows == [] + assert delivery.delivered == [] + + +@pytest.mark.asyncio +async def test_two_stops_in_a_row_collapse_onto_one_command(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + ) + + first = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + second = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + idempotency_key="a-different-key", + ) + + assert len(dao.rows) == 1, "one intent, one command" + assert second.command.id == first.command.id + assert second.accepted is True + + +@pytest.mark.asyncio +async def test_reused_idempotency_key_replays_the_original_turn_without_redelivery( + lock_engine, +): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + svc = _service( + lock_engine, + dao=dao, + streams=streams, + delivery=delivery, + ) + + first = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + idempotency_key="same-request", + ) + dao.rows[0] = dao.rows[0].model_copy( + update={ + "state": SessionCommandState.applied, + "outcome": SessionCommandOutcome.stopped, + } + ) + await release_running( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-A", + ) + await acquire_running( + lock_engine, + project_id=str(_PROJECT), + session_id=_SESSION, + turn_id="turn-B", + ) + streams.stream = _stream( + "turn-B", datetime.now(timezone.utc) - timedelta(seconds=5) + ) + + replay = await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + idempotency_key="same-request", + ) + + assert replay.command.id == first.command.id + assert replay.command.state == SessionCommandState.applied + assert replay.execution_id == "turn-A" + assert replay.accepted is True + assert len(delivery.delivered) == 1, "an idempotent replay must not target turn-B" + + +@pytest.mark.asyncio +async def test_reused_idempotency_key_rejects_a_different_expected_execution( + lock_engine, +): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + delivery = _RecordingDelivery() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + delivery=delivery, + ) + await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-A", + idempotency_key="same-key-different-request", + ) + + with pytest.raises(SessionCommandIdempotencyConflict): + await svc.request_cancel( + project_id=_PROJECT, + user_id=_USER, + session_id=_SESSION, + expected_execution_id="turn-B", + idempotency_key="same-key-different-request", + ) + + assert len(dao.rows) == 1 + assert len(delivery.delivered) == 1 + + +@pytest.mark.asyncio +async def test_a_reachable_runner_that_does_not_hold_the_session_settles_at_once( + lock_engine, +): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + # A row that has not beaten for a long time: the session really did end, so `not_running` + # is the honest answer rather than the wrong-replica failure. + streams.stream.updated_at = datetime.now(timezone.utc) - timedelta(minutes=30) + interactions = _FakeInteractionsService() + svc = _service( + lock_engine, + dao=dao, + streams=streams, + interactions=interactions, + delivery=_RecordingDelivery(status="not_held"), + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is True, "the caller still gets a durable command" + assert dao.rows[0].state == SessionCommandState.obsolete + assert dao.rows[0].outcome == SessionCommandOutcome.not_running + assert interactions.cancelled == ["turn-A"] + assert streams.ended == [_SESSION] + + +@pytest.mark.asyncio +async def test_not_held_on_a_beating_session_is_reported_as_lost_not_finished( + lock_engine, +): + # The wrong-replica failure. The user must be told the Stop failed, never that the work had + # already finished. `_run_turn` holds `running`, which is the discriminator: an execution is + # being run somewhere, and it is not by the process we called. + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + svc = _service( + lock_engine, + dao=dao, + streams=streams, + delivery=_RecordingDelivery(status="not_held"), + ) + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + + assert dao.rows[0].outcome == SessionCommandOutcome.lost + + +@pytest.mark.asyncio +async def test_not_held_on_a_turn_that_just_ended_is_not_running_not_lost(lock_engine): + """The everyday late Stop: the answer landed, the user pressed Stop a moment after. + + The turn released `running` and left `alive` and a fresh heartbeat behind it, exactly as a + RUNNING turn would, so a beating-row test calls this a failed Stop and tells the user their + Stop was lost. Nothing was lost: the work finished. `running` is what separates the two, + because a session nobody is executing has no `running` owner at all. + """ + # `alive` only, which is what a turn leaves when it ends. + await acquire_alive( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION, turn_id="turn-A" + ) + dao = _FakeCommandsDAO() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + # Beating, and recently: the turn ended seconds ago, not half an hour ago. + streams.stream.updated_at = datetime.now(timezone.utc) + svc = _service( + lock_engine, + dao=dao, + streams=streams, + delivery=_RecordingDelivery(status="not_held"), + ) + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + + assert dao.rows[0].state == SessionCommandState.obsolete + assert dao.rows[0].outcome == SessionCommandOutcome.not_running + + +@pytest.mark.asyncio +async def test_an_unreachable_runner_leaves_the_command_open(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + delivery=_RecordingDelivery(status="unreachable"), + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + # Admission still succeeded. The command is durable, so a later delivery or the settlement + # sweep gives the user a terminal state instead of a Stop that vanished. + assert admission.accepted is True + assert dao.rows[0].state == SessionCommandState.pending + + +@pytest.mark.asyncio +async def test_settlement_releases_running_and_leaves_alive_alone(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + interactions = _FakeInteractionsService() + svc = _service(lock_engine, dao=dao, streams=streams, interactions=interactions) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert dao.rows[0].state == SessionCommandState.applied + assert dao.rows[0].outcome == SessionCommandOutcome.stopped + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + is None + ), "running is released under an owner check" + # THE assertion that pins warm resume. Force-deleting `alive` is what makes today's cancel + # read as a session teardown; Stop must leave the session as a finished turn leaves it. + assert ( + await get_alive_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-A" + ) + assert interactions.cancelled == ["turn-A"] + assert interactions.command_ids == [admission.command.id] + assert streams.ended == [_SESSION] + + +@pytest.mark.asyncio +async def test_settlement_writes_the_row_as_alive_and_not_running(lock_engine): + """The ROW, not only Redis — the row is the only thing the product's liveness polls read. + + Redis is already right the moment settlement returns, and the test above pins that. The row + is a separate write, and nothing else performs it: settlement tombstones the execution first, + so the runner's own final `is_running=false` heartbeat is refused before it reaches the + heartbeat's mirror write. Left unwritten, the row says `is_running: true` until the orphan + sweep collapses it minutes later, and the tab that pressed Stop shows its own session as + running somewhere else for that whole time. + """ + await _run_turn(lock_engine, "turn-A") + streams = _FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ) + svc = _service(lock_engine, streams=streams) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + # Written once, and written AFTER `running` was released — a mirror taken before the release + # would have recorded `is_running: True` and been exactly the bug. + assert streams.mirrored == [ + {"is_alive": True, "is_running": False, "is_attached": False} + ] + # And the mirror is the state a normally finished turn leaves behind, which is what makes + # the session read as resumable rather than as torn down. + assert streams.mirrored[-1]["is_alive"] is True + + +@pytest.mark.asyncio +async def test_a_settlement_that_stops_nothing_does_not_touch_the_row(lock_engine): + """`not_running` changes no lock, so it must not write the row either. + + An obsolete Stop lands here: the turn it named had already finished, a NEWER turn may hold + the nest, and a mirror write from this path would be a write the settlement has no business + making. The row is left to the live turn's own heartbeats. + """ + dao = _FakeCommandsDAO() + streams = _FakeStreamsService(None) + svc = _service(lock_engine, dao=dao, streams=streams) + + # Nothing running and nothing parked: admission settles the command at insert. + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + + assert admission.accepted is False + assert dao.rows[0].outcome == SessionCommandOutcome.not_running + assert streams.mirrored == [] + + +@pytest.mark.asyncio +async def test_an_outcome_that_beats_the_claim_still_settles(lock_engine): + """The race the runner wins on a fast abort, driven at the exact instant it happens. + + Admission inserts the command `pending`, hands it to the runner, and writes `claimed` only + after the runner answers. A runner that aborts inside that window reports its outcome while + the row still says `pending`. Guarded on `claimed` alone that report was refused with a + conflict, the command sat open, and the sweep later recorded a Stop that actually worked as + lost — with the user watching "stopping" for the whole sweep window. + + The delivery double below reports from inside `deliver`, which is precisely where the real + runner's report lands relative to the claim. + """ + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + holder: Dict[str, SessionCommandsService] = {} + + class _ReportsBeforeTheClaimCommits: + def __init__(self) -> None: + self.delivered: List[SessionCommand] = [] + self.state_at_report: Optional[SessionCommandState] = None + + async def deliver(self, *, command): + self.delivered.append(command) + # The window. Nothing has written `claimed` yet, and the runner is already done. + self.state_at_report = dao.rows[0].state + await holder["svc"].report_outcome( + command_id=command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + return DeliveryReceipt(status="accepted", replica_id="runner-1") + + async def acknowledge(self, *, command_id, replica_id): + return None + + delivery = _ReportsBeforeTheClaimCommits() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=5)) + ), + delivery=delivery, + ) + holder["svc"] = svc + + await svc.request_cancel(project_id=_PROJECT, user_id=_USER, session_id=_SESSION) + + assert delivery.state_at_report == SessionCommandState.pending, ( + "the test is only meaningful if the report really did beat the claim" + ) + assert dao.rows[0].state == SessionCommandState.applied + assert dao.rows[0].outcome == SessionCommandOutcome.stopped + # And the claim that arrives afterwards must not resurrect a settled command. + assert dao.rows[0].state == SessionCommandState.applied + + +@pytest.mark.asyncio +async def test_an_outcome_from_a_replica_that_does_not_hold_the_claim_is_refused( + lock_engine, +): + """Widening the guard to `pending` must not weaken it for a row that IS claimed. + + A claimed row names its holder, and only that holder may write the outcome. The null + `claimed_by` this change now admits exists solely for the unclaimed row. + """ + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=5)) + ), + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + assert dao.rows[0].state == SessionCommandState.claimed + + from oss.src.core.sessions.commands.types import SessionCommandNotClaimable + + with pytest.raises(SessionCommandNotClaimable): + await svc.report_outcome( + command_id=admission.command.id, + replica_id="a-different-replica", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert dao.rows[0].state == SessionCommandState.claimed + + +@pytest.mark.asyncio +async def test_a_second_outcome_report_changes_nothing(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + interactions = _FakeInteractionsService() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + interactions=interactions, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + from oss.src.core.sessions.commands.types import SessionCommandNotClaimable + + with pytest.raises(SessionCommandNotClaimable): + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="applied", + execution_id="turn-A", + execution_state="stopped", + ) + + assert interactions.cancelled == ["turn-A"], "the side effects run exactly once" + + +@pytest.mark.asyncio +async def test_a_superseded_report_leaves_the_newer_turns_locks_alone(lock_engine): + await _run_turn(lock_engine, "turn-A") + dao = _FakeCommandsDAO() + interactions = _FakeInteractionsService() + svc = _service( + lock_engine, + dao=dao, + streams=_FakeStreamsService( + _stream("turn-A", datetime.now(timezone.utc) - timedelta(seconds=30)) + ), + interactions=interactions, + ) + + admission = await svc.request_cancel( + project_id=_PROJECT, user_id=_USER, session_id=_SESSION + ) + await svc.report_outcome( + command_id=admission.command.id, + replica_id="runner-1", + result="obsolete", + execution_id="turn-A", + execution_state="superseded_by_newer_turn", + ) + + assert dao.rows[0].outcome == SessionCommandOutcome.superseded_by_newer_turn + # Nothing was stopped, so nothing is released and no gate is cancelled. + assert ( + await get_running_owner( + lock_engine, project_id=str(_PROJECT), session_id=_SESSION + ) + == "turn-A" + ) + assert interactions.cancelled == [] diff --git a/api/oss/tests/pytest/unit/sessions/test_session_cancel_feature_flag.py b/api/oss/tests/pytest/unit/sessions/test_session_cancel_feature_flag.py new file mode 100644 index 00000000000..b8d4283b071 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_session_cancel_feature_flag.py @@ -0,0 +1,107 @@ +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest +from fastapi import HTTPException + +from oss.src.apis.fastapi.sessions import router as router_module +from oss.src.apis.fastapi.sessions.models import SessionCancelRequest +from oss.src.apis.fastapi.sessions.router import SessionControlRouter +from oss.src.core.sessions.commands.dtos import SessionCommandState +from oss.src.core.sessions.streams.dtos import CommandMode, SessionStreamCommandResponse +from oss.src.utils.env import env + + +_PROJECT = UUID("00000000-0000-0000-0000-0000000000aa") +_USER = UUID("00000000-0000-0000-0000-0000000000bb") + + +def _request(): + return SimpleNamespace( + state=SimpleNamespace(project_id=_PROJECT, user_id=_USER), + headers={}, + ) + + +async def test_cancel_route_uses_legacy_path_when_durable_stop_is_off(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", False) + monkeypatch.setattr( + router_module, "check_action_access", AsyncMock(return_value=True) + ) + service = SimpleNamespace( + request_cancel_legacy=AsyncMock( + return_value=SessionStreamCommandResponse( + mode=CommandMode.cancel, + session_id="session-1", + turn_id="turn-1", + detached=True, + ) + ), + request_cancel=AsyncMock(), + ) + + response = await SessionControlRouter( + commands_service=service + ).cancel_session_execution( + _request(), + "session-1", + SessionCancelRequest(expected_execution_id="turn-1"), + ) + + service.request_cancel_legacy.assert_awaited_once_with( + project_id=_PROJECT, + user_id=_USER, + session_id="session-1", + expected_execution_id="turn-1", + ) + service.request_cancel.assert_not_awaited() + assert response.status_code == 200 + assert json.loads(response.body) == { + "mode": "cancel", + "session_id": "session-1", + "turn_id": "turn-1", + "watcher_id": None, + "detached": True, + "cancelled_turn_ids": [], + } + + +async def test_cancel_route_uses_durable_path_when_flag_is_on(monkeypatch): + monkeypatch.setattr(env.agenta.sessions, "durable_stop", True) + monkeypatch.setattr( + router_module, "check_action_access", AsyncMock(return_value=True) + ) + command = SimpleNamespace( + id=UUID("00000000-0000-0000-0000-0000000000cc"), + state=SessionCommandState.pending, + ) + service = SimpleNamespace( + request_cancel_legacy=AsyncMock(), + request_cancel=AsyncMock( + return_value=SimpleNamespace( + command=command, + execution_id="turn-1", + accepted=True, + ) + ), + ) + + response = await SessionControlRouter( + commands_service=service + ).cancel_session_execution(_request(), "session-1") + + service.request_cancel.assert_awaited_once() + service.request_cancel_legacy.assert_not_awaited() + assert response.status_code == 202 + + +def test_runner_token_rejects_non_ascii_credentials_as_unauthorized(monkeypatch): + monkeypatch.setattr(env.runner, "token", "shared-secret") + request = SimpleNamespace(headers={"X-Agenta-Runner-Token": "nøt-the-token"}) + + with pytest.raises(HTTPException) as exc_info: + router_module._assert_runner_token(request) + + assert exc_info.value.status_code == 401 diff --git a/api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py b/api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py new file mode 100644 index 00000000000..99f4e22555a --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py @@ -0,0 +1,545 @@ +"""The compare-and-set rules that make a command safe under concurrency. + +These run against a real Postgres, because what is being tested IS the database's behaviour: +a unique constraint, a partial index's predicate, `FOR UPDATE SKIP LOCKED`, and an `UPDATE ... +WHERE RETURNING *` that must be won by exactly one caller. + +The rule every one of them protects: one execution reaches exactly one terminal outcome, +written by exactly one writer. +""" + +import asyncio +import uuid +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import text + +from oss.src.core.sessions.commands.dtos import ( + SessionCommandCreate, + SessionCommandKind, + SessionCommandOutcome, + SessionCommandSettle, + SessionCommandState, +) +from oss.src.core.sessions.commands.interfaces import SessionScope +from oss.src.dbs.postgres.sessions.commands.dao import SessionCommandsDAO +import oss.src.dbs.postgres.shared.engine as engine_module +from oss.src.dbs.postgres.shared.engine import get_transactions_engine +import oss.src.models.db_models # noqa: F401 + + +pytestmark = pytest.mark.integration + + +@pytest.fixture(autouse=True) +async def _fresh_engine_per_test(): + if engine_module._transactions_engine is not None: + await engine_module._transactions_engine.close() + engine_module._transactions_engine = None + yield + if engine_module._transactions_engine is not None: + await engine_module._transactions_engine.close() + engine_module._transactions_engine = None + + +@pytest.fixture +async def command_scope(): + engine = get_transactions_engine() + user_id = uuid.uuid4() + organization_id = uuid.uuid4() + workspace_id = uuid.uuid4() + project_id = uuid.uuid4() + session_id = f"cmd-dao-{project_id.hex[:12]}" + + async with engine.session() as session: + await session.execute( + text( + "INSERT INTO users (id, uid, username, email) " + "VALUES (:id, :uid, :username, :email)" + ), + { + "id": user_id, + "uid": str(user_id), + "username": "command-dao-test", + "email": f"command-dao-{user_id.hex[:8]}@example.com", + }, + ) + await session.execute( + text( + "INSERT INTO organizations (id, name, owner_id) " + "VALUES (:id, :name, :owner_id)" + ), + { + "id": organization_id, + "name": "command-dao-test-org", + "owner_id": user_id, + }, + ) + await session.execute( + text( + "INSERT INTO workspaces (id, name, organization_id) " + "VALUES (:id, :name, :organization_id)" + ), + { + "id": workspace_id, + "name": "command-dao-test-workspace", + "organization_id": organization_id, + }, + ) + await session.execute( + text( + "INSERT INTO projects " + "(id, project_name, workspace_id, organization_id) " + "VALUES (:id, :project_name, :workspace_id, :organization_id)" + ), + { + "id": project_id, + "project_name": "command-dao-test-project", + "workspace_id": workspace_id, + "organization_id": organization_id, + }, + ) + # The session row the command's `stopping_turn_id` is stamped on. + await session.execute( + text( + "INSERT INTO session_streams (id, project_id, session_id, turn_id) " + "VALUES (:id, :project_id, :session_id, :turn_id)" + ), + { + "id": uuid.uuid4(), + "project_id": project_id, + "session_id": session_id, + "turn_id": "turn-A", + }, + ) + await session.commit() + + yield { + "engine": engine, + "project_id": project_id, + "user_id": user_id, + "session_id": session_id, + } + + +def _create(scope, **overrides) -> SessionCommandCreate: + payload = dict( + project_id=scope["project_id"], + session_id=scope["session_id"], + kind=SessionCommandKind.cancel, + target_turn_id="turn-A", + state=SessionCommandState.pending, + created_at=datetime.now(timezone.utc), + ) + payload.update(overrides) + return SessionCommandCreate(**payload) + + +async def _stopping_turn_id(scope) -> str: + async with scope["engine"].session() as session: + result = await session.execute( + text( + "SELECT stopping_turn_id FROM session_streams " + "WHERE project_id = :project_id AND session_id = :session_id" + ), + {"project_id": scope["project_id"], "session_id": scope["session_id"]}, + ) + return result.scalar() + + +async def test_the_command_and_the_stopping_marker_are_written_together(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + + command = await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope), + stopping_turn_id="turn-A", + ) + + assert command.state == SessionCommandState.pending + # A session that renders as plainly running while a command exists to stop it is a session + # nothing later reconciles, so the two writes share one transaction. + assert await _stopping_turn_id(command_scope) == "turn-A" + + +async def test_a_repeated_idempotency_key_returns_the_first_row(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + + first = await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope, idempotency_key="retry-me"), + ) + second = await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope, idempotency_key="retry-me"), + ) + + assert second.id == first.id + assert ( + await dao.count_open( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + ) + == 1 + ) + + +async def test_two_open_commands_for_one_execution_collapse_to_one(command_scope): + # Two Stops for the same execution are one intent, even with no idempotency key and even + # when admission's own read cannot see the other because it has not committed yet. The + # database refuses the second insert and the DAO answers with the command that exists. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + + first = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + second = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + assert second.id == first.id + assert ( + await dao.count_open( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + ) + == 1 + ) + + +async def test_two_concurrent_admissions_still_yield_one_command(command_scope): + # The race the unique index exists for: both inserts run before either commits. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + + first, second = await asyncio.wait_for( + asyncio.gather( + dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ), + dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ), + return_exceptions=True, + ), + timeout=30, + ) + + ids = {r.id for r in (first, second) if not isinstance(r, Exception)} + assert len(ids) == 1, f"expected one command, got {first!r} and {second!r}" + + +async def test_a_settled_command_does_not_block_a_new_one(command_scope): + # The unique index is partial on the OPEN states, so once a Stop has settled the next Stop + # against the same execution is a fresh command, not a constraint violation. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + first = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=first.id, + state=SessionCommandState.applied, + outcome=SessionCommandOutcome.stopped, + expected_states=[SessionCommandState.pending], + replica_id=None, + ) + ) + + second = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + assert second.id != first.id + + +async def test_the_open_command_read_finds_only_the_same_target(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope, target_turn_id="turn-A"), + ) + + same = await dao.fetch_open_command( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + kind=SessionCommandKind.cancel, + target_turn_id="turn-A", + ) + other = await dao.fetch_open_command( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + kind=SessionCommandKind.cancel, + target_turn_id="turn-B", + ) + + assert same is not None + assert other is None, "a different execution is a different intent" + + +async def test_a_settled_command_is_no_longer_open(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], + command=_create( + command_scope, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.not_running, + settled_at=datetime.now(timezone.utc), + ), + ) + + assert command.state == SessionCommandState.obsolete + assert ( + await dao.fetch_open_command( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + kind=SessionCommandKind.cancel, + target_turn_id="turn-A", + ) + is None + ) + + +async def test_two_concurrent_claims_of_one_command_yield_exactly_one_winner( + command_scope, +): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + scopes = [ + SessionScope( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + ) + ] + + # Bounded: both calls contend for the same row on separate pooled connections, so a + # regression that drops SKIP LOCKED would hang the run rather than fail it. + first, second = await asyncio.wait_for( + asyncio.gather( + dao.claim_commands( + sessions=scopes, replica_id="replica-1", lease_seconds=90, limit=10 + ), + dao.claim_commands( + sessions=scopes, replica_id="replica-2", lease_seconds=90, limit=10 + ), + ), + timeout=30, + ) + + assert len(first) + len(second) == 1, ( + "a command is delivered to one replica, not two" + ) + + +async def test_a_claim_ignores_sessions_the_caller_did_not_declare(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + claimed = await dao.claim_commands( + sessions=[ + SessionScope( + project_id=command_scope["project_id"], session_id="a-different-session" + ) + ], + replica_id="replica-1", + lease_seconds=90, + limit=10, + ) + + assert claimed == [] + + +async def test_the_claim_records_the_lease_and_counts_the_delivery(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + claimed = await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-1", + lease_seconds=90, + ) + + assert claimed is not None + assert claimed.state == SessionCommandState.claimed + assert claimed.claimed_by == "replica-1" + assert claimed.claim_count == 1 + assert claimed.claim_expires_at is not None + assert claimed.claim_expires_at > datetime.now(timezone.utc) + timedelta(seconds=60) + + +async def test_a_second_delivery_claim_finds_nothing_to_take(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-1", + lease_seconds=90, + ) + + again = await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-2", + lease_seconds=90, + ) + + assert again is None + + +async def test_only_the_replica_holding_the_claim_may_settle(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-1", + lease_seconds=90, + ) + + wrong = await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.applied, + outcome=SessionCommandOutcome.stopped, + replica_id="replica-2", + ) + ) + + assert wrong is None + stored = await dao.fetch_command(command_id=command.id) + assert stored.state == SessionCommandState.claimed, "the stored state is unchanged" + + +async def test_settling_an_already_terminal_command_changes_nothing(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=command.id, + replica_id="replica-1", + lease_seconds=90, + ) + settled = await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.applied, + outcome=SessionCommandOutcome.stopped, + replica_id="replica-1", + ) + ) + assert settled is not None + + repeat = await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.failed, + replica_id="replica-1", + ) + ) + + assert repeat is None, "one execution, one terminal outcome, one writer" + stored = await dao.fetch_command(command_id=command.id) + assert stored.outcome == SessionCommandOutcome.stopped + + +async def test_the_api_can_settle_a_pending_command_nobody_took(command_scope): + # The `not_held` case: a reachable runner said it does not hold the session, so there is no + # claim to guard on and the API settles it itself. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + settled = await dao.settle_command( + settle=SessionCommandSettle( + project_id=command_scope["project_id"], + command_id=command.id, + state=SessionCommandState.obsolete, + outcome=SessionCommandOutcome.not_running, + expected_states=[SessionCommandState.pending], + replica_id=None, + ) + ) + + assert settled is not None + assert settled.outcome == SessionCommandOutcome.not_running + + +async def test_the_runner_can_find_a_command_without_a_project_id(command_scope): + # The runner reports an outcome with the command id alone; it holds no project credential. + dao = SessionCommandsDAO(engine=command_scope["engine"]) + command = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + + found = await dao.fetch_command(command_id=command.id) + + assert found is not None + assert found.project_id == command_scope["project_id"] + + +async def test_clearing_the_stopping_marker_is_scoped_to_the_turn_it_set(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + await dao.create_command( + user_id=command_scope["user_id"], + command=_create(command_scope), + stopping_turn_id="turn-A", + ) + + # A settlement for an OLDER turn must not clear a newer Stop's marker. + await dao.clear_stopping_turn( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + turn_id="turn-older", + ) + assert await _stopping_turn_id(command_scope) == "turn-A" + + await dao.clear_stopping_turn( + project_id=command_scope["project_id"], + session_id=command_scope["session_id"], + turn_id="turn-A", + ) + assert await _stopping_turn_id(command_scope) is None + + +async def test_expire_claims_returns_only_leases_that_have_passed(command_scope): + dao = SessionCommandsDAO(engine=command_scope["engine"]) + fresh = await dao.create_command( + user_id=command_scope["user_id"], command=_create(command_scope) + ) + await dao.claim_for_delivery( + project_id=command_scope["project_id"], + command_id=fresh.id, + replica_id="replica-1", + lease_seconds=90, + ) + + # The sweep is deliberately NOT project-scoped: it settles every abandoned claim in the + # deployment, so assert on this command's presence rather than on the whole result. + now = datetime.now(timezone.utc) + assert fresh.id not in { + row.id for row in await dao.expire_claims(now=now, max_deliveries=3) + }, "a lease that has not passed is not swept" + # An hour later the same lease has passed, and the settlement sweep sees it. + later = await dao.expire_claims(now=now + timedelta(hours=1), max_deliveries=3) + assert fresh.id in {row.id for row in later} diff --git a/api/oss/tests/pytest/unit/sessions/test_watch_interactions_publish.py b/api/oss/tests/pytest/unit/sessions/test_watch_interactions_publish.py index 9835cb98452..790190c38a1 100644 --- a/api/oss/tests/pytest/unit/sessions/test_watch_interactions_publish.py +++ b/api/oss/tests/pytest/unit/sessions/test_watch_interactions_publish.py @@ -115,7 +115,12 @@ async def test_failed_transition_publishes_nothing(): @pytest.mark.asyncio async def test_cancel_sweep_publishes_resolved_only_when_it_cancelled(): dao = AsyncMock() - dao.cancel_session_pending = AsyncMock(return_value=2) + dao.cancel_session_pending = AsyncMock( + return_value=[ + _interaction("sess-1"), + _interaction("sess-1").model_copy(update={"token": "tok-2"}), + ] + ) svc, publisher = _service(dao) cancelled = await svc.cancel_session_pending( @@ -125,7 +130,7 @@ async def test_cancel_sweep_publishes_resolved_only_when_it_cancelled(): assert publisher.interaction_calls == [(str(_PROJECT), "sess-1", "resolved")] # No-op sweep: nothing was pending, nothing changed, nothing to notify. - dao.cancel_session_pending = AsyncMock(return_value=0) + dao.cancel_session_pending = AsyncMock(return_value=[]) publisher.interaction_calls.clear() await svc.cancel_session_pending(project_id=_PROJECT, session_id="sess-1") assert publisher.interaction_calls == [] diff --git a/api/oss/tests/pytest/unit/sessions/test_watch_lifecycle_publish.py b/api/oss/tests/pytest/unit/sessions/test_watch_lifecycle_publish.py index 1859c655f30..00603bf3464 100644 --- a/api/oss/tests/pytest/unit/sessions/test_watch_lifecycle_publish.py +++ b/api/oss/tests/pytest/unit/sessions/test_watch_lifecycle_publish.py @@ -139,6 +139,16 @@ async def test_cancel_publishes_lifecycle_ended(lock_engine): svc, publisher = _service(lock_engine) session_id = _session_id() + await svc.command( + project_id=_PROJECT, + user_id=_USER, + request=SessionStreamCommandRequest( + session_id=session_id, + data=WorkflowServiceRequestData(inputs={"messages": ["hi"]}), + ), + ) + publisher.lifecycle_calls.clear() + await svc.command( project_id=_PROJECT, user_id=_USER, diff --git a/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py b/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py index 5f44f2b18f3..2b4a36438b6 100644 --- a/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py +++ b/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py @@ -226,6 +226,58 @@ async def test_interaction_transition_preserves_data_and_optionally_adds_resolut assert transitioned_without_resolution.data.resolution is None +async def test_cancel_pending_returns_exactly_the_rows_it_transitioned( + interactions_dao, project +): + project_id = project["project_id"] + session_id = f"interaction-cancel-returning-{uuid.uuid4().hex[:8]}" + + for token in ("pending-1", "pending-2", "already-answered"): + await interactions_dao.create_interaction( + project_id=project_id, + user_id=None, + interaction=SessionInteractionCreate( + project_id=project_id, + session_id=session_id, + turn_id="turn-1", + token=token, + kind=SessionInteractionKind.user_approval, + ), + ) + + await interactions_dao.transition_interaction( + transition=SessionInteractionTransition( + project_id=project_id, + session_id=session_id, + token="already-answered", + status=SessionInteractionStatus.responded, + ) + ) + + cancelled = await interactions_dao.cancel_session_pending( + project_id=project_id, + session_id=session_id, + only_turn_id="turn-1", + ) + + assert {interaction.token for interaction in cancelled} == { + "pending-1", + "pending-2", + } + assert all( + interaction.status == SessionInteractionStatus.cancelled + for interaction in cancelled + ) + assert ( + await interactions_dao.cancel_session_pending( + project_id=project_id, + session_id=session_id, + only_turn_id="turn-1", + ) + == [] + ) + + # --------------------------------------------------------------------------- # SessionInteractionsDAO.delete_by_session_id — new hard delete # --------------------------------------------------------------------------- diff --git a/docs/design/session-control-and-live-events/api-design.md b/docs/design/session-control-and-live-events/api-design.md new file mode 100644 index 00000000000..76e6fef6095 --- /dev/null +++ b/docs/design/session-control-and-live-events/api-design.md @@ -0,0 +1,469 @@ +# API design: the routes version one exposes + +> AGENT-GENERATED, low weight. Draft for discussion. Mahmoud makes final decisions. + +This file holds the route contracts considered for the durable-command work. Version one adds the +public Cancel route, the internal outcome route, and the runner's direct Cancel route; the +long-poll claim contract is explicitly deferred. Everything else in the RFC's public interface +section stays in [the RFC](rfc.md). + +The design behind these routes is in +[the durable command design](spike-b-durable-commands-design.md). Read that first for the state +machine, the lease, the settlement rule and the failure cases. + +Version one ships the direct-call adapter behind the replaceable control-delivery port. The two +long-poll routes remain future contracts; selecting `long_poll` currently fails startup rather +than silently choosing an unimplemented transport. + +Conventions taken from the existing code, not invented here: + +- Request and response models live in `api/oss/src/apis/fastapi/sessions/models.py`, are plain + Pydantic models, and set `model_config = ConfigDict(extra="forbid")` on new request bodies + (`SessionQueryRequest`, `models.py:59`). +- List responses carry `count` plus the list (`SessionsResponse`, `models.py:105`). +- Domain errors are typed exceptions in a `types.py`, mapped to status codes by one decorator on the + router (`_handle_session_exceptions`, `router.py:181`). +- Field names are `lower_snake_case`. Header names keep their standard spelling. The runner's own + HTTP surface uses `camelCase`, matching its existing `/kill` body + (`services/runner/src/server.ts:704`). + +--- + +## 1. Interface review + +Every field is classified before it is written down, as the `design-interfaces` skill requires. The +architecture review's section 4 fixed four of these shapes; where it did, that is noted. + +### Public Cancel request + +| Field | Concretely | Owner | Changes | Role | Placement | +|---|---|---|---|---|---| +| `session_id` | Which session to act on | Caller | Per call | routing | Path parameter, because it names the resource | +| `expected_execution_id` | The execution the caller believes is running | Caller | Per call | precondition | Body, flat | +| `Idempotency-Key` | Retry identity for this request | Caller | Per call | protocol context | Header | + +Three decisions fall out of that table. + +- **The public Cancel body stays flat.** The review examined this exact shape and ruled that it is + correct and should not change: `expected_execution_id` is per-call context named as the guard it + is, in the style of an HTTP `If-Match`. The grouping under `target` applies to the internal + envelope, where a resolved `target.turn_id` needs a home next to the asserted one. A public body + with one field does not. +- **`Idempotency-Key` stays a header** with its standard spelling. It describes the delivery of the + request, not the intent inside it. The stored column is `idempotency_key`, matching + `session_attachments.idempotency_key` (`api/oss/src/dbs/postgres/sessions/attachments/dbas.py:25`). +- **No `force` flag.** `force` on the current stream endpoint is what makes one route mean four + things (`api/oss/src/core/sessions/streams/service.py:7`). Cancel means cancel. + +The field stays optional, as decision D-010 requires, and first-party clients must always send it. +Today the desktop sends nothing (`web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts:505`, +verified), which is the third guard of the design document's section 4 left switched off. + +### Public Cancel response + +| Field | Concretely | Role | +|---|---|---| +| `command.id` | The durable command's id | identity, for the caller's own retries and logs | +| `command.state` | `pending` or `obsolete` at admission time | delivery | +| `execution.id` | The execution this Cancel targets, null when nothing ran | routing | +| `execution.state` | `stopping` or `idle` | result | + +`command` and `execution` are separate objects because they answer different questions and settle at +different times. A client drawing a button reads `execution`. A client retrying safely reads +`command.id`. This is decision D-016 expressed in the response shape. + +### The internal command envelope + +The review's corrected shape, adopted here: + +| Group | Fields | Role | +|---|---|---| +| top level | `id`, `project_id`, `session_id`, `kind`, `created_at` | identity, routing, metadata | +| `target` | `turn_id` (resolved at admission), `expected_turn_id` (as the caller sent it) | context | +| `input` | `text`, `attachments` | input data, absent for `cancel` | +| `policy` | `on_busy` | policy, absent for `cancel` | +| `delivery` | `claimed_by`, `claim_expires_at`, `attempt` | delivery bookkeeping | + +Four rules this applies. + +- **Delivery bookkeeping is grouped and never merged with the result.** That is decision D-016, and + it is easier to hold when the shapes are separate objects. +- **`replica_id` is not a top-level routing field.** It is delivery bookkeeping, it is logical rather + than an address, and it lives under `delivery` as `claimed_by`. +- **There is no `runner_url` field of any kind.** An address in a durable record is an + implementation detail with a longer lifetime than the thing it points at. +- **`input` is an object from the start**, not a bare `message` string. A turn already carries text + plus attachments (`services/runner/src/server.ts:565`), so a string could not grow into that + without a breaking change. `cancel` omits the group entirely rather than sending it empty. + +`created_at` is on the envelope because the runner needs it: it refuses to abort an execution that +started after the command was created. + +### Internal claim request + +| Field | Concretely | Owner | Role | +|---|---|---|---| +| `replica_id` | Which runner is asking, for `claimed_by` | Runner | delivery bookkeeping | +| `sessions` | The sessions this runner holds warm right now | Runner | routing | +| `wait_seconds` | How long the caller accepts being held | Runner | protocol context of this call | +| `limit` | How many commands to return at most | Runner | protocol context of this call | + +`sessions` is the routing input, not `replica_id`. The runner declares what it holds, so the API +never has to guess from an expiring Redis key, and a parked session keeps receiving commands after +its heartbeat stops. A claim is a query over durable state, never a cursor or a stream position. + +### Internal outcome request + +| Field | Concretely | Owner | Role | +|---|---|---|---| +| `replica_id` | Which runner is reporting | Runner | delivery bookkeeping, and the claim guard | +| `result` | The command's terminal state | Runner | delivery | +| `execution.id` | Which execution the runner acted on | Runner | routing | +| `execution.state` | What happened to it | Runner | result | +| `execution.error` | Why it failed, when it did | Runner | result | + +`execution.error` sits under `execution` because it explains one field of that object. + +--- + +## 2. Public: cancel the current execution + +```http +POST /sessions/{session_id}/cancel +Idempotency-Key: 0199a3f2-0000-7000-8000-000000000001 + +{ + "expected_execution_id": "0199a3f1-0000-7000-8000-00000000000a" +} +``` + +Permission: `Permission.RUN_SESSIONS`, the same permission the current cancel path checks +(`api/oss/src/apis/fastapi/sessions/router.py:377`). + +```python +class SessionCancelRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + # Optional stale-request guard (decision D-010). When present, the API cancels only this + # execution and rejects the request if another one is running. When absent, it cancels + # whichever execution is active when the request is applied. A person never types this; + # the browser fills it from the session snapshot, and a first-party client always sends it. + expected_execution_id: Optional[str] = None + + +class SessionCommandRef(BaseModel): + """The durable command an accepted request created. Identity and delivery state only. + A client must not infer execution state from it (decision D-016).""" + + id: UUID + state: Literal["pending", "claimed", "applied", "obsolete"] + + +class SessionExecutionRef(BaseModel): + """What the caller should render. `id` is null when the session was idle.""" + + id: Optional[str] = None + state: Literal["stopping", "idle"] + + +class SessionCancelResponse(BaseModel): + command: SessionCommandRef + execution: SessionExecutionRef +``` + +Responses: + +| Status | When | Body | +|---|---|---| +| 202 Accepted | An execution was running or parked. The command is durable and on its way | `command.state = "pending"`, `execution.state = "stopping"` | +| 200 OK | Nothing was running and no `expected_execution_id` was sent | `command.state = "obsolete"`, `execution.state = "idle"`, `execution.id = null` | +| 200 OK | The running execution started **after** this request arrived, and no `expected_execution_id` was sent | `command.state = "obsolete"`, `execution.state = "idle"`, `execution.id = null`. The newer execution is not touched. See the stale-Stop guard in section 4 of the design document | +| 409 Conflict | `expected_execution_id` does not name the running execution | `detail: {"message": ..., "current_execution_id": }` | +| 422 | The session id fails the allowlist (`SessionIdInvalid`) | `detail: ` | +| 403 | The caller lacks `RUN_SESSIONS` | `FORBIDDEN_EXCEPTION` | + +The two 200 cases are deliberately indistinguishable to the client. Both mean "there is nothing of +yours left to stop", and a client that needs to know which one it hit is reading the wrong signal: +it should read the session's execution state, not this response. The command row keeps the exact +reason in `outcome` for anyone debugging afterwards. + +202 and not 200 for the accepted case, because the work is not done when the response returns. The +caller learns the outcome from the session's own state, not from this response. **A delivery failure +does not change the status**: the command is inserted and committed before any adapter is called, so +an unreachable runner still yields 202 and the watchdog settles the command. + +Repeating the request with the same `Idempotency-Key` returns the same `command.id` and the same +status. Repeating it without a key also returns the same command while one is still open, because +admission collapses onto an open command for the same target execution. + +New domain exceptions in `api/oss/src/core/sessions/commands/types.py`, mapped by a +`_handle_command_exceptions()` decorator alongside the existing one: + +```python +class SessionCommandError(Exception): ... + +class ExecutionExpectationFailed(SessionCommandError): + """expected_execution_id does not name the running execution.""" + def __init__(self, session_id: str, expected: str, current: Optional[str]): ... +``` + +--- + +## 3. Deferred: claim commands (future long-poll adapter) + +```http +POST /sessions/control/commands/claim +X-Agenta-Runner-Token: + +{ + "replica_id": "runner-7f3c", + "sessions": [ + {"project_id": "1f0a4b2c-0000-4000-8000-000000000002", "session_id": "sess-42"} + ], + "wait_seconds": 25, + "limit": 10 +} +``` + +Not a product API. It is excluded from the public schema with `include_in_schema=False`, the +treatment the admin routers already get (`api/entrypoints/routers.py:1502`). + +Authentication is the shared runner token, not a user credential: the loop belongs to the process +and spans many projects, and a run's credential expires while the process keeps polling. The path +prefix `/sessions/control/` is added to `_PUBLIC_ENDPOINTS` (`api/oss/src/middlewares/auth.py:52`) +so the project-scoped middleware does not reject a request with no user credential, and the route +then compares the presented token to `env.runner.token` in constant time. If that setting is unset +the route answers 503 and serves nothing. Scope comes from the declared `(project_id, session_id)` +pairs and the rows themselves, never from a header. + +```python +class SessionScope(BaseModel): + model_config = ConfigDict(extra="forbid") + + project_id: UUID + session_id: SessionId + + +class SessionControlClaimRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + # Delivery bookkeeping: this becomes `claimed_by` so a settle can be matched to its claim. + # Not routing, and not an address. + replica_id: str = Field(min_length=1, max_length=128) + # The routing input: every session this runner holds warm right now, including sessions + # parked awaiting an approval. Most recently used first. + sessions: List[SessionScope] = Field(min_length=1, max_length=200) + # How long the API may hold this request. Clamped server-side to the configured hold. + wait_seconds: int = Field(default=25, ge=0, le=60) + limit: int = Field(default=10, ge=1, le=50) + + +class SessionCommandTarget(BaseModel): + # Resolved once at admission; the runner aborts only this execution. + turn_id: Optional[str] = None + # What the caller asserted, kept so a 409 stays explainable after the fact. + expected_turn_id: Optional[str] = None + + +class SessionCommandDelivery(BaseModel): + claimed_by: str + claim_expires_at: datetime + attempt: int + + +class SessionCommandEnvelope(BaseModel): + """One command as the runner receives it. Every transport delivers this same shape, + so the runner has one parser, one set of guards and one applier.""" + + id: UUID + project_id: UUID + session_id: str + kind: Literal["cancel"] + target: SessionCommandTarget + delivery: SessionCommandDelivery + # The runner refuses to abort an execution that started after this time. + created_at: datetime + # Absent for `cancel`. Present for the kinds that carry them, so a reader never has to + # interpret an empty object. + input: Optional[SessionCommandInput] = None + policy: Optional[SessionCommandPolicy] = None + + +class SessionControlClaimResponse(BaseModel): + count: int = 0 + commands: List[SessionCommandEnvelope] = Field(default_factory=list) +``` + +Responses: + +| Status | When | +|---|---| +| 200 OK | At least one command was claimed. The body is never an empty list | +| 204 No Content | The hold expired with nothing to deliver | +| 401 Unauthorized | The token is absent or wrong | +| 422 | `sessions` is empty or over the cap | +| 503 Service Unavailable | `AGENTA_RUNNER_TOKEN` is not configured on the API | + +204 rather than an empty 200 keeps the common case cheap and gives the runner an unambiguous "claim +again now" signal. + +--- + +## 4. Internal: report a command's outcome + +Used by **both** adapters. Settlement has one path on every transport. + +```http +POST /sessions/control/commands/{command_id}/outcome +X-Agenta-Runner-Token: + +{ + "replica_id": "runner-7f3c", + "result": "applied", + "execution": { + "id": "0199a3f1-0000-7000-8000-00000000000a", + "state": "stopped" + } +} +``` + +```python +class SessionExecutionOutcome(BaseModel): + model_config = ConfigDict(extra="forbid") + + # The execution the runner acted on. Null when it held none. + id: Optional[str] = None + # stopped: cancelled as asked. not_running: no such execution here. + # superseded_by_newer_turn: the held execution started after the command arrived. + # failed: the cancel itself failed. + state: Literal["stopped", "failed", "not_running", "superseded_by_newer_turn"] + # Short, human-readable, present only when `state` is "failed". + error: Optional[str] = Field(default=None, max_length=2000) + + +class SessionControlOutcomeRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + replica_id: str = Field(min_length=1, max_length=128) + # The command's terminal state. `applied` means the runner did the work; `obsolete` + # means there was nothing to do. + result: Literal["applied", "obsolete"] + execution: SessionExecutionOutcome + + +class SessionCommandSettlement(BaseModel): + id: UUID + state: Literal["applied", "obsolete"] + outcome: Literal["stopped", "not_running", "superseded_by_newer_turn", "failed", "lost"] + settled_at: datetime + + +class SessionControlOutcomeResponse(BaseModel): + command: SessionCommandSettlement +``` + +Responses: + +| Status | When | Body | +|---|---|---| +| 200 OK | The command was `claimed` by this replica and is now settled | The settlement | +| 409 Conflict | The claim expired, or another actor settled the command | The stored settlement, so the runner stops instead of retrying | +| 404 Not Found | No command with that id in any project | `detail` | +| 401, 503 | As for the claim route | | + +The API does the settlement side effects inside the same request: it clears +`session_streams.stopping_turn_id`, tombstones the stopped execution, releases the Redis `running` +key under an owner check, leaves `alive` to its own time to live, cancels that execution's pending +interactions, and publishes the existing `lifecycle: ended` watch notification. The full ordering is +in section 7 of the design document. + +--- + +## 5. Internal: the runner's cancel route (direct-call adapter) + +This is the runner's own HTTP surface, not the API's. It sits beside the existing `POST /kill` +(`services/runner/src/server.ts:704`, verified) and shares its token gate, its capped body reader and +its scoping rule. The API calls it the way `kill_runner_sandbox` already calls `/kill` +(`api/oss/src/core/sessions/streams/runner_client.py:30`, verified). + +```http +POST /cancel +Authorization: Bearer + +{ + "commandId": "0199a3f2-0000-7000-8000-000000000001", + "projectId": "1f0a4b2c-0000-4000-8000-000000000002", + "sessionId": "sess-42", + "targetTurnId": "0199a3f1-0000-7000-8000-00000000000a", + "createdAt": "2026-09-02T22:09:01Z" +} +``` + +`camelCase` because the runner's existing routes use it. `projectId` and `sessionId` are both +required, for the same reason `/kill` requires both: a pool key is always project-scoped, so a +single-tenant scope needs the pair. + +Responses: + +| Status | When | Meaning to the API adapter | +|---|---|---| +| 202 Accepted | The runner holds this session and accepted the command | `accepted`; the outcome will arrive on the outcome route | +| 404 Not Found | The runner does not hold this session | `not_held`; the service settles the command at once | +| 400 | `sessionId` or `projectId` missing | `unreachable`, and a bug to fix | +| 401 | Token mismatch | `unreachable`, and a deployment error to log loudly | + +**The response is an acknowledgement, not an outcome.** The runner reports what happened to the +execution through the outcome route in section 4, so both adapters settle through one path. + +**404 is ambiguous, and the API must disambiguate it.** `not_held` is the honest answer both when the +session really has ended and when the call reached the wrong replica. The API tells them apart with +data it already has: a `not_held` for a session whose row says `is_alive` with a heartbeat younger +than one interval is the wrong-replica failure. It is logged at error level, counted, and settled as +`lost` rather than `not_running`, so the user is told the Stop failed instead of being told the work +had already finished. Section 9 of the design document has the rule and the optional preventive +configuration check. + +**The runner resolves a parked session through the pool, not the execution registry.** A Stop against +a parked approval has no in-flight execution, so `/cancel` falls back to +`SessionPool.awaitingApproval(sessionId)` +(`services/runner/src/engines/sandbox_agent/session-pool.ts:117`, verified) before answering 404. + +--- + +## 6. One field added to an existing contract + +The heartbeat response grows one field. Nothing else about `POST /sessions/streams/heartbeat` +changes. + +```python +class SessionHeartbeatResult(BaseModel): + stream: Optional[SessionStream] = None + replica_id: str + is_current_turn: bool = True + # Commands for THIS session only, claimed by this beat under the same compare-and-set + # the claim route uses. Empty in the normal case. + commands: List[SessionCommandEnvelope] = Field(default_factory=list) +``` + +The field is additive and defaults to an empty list, so a runner build that does not know about it is +unaffected. + +This fallback reaches only a session with a live turn. The heartbeat stops when a turn ends or parks +(`services/runner/src/server.ts:618` and `services/runner/src/sessions/alive.ts:241`, verified), so +it is not the delivery path for a parked session and must not be relied on as one. + +--- + +## 7. What does not change in version one + +- `POST /sessions/streams/` keeps its current four-mode behavior until the last migration step, when + its cancel branch becomes a thin wrapper over the same command. See section 10 of the design + document. +- `DELETE /sessions/streams/` (kill) is untouched. Stop and Delete stay different operations + (decision D-008). +- `POST /sessions/interactions/{interaction_id}/respond` is untouched. Turning interaction responses + into commands is later work, and so is the `continuation` field the architecture review asks for on + its response. +- No new public read route. Clients keep using `GET /sessions/streams/` and the watch stream. +- Steer stays out. The `input` and `policy` groups are reserved in the envelope so it does not need a + breaking change later, but no route accepts them in version one. diff --git a/docs/design/session-control-and-live-events/decisions.md b/docs/design/session-control-and-live-events/decisions.md index 47fc04bf3a7..ead98a2148c 100644 --- a/docs/design/session-control-and-live-events/decisions.md +++ b/docs/design/session-control-and-live-events/decisions.md @@ -148,10 +148,10 @@ It does not add Postgres execution authority, ownership generations, or full sta Those changes have low current value because Agenta operates one runner and does not plan near-term runner scaling. -Durable commands and runner-initiated long polling remain in scope. Stop delivery no longer depends -on deleting ownership and waiting for a heartbeat. The current execution keeps its Redis ownership -while stopping and releases it after cancellation settles. Heartbeat command discovery remains a -fallback if long polling is unavailable. +Durable commands and direct API-to-runner delivery are in scope. Stop no longer depends on deleting +ownership and waiting for a heartbeat. The current execution keeps its Redis ownership while +stopping and releases it after cancellation settles. Long polling is deferred behind the same +control-delivery port. ### D-018: Use runner-initiated HTTP long polling for immediate control @@ -235,6 +235,14 @@ reuses a `record_id`. Separate exact delivery retries from progressive updates a re-emissions. Add regression tests for the final state of tools, interactions, terminal events, and harness reconstruction. +### O-006: Immediate runner control + +**Status:** Resolved for version one on 2026-09-03. + +Use a direct API-to-runner HTTP call through the replaceable control-delivery port. Durable storage +precedes the call, so transport failure costs promptness rather than command correctness. Defer +runner-initiated long polling until multi-runner or user-operated routing requires it. + ### O-007: Command boundary Decide which actions enter a general command inbox. The working boundary is execution-affecting diff --git a/docs/design/session-control-and-live-events/slice-durable-cancel.md b/docs/design/session-control-and-live-events/slice-durable-cancel.md new file mode 100644 index 00000000000..cd9478acb79 --- /dev/null +++ b/docs/design/session-control-and-live-events/slice-durable-cancel.md @@ -0,0 +1,255 @@ +# Slice: the durable Stop command, with the direct-call adapter + +> AGENT-GENERATED, low weight. Built and verified live. Mahmoud makes final decisions. + +Branch `feat/session-durable-cancel`, rebased onto `spike/session-cancel-warm` at `f5b1ae6244`. +It implements [the durable command design](spike-b-durable-commands-design.md) at `86281fa313` +and [the route contracts](api-design.md), with the direct-call adapter of that design's +section 9. The long-poll adapter is not built. + +Every claim below is marked **verified** (observed on the running stack, or read in this +branch's code with a `path:line`) or **reported** (taken from a document). + +--- + +## What a Stop does now + +**Verified live.** A user Stop reaches the running turn in 82 milliseconds, ends it, and leaves +the sandbox and the native harness session warm. Before this branch it reached the runner on the +next heartbeat, up to 30 seconds later. + +| Step | Observed at | After the Stop request | +|---|---|---| +| The browser's request arrives, the command row commits, the API calls the runner | 00:12:45.624 | 0 | +| The runner aborts the execution | 00:12:45.706 | 82 ms | +| The harness confirms it stopped | 00:12:45.730 | 106 ms | +| The runner reports, and the API settles the command and the execution | 00:12:45.750 | 126 ms | +| The sandbox is parked warm, not deleted | 00:12:46.617 | 993 ms | + +The 5 second budget in the design is met with two orders of magnitude to spare. The next message +on that session recalled a codeword from the stopped turn, which is warm resume measured from +the product rather than from a timer. + +--- + +## What changed, with references + +### The record + +`session_commands` holds one row per durable request to change an execution +(`api/oss/src/dbs/postgres/sessions/commands/dbes.py:14`, migration +`api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py`). +Two columns are never merged: `state` says where the COMMAND is (`pending`, `claimed`, +`applied`, `obsolete`) and `outcome` says what happened to the EXECUTION (`stopped`, +`not_running`, `superseded_by_newer_turn`, `failed`, `lost`). + +`session_streams` gains `stopping_turn_id` and `turn_started_at` +(`api/oss/src/dbs/postgres/sessions/streams/dbes.py:73` and `:82`). The start time is stamped +only when the turn id actually changes +(`api/oss/src/dbs/postgres/sessions/streams/mappings.py`, the edit mapper), so the heartbeat that +restamps the same id every 30 seconds never moves it. + +Every transition is one `UPDATE ... WHERE RETURNING *` decided by +`scalar_one_or_none()` (`api/oss/src/dbs/postgres/sessions/commands/dao.py`). Two API replicas +cannot both win a claim or both write a terminal outcome. + +### Admission + +`SessionCommandsService.request_cancel` +(`api/oss/src/core/sessions/commands/service.py:111`) stamps the arrival time before it reads +anything, resolves the target once from Redis `running` falling back to `alive` +(`service.py:217`), applies the three late-Stop guards, then writes the command and the session's +`stopping_turn_id` in one transaction. **Redis is not written at admission**, so the stopping +execution keeps both locks while it stops, which is what prevents a second message from starting +underneath it. + +### Settlement + +`SessionCommandsService.settle` (`service.py:419`) settles the command and the execution +together, guarded on the command's state so a repeat changes nothing. For a `stopped` outcome it +tombstones the turn, then releases `running` under an owner check, then cancels that execution's +pending interactions, then publishes the existing `lifecycle: ended` notification. **It leaves +`alive` to its own time to live**, exactly as the end of an ordinary turn does. That single +decision is what makes Stop a stop rather than a session teardown. + +### Delivery + +`ControlDeliveryPort` (`api/oss/src/core/sessions/commands/interfaces.py`) is the port. The one +adapter is `DirectControlDelivery` +(`api/oss/src/dbs/http/sessions/control_delivery_direct.py`), which posts to the runner's own +`/cancel` beside the existing `kill_runner_sandbox` +(`api/oss/src/core/sessions/streams/runner_client.py`). The command row is committed BEFORE the +runner is called, and a delivery failure never fails the request. + +### The runner + +`POST /cancel` sits beside `POST /kill` behind the same token gate +(`services/runner/src/server.ts:821`). It resolves a live execution through a module-level +registry (`services/runner/src/sessions/execution-registry.ts`), falls back to the keep-alive +pool for a parked approval, and answers 404 when it holds neither. + +**The abort carries the user-stop label.** `shouldPark` parks only an abort the runner can prove +was a cooperative Stop (`services/runner/src/sessions/stop-signal.ts`, from Spike A), so the +registry aborts with `USER_STOP_ABORT_REASON`. Without it a Stop delivered as a command ends the +turn `cancelled` and then DESTROYS the sandbox, which is the failure Stop exists to avoid. Two +tests pin both directions, and the live run after the rebase logs `park-cancelled`. The applier +sits above the transport (`services/runner/src/sessions/control-channel.ts`) with the +deduplication set beside the session pool (`services/runner/src/sessions/applied-commands.ts`), +so a long-poll loop would reuse every guard unchanged. + +### The routes + +`POST /sessions/{session_id}/cancel` and +`POST /sessions/control/commands/{command_id}/outcome`, both on `SessionControlRouter` +(`api/oss/src/apis/fastapi/sessions/router.py:1909`). The public route checks +`Permission.RUN_SESSIONS` and is deliberately **not** behind `check_runner_concurrency_limit`: +refusing to STOP work because a project is at its run limit is the wrong answer to a busy +project. The internal route authenticates with the shared runner token +(`router.py:2027`) and resolves the project from the command id, so the auth exemption +(`api/oss/src/middlewares/auth.py`, the `/sessions/control/` prefix) widens no tenant boundary. + +`POST /sessions/streams/` is untouched. Its cancel branch becomes a thin wrapper over this +command in a later change, together with the mobile client; do both in one change so one revert +restores one behaviour. + +### The desktop + +The Stop button posts the new route +(`web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts`, `stopCurrentExecution`), +awaits it, and refreshes the session state on the answer. It names the execution it means, read +FRESH from the session row rather than from the project-wide liveness poll, which is up to +15 seconds stale; a stale id is refused with a conflict and the Stop would silently do nothing. +The client is `cancelSessionExecution` +(`web/packages/agenta-entities/src/session/api/api.ts`), written against raw axios because the +Fern client does not know the route yet. Mobile is untouched. + +--- + +## Four defects the live run and the rebase found + +None were visible in unit tests. The first three were found by pressing Stop against a real +agent turn; the fourth by rebasing onto Spike A's final tip. Each is committed with its own fix. + +1. **The execution registry never held the session, so every Stop got a 404 and the turn ran to + completion.** The entry was keyed by `:`, but the project scope is not + known when a run starts: `runContext.project.id` is empty on the live invoke path and the + scope that forms the pool key comes from the signed mount, which the coordinator resolves + after the run is in flight (`services/runner/src/lifecycle/session-coordinator.ts:281`, + verified). The registry is now keyed by session id and the coordinator fills the project in + through `onScopeResolved`. A lookup with a disagreeing project is refused; an entry whose + project is not known yet matches, because refusing every Stop in the first moments of a run is + the bug being replaced. +2. **The outcome report was refused with a 409, leaving the command `claimed` and the session + marked stopping forever.** The API claimed on the runner's behalf under a placeholder while + the runner reported under its own replica id, and the settle guard compares the two. The + runner's acknowledgement now carries its replica id and the API claims under that. +3. **The multi-replica census refused delivery for five minutes after every runner restart.** A + runner mints a fresh replica id at boot when `AGENTA_RUNNER_REPLICA_ID` is unset + (`services/runner/src/sessions/alive.ts:31`, verified), so its previous id is still inside the + window and the count reads two, which broke Stop after every ordinary deploy. **The census is + now removed entirely**, on the revised design's guidance that it is optional and the exact + detector is the one to build. That deletes a Redis write on every heartbeat, two settings and + a module. What remains is the detector that cannot be fooled: a `not_held` for a session whose + row says alive with a heartbeat younger than one interval means some process is running that + session and it is not the one we called. It logs at error level naming the owner replica from + the Redis `owner` key, and settles the command `lost` rather than `not_running`, so the user + is told the Stop failed instead of that the work had already finished + (`api/oss/src/core/sessions/commands/service.py`, `_settle_not_held`). + +4. **The control-plane abort carried no label, so after the rebase every Stop would have + destroyed the sandbox.** Spike A's `96012e8d8e` made `shouldPark` require proof that an abort + was a cooperative Stop, because inferring it from the stop reason alone would let any future + `controller.abort()` park a sandbox nobody had checked. The registry handed the applier a bare + `controller.abort()`. It now aborts with `USER_STOP_ABORT_REASON`, and two tests pin both + directions of the contract. + +A fifth, smaller one: two Stops **in the same instant** both inserted, because admission reads +for an open command and then inserts and neither request can see a row the other has not +committed. Sequential Stops always collapsed. A unique partial index over the open states now +makes the database decide, and the losing insert reads the winner back. + +--- + +## Live verification + +Stack: `http://144.76.237.122:9180`, project `agenta-ee-dev-session-cancel`, EE, dev images, +built from this worktree. The agent ran the `pi_core` harness on the local sandbox with an +OpenAI model. + +| Scenario | Result | Evidence | +|---|---|---| +| 1. Stop during a 60 s tool call | **Pass**, re-verified after the rebase. Turn ends at 26.2 s instead of 77.6 s. Command `pending` to `applied`, outcome `stopped`, settled 116 ms after the request. Runner logs `aborted`, then `harness_cancel sent=true settled=true elapsed_ms=17`, then `park-cancelled`. Next message recalled the codeword. | command `01a0641f-b775-75c1-bfe1-32a80e85f85e` | +| 2. Stop when nothing runs | **Pass.** 200, one row inserted already settled: `obsolete` with outcome `not_running`, no target, no Redis write. | command `01a0641f-5535-7130-a6be-537d287b6d9b` | +| 3. Stop with a stale `expected_execution_id` | **Pass.** 409 naming the current execution, and no row inserted. | `detail.current_execution_id` returned the live turn | +| 4. Two Stops in a row | **Pass.** Two simultaneous requests return the same command id and one row exists. Sequentially, the second now correctly reports nothing running, because a Stop settles in about 100 ms. | command `01a06423-c067-7c80-9b68-636953655698` returned to both | +| 5. Stop a turn parked for approval | **Pass.** The interaction goes `pending` to `cancelled`, the command settles `applied` with `not_running` in 68 ms, the pool keeps the entry, and the next message recalled the codeword. | command `01a06424-102b-76d0-a7cf-9e7d25c88041` | +| 6. Runner gone while a command is open | **Not settled, as expected.** No sweep exists in this slice. | see below | + +**Redis after a Stop, verified by direct inspection:** `running` gone, `alive` still present and +by then held by the resuming turn, and `superseded::session::turn:` +written. That is the same shape an ordinary turn end leaves, which is the point. + +**Scenario 6 in detail.** A command that is claimed and never reported stays `claimed`, and the +session's `stopping_turn_id` stays set, indefinitely. Observed directly: command +`01a0641a-d3c0-7980-8675-5349d0e3a118` sat `claimed` for over ten minutes with nothing to settle +it, and two session rows were left marked stopping. **This slice does not build the settlement +sweep.** The DAO exposes `expire_claims(now, max_deliveries)` and `settle_command` for it. The +handoff is to the branch `feat/session-execution-watchdog`, and the rule both sides must obey is +that one execution reaches exactly one terminal outcome from exactly one writer. It has to be +agreed before either lands; a second sweep racing the first is a worse bug than the one being +fixed. + +### Tests + +| Suite | Result | +|---|---| +| `api/oss/tests/pytest/unit/sessions/test_session_cancel_admission.py` | 15 pass. Admission guards, the arrival-time stamp, the collapse, the settlement, and the assertion that pins warm resume: `alive` survives a Stop. | +| `api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py` | 17 pass against a real Postgres. Two concurrent claims yield one winner, two concurrent admissions yield one command, the settle guard refuses a foreign replica, a terminal command cannot be settled twice. | +| `api/oss/tests/pytest/unit/sessions` (whole directory) | 553 pass. Four failures in `test_records_turn_span_dao.py` are a DNS failure reaching the tracing database from the host, unrelated to this branch. | +| `cd services/runner && pnpm test` | 2663 pass, 4 fail. All four are `gateway-run-turn-composition.test.ts`, verified failing on the base commit before any change here. | +| `cd web && pnpm lint-fix` | 25 tasks, no errors. | +| `ruff format` and `ruff check` in `api/` | Clean, run with the CI-pinned 0.15.12. | + +--- + +## What is left + +- **The settlement sweep.** Named above. It is the difference between "a Stop the runner missed + settles in two minutes" and "it never settles". +- **The long-poll adapter.** Not built. `AGENTA_SESSIONS_CONTROL_ADAPTER` defaults to `direct` + and any other value refuses to boot (`api/entrypoints/routers.py`) rather than falling back + silently to a transport the operator did not choose. Building it changes one file plus one + runner module, and no route, data shape, or transition. +- **The wrapper.** `POST /sessions/streams/` still does what it always did. Its cancel branch + becomes a call to `request_cancel` in the same change that flips mobile, so released clients + get the new behaviour with no client change. +- **The Fern client.** The desktop calls the new route through raw axios. Move it when the API + client is next regenerated. +- **Mobile.** Untouched, as the brief asked. + +--- + +## Open questions for Mahmoud + +1. **Is the exact `not_held` detector enough on its own, with no replica census?** Settled in + the revised design and built that way. Recommendation: **yes**. Reason: the census could not + tell two live replicas from one that had restarted and broke Stop after every deploy, while + the `not_held` condition is produced by nothing but the wrong-replica failure. Listed here + only so the removal is on the record. +2. **Who owns settling an abandoned command?** Recommendation: **the execution watchdog**, using + the DAO methods this slice exposes. Reason: one execution must reach exactly one terminal + outcome from one writer, and two sweeps racing to write `lost` is worse than the bug. Until + it exists, a Stop the runner never reports leaves the session reading "stopping" forever. +3. **Does the desktop read the execution id with an extra request?** Recommendation: **yes, as + built.** Reason: the cached liveness poll is up to 15 seconds stale and a stale id is refused + with a conflict, which would make Stop silently do nothing. The extra read costs about 30 + milliseconds inside a budget of five seconds. The alternative is to send no expectation, which + switches off the cheapest late-Stop guard. +4. **Should a Stop settle before the sandbox has finished parking?** Recommendation: **yes, as + built.** The runner reports as soon as it has issued the abort, about 70 milliseconds in, + while the park completes around a second later. Reason: the command's job is to deliver the + Stop, and waiting for the teardown would make a Stop that worked look stuck. The cost is that + `outcome = stopped` means "the cancel was delivered", not "the sandbox is parked". +5. **Do we keep `session_commands` rows forever?** Recommendation: **delete settled rows seven + days after `settled_at`**, as the design says. Not built here, because it belongs with the + sweep. Commands are operational state; durable history stays in `session_records`. diff --git a/docs/design/session-control-and-live-events/spike-b-durable-commands-design.md b/docs/design/session-control-and-live-events/spike-b-durable-commands-design.md new file mode 100644 index 00000000000..65f82bd3806 --- /dev/null +++ b/docs/design/session-control-and-live-events/spike-b-durable-commands-design.md @@ -0,0 +1,1355 @@ +# Spike B: durable commands and control delivery + +> AGENT-GENERATED, low weight. Implementation-ready design for discussion. Mahmoud makes final +> decisions. + +Scope: reliable API-to-runner commands, version one. The only command kind in version one is +Cancel, which the product calls Stop. The design keeps Redis execution ownership as it is, adds no +Postgres execution authority, no ownership generations, no stale-writer fencing, and no +multi-runner routing. + +Every claim below is marked **verified** (read in the code of this worktree, with `path:line`) or +**reported** (taken from a document, named at the point of use). + +This revision answers the architecture review at `review-architecture.md`, sections 3 and 4. The +holes it names are addressed here: H-2 in sections 5 and 7, H-3 in sections 4 and 7, H-4 in section +4, H-5 in section 4, H-6 in section 5, and the interface corrections in sections 2, 5 and 9. H-1, +the `shouldPark` change, belongs to Work package A and is named as a dependency in section 7. + +Terms used here: + +- **Execution:** one runner attempt at one user message. In the code today its identifier is the + `turn_id` the runner mints (`services/runner/src/server.ts:190`). This design does not rename it. +- **Command:** one durable request to change an execution. +- **Held session:** a session this runner process holds warm, whether it is running a turn, idle in + the keep-alive pool, or parked awaiting an approval. + +--- + +## 1. What happens today when a user presses Stop + +**Verified.** The browser stops its own stream at once. The runner learns nothing until its next +heartbeat, which is up to 30 seconds later. The sandbox is then deleted, so the next message is a +cold start. + +The chain, in order: + +1. `handleStop` marks the turn stopped locally and aborts the client fetch + (`web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts:480`). +2. The browser posts `POST /sessions/streams/` with no inputs and no `force`, and **with no + execution id** (`useAgentChatSession.ts:505`, which passes only `{sessionId, projectId}`). + Mobile posts the same call (`web/mobile/src/features/chat/StopButton.tsx:18`). +3. The route runs `set_session_stream` (`api/oss/src/apis/fastapi/sessions/router.py:369`), which + calls `SessionStreamsService.command` (`api/oss/src/core/sessions/streams/service.py:229`). +4. No inputs and no `force` resolves to `CommandMode.cancel` + (`api/oss/src/core/sessions/streams/service.py:288`). +5. Cancel calls `_displace_turns` (`api/oss/src/core/sessions/streams/service.py:169`). It writes a + supersession tombstone for the current `alive` and `running` owners, then force-deletes both keys + (`service.py:190` and `service.py:193`). It marks the row ended and publishes the `ended` + lifecycle event. **The API never contacts the runner.** +6. The runner finds out on its next heartbeat. The beat runs on a 30 second interval + (`services/runner/src/sessions/alive.ts:221`, `HEARTBEAT_INTERVAL_SECONDS = 30` in + `services/runner/src/sessions/contract.ts:18`). +7. The beat returns `is_current_turn: false` + (`api/oss/src/core/sessions/streams/service.py:452`), the runner reads it as `interrupted` + (`services/runner/src/sessions/alive.ts:105`), and the watchdog fires `onInterrupted` once + (`alive.ts:207`), which `server.ts:519` wires to `controller.abort()`. +8. The abort makes `shouldPark` return false, so the environment is destroyed rather than parked + (`services/runner/src/engines/sandbox_agent/engine.ts:26`). The sandbox and the native harness + session are gone. + +### The delay chain + +| Step | Where | Cost | +|---|---|---| +| Browser aborts its own stream | `useAgentChatSession.ts:480` | immediate | +| Cancel request returns | `router.py:369` | one API round trip | +| Redis keys cleared, row marked ended | `streams/service.py:190` | inside that call | +| Runner notices | `alive.ts:221` | **0 to 30 seconds** | +| Run aborts | `server.ts:519` | immediate after the beat | +| Harness cancel and sandbox teardown | `engine.ts:26` | seconds, and the sandbox is deleted | + +The 30 second wait is the whole problem. Four further defects ride on it: + +- **A Stop can be lost silently.** A heartbeat that returns a non-2xx status yields + `interrupted: false` by design (`services/runner/src/sessions/alive.ts:92`). A run whose platform + credential expired or was dropped can never be stopped. The credential states are logged at + `services/runner/src/server.ts:445`. +- **A parked session has no channel at all.** When a turn parks awaiting an approval, the request + handler's `finally` calls `aliveWatchdog.release()` (`services/runner/src/server.ts:618`), which + clears the heartbeat interval and sends one last beat with `is_running: false` + (`services/runner/src/sessions/alive.ts:241`). From that moment the runner sends no heartbeat for + that session, so the only existing control channel is gone. This is review hole H-2, and it is why + section 5 makes the poll session-scoped rather than turn-scoped. +- **A late Stop can kill the next turn.** `_displace_turns` reads whoever holds `alive` and + `running` at the moment it runs, so a Stop applied 300 ms after the turn ended tombstones the turn + that started in between. The tombstone lasts an hour and every read refreshes it + (`api/oss/src/dbs/redis/sessions/locks.py:147`). This is review hole H-3. +- **Stop is not free.** Because the abort path destroys the environment, Stop today costs the warm + sandbox and the native harness session. Work package A owns the fix. This design assumes it + delivers a warm park on Stop. + +--- + +## 2. The command record + +### Placement + +| Question | Answer | +|---|---| +| Database | Core Postgres (`env.postgres.uri_core`, `TransactionsEngine`), the same database as `session_streams`, `session_turns`, `session_interactions`. Verified at `api/oss/src/dbs/postgres/shared/engine.py:29`. | +| Table | `session_commands` | +| Core module | `api/oss/src/core/sessions/commands/` with `dtos.py`, `interfaces.py`, `service.py`, `types.py`, matching the layout of `core/sessions/interactions/` | +| Storage module | `api/oss/src/dbs/postgres/sessions/commands/` with `dbas.py`, `dbes.py`, `dao.py`, `mappings.py` | +| Migration | `api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py`, revising `oss000000021` (verified: `oss000000021_add_session_streams_references.py` is the current head of that chain) | + +Not tracing. The tracing database holds spans, and a command is coordination state that the +sessions plane owns. + +### Columns + +The mixins are the house ones from `api/oss/src/dbs/postgres/shared/dbas.py`: `ProjectScopeDBA`, +`IdentifierDBA`, `LifecycleDBA`, `DataDBA`, `FlagsDBA`, `TagsDBA`, `MetaDBA`. That is the same set +`SessionInteractionDBA` uses (`api/oss/src/dbs/postgres/sessions/interactions/dbas.py:14`). + +| Column | Type | Role | Meaning | +|---|---|---|---| +| `project_id` | UUID, not null | scope | Tenant boundary. Foreign key to `projects.id`, `ON DELETE CASCADE`. | +| `id` | UUID, not null, uuid7 | identity | The `command_id`. The API mints it. | +| `session_id` | String, not null | routing | Which session the command acts on. A bare correlator, not a foreign key, like every other sessions table. | +| `kind` | String, not null | routing | `cancel` in version one. | +| `target_turn_id` | String, null | target | The execution this command must reach, resolved once at admission. Null only when nothing was running or parked. | +| `expected_turn_id` | String, null | target | The caller's `expected_execution_id`, stored as sent. Null when the caller supplied none. | +| `data` | JSON, null | input | The command's own arguments, shaped `{"input": {"text": ..., "attachments": [...]}, "policy": {"on_busy": ...}}`. Empty for `cancel`. | +| `state` | String, not null | delivery | `pending`, `claimed`, `applied`, `obsolete`. | +| `claimed_by` | String, null | delivery | The replica that holds the current claim. Bookkeeping, not an address. | +| `claim_expires_at` | TIMESTAMP tz, null | delivery | When the claim may be delivered again. | +| `claim_count` | Integer, not null, default 0 | delivery | Deliveries so far. Caps re-delivery. | +| `outcome` | String, null | result | What happened to the execution: `stopped`, `not_running`, `superseded_by_newer_turn`, `failed`, `lost`. Null while open. | +| `idempotency_key` | String, null | context | The caller's `Idempotency-Key` header, stored verbatim. | +| `settled_at` | TIMESTAMP tz, null | metadata | When the command reached a terminal state. | +| `flags`, `tags`, `meta` | JSONB / JSON, null | metadata | House mixins. Unused in version one, present for consistency. | +| `created_at`, `updated_at`, `deleted_at`, `created_by_id`, `updated_by_id`, `deleted_by_id` | `LifecycleDBA` | metadata | House lifecycle columns. `created_at` carries a guard: it is the "do not supersede a newer turn" comparison of section 4. | + +Four grouping rules from the interface review are applied here. + +- **Delivery bookkeeping is one group.** `state`, `claimed_by`, `claim_expires_at` and `claim_count` + are the delivery record. On the wire they are nested under `delivery`. In the table they are flat + columns because a claim query filters and orders on them, and a JSON blob cannot be indexed for + that. The names carry the grouping. +- **Delivery is never merged with the result.** `state` says where the command is; `outcome` says + what happened to the execution. That separation is the whole point of decision D-016. +- **The target has its own two columns.** `expected_turn_id` is what the caller asserted; + `target_turn_id` is what the API resolved. Keeping both makes a 409 explainable after the fact and + gives a future `target.execution_id` an obvious home. +- **There is no `owner_replica_id` and no `runner_url`.** The first revision routed commands by the + owner replica. Section 5 replaces that with session-scoped claims, so the record needs no routing + identity at all, and an address in a durable record would be an implementation detail with a + lifetime longer than the thing it points at. + +### Two columns added to `session_streams` + +**`stopping_turn_id`**, String, nullable. It names the execution that an accepted Stop is waiting on. +It is written in the same transaction as the command insert, and cleared at settlement. + +**`turn_started_at`**, TIMESTAMP tz, nullable. It records when the row's current `turn_id` started. +It exists for one reason: the stale-Stop guard in section 4 needs to compare a command's arrival +time with the current execution's start time, and **there is nowhere to read that today**. The +options were checked, and none of them works: + +| Candidate | Why it does not serve | +|---|---| +| `session_streams.updated_at` | It is the heartbeat timestamp and moves every 30 seconds. Verified: the mirror write is unconditional (`api/oss/src/core/sessions/streams/service.py:618`). | +| The turn id itself | API-minted turns use uuid7 and are time-ordered (`streams/service.py:940`), but the runner mints its own with `randomUUID()`, which is uuid4 and carries no time (`services/runner/src/server.ts:190`, verified). Every browser turn today is runner-minted. | +| Redis `running` or `alive` | The value is the bare turn id, and the release-if-owner script compares the whole value (`api/oss/src/dbs/redis/sessions/contract.py:153`). Packing a timestamp into it would break that compare and the golden fixture the runner shares. | +| `session_turns.start_time` | It is written, from `turnStartedAt` captured at `services/runner/src/engines/sandbox_agent/run-turn.ts:192` and sent at `:469`. But the append is fire-and-forget (`.catch(() => {})`) and it needs a stream id and a continuity index, so a turn can be running with no row at all. It is a good secondary source, not a guard. | + +So add the column. It is written wherever `turn_id` is written, in the same statement, and only when +the id actually changes: + +```sql +UPDATE session_streams + SET turn_id = :turn_id, + turn_started_at = CASE + WHEN turn_id IS DISTINCT FROM :turn_id THEN now() + ELSE turn_started_at + END, + ... +``` + +That form is idempotent under the repeated heartbeats that stamp the same id every 30 seconds, and +it needs no new writer: both `_start_turn` (`streams/service.py:940`) and the heartbeat's +`durable_turn_id` stamp already go through `SessionStreamEdit`. + +Both are columns and not bits inside `flags` because `flags` is the Redis mirror. Every heartbeat +rewrites it (`api/oss/src/core/sessions/streams/service.py:618`), so a value stored there would be +erased on the next beat. `SessionStreamEdit` carries only `flags`, `tags`, `meta` and `turn_id` +(`api/oss/src/core/sessions/streams/dtos.py:73`), so the heartbeat path cannot touch +`stopping_turn_id` by accident, and it touches `turn_started_at` only through the guarded `CASE`. + +### Indexes and constraints + +```python +__table_args__ = ( + ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + PrimaryKeyConstraint("project_id", "id"), + UniqueConstraint( + "project_id", "session_id", "idempotency_key", + name="uq_session_commands_idempotency", + ), + CheckConstraint("kind IN ('cancel')", name="ck_session_commands_kind"), + CheckConstraint( + "state IN ('pending', 'claimed', 'applied', 'obsolete')", + name="ck_session_commands_state", + ), + Index( + "ix_session_commands_open", + "project_id", "session_id", "created_at", + postgresql_where=text("state IN ('pending', 'claimed') AND deleted_at IS NULL"), + ), + Index( + "ix_session_commands_claims", + "claim_expires_at", + postgresql_where=text("state = 'claimed' AND deleted_at IS NULL"), + ), + Index( + "ix_session_commands_project_session", + "project_id", "session_id", "created_at", + ), +) +``` + +`ix_session_commands_open` is the claim query's index. It leads with `(project_id, session_id)` +because a claim asks for the commands of a named set of sessions, and it is partial on the open +states because a settled command is never claimed again. It also serves the open-command collapse +read at admission. + +The check constraints copy the shape of `ck_session_attachments_state` +(`api/oss/src/dbs/postgres/sessions/attachments/dbes.py:35`). + +### Idempotency, in two layers + +1. **Client key.** `uq_session_commands_idempotency` on `(project_id, session_id, + idempotency_key)`, the same triple `uq_session_attachments_idempotency` uses + (`api/oss/src/dbs/postgres/sessions/attachments/dbes.py:29`). An insert that hits the constraint + is caught, the existing row is read back, and it is returned to the caller. That is the pattern + `SessionInteractionsDAO.create_interaction` already uses + (`api/oss/src/dbs/postgres/sessions/interactions/dao.py:60`). A null key never collides, because + Postgres treats nulls as distinct in a unique index. +2. **Open-command collapse.** Even with no client key, admission first looks for an open command + (`state IN ('pending','claimed')`) of the same `kind` for the same `(project_id, session_id, + target_turn_id)`. If one exists, the API returns it instead of creating a second. This is what + makes "two Stops in a row" correct without asking the browser to send a key. + +The server `command_id` is the idempotency identity of every later step. A settle for a command that +already reached a terminal state returns the stored state and changes nothing. + +### Retention + +Settled rows (`state IN ('applied','obsolete')`) are deleted 7 days after `settled_at` by the sweep +described in section 4. Commands are operational state, not session history. Durable session history +stays in `session_records`. Open rows are never deleted by the sweep; the watchdog settles them +first. + +--- + +## 3. The state machine + +```text + admission + | + v + +------------> pending ------------------------+ + | | | + | claim expired, | claim (poll, direct call, | nothing to do + | session still | or heartbeat) | + | beating v v + | claimed --------> applied obsolete + | | runner + +-----------------+ reports + | + | claim expired and the session stopped beating + v + obsolete (outcome = lost) +``` + +`applied` and `obsolete` are terminal. There is no transition out of either. + +Every transition is one `UPDATE ... WHERE ... RETURNING *` whose `WHERE` names the state it expects. +`scalar_one_or_none()` decides the winner, so two API replicas cannot both win. This is exactly the +pattern `SessionInteractionsDAO.transition_interaction` already uses +(`api/oss/src/dbs/postgres/sessions/interactions/dao.py:120`). Verified. + +| Transition | Who does it | Guard | +|---|---|---| +| none to `pending` | The API, on an accepted Cancel | `INSERT`, protected by `uq_session_commands_idempotency` and by the open-command collapse read in the same transaction | +| none to `obsolete` | The API, when nothing is running or parked | Same insert, with `state='obsolete'`, `outcome='not_running'`, `settled_at=now()` | +| `pending` to `claimed` | The API, serving a claim, a direct call, or a heartbeat | `WHERE state = 'pending'` | +| `claimed` to `pending` | The command sweep, when a lease expired and the session is still beating | `WHERE state = 'claimed' AND claim_expires_at < now() AND claim_count < :max_deliveries` | +| `claimed` to `applied` | The API, on the runner's outcome report | `WHERE state = 'claimed' AND claimed_by = :replica_id` | +| `claimed` to `obsolete` | The API, on a report of `not_running` or `superseded_by_newer_turn` | Same guard | +| `claimed` to `obsolete` (`lost`) | The command sweep, when the lease expired and the session stopped beating | `WHERE state = 'claimed' AND claim_expires_at < now()`, plus the heartbeat-age test of section 4 | +| `pending` to `obsolete` (`lost`) | The command sweep, when nobody ever claimed it | `WHERE state = 'pending' AND created_at < :admission_deadline` | + +The claim statement, in the form the DAO writes it: + +```sql +UPDATE session_commands + SET state = 'claimed', + claimed_by = :replica_id, + claim_expires_at = now() + make_interval(secs => :lease_seconds), + claim_count = claim_count + 1, + updated_at = now() + WHERE (project_id, id) IN ( + SELECT project_id, id + FROM session_commands + WHERE state = 'pending' + AND deleted_at IS NULL + AND (project_id, session_id) IN :held_sessions + ORDER BY created_at + LIMIT :limit + FOR UPDATE SKIP LOCKED + ) +RETURNING *; +``` + +`:held_sessions` is the set of sessions the calling runner holds warm, sent with the request. See +section 5. `FOR UPDATE SKIP LOCKED` is what lets two API replicas serve two claims at the same time +without either blocking or double-claiming. + +The settle statement: + +```sql +UPDATE session_commands + SET state = :result, outcome = :outcome, settled_at = now(), updated_at = now() + WHERE project_id = :project_id + AND id = :command_id + AND state = 'claimed' + AND claimed_by = :replica_id +RETURNING *; +``` + +Zero rows means the claim had already expired or another actor settled it. The route then reads the +row and answers 409 with its stored state, so the runner learns the truth instead of retrying. + +--- + +## 4. The claim lease and the settlement rule + +| Setting | Value | Reason | Environment variable | +|---|---|---|---| +| Lease duration | 90 seconds | Three heartbeat intervals, the window the review picked for H-4 | `AGENTA_SESSIONS_COMMAND_LEASE_SECONDS` | +| Maximum deliveries | 3 | Bounds a delivery loop when a runner accepts but never reports | `AGENTA_SESSIONS_COMMAND_MAX_DELIVERIES` | +| Sweep interval | 10 seconds | Fine enough that a lost Stop settles inside two minutes | `AGENTA_SESSIONS_COMMAND_SWEEP_SECONDS` | +| Admission deadline | 90 seconds | A command nobody ever claimed is a runner that is not there | `AGENTA_SESSIONS_COMMAND_ADMISSION_TIMEOUT_SECONDS` | + +All four go in a new `SessionsCommandsConfig` block in `api/oss/src/utils/env.py`, read through the +shared `env` object. Do not call `os.getenv` in the service (`AGENTS.md`, "Environment config"). + +**Renewal: none in version one.** A claim is not renewed while the runner works. It expires and is +either delivered again or settled. This is safe because applying a Cancel is idempotent, and because +the runner deduplicates. A renewal route is the first thing to add if a harness cancel is ever +slower than the lease, and the column that would carry it (`claim_expires_at`) already exists. + +### The settlement rule when the runner is gone (H-4) + +**The Redis time to live cannot be the signal.** `alive` and `running` both hold 3600 seconds +(`api/oss/src/utils/env.py:1417` and `:1421`, verified). A `stopping` state that waits for those +keys to expire is a `stopping` state that lasts an hour. Settlement must key off **heartbeat age**, +which is `session_streams.updated_at`, the column the heartbeat writes on every beat and the one the +orphan sweep already filters on (`api/oss/src/tasks/asyncio/sessions/orphan_sweep.py:57`, verified). + +The rule, evaluated by the sweep for every command whose `claim_expires_at` has passed: + +| Heartbeat age for that session | Attempts left | Action | +|---|---|---| +| Under 90 seconds (the runner is alive, the report was lost) | yes | Re-arm to `pending` and deliver again | +| Under 90 seconds | no | Settle `obsolete`, `outcome='lost'`, and run the settlement side effects | +| 90 seconds or more (the runner is gone) | either | Settle `obsolete`, `outcome='lost'`, and run the settlement side effects | + +A session parked awaiting an approval stops beating on purpose (`server.ts:618`), so it would look +"gone" by heartbeat age alone. Exclude it: a command whose target session has an open interaction, +or whose stream row is `alive` but not `running`, uses the admission deadline rather than the +heartbeat-age test. That is the same distinction the orphan sweep already draws between its 300 +second running threshold and its 1800 second idle threshold +(`api/oss/src/tasks/asyncio/sessions/orphan_sweep.py:33` and `:37`, verified). + +**The watchdog owns settlement, not this design.** A separate agent is building the execution +watchdog on branch `feat/session-execution-watchdog`. This design does not build a second one. The +command sweep described here is either that watchdog with the command rules folded in, or a caller +of it. The single rule both must obey: **one execution reaches exactly one terminal outcome, written +by exactly one writer.** If the watchdog marks an execution `lost`, it must settle that execution's +open commands in the same transaction, and vice versa. Decide the ownership before either lands. + +The side effects of a `lost` settlement are the same as a normal settlement (section 7, step 10 to +13), with one difference: Redis keys are cleared with the force variants rather than the +owner-checked ones, because the owning process is gone. + +### Deduplication on the runner (H-5) + +The applied-command set must outlive the poll loop, because a loop restart with an empty set would +apply a Stop a second time, and by then the session may be running a newer turn. + +- The set lives in the same module as the session state the runner already keeps across turns, next + to `SessionPool` (`services/runner/src/engines/sandbox_agent/session-pool.ts:90`), keyed by + `${projectId}:${sessionId}` with a bounded list of applied command ids and their apply times, kept + for 30 minutes. It is not owned by the poll loop and does not reset when the loop restarts. +- Both delivery paths call one `applyCommand(command)` entry point that consults the set first. +- **Applying an already-applied command is a no-op that re-sends the acknowledgement.** It does not + abort anything, and it does report the stored outcome, so a lost acknowledgement is repaired + without a second abort. + +### The three guards on the target execution (H-3) + +A Stop that arrives after its turn ended must not touch the next turn. Three guards, in order of +strength: + +1. **The API compares arrival time with the current turn's start time.** This is the guard that + closes the reported race, so it is spelled out below. +2. **The target is pinned at admission.** The API resolves `target_turn_id` once and never + re-resolves it. A turn that starts later has a different id, so a pinned command cannot reach it. +3. **The runner repeats the comparison locally.** The envelope carries the command's arrival time. + The runner refuses to abort an execution that started after it, and settles the command + `obsolete` with `outcome='superseded_by_newer_turn'`. The runner holds its own execution's start + time in memory, so this check is exact even when the API's is not. +4. **First-party clients always send `expected_execution_id`.** The field stays optional in the + contract, as decision D-010 requires, but the desktop and mobile Stop buttons must send it. Today + the desktop sends nothing (`useAgentChatSession.ts:505`, verified). Treat an omitted id from a + first-party client as a bug, not as a supported mode. + +#### The arrival-time comparison, when no expected execution id was sent + +The race: the user presses Stop at t=0 while turn one is running. Turn one ends at t=0.1. Turn two +starts at t=0.2. The request is applied at t=0.3, reads Redis, finds turn two, and targets a turn the +user never meant to stop. + +The rule, applied at admission before anything is inserted: + +1. The service stamps `received_at = now()` as its **first** action, before it reads Redis. It later + writes that same value as the row's `created_at` rather than letting the server default fill it, + so the value it compared is the value it stored. +2. It reads the current running owner from Redis and the session's row, which gives `turn_id` and + `turn_started_at` in one query the admission path already makes. +3. If `turn_started_at > received_at`, the current execution began after the user pressed Stop. + Insert the command already settled: `state='obsolete'`, + `outcome='superseded_by_newer_turn'`, `settled_at=now()`, `target_turn_id=null`. Return 200 with + `execution.state = "idle"`. **Do not target that turn and do not touch Redis.** +4. Otherwise proceed normally. + +This runs only when `expected_execution_id` is absent. When the caller sent one, the 409 comparison +already settles the question and is stricter. + +**When `turn_started_at` is null, the guard does not fire.** A row written before this column +existed, or a turn whose stamp was lost, yields no comparison. The API then targets the turn as it +does today and leaves the decision to guard 3, which is exact because the runner reads its own +memory. Failing this way round is deliberate: a guard that refuses to Stop whenever it lacks data +would break the common case to protect a rare one. + +`session_turns.start_time` is a useful secondary source when the row exists, but the design does not +depend on it, for the reasons in the table in section 2. + +The `expected_execution_id` check itself happens twice, for two different reasons. At admission the +API compares it to the Redis running owner and answers 409 if they differ. At application the runner +applies the command only to a local execution whose `turnId` equals `target_turn_id`, and settles +`obsolete` with `outcome='not_running'` when it holds no such execution. + +--- + +## 5. The claim contract + +### The loop is session-scoped and lives as long as the session is warm (H-2) + +This is the single most important correction from the review. The first revision started one poll +per runner process and routed by owner replica. That has two faults: it cannot say which sessions +the runner actually holds, and a per-turn loop would go silent exactly when a turn parks. + +The rule: + +- **One loop per runner process.** Not one per turn and not one per session. +- **The loop declares the sessions it holds.** Every claim carries the current set. That set is the + union of the execution registry (turns in flight) and the keep-alive pool keys, which are already + `${projectId}:${sessionId}` strings and already include parked entries + (`SessionPool.keys()` and `SessionPool.snapshot()`, + `services/runner/src/engines/sandbox_agent/session-pool.ts:108` and `:127`, verified; a parked + entry is seated as `awaiting_approval` at + `services/runner/src/lifecycle/session-coordinator.ts:764`, verified). +- **A session leaves the set only when the runner stops holding it warm.** A parked approval stays + in the set, so a Stop reaches it. That is H-2 closed. +- **Claims are queries over durable state, never a stream position** (H-6). The request declares a + set of sessions and the API answers with whatever is pending for them right now. There is no + cursor, no offset and no resume token, so a command created while the connection was down is + picked up by the next claim like any other. + +### Routes + +| Route | Method | Caller | Purpose | +|---|---|---|---| +| `/sessions/control/commands/claim` | POST | Runner | Claim the pending commands for the sessions this runner holds, waiting up to the hold if there are none | +| `/sessions/control/commands/{command_id}/outcome` | POST | Runner | Report the terminal outcome | + +Both live on a new `SessionControlRouter` in `api/oss/src/apis/fastapi/sessions/router.py`, included +with no prefix like the streams router (`api/entrypoints/routers.py:1354`, verified), and excluded +from the public schema. + +### Authentication + +The runner authenticates its per-run calls as the invoke caller, using the ephemeral platform +credential from the run (`services/runner/src/sessions/alive.ts:60`, verified). That credential +cannot carry these routes: the loop belongs to the process and spans many projects, and a run's +credential expires while the process keeps polling. + +So both routes use the shared runner token, `AGENTA_RUNNER_TOKEN`, which both sides already hold +(`api/oss/src/utils/env.py:1161` as `env.runner.token`, and `services/runner/src/server.ts:104`). +Verified. It is the same secret the existing API-to-runner hop uses in the other direction +(`api/oss/src/core/sessions/streams/runner_client.py:44`). + +Mechanics: + +- Add the prefix `/sessions/control/` to `_PUBLIC_ENDPOINTS` + (`api/oss/src/middlewares/auth.py:52`), so the project-scoped auth middleware does not reject a + request that carries no user credential. This is the same treatment the OAuth callback and the + Composio event routes already get. +- The route then does its own check, with a constant-time comparison against `env.runner.token`, + accepting `X-Agenta-Runner-Token: ` first and `Authorization: Bearer ` second. That + is the header pair and the comparison the runner itself already implements + (`services/runner/src/server.ts:127`). +- **Fail closed.** If `env.runner.token` is unset or blank, both routes answer 503 and serve nothing. + Being exempt from the middleware makes the route's own check the only gate, so it must never + default to open. +- The project scope of every command comes from the row and from the declared session set, never + from a header. A runner can only receive commands for sessions it named, and a session id is + meaningful only inside its project, so the pair is the scope. + +### Request and response bodies + +Claim request: + +```json +{ + "replica_id": "runner-7f3c", + "sessions": [ + {"project_id": "1f0a4b2c-0000-4000-8000-000000000002", "session_id": "sess-42"}, + {"project_id": "1f0a4b2c-0000-4000-8000-000000000002", "session_id": "sess-77"} + ], + "wait_seconds": 25, + "limit": 10 +} +``` + +`replica_id` is delivery bookkeeping: it becomes `claimed_by` so a settle can be matched to its +claim. It is not routing, and it is not an address. `sessions` is the routing input, capped at 200 +entries and ordered most recently used first. `wait_seconds` is bounded server-side to +`[0, AGENTA_SESSIONS_CONTROL_POLL_HOLD_SECONDS]`, default 25. `limit` is bounded to `[1, 50]`, +default 10. + +Claim response, 200: + +```json +{ + "count": 1, + "commands": [ + { + "id": "0199a3f2-0000-7000-8000-000000000001", + "project_id": "1f0a4b2c-0000-4000-8000-000000000002", + "session_id": "sess-42", + "kind": "cancel", + "target": { + "turn_id": "0199a3f1-0000-7000-8000-00000000000a", + "expected_turn_id": "0199a3f1-0000-7000-8000-00000000000a" + }, + "delivery": { + "claimed_by": "runner-7f3c", + "claim_expires_at": "2026-09-02T22:10:31Z", + "attempt": 1 + }, + "created_at": "2026-09-02T22:09:01Z" + } + ] +} +``` + +`count` plus a list is the house response envelope (`SessionsResponse`, +`api/oss/src/apis/fastapi/sessions/models.py:105`). A `cancel` carries no `input` and no `policy`; +both appear only for the kinds that have them, so a reader never has to interpret an empty object. +`created_at` is on the envelope because the runner needs it for guard 3 of section 4. + +Claim response, 204: the hold expired with nothing to deliver. No body. + +Outcome request: + +```json +{ + "replica_id": "runner-7f3c", + "result": "applied", + "execution": { + "id": "0199a3f1-0000-7000-8000-00000000000a", + "state": "stopped" + } +} +``` + +`result` is the command's terminal state, `applied` or `obsolete`. `execution.state` is one of +`stopped`, `failed`, `not_running`, `superseded_by_newer_turn`. `execution.error` is a short string, present only +when the state is `failed`. The two objects are separate because they answer different questions and +have different owners: `result` is delivery bookkeeping the runner controls, `execution` is a +product fact the user sees. + +Outcome response, 200: + +```json +{ + "command": { + "id": "0199a3f2-0000-7000-8000-000000000001", + "state": "applied", + "outcome": "stopped", + "settled_at": "2026-09-02T22:09:12Z" + } +} +``` + +Outcome response, 409: the claim was not held by this replica. The body carries the same `command` +object with its stored state, so the runner can stop and move on rather than retry. + +### How the hold works + +The route subscribes to one Redis Pub/Sub channel per declared session on the durable plane, then +loops: + +1. Claim once, without waiting. Return 200 if anything came back. +2. Wait on the subscription with a one second timeout, so the loop can re-check the shutdown flag. +3. On a message, or every second, try the claim again. +4. When the hold budget runs out, return 204. + +Three details are not optional: + +- **Add `control_channel(project_id, session_id)` to the Redis contract** + (`api/oss/src/dbs/redis/sessions/contract.py`), with the payload `{"type": "command-pending"}` and + nothing else. It is project-scoped like every other key in that file, and it carries no tenant data + because the claim re-queries Postgres, which is the authority. +- **Reuse the watch endpoint's shutdown release.** `api/oss/src/apis/fastapi/sessions/watch.py:50` + installs a hook on uvicorn's exit path because a held response blocks graceful shutdown for ever. + A held claim has exactly the same failure. Import `request_shutdown` and the same threading event, + or move both into a small shared helper. +- **A new session mid-hold ends the hold.** When the runner starts holding a session that was not in + the declared set, the loop aborts its in-flight request locally and re-issues the claim with the + new set. That is one in-process event, not a server concern. + +### What the runner does + +| Result | What the runner does | +|---|---| +| 200 with commands | Apply each through `applyCommand`, report each outcome, then claim again at once | +| 204 | Claim again at once | +| Read timeout with no response | Claim again after the backoff floor | +| Network error, 502, 503, 504 | Back off: 1 s, 2 s, 4 s, 8 s, 16 s, then 30 s, with 20 percent jitter. Reset on the first success | +| 401 or 403 | Log once at error level and retry every 60 s. This is a deployment misconfiguration and must be loud, not a tight loop | +| 429 | Back off as for a network error | +| API restart | The held connection closes. This is the network error case. Nothing is lost, and the next claim is a fresh query over durable state, not a resumed cursor | +| Empty session set | Do not call. Wait for the next session to be held | + +The client timeout must exceed the hold: set the fetch timeout to `hold_seconds + 10`. + +### After a reconnect, the runner asks again; it never resumes a position + +This is worth stating on its own, because getting it wrong loses commands silently. + +A claim is a **query over durable state**. The runner sends the sessions it currently holds and the +API answers with whatever is pending for them at that moment. There is no cursor, no offset, no +sequence number, no resume token and no server-side per-runner queue position. + +So after any break, whether the connection dropped, the API replica restarted, the runner process +restarted, or the loop was switched off and on, the runner simply issues the next claim with its +current session set. A command created while nothing was listening is `pending` in Postgres, and the +next claim returns it like any other. Nothing has to be replayed, and nothing can be skipped by +starting from the wrong place, because there is no place to start from. + +The one thing this requires: the session set must be rebuilt from what the process actually holds, +not cached from before the break. After a runner restart the set comes from the rebuilt pool and the +live execution registry, both of which reflect reality rather than history. + +--- + +## 6. The heartbeat fallback + +One field is added to the heartbeat response DTO `SessionHeartbeatResult` +(`api/oss/src/core/sessions/streams/dtos.py:180`): + +```python +class SessionHeartbeatResult(BaseModel): + stream: Optional[SessionStream] = None + replica_id: str + is_current_turn: bool = True + # Commands for THIS session, claimed by this beat under the same compare-and-set the + # claim route uses. Empty when there is nothing to deliver, which is the normal case. + commands: List[SessionCommandEnvelope] = Field(default_factory=list) +``` + +`SessionCommandEnvelope` is the same model the claim route returns, so the runner has one parser and +one applier. + +Rules: + +- The beat serves only commands for its own `(project_id, session_id)`, and only those whose + `target.turn_id` matches the beat's `turn_id` or is null. It never serves another session's + commands, because the beat is authenticated with the run's project-scoped credential. +- It claims them under the same statement as the claim route, so a command cannot be delivered by + both paths at once. One of the two wins the compare-and-set; the other sees zero rows. +- The runner deduplicates by `command_id` in the set described in section 4, so a command delivered + by the claim route and offered again by a beat is acknowledged again but applied once. + +**Know what this fallback cannot do.** It covers only a session with a live turn, because the +heartbeat stops when a turn ends or parks (`services/runner/src/server.ts:618` and +`services/runner/src/sessions/alive.ts:241`, verified). It is not a substitute for the session-scoped +loop, and it must not be treated as the delivery path for a parked session. It exists for two cases: +the primary adapter is switched off, and the primary adapter is failing while the run's own +heartbeat still works. + +The runner reads the new field in `sendHeartbeat` (`services/runner/src/sessions/alive.ts:96`) and +hands each entry to `applyCommand`. The existing fail-open rule at `alive.ts:92` is unchanged: a +non-2xx beat returns nothing. That is one more reason the primary path does not depend on a run's +credential. + +--- + +## 7. Stop, end to end + +### Case 1: the normal Stop + +1. The browser posts `POST /sessions/{session_id}/cancel` with `expected_execution_id` filled in + from its own state, and an optional `Idempotency-Key` header. It marks its own view "stopping" + and stops rendering. It does not abort anything server-side by itself. +2. The API authorizes the caller with `Permission.RUN_SESSIONS`, the same permission the current + cancel path uses (`api/oss/src/apis/fastapi/sessions/router.py:377`). +3. The API resolves the target once. It stamps `received_at` first, then reads + `get_running_owner`, falling back to `get_alive_owner`, both already imported by the streams + service (`api/oss/src/core/sessions/streams/service.py:39`), and reads the session row for + `turn_started_at`. Call the result `turn_id`. Three outcomes: if `expected_execution_id` was sent + and differs, stop with 409; if no expected id was sent and `turn_started_at > received_at`, stop + with a settled `superseded_by_newer_turn` command and 200 (section 4); otherwise continue. +4. **One transaction.** Insert the command with `state='pending'`, `kind='cancel'`, + `target_turn_id=turn_id`, `expected_turn_id=`, and set + `session_streams.stopping_turn_id = turn_id` on the same session's row. The DAO method takes an + optional `AsyncSession` so both writes share one session, the pattern `RecordsDAO.append` already + uses (`api/oss/src/dbs/postgres/sessions/records/dao.py:33`). +5. **Redis is not touched.** No tombstone, no `force_cancel_alive`, no `clear_running`. The current + execution keeps `alive` and `running` while it stops, which is what stops a second message from + starting underneath it. This is decision D-017. +6. The API delivers through the configured adapter: the direct call posts to the runner (section 9), + the long-poll adapter publishes on the session's control channel. Either way the API then returns + 202 with the command id and the target execution id. **Delivery failure does not fail the + request**, because the command is already durable. +7. The runner receives the command, on its held claim or on the direct route. +8. `applyCommand` checks the deduplication set, checks that it holds an execution with + `target.turn_id`, checks that the execution did not start after `created_at`, and then aborts it. + The abort must be a harness cancel that keeps the sandbox and the native harness session warm. + **This step is Work package A's deliverable, and it is not free today.** `shouldPark` returns + false whenever the signal is aborted (`services/runner/src/engines/sandbox_agent/engine.ts:26`, + verified), so the environment is destroyed. The review's proposed fix, which this design assumes: + thread a cancel reason to the runner so a user Stop is distinguishable from a disconnect abort, + and let `shouldPark` park when the result is a clean cancellation caused by a user Stop. Nothing + in this design can deliver a warm Stop without that change. +9. The runner posts `POST /sessions/control/commands/{command_id}/outcome` with + `result: "applied"` and `execution: {"id": turn_id, "state": "stopped"}`. +10. The API settles both, in one transaction: + - Command: `state='applied'`, `outcome='stopped'`, `settled_at=now()`, guarded on + `state='claimed' AND claimed_by=`. + - Stream row: clear `stopping_turn_id`. +11. The API releases ownership, in this order: + - `mark_turn_superseded(turn_id)`, so a late beat from the stopped execution cannot re-arm the + locks. + - `release_running(turn_id)`, owner-checked, so it can only release its own execution's key. + - **`alive` is left alone.** It expires on its own time to live, exactly as it does at the end + of a normal turn (`api/oss/src/core/sessions/streams/service.py:590`, verified). This is the + deliberate difference from today's cancel, which force-deletes `alive` and is a large part of + why Stop currently reads as a session teardown. Warm resume is the required outcome, so Stop + must leave the session in the state a finished turn leaves it in. +12. The API cancels the stopped execution's pending interactions, the same call the kill route + already makes (`api/oss/src/apis/fastapi/sessions/router.py:441`), scoped with `only_turn_id` so + it touches only this execution's gates. +13. The API publishes the existing watch notification `lifecycle: ended` on the session channel + (`api/oss/src/core/sessions/streams/service.py:202`), which every open browser already listens + to. +14. Browsers refetch through their current query paths and show the turn as stopped. + +Steps 1 to 8 are the five second budget. Steps 9 to 14 follow the runner's own cancel time. + +### Case 2: Stop when nothing runs + +At step 3 there is no running owner and no alive owner. + +- If the caller sent no `expected_execution_id`: the API inserts the command already settled, + `state='obsolete'`, `outcome='not_running'`, `settled_at=now()`, and returns 200. No Redis write, + no delivery. The caller gets a stable command id, so a retry with the same idempotency key returns + the same record. +- The stream row is not touched, because nothing is stopping. + +### Case 3: Stop with a stale `expected_execution_id` + +The caller sent an execution id that is not the current running owner. The API returns 409 with a +body naming the current execution id, or null when nothing runs. Nothing is inserted and nothing is +delivered. The browser learns that the run it was looking at already ended and refreshes. + +### Case 4: Stop while an interaction is pending and the sandbox is parked + +This is the case with no channel today. A parked approval means the runner is running no turn: the +coordinator seats the environment as `awaiting_approval` +(`services/runner/src/lifecycle/session-coordinator.ts:764`, verified) and the request handler's +`finally` has already released the alive watchdog (`services/runner/src/server.ts:618`, verified), +so the heartbeat has stopped. Redis holds `alive` but not `running`, because the last beat carried +`is_running: false` (`api/oss/src/core/sessions/streams/service.py:590`, verified). + +1. Step 3 finds no `running` owner and does find an `alive` owner. `target_turn_id` takes the alive + owner's value. +2. The command is created `pending` and delivered. **The session is in the runner's declared set**, + because the parked pool entry is one of `SessionPool.keys()`, so the held claim delivers it. With + the direct adapter the process is reachable regardless. +3. `applyCommand` finds no live execution for that turn. It resolves the parked entry instead, + settles the command `applied` with `execution.state = "not_running"`, and leaves the parked + environment in the pool so the session stays warm. It does not destroy the park: Stop ends the + work, not the session. +4. The API settles as in case 1. **Step 12 is the visible part here:** the pending interaction is + cancelled, so the approval card stops rendering as actionable. That closes the class of bugs where + an approval survives a Stop and its buttons do nothing. + +### Case 5: two Stops in a row + +The second request finds an open command for the same `(project_id, session_id, target_turn_id)` +and returns it unchanged, with the same command id. If the second request carries a different +`Idempotency-Key`, the open-command collapse still wins, because it runs before the insert. If the +first command has already settled and a new execution has started, the second Stop is a fresh +command against the new execution, which is what the user meant. + +### Case 6: a Stop that arrives after its turn ended + +The user presses Stop at t=0 while turn one runs. Turn one ends at t=0.1, turn two starts at t=0.2, +and the request is applied at t=0.3. Today `_displace_turns` would tombstone turn two before its +first output, and that tombstone lasts an hour because every read refreshes it +(`api/oss/src/dbs/redis/sessions/locks.py:147`, verified). The four guards of section 4 answer this +case in order. + +1. **Guard 1, at admission.** The API compares `received_at` with the row's `turn_started_at`. Turn + two started after the request arrived, so the API inserts a command that is already settled, + `state='obsolete'` with `outcome='superseded_by_newer_turn'`, targets nothing, touches no Redis + key, and returns 200 with `execution.state = "idle"`. **Turn two never hears about it.** This is + the guard that closes the case; the rest are for what it cannot see. +2. **Guard 2** covers the ordinary late Stop, where turn one simply ended and nothing replaced it. + The command names a turn that no longer exists, so the runner settles `obsolete` with + `not_running`. +3. **Guard 3** covers the residual window where turn two took over between the API's Redis read and + its insert, or where `turn_started_at` was null and guard 1 could not fire. The runner sees an + execution that started after the command's arrival time and settles `obsolete` with + `superseded_by_newer_turn` rather than aborting it. This check is exact, because the runner reads + its own memory. +4. **Guard 4** removes the whole class for first-party clients, which send `expected_execution_id` + and get a 409 naming the current execution. + +No guard writes a Redis tombstone, so nothing can be killed for an hour the way `_displace_turns` +can today. + +### Case 7: the runner is gone + +No claim arrives, or the command was claimed and never settled. The sweep applies the table in +section 4, keyed off heartbeat age rather than the 3600 second Redis time to live. It settles the +command `obsolete` with `outcome='lost'`, force-clears the Redis keys, cancels the pending +interactions, and publishes `ended`. The user sees a terminal state within about two minutes instead +of an hour of "stopping". + +--- + +## 8. The control-delivery port + +There are two ports, one on each side. They are named separately because they are implemented in +different languages by different components, and only one of them is the RFC's `deliver / +acknowledge / recover`. + +### API side, Python + +`api/oss/src/core/sessions/commands/interfaces.py`: + +```python +class ControlDeliveryPort(ABC): + """How the API reaches the runner that holds a session. Transport only. + + Durability, authorization, idempotency, the state machine, and terminal settlement + live in SessionCommandsService and must not move into an adapter. + """ + + @abstractmethod + async def deliver(self, *, command: SessionCommand) -> DeliveryReceipt: + """Make `command` reachable by whoever holds its session, promptly. + + Best effort: a failure here never fails admission, because the command is already + durable and both the sweep and the fallback recover it. The receipt says only what + the transport learned, never what happened to the execution. + """ + ... + + @abstractmethod + async def acknowledge(self, *, command_id: UUID, replica_id: str) -> None: + """Record that a replica took the command, for adapters that keep their own + delivery bookkeeping.""" + ... + + @abstractmethod + async def recover( + self, *, sessions: List[SessionScope], limit: int + ) -> List[SessionCommand]: + """Open commands for these sessions. The claim route, the direct-call retry and the + heartbeat fallback all go through this.""" + ... +``` + +```python +class DeliveryReceipt(BaseModel): + # What the transport learned. Not an execution outcome. + status: Literal["accepted", "unreachable", "not_held"] +``` + +`accepted` means a runner took the command and will report. `unreachable` means the transport +failed, so the sweep or a later claim will handle it. `not_held` means a reachable runner said it +does not hold that session, which lets the service settle the command at once instead of waiting for +the deadline. + +A later adapter must provide prompt, at-least-once delivery to whoever holds the named session. It +may reorder. It may deliver twice. It must not transform or interpret a command, must not settle +one, and must not be the only record that a command exists. Replacing it must change no route, no +DTO, and no state transition. + +### Runner side, TypeScript + +`services/runner/src/sessions/control-channel.ts`: + +```ts +/** One command as the API delivers it. The same shape arrives on every transport. */ +export interface ControlCommand { + id: string; + projectId: string; + sessionId: string; + kind: "cancel"; + target: { turnId: string | null; expectedTurnId: string | null }; + createdAt: string; +} + +export interface ControlOutcome { + /** The command's terminal state. */ + result: "applied" | "obsolete"; + execution: { + id: string | null; + state: "stopped" | "failed" | "not_running" | "superseded_by_newer_turn"; + error?: string; + }; +} + +/** The transport. `control-poll.ts` implements it over long polling; the direct route + * in `server.ts` feeds the same applier without implementing this at all. */ +export interface ControlChannel { + /** Block until a command arrives for one of `sessions`, or the hold expires. */ + receive(sessions: SessionScope[], signal: AbortSignal): Promise; + settle(command: ControlCommand, outcome: ControlOutcome): Promise; +} +``` + +`applyCommand(command)` sits above the channel, not inside it, so every path shares one applier, one +set of guards and one deduplication set. + +The runner also needs an execution registry, because the abort controller is a local variable inside +`runAndStreamWithApiBaseResolved` today (`services/runner/src/server.ts:450`, verified). Add a +module-level map from `${projectId}:${sessionId}` to `{ turnId, startedAt, abort(): void }`, +registered when the run starts and removed in the same `finally` that releases the watchdog +(`services/runner/src/server.ts:618`). `startedAt` is what guard 3 of section 4 compares. This +mirrors `inFlightSandboxes` (`services/runner/src/engines/sandbox_agent/environment.ts:239`). + +--- + +## 9. The direct-call adapter as an alternative first adapter + +This is the section the architecture review asked for as 8b. It sits here, directly after the port, +because that is what it is: the second adapter behind the same port, and a candidate for being the +**first** one built. + +The product review argues that with one runner, the authenticated API-to-runner hop that already +carries hard kill can carry Cancel today, and that long polling is machinery for a second runner that +does not exist. The RFC's own text agrees that direct managed-runner routing is a legitimate adapter +behind the port (`rfc.md`, "Control delivery must sit behind an internal port"). That argument is +correct on its own terms, and this design makes both adapters cheap so Mahmoud can pick either in +the morning without changing anything else. + +### What already exists + +- **The API side.** `kill_runner_sandbox` posts `{sessionId, projectId}` with + `Authorization: Bearer ` to `env.runner.internal_url` and swallows every + failure (`api/oss/src/core/sessions/streams/runner_client.py:30`, verified). It is 33 lines. +- **The runner side.** `POST /kill` sits behind the same token gate, reads a capped body, resolves + the pool scope and tears the session down (`services/runner/src/server.ts:704`, verified). + +### What the direct adapter adds + +**The runner: `POST /cancel`, beside `/kill`.** Same token gate, same capped body reader, same +scoping rule. Body: + +```json +{ + "commandId": "0199a3f2-0000-7000-8000-000000000001", + "projectId": "1f0a4b2c-0000-4000-8000-000000000002", + "sessionId": "sess-42", + "targetTurnId": "0199a3f1-0000-7000-8000-00000000000a", + "createdAt": "2026-09-02T22:09:01Z" +} +``` + +It builds a `ControlCommand` from that body and hands it to the same `applyCommand`. It answers 202 +when it holds the session and has accepted the command, and 404 when it does not. It does **not** +return the execution outcome: the runner reports that through the settle route, so settlement has +one path on every transport. Roughly 40 lines beside the existing kill branch. + +**The API: `cancel_runner_execution`, beside `kill_runner_sandbox`.** The same 30 lines with a +different path and body. The adapter maps the response: 202 to `accepted`, 404 to `not_held`, +anything else and every exception to `unreachable`. One file, +`api/oss/src/dbs/http/sessions/control_delivery_direct.py`, implementing `ControlDeliveryPort`. +`acknowledge` is a no-op, because the claim compare-and-set is the acknowledgement. `recover` runs +the same query the claim route runs, and the service calls it from the sweep. + +**The durable command is still inserted first.** The order is not negotiable and it is the whole +difference between this adapter and a bare remote call: + +1. Admit and insert the command, with `stopping_turn_id`, in one transaction. Commit. +2. Only then call the runner. +3. Whatever the call returns, the user's request has already succeeded. A `not_held` lets the + service settle at once; an `unreachable` leaves the command `pending` for the sweep or for a + later retry. **Neither changes the 202.** + +Inverting those two steps, calling first and recording afterwards, would give back every failure the +record exists to close, because a crash between the call and the insert leaves an aborted execution +with no terminal outcome written anywhere. + +### What it cannot do + +- **Reach a session it cannot resolve locally.** A Stop against a parked approval has no entry in the + execution registry, because no turn is running. The runner must fall back to the keep-alive pool, + which already has the lookup for exactly this: `SessionPool.awaitingApproval(sessionId)` + (`services/runner/src/engines/sandbox_agent/session-pool.ts:117`, verified). That is a few lines, + but it is not free, and it is needed by both adapters. Do not treat the parked case as covered + just because the process is reachable. +- **Survive a second runner replica.** `env.runner.internal_url` is one service address + (`api/oss/src/core/sessions/streams/runner_client.py:44`, verified). Behind a load balancer the + call lands on whichever replica answers, which is the right one only by luck. +- **Reach a user-operated runner.** It needs inbound reachability from the API to the runner. A + runner behind a firewall cannot be called at all. The RFC treats that deployment as a + consideration rather than a requirement, so this is a real but not yet binding limit. + +### Making the wrong-replica failure loud + +The silent-failure worry is fair, and there are two ways to close it. Build the first; the second is +optional. + +**Primary, and exact: treat a contradictory `not_held` as an error.** A mis-routed call is not +actually silent at the protocol level. The runner answers 404 `not_held` when it does not hold the +session, so the API always learns that delivery did not land. What makes it dangerous is that +`not_held` is also the **legitimate** answer when the session really has ended, so the two cases look +alike. They are easy to tell apart with data the API already has: + +> A `not_held` for a session whose `session_streams` row says `is_alive` **and** whose heartbeat age +> is under one interval means some process is running that session and it is not the one we just +> called. That is the wrong-replica failure, and nothing else produces it. + +On that condition, log at error level with the session id, the target turn id and the replica id +from the Redis `owner` key, count it on a metric, and settle the command `obsolete` with +`outcome='lost'` rather than `not_running`, so the user is told the Stop failed instead of being +told the work had already finished. This needs no new storage and no census. + +**Optional, preventive: refuse the configuration.** Two parts, both cheap: + +- A required flag. The direct adapter refuses to start unless + `AGENTA_SESSIONS_CONTROL_DIRECT_SINGLE_REPLICA=true` is set, so choosing it is a deliberate + statement about the deployment rather than a default someone inherited. Optionally let the operator + name the replica instead, `AGENTA_SESSIONS_CONTROL_DIRECT_REPLICA_ID=`, and refuse delivery + when the session's owner key names a different one. +- A replica census. The heartbeat handler already computes the owning `replica_id` on every beat + (`api/oss/src/core/sessions/streams/service.py:458`). Have it also run one `ZADD` into a sorted set + keyed by replica id and scored by timestamp. The sweep then reads `ZCOUNT` over the last 10 + minutes and, if the direct adapter is configured and the count exceeds one, logs an error every + pass naming the replicas it saw. One write per beat, one read per sweep, no key scan. + +Do not add a retry across the load balancer in the hope of hitting the right process. It converts a +diagnosable failure into a lottery, and it multiplies load exactly when a deployment is already +misconfigured. + +### What the durable command record adds beyond a bare direct call + +The direct call alone would be an HTTP request with no memory. The record buys four things, and each +one is a bug the current system has: + +1. **Recovery.** The runner can be restarting, deploying, or briefly unreachable. A bare call fails + and the Stop is gone; the user pressed a button and nothing happened. With the record the command + survives, the sweep settles it as `lost` with a terminal outcome the user sees, and a returning + runner picks it up on its next claim. +2. **Idempotency.** Two Stops, a retried request, or a browser that resends on reconnect all collapse + onto one command. A bare call would abort twice, and the second abort can land on a newer turn. + That is review hole H-3 in its cheapest form. +3. **One terminal outcome per execution.** The record is where `stopped`, `not_running`, + `superseded_by_newer_turn`, `failed` and `lost` are written down, and where the watchdog and the runner agree + on who wrote it. A bare call has nowhere to record that the execution really ended. +4. **Audit and the next command kinds.** Who stopped what, when, and what happened. Steer and Queue + need exactly this record, so building it now is not speculative: it is the part of version one + that version two does not have to redo. + +The honest counter-argument, stated plainly: for a single Stop that succeeds on the first try, the +record adds a table and two writes and changes nothing the user sees. Its value is entirely in the +failure cases. + +### Choosing the adapter + +One setting, `AGENTA_SESSIONS_CONTROL_ADAPTER`, with values `direct` and `long_poll`, read through +`env`. The service depends only on the port. Neither adapter changes a route, a DTO, or a state +transition. + +| | Direct call | Long poll | +|---|---|---| +| New code | One runner route, one API client, both small | A runner loop, an API route with a hold, a Redis channel | +| Reaches a parked session | Yes, with the pool lookup above | Yes, the parked session is in the declared set | +| Two or more runner replicas | Wrong process gets the call. Loud with the `not_held` rule above, silent without it | Correct, because the runner declares what it holds | +| Runner behind a firewall | Impossible | Works | +| Runner restarting | The call fails, the sweep settles or a later claim delivers | The claim resumes on reconnect | +| Held connections | None | One per runner process | + +**If `direct` is the default, PR 3b in section 10 is deferred** and the session-scoped loop is not +built at all. H-2 is then closed by the direct route plus the pool lookup rather than by the loop, +and the heartbeat fallback stays as the second path for a session with a live turn. Everything else +in this design is unchanged, which is the point of the port. + +### Recommendation + +**Build the direct adapter first.** Three reasons, in order of weight: + +1. **It removes the largest piece of new machinery from the first release.** No held connection, no + poll loop, no per-session Redis channel, no uvicorn shutdown interaction. The parts that carry the + correctness, the record, the state machine, the guards and the settlement rule, are identical + either way, and they are the parts worth reviewing carefully. +2. **The deployment it fails on does not exist yet.** Agenta runs one runner. The failure mode is + real, and the `not_held` rule above makes it loud rather than silent, which is what turns a + dangerous limitation into a known one. +3. **The port makes the switch small.** Long polling stays one file plus one runner module. When a + second replica or a user-operated runner becomes real, the change is a configuration value and a + module, not a redesign. + +The cost of being wrong is bounded and visible: if a second replica appears before the long-poll +adapter is built, Stop starts failing loudly on the wrong-replica condition and the fix is already +designed. The cost of building long polling first is a larger first release for a deployment that +does not exist. Take the smaller one. + +--- + +## 10. Migration sequence + +Eight pull requests. Each names the files it touches so parallel agents do not collide. Ordering +constraints are stated; anything not constrained can go in any order. + +| PR | Title | Files | Depends on | +|---|---|---|---| +| 1 | Add the session command record | `api/oss/databases/postgres/migrations/core_oss/versions/oss000000022_add_session_commands.py`, `api/oss/src/dbs/postgres/sessions/commands/{dbas,dbes,dao,mappings}.py`, `api/oss/src/core/sessions/commands/{dtos,interfaces,service,types}.py`, `api/oss/src/utils/env.py`, `api/entrypoints/routers.py` (wiring only), `api/oss/tests/pytest/unit/sessions/test_session_commands_dao.py` | none | +| 2 | Runner execution registry and applier | `services/runner/src/sessions/control-channel.ts`, `services/runner/src/sessions/execution-registry.ts`, `services/runner/src/sessions/applied-commands.ts`, `services/runner/src/server.ts` (register and unregister), runner unit tests | none | +| 3a | Direct-call adapter | `services/runner/src/server.ts` (the `/cancel` route and the parked-pool lookup), `api/oss/src/dbs/http/sessions/control_delivery_direct.py` (including the wrong-replica detector of section 9) | 1, 2 | +| 3b | Long-poll adapter | `api/oss/src/apis/fastapi/sessions/router.py` (`SessionControlRouter`), `api/oss/src/apis/fastapi/sessions/models.py`, `api/oss/src/middlewares/auth.py` (one prefix), `api/oss/src/dbs/redis/sessions/contract.py`, `api/oss/src/dbs/redis/sessions/control_delivery.py`, `services/runner/src/sessions/control-poll.ts` | 1, 2 | +| 4 | Public Cancel creates a command | `api/oss/src/apis/fastapi/sessions/router.py`, `api/oss/src/apis/fastapi/sessions/models.py`, `api/oss/src/core/sessions/commands/service.py`, migration `oss000000023` for `session_streams.stopping_turn_id` **and** `session_streams.turn_started_at`, `api/oss/src/dbs/postgres/sessions/streams/{dbas,dbes,dao}.py` (the `CASE` that stamps the start time), `api/oss/src/core/sessions/streams/service.py` (`_start_turn` and the heartbeat stamp) | 1 | +| 5 | Heartbeat command discovery | `api/oss/src/core/sessions/streams/{dtos,service}.py`, `services/runner/src/sessions/alive.ts` | 3a or 3b, and 4 | +| 6 | Command settlement in the watchdog | `api/oss/src/tasks/asyncio/sessions/command_sweep.py` or the equivalent file on `feat/session-execution-watchdog`, `api/entrypoints/routers.py` (lifespan) | 1, and agreement with the watchdog author | +| 7 | Point the clients at the command | `web/packages/agenta-entities/src/session/api/api.ts`, `web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts` (send `expected_execution_id`), `web/mobile/src/features/chat/StopButton.tsx`, `api/oss/src/core/sessions/streams/service.py` (the cancel branch becomes a wrapper) | 4, 5 | + +3a and 3b are alternatives, not a sequence. Build whichever Mahmoud picks; the other becomes optional +later work. + +Conflict notes: + +- PRs 1, 3b, 4 and 6 touch `api/entrypoints/routers.py`. Keep each edit to its own block and land + them in order. +- PRs 3b and 4 both touch `router.py` and `models.py`. Land 3b first; 4 adds a separate router class. +- PR 2 must land before 3a or 3b, because both need the registry and the applier. +- PRs 1 and 4 each add a migration and must not both claim `oss000000022`. +- PR 6 must be agreed with the agent on `feat/session-execution-watchdog` before either lands. Two + independent writers of an execution's terminal outcome is a worse bug than the one being fixed. +- **Work package A's `shouldPark` change is a hard dependency of the user-visible result.** Landing + PRs 1 to 7 without it gives a fast Stop that still destroys the sandbox. + +**Keeping the current Stop working.** `POST /sessions/streams/` with no inputs and no `force` keeps +its exact current behavior through PRs 1 to 6. Nothing about `CommandMode.cancel` changes. Released +browsers and the current mobile build keep working unchanged. + +**When it becomes a wrapper.** In PR 7. At that point `SessionStreamsService.command`'s cancel +branch (`api/oss/src/core/sessions/streams/service.py:288`) stops calling `_displace_turns` and +instead calls `SessionCommandsService.request_cancel(...)` with no expected execution id, then +returns the same `SessionStreamCommandResponse` shape it returns today. That gives every old client +the new behavior with no client change, and it is also the point at which the old teardown of +`alive` and the hour-long tombstone disappear. Do it in the same PR that flips the browser, so one +revert restores one consistent behavior. + +--- + +## 11. Test plan + +### Unit tests + +| Component | Test | Passes when | +|---|---|---| +| Commands DAO | Two concurrent claims of one pending command | Exactly one returns a row; the other returns none | +| Commands DAO | Insert with a repeated `Idempotency-Key` | The second insert returns the first row, and one row exists | +| Commands DAO | Settle with the wrong `replica_id` | Returns no row; the stored state is unchanged | +| Commands DAO | Settle a command that is already `applied` | Returns no row; the caller reads the terminal state | +| Commands DAO | Claim with a session set that excludes the command's session | Returns nothing | +| Commands service | Admission with a stale `expected_execution_id` | Raises the conflict type; no row inserted | +| Commands service | Admission with nothing running or parked | One row, `state='obsolete'`, `outcome='not_running'` | +| Commands service | Admission when `turn_started_at` is later than `received_at` | One row, `state='obsolete'`, `outcome='superseded_by_newer_turn'`, `target_turn_id` null, no Redis write | +| Commands service | Admission when `turn_started_at` is null | The guard does not fire; the command targets the current turn | +| Commands service | Admission when `turn_started_at` is earlier than `received_at` | Normal admission, `state='pending'` | +| Commands service | The stored `created_at` equals the `received_at` that was compared | The two values match exactly, not merely closely | +| Streams DAO | The same `turn_id` stamped by ten heartbeats | `turn_started_at` is written once and never moves | +| Streams DAO | A new `turn_id` stamped over an old one | `turn_started_at` moves to the new turn's time | +| Commands service | Admission twice with no idempotency key | One row; the second call returns the first | +| Commands service | Admission writes the command and `stopping_turn_id` | Both are visible after one commit, neither after a rollback | +| Command sweep | Claim expired, session beating, attempts left | Back to `pending` | +| Command sweep | Claim expired, session silent for 90 s | `obsolete`, `outcome='lost'`, keys force-cleared, `ended` published | +| Command sweep | Claim expired, session parked with an open interaction | Not settled as lost; the admission deadline applies instead | +| Command sweep | Redis `alive` still holds its 3600 s value | Settlement still happens, because the rule reads heartbeat age, not the key | +| Direct adapter | Runner answers 404 for a session whose row is not alive | Receipt is `not_held`; the command settles `obsolete` with `not_running` | +| Direct adapter | Runner answers 404 for a session that is alive and beating | Logged at error level, counted, and settled `obsolete` with `lost`, never `not_running` | +| Direct adapter | Runner unreachable | Receipt is `unreachable`; admission still succeeded and returned 202 | +| Direct adapter | The command row exists before the runner is called | A crash injected between the two leaves a `pending` command, never an aborted execution with no record | +| Long-poll adapter | `deliver` when Redis is down | Admission still succeeds; the failure is logged, not raised | +| Runner claim loop | 204, then 200, then a network error | Immediate re-claim, apply, then the backoff sequence with jitter | +| Runner claim loop | 401 | One error log, then a 60 second retry, no tight loop | +| Runner claim loop | Session set includes a parked pool entry | The parked session appears in the request body | +| Runner applier | A command for a `turnId` this process does not hold | Settles `obsolete` with `not_running`; nothing is aborted | +| Runner applier | The held execution started after the command's `created_at` | Settles `obsolete` with `superseded_by_newer_turn`; nothing is aborted | +| Runner applier | The same `command_id` delivered twice | Aborted once, acknowledged twice | +| Runner applier | The deduplication set survives a loop restart | A command applied before the restart is not applied again | +| Runner registry | The run's `finally` runs | The entry is removed even when the run threw | + +The runner suite is `cd services/runner && pnpm test` (vitest). The API unit tests sit under +`api/oss/tests/pytest/unit/sessions/`, next to `test_command_matrix_inputs_data.py`. + +### One API integration test + +`api/oss/tests/pytest/integration/sessions/test_stop_command_delivery.py`, against a real Postgres +and a real Redis, with a fake runner: + +1. Establish a session with `alive` and `running` held by `turn-A`, exactly as a heartbeat does. +2. Call the public Cancel route with `expected_execution_id = 'turn-A'`. Assert 202, one `pending` + row, and `session_streams.stopping_turn_id = 'turn-A'`. +3. Call the claim route as `replica-1`, declaring that session. Assert 200, one command, + `state='claimed'`. +4. Call the claim route again. Assert 204 within the hold. +5. Post the outcome with `result='applied'` and `execution.state='stopped'`. Assert 200. +6. Assert: the command is `applied` with `outcome='stopped'`; `stopping_turn_id` is null; the Redis + `running` key is gone; **the Redis `alive` key is still present**; `superseded:...:turn-A` exists; + the session's pending interactions are cancelled; one `lifecycle: ended` message was published on + the session watch channel. + +Step 6's `alive` assertion is the one that pins warm resume at the API layer. If a later change +starts clearing `alive` on Stop, this test fails. + +Add a second integration case for the parked path: park the session (no `running`, `alive` held, one +pending interaction), Stop it, and assert the command is delivered, the interaction is cancelled, and +`alive` still holds. + +### One live-stack wire test + +Add a cell to the agent release gate, next to the existing W5 steer cell +(`.agents/skills/agent-release-gate/resources/`), driving a deployed stack over the product +endpoints only: + +1. Start a turn with a prompt that runs for at least 60 seconds. +2. Wait for the first agent output frame, then record the wall clock and press Stop through + `POST /sessions/{id}/cancel`. +3. **Pass criterion one:** the runner reports the outcome, and the session's `running` flag goes + false, within **5 seconds** of the Stop request. Measure from the request, not from the frame. +4. **Pass criterion two:** `session_turns` for the stopped turn still names the same `sandbox_id` + and `agent_session_id` as before the Stop, and the session's `alive` flag is still true. +5. Send a second message on the same session. +6. **Pass criterion three:** the second turn reuses the same `sandbox_id` and `agent_session_id`. + That is warm resume, measured from stored rows rather than from timing. +7. **Pass criterion four:** the stopped turn's records end with a cancelled outcome, not an error + record. + +A second cell for the parked path: run a prompt that triggers an approval, wait for the gate, press +Stop, and assert that the outcome lands within 5 seconds, the interaction reads `cancelled`, and the +next message still resumes warm. That cell is the regression test for H-2 and it fails on today's +code for a reason no timing change can fix. + +Criteria 2, 3 and 4 depend on Work package A. Criterion 1 does not, and can be gated as soon as PR 7 +lands. + +--- + +## 12. Rejected alternatives + +**Shorten the heartbeat interval.** Dropping `HEARTBEAT_INTERVAL_SECONDS` from 30 to 2 would cut the +Stop delay with no new machinery. It fails on four counts. It multiplies heartbeat load by fifteen +for every live session, and each beat is a Postgres write plus four Redis operations +(`api/oss/src/core/sessions/streams/service.py:406`). It cannot deliver a Stop to a run whose +credential was dropped, because the beat itself is what fails (`alive.ts:92`). It cannot deliver a +Stop to a parked session at any interval, because the heartbeat has stopped (`server.ts:618`). And it +leaves the control signal encoded as the absence of a lock, which is what makes today's cancel a +session teardown rather than an execution cancel. + +**Route commands by owner replica instead of by declared session.** This was the first revision's +design and it is worse. The Redis `owner` key expires after 120 seconds +(`api/oss/src/dbs/redis/sessions/contract.py:40`), so a parked session's owner can lapse and its +commands become unroutable. It also cannot tell whether the named replica still holds the session, +which is exactly the question delivery needs answered. Letting the runner declare what it holds +turns a guess into a fact, and it removes a column from the durable record. + +**Subscribe the runner to Redis directly.** The runner could subscribe to a per-session Pub/Sub +channel and skip the claim. It is the least code. It fails on the boundary the codebase already +enforces: the API is the single Redis writer and the runner reaches the coordination plane only over +HTTP (`services/runner/src/sessions/alive.ts:13` and `sessions/contract.ts:25`, both explicit about +this). Handing the runner Redis credentials reverses a deliberate decision, and Pub/Sub has no +replay, so a disconnected runner loses every command sent while it was away. + +**A persistent WebSocket or bidirectional stream.** It removes the repeated request and can carry +richer runner status. It is deferred, not wrong. It needs connection lifecycle handling, ping and +pong, reconnect with backoff, and a message framing contract, none of which the command state +machine needs to be correct. Because delivery sits behind the port in section 8, it becomes a later +adapter rather than a rewrite. + +**Skip the durable record and make Stop a bare direct call.** This is the product review's position +and it is the strongest alternative. Note what is and is not rejected here. The **direct call** is +not rejected at all: it is section 9, it is a first-class adapter behind the port, and it is the +recommended first adapter. What is rejected is dropping the **record**, for the four reasons set out +in section 9: no recovery when the runner is unreachable, no idempotency against a double Stop +landing on a newer turn, no place to write the one terminal outcome the watchdog and the runner must +agree on, and no foundation for Steer and Queue. Insert first, then call. + +--- + +## 13. Open questions for Mahmoud + +1. **Which adapter is the default, `direct` or `long_poll`?** Recommendation: **`direct`** for + version one, with the wrong-replica detector from section 9 built in the same PR. Reason: you run + one runner, the hop is authenticated and in production today, it reaches a parked session once the + pool lookup is added, and it removes a held connection and a poll loop from the first release. The + port keeps long polling one file away for the day a second replica or a user-operated runner is + real. The condition on the recommendation: the detector is not optional, because without it the + two-replica failure is silent, and with it the choice is reversible on a metric rather than on a + bug report. + +2. **Who owns execution settlement, this design or the watchdog branch?** Recommendation: **the + watchdog owns it, and the command rules move into it.** Reason: one execution must reach exactly + one terminal outcome from exactly one writer, and two sweeps racing to write `lost` is a worse + bug than the one being fixed. This needs deciding before PR 6 and before the watchdog branch + lands. + +3. **Does Stop leave the Redis `alive` key in place?** Recommendation: **yes, leave it**, exactly as + a normal turn end does. Reason: force-deleting `alive` is what makes today's cancel read as a + session teardown, and warm resume is the required outcome. This is a deliberate deviation from + the phrase "Redis `running` and `alive` released" in the work package brief, so it needs an + explicit yes or no. + +4. **Do first-party clients always send `expected_execution_id`?** Recommendation: **yes, and treat + an omission as a bug.** Reason: it is the cheapest of the three H-3 guards and the only one that + works before the request reaches the server. The field stays optional in the contract for + external callers, as decision D-010 requires. + +5. **Do we cancel the pending interaction when Stop hits a parked session?** Recommendation: + **yes, cancel it, and keep the parked environment.** Reason: an approval card whose execution was + stopped is exactly the "actionable card whose buttons do nothing" bug, and the kill route already + makes this call (`api/oss/src/apis/fastapi/sessions/router.py:441`). Keeping the environment is + what makes the next message warm, and it is what distinguishes Stop from Delete. diff --git a/docs/design/session-control-and-live-events/status.md b/docs/design/session-control-and-live-events/status.md index dc100b335be..da73284e795 100644 --- a/docs/design/session-control-and-live-events/status.md +++ b/docs/design/session-control-and-live-events/status.md @@ -38,12 +38,12 @@ - Kept the public Stop execution guard optional. - Added possible future user-operated runners as a control-transport consideration, not a requirement. -- Recorded long polling as the current control-transport preference behind a replaceable adapter. +- Implemented direct control delivery behind a replaceable adapter for version one. - Recorded warm sandbox and harness resume as the required Stop outcome. - Confirmed the minimal internal command lifecycle and its separation from public execution state. - Left the Stop settlement timeout for the sandbox cancellation spike. - Confirmed that the first version keeps current Redis execution ownership. -- Kept durable commands and long polling in scope; deferred Postgres ownership and full fencing. +- Kept durable commands and direct delivery in scope; deferred long polling and full fencing. ## Branch @@ -56,7 +56,7 @@ Start with **Stop and ownership**: 1. Start the sandbox-agent capability investigation. 2. Confirm the user-visible Stop requirements and latency target. -3. Specify the runner-initiated long-poll claim and acknowledgement contract. +3. Validate the direct runner-control transport and its failure behavior. 4. Define terminal settlement and watchdog responsibility. 5. Decide which current issues this track is expected to close. diff --git a/docs/design/session-control-and-live-events/tonight-handoff.md b/docs/design/session-control-and-live-events/tonight-handoff.md index b6b4baa49f0..3b003732c35 100644 --- a/docs/design/session-control-and-live-events/tonight-handoff.md +++ b/docs/design/session-control-and-live-events/tonight-handoff.md @@ -6,10 +6,10 @@ - Keep current Redis execution ownership for version one. - Add durable commands with `pending`, `claimed`, `applied`, and `obsolete` states. -- Use runner-initiated HTTP long polling behind a replaceable control-delivery port. +- Use direct API-to-runner HTTP behind a replaceable control-delivery port for version one. - Keep `expected_execution_id` optional on public Stop. - Keep the Redis ownership lock until Stop settles. -- Use heartbeat command discovery as delivery fallback. +- Keep durable storage and settlement independent of the delivery transport. - Require Stop followed by warm resume of the same sandbox and native harness session. Run this release-gate cell for every supported harness and sandbox-provider pair. - Keep live-frame work independent from Stop work. @@ -32,14 +32,13 @@ Deliver a code-traced report, a characterization test, the smallest patch propos plan for start, Stop, and resume in the same sandbox and native session. Do not redesign ownership, commands, or public endpoints. -## Work package B: durable command and long-poll design +## Work package B: durable command and direct-delivery design **Goal:** Produce an implementation-ready design for reliable API-to-runner commands. -Define the command schema, claim lease, idempotency, long-poll claim and acknowledgement behavior, -heartbeat fallback, failure recovery, adapter boundary, and how Redis ownership remains held until -Stop settles. Deliver a short design and migration sequence. Do not implement a new execution -ownership model. +Define the command schema, idempotency, direct-delivery acknowledgement, failure recovery, adapter +boundary, and how Redis ownership remains held until Stop settles. Keep long-poll claim semantics +as a deferred transport. Do not implement a new execution ownership model. ## Work package C: current Stop implementation map @@ -62,7 +61,7 @@ or a separate event table. ## First implementation after the spikes 1. Add the durable command repository and service behind interfaces. -2. Add the runner long-poll claim loop and API adapter. +2. Add the direct API-to-runner adapter and authenticated runner route. 3. Let Stop create a durable command with an optional expected-execution guard. 4. Let the runner apply Stop through its active abort controller. 5. Preserve Redis ownership until cancellation settles. @@ -79,3 +78,4 @@ or a separate event table. - Final records versus event-table selection. - Final public endpoint naming. - WebSocket or gRPC control transport. +- Runner-initiated long-poll control transport. diff --git a/hosting/docker-compose/ee/env.ee.dev.example b/hosting/docker-compose/ee/env.ee.dev.example index 411db1efddf..b9173766436 100644 --- a/hosting/docker-compose/ee/env.ee.dev.example +++ b/hosting/docker-compose/ee/env.ee.dev.example @@ -136,6 +136,8 @@ AGENTA_RUNNER_DEFAULT_SANDBOX_PROVIDER=local # Smart truncation preserves the structure of a record whose body exceeds the API size # cap (higher-fidelity reconstruction). Still opt-in, default off. # AGENTA_RECORDS_SMART_TRUNCATION=true +# Durable Stop is exercised in development; production keeps the API default off. +AGENTA_SESSIONS_DURABLE_STOP=true # --- Attachment limits (files attached to an agent chat turn) --- # Per-file caps in bytes, by kind: 10 MB, except audio at 15 MB. Read by the api. diff --git a/hosting/docker-compose/oss/env.oss.dev.example b/hosting/docker-compose/oss/env.oss.dev.example index 6f65ca6c913..1c559feb549 100644 --- a/hosting/docker-compose/oss/env.oss.dev.example +++ b/hosting/docker-compose/oss/env.oss.dev.example @@ -142,6 +142,8 @@ NEXT_PUBLIC_AGENT_FILE_UPLOADS=true # Smart truncation preserves the structure of a record whose body exceeds the API size # cap (higher-fidelity reconstruction). Still opt-in, default off. # AGENTA_RECORDS_SMART_TRUNCATION=true +# Durable Stop is exercised in development; production keeps the API default off. +AGENTA_SESSIONS_DURABLE_STOP=true # --- Attachment limits (files attached to an agent chat turn) --- # Per-file caps in bytes, by kind: 10 MB, except audio at 15 MB. Read by the api. diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts index 9d7c9a8ea13..548c66af650 100644 --- a/services/runner/src/engines/sandbox_agent/run-turn.ts +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -67,6 +67,8 @@ import { CREDENTIAL_RACE_REPORTS_PER_SESSION, withinCredentialPropagationWindow, } from "./errors.ts"; +import { noteExecutionSettled } from "../../sessions/execution-registry.ts"; +import { isUserStopAbort } from "../../sessions/stop-signal.ts"; import { cancelHarnessTurn } from "./cancel-turn.ts"; import { reapLeakedExecChildren } from "./reap-exec.ts"; import { sandboxAgentServerPort } from "./provider.ts"; @@ -1207,12 +1209,22 @@ export async function runTurn( if (raced === RUN_LIMIT_TRIPPED) { throw new Error(runLimitReason ?? "run limit tripped"); } - const stopReason = + let stopReason = raced === CANCELLED ? "cancelled" : raced === PAUSED || pause.active ? "paused" : (raced as any)?.stopReason; + // THE TURN'S OWN WORK IS OVER HERE. Everything below is teardown: draining gates, writing + // the transcript, exporting the trace, deciding whether to park. That takes hundreds of + // milliseconds, and the execution stays registered for all of it, so a Stop arriving now + // would abort a run that has already finished. The abort would change no outcome and would + // still make the teardown treat the run as aborted, which DESTROYS the warm environment + // instead of parking it. Marked here rather than where the caller awaits this function, + // because that window is precisely what lies between the two. + if (stopReason !== "paused" && request.sessionId && request.turnId) { + noteExecutionSettled(request.sessionId, request.turnId); + } // Terminalization drains queued gates, classifies pause-time completions, and gives allowed // executions their original per-call bound before the orphan sweep closes the turn. if (stopReason === "paused") { @@ -1280,8 +1292,19 @@ export async function runTurn( unexpectedOpenToolCallIds.join(","), ); } + + if (isUserStopAbort(signal)) { + stopReason = "cancelled"; + } + if (request.sessionId && request.turnId) { + noteExecutionSettled(request.sessionId, request.turnId); + } } if (stopReason === "cancelled") { + env.parkedApprovals.clear(); + env.parkedApproval = undefined; + env.approvalGateCount = 0; + parkedApprovedExecutions.clear(); // Tell the HARNESS to stop before anything else. The abort only made the runner stop // waiting; without this the harness still holds an open prompt and a running tool, and the // sandbox could never be parked. A settled cancel is what earns the warm park below; see diff --git a/services/runner/src/lifecycle/session-coordinator.ts b/services/runner/src/lifecycle/session-coordinator.ts index 13d26c6d758..c707570f424 100644 --- a/services/runner/src/lifecycle/session-coordinator.ts +++ b/services/runner/src/lifecycle/session-coordinator.ts @@ -193,6 +193,14 @@ export interface KeepaliveContext { clientGone?: () => boolean; /** Latest session credential accessor supplied by the alive watchdog. */ credential?: () => string; + /** + * Called once with this run's project scope, as soon as it is known. + * + * The scope can only be resolved here: `runContext.project.id` is empty on the live invoke + * path, so the project comes from the signed mount, which is signed inside this function. The + * transport needs it to route a control command to the right tenant's session. + */ + onScopeResolved?: (projectId: string) => void; /** * Test seam for the credential-propagation hold. Production waits for real: the hold is what * keeps applied state from advancing over a value the provider's egress layer has probably not @@ -293,6 +301,10 @@ export async function runWithKeepalive( } const key = scope.key; klog(`scope=${scope.source} key=${key} session=${sessionId}`); + // Tell the transport which project this run belongs to. Until this lands, a control command + // cannot tell one tenant's session from another's, because the request itself often carries + // no project and the scope was only just derived from the signed mount. + ctx.onScopeResolved?.(scope.key.slice(0, scope.key.lastIndexOf(":"))); // The mount may be null here (store unconfigured, 503, ephemeral fallback) or undefined (the // sign attempt threw) when the run-context scope produced the key. A mount-less session still diff --git a/services/runner/src/server.ts b/services/runner/src/server.ts index 2015077e4d5..d6de4638eb4 100644 --- a/services/runner/src/server.ts +++ b/services/runner/src/server.ts @@ -8,6 +8,7 @@ * GET /subscription-status -> one login state per harness (no paths, no credentials) * POST /stream -> body is an AgentRunRequest, NDJSON event stream (alias: POST /run) * POST /kill -> best-effort, idempotent teardown, scoped to one { sessionId, projectId } + * POST /cancel -> stop the CURRENT TURN of one session and keep it warm * * Uses Node's built-in http server (no framework dependency). * @@ -58,6 +59,7 @@ import type { TeardownReason } from "./engines/sandbox_agent/teardown.ts"; import { approvalDecisionForToolCall, poolKeyFor, + projectScopeFor, readKeepaliveConfig, tailIsFreshUserMessage, type KeepaliveConfig, @@ -84,7 +86,21 @@ import { SESSION_TURN_IN_USE_CODE, SESSION_TURN_IN_USE_MESSAGE, } from "./sessions/admission.ts"; -import { releaseOwnedSessions, startAliveWatchdog } from "./sessions/alive.ts"; +import { + REPLICA_ID, + releaseOwnedSessions, + startAliveWatchdog, +} from "./sessions/alive.ts"; +import { + applyCommand, + holdsSession, + type ControlCommand, +} from "./sessions/control-channel.ts"; +import { + noteExecutionProject, + registerExecution, + unregisterExecution, +} from "./sessions/execution-registry.ts"; import { buildWorkflowReferenceList, cancelStaleInteractions, @@ -358,6 +374,15 @@ const runAgent: RunAgent = (request, emit, signal, options) => { config, clientGone: options?.clientGone, credential: options?.credential, + // The coordinator is the first place that knows this run's project, because the scope can + // come from the signed mount rather than the request. A control command needs it to tell + // one tenant's session from another's. + onScopeResolved: (projectId) => { + const sessionId = request.sessionId?.trim(); + const turnId = request.turnId?.trim(); + if (sessionId && turnId) + noteExecutionProject(sessionId, turnId, projectId); + }, }); }; @@ -505,149 +530,165 @@ async function runAndStreamWithApiBaseResolved( } | undefined; - if (sessionOwned) { - // The request's api base (if any) is already scoped for this call via - // runWithRequestApiBase in the outer runAndStream — apiBase() below sees it. - // The runner authenticates session calls AS the invoke caller (the run credential), - // refreshing it for the turn's lifetime — never the admin key. Project scope is - // resolved server-side from the credential, so no project_id rides the request. - // - // onInterrupted (W7.4): a cancel/steer/kill against this session (via - // `POST /sessions/streams/` or the runner's own `/kill`) drops this turn's alive lock. - // The next heartbeat surfaces that as `is_current_turn: false`; wiring it to - // `controller.abort()` is what makes the control-plane signal actually reach this - // in-flight run — before this, a session-owned run's controller was never aborted. - // Awaited (WP3) so the first heartbeat's stream_id is ready before the turn starts. - // - // The beat also proposes the two things a headless session otherwise never gets: a name - // (no browser ever renders it, and the browser is the only other title writer) and the - // run's workflow references (they ride only a fire-and-forget turn append today, so a - // dropped append leaves a row the UI cannot open). Both are fill-once server-side. - const watchdog = await startAliveWatchdog( - sessionId, - turnId, - platformCredentialForRequest(request), - // LABELLED, not a bare abort: `shouldPark` parks only an abort it can prove was a - // cooperative Stop. See `sessions/stop-signal.ts`. - () => controller.abort(USER_STOP_ABORT_REASON), - { - name: proposeSessionName(request), - references: buildWorkflowReferenceList(request.runContext?.workflow), - }, - ); - aliveWatchdog = watchdog; - // The heartbeat response already carries the session_streams row id — free, no extra - // round-trip. Thread it onto the request so the engine's turn-append write has it. - request.streamId = watchdog.streamId(); - - // ADMISSION. That first beat asked the platform's atomic `nx` acquire whether this turn may - // run, and `admitted: false` means a DIFFERENT turn already holds the session. Stop here. - // - // Everything below this point has a side effect that a refused turn must not have: - // `cancelStaleInteractions` would cancel the LIVE turn's unanswered approval gate, the - // persisting emitter would write this message into the durable transcript, and `run()` would - // reach the keepalive pool and destroy the live turn's warm environment. That last one is - // the double-send bug (#6417, #5539, #5538): the arbiter's answer was already correct, the - // runner simply never read it before acting. - // - // The refusal travels as an `error` EVENT with a stable code plus a failed terminal result, - // which is the path every runner failure already takes to the browser. Nothing is persisted, - // so the refused message never appears in the session's history — the client keeps the text. - if (!watchdog.admitted) { - process.stderr.write( - `[sessions] admission REFUSED session=${sessionId} turn=${turnId}; ` + - `another turn owns this session. No pool resolve, no eviction.\n`, + try { + if (sessionOwned) { + // The request's api base (if any) is already scoped for this call via + // runWithRequestApiBase in the outer runAndStream — apiBase() below sees it. + // The runner authenticates session calls AS the invoke caller (the run credential), + // refreshing it for the turn's lifetime — never the admin key. Project scope is + // resolved server-side from the credential, so no project_id rides the request. + // + // onInterrupted (W7.4): a cancel/steer/kill against this session (via + // `POST /sessions/streams/` or the runner's own `/kill`) drops this turn's alive lock. + // The next heartbeat surfaces that as `is_current_turn: false`; wiring it to + // `controller.abort()` is what makes the control-plane signal actually reach this + // in-flight run — before this, a session-owned run's controller was never aborted. + // Awaited (WP3) so the first heartbeat's stream_id is ready before the turn starts. + // + // The beat also proposes the two things a headless session otherwise never gets: a name + // (no browser ever renders it, and the browser is the only other title writer) and the + // run's workflow references (they ride only a fire-and-forget turn append today, so a + // dropped append leaves a row the UI cannot open). Both are fill-once server-side. + const watchdog = await startAliveWatchdog( + sessionId, + turnId, + platformCredentialForRequest(request), + // LABELLED, not a bare abort: `shouldPark` parks only an abort it can prove was a + // cooperative Stop. See `sessions/stop-signal.ts`. + () => controller.abort(USER_STOP_ABORT_REASON), + { + name: proposeSessionName(request), + references: buildWorkflowReferenceList(request.runContext?.workflow), + }, ); - // Stops the heartbeat interval and releases the credential lease. Its final - // `is_running: false` beat is owner-scoped server-side, so it cannot clear the live - // turn's `running` lock or stamp its own turn id on the session row. - await watchdog.release().catch(() => {}); - liveEmit({ - type: "error", - message: SESSION_TURN_IN_USE_MESSAGE, - code: SESSION_TURN_IN_USE_CODE, - }); - writeRecord({ - kind: "result", - result: { ok: false, error: SESSION_TURN_IN_USE_MESSAGE, events: [] }, + aliveWatchdog = watchdog; + // The heartbeat response already carries the session_streams row id — free, no extra + // round-trip. Thread it onto the request so the engine's turn-append write has it. + request.streamId = watchdog.streamId(); + + // ADMISSION. That first beat asked the platform's atomic `nx` acquire whether this turn may + // run, and `admitted: false` means a DIFFERENT turn already holds the session. Stop here. + // + // Everything below this point has a side effect that a refused turn must not have: + // `cancelStaleInteractions` would cancel the LIVE turn's unanswered approval gate, the + // persisting emitter would write this message into the durable transcript, and `run()` would + // reach the keepalive pool and destroy the live turn's warm environment. That last one is + // the double-send bug (#6417, #5539, #5538): the arbiter's answer was already correct, the + // runner simply never read it before acting. + // + // The refusal travels as an `error` EVENT with a stable code plus a failed terminal result, + // which is the path every runner failure already takes to the browser. Nothing is persisted, + // so the refused message never appears in the session's history — the client keeps the text. + if (!watchdog.admitted) { + process.stderr.write( + `[sessions] admission REFUSED session=${sessionId} turn=${turnId}; ` + + `another turn owns this session. No pool resolve, no eviction.\n`, + ); + // Stops the heartbeat interval and releases the credential lease. Its final + // `is_running: false` beat is owner-scoped server-side, so it cannot clear the live + // turn's `running` lock or stamp its own turn id on the session row. + await watchdog.release().catch(() => {}); + unregisterExecution(sessionId, turnId); + liveEmit({ + type: "error", + message: SESSION_TURN_IN_USE_MESSAGE, + code: SESSION_TURN_IN_USE_CODE, + }); + writeRecord({ + kind: "result", + result: { ok: false, error: SESSION_TURN_IN_USE_MESSAGE, events: [] }, + }); + res.end(); + return; + } + + // A refused contender must never replace the admitted execution's Stop handle. + registerExecution({ + projectId: projectScopeFor(request, undefined)?.id, + sessionId, + turnId, + startedAt: Date.now(), + abort: () => controller.abort(USER_STOP_ABORT_REASON), }); - res.end(); - return; - } - // Admitted. Tell the client which execution it is watching, before anything else streams. - // - // The runner mints the turn id (`resolveTurnId`), and until now it never told anyone: the - // client's `start` frame is built and sent before the runner replies at all, so it cannot - // carry a runner-minted id. That is why `expected_execution_id` on the public Cancel has had - // no first-party caller able to fill it — a Stop could only mean "whatever is running now", - // never "the turn I was watching". This is the earliest frame that can carry it. - // - // Deliberately on `liveEmit`, not the persisting emitter that replaces it below: this is - // transport correlation, not conversation, and it must never become a session record. - liveEmit({ type: "turn", turnId }); + // Admitted. Tell the client which execution it is watching, before anything else streams. + // + // The runner mints the turn id (`resolveTurnId`), and until now it never told anyone: the + // client's `start` frame is built and sent before the runner replies at all, so it cannot + // carry a runner-minted id. That is why `expected_execution_id` on the public Cancel has had + // no first-party caller able to fill it — a Stop could only mean "whatever is running now", + // never "the turn I was watching". This is the earliest frame that can carry it. + // + // Deliberately on `liveEmit`, not the persisting emitter that replaces it below: this is + // transport correlation, not conversation, and it must never become a session record. + liveEmit({ type: "turn", turnId }); - // A new turn supersedes any prior turn's unanswered gate: cancel stale pending - // interactions (sparing this turn's own, plus a parked gate this turn answers in-band — - // the resume resolves that one). Best-effort, never blocks the turn. - const answeredTokens = inBandAnswerTokens(request); - void cancelStaleInteractions( - sessionId, - turnId, - answeredTokens, - watchdog.credential, - ); - // Deny-set from THIS run's typed credential material (model connection credentials + - // materialized environment values + MCP connection credentials) and the run credential — - // not process env, which never holds them. A credential value a model echoes back must - // never reach the durable session records unredacted. - const { - emit: persistingEmit, - persist, - flush, - } = buildPersistingEmitter( - sessionId, - watchdog.credential, - liveEmit, - seedForRun(request), - turnId, - request.runContext?.trace?.span_id, - ); - // Record the inbound user turn first so the session record is the full conversation, not just - // agent output. Guard on `tailIsFreshUserMessage`: an approval RESUME's tail is the tool_result - // envelope, so it must not re-persist the ORIGINAL prompt as a duplicate user row. The guard - // writes the prompt only on the turn that first introduced it. - if (tailIsFreshUserMessage(request)) { - persist( - { type: "message", text: turn.text, attachments: turn.attachments }, - "user", + // A new turn supersedes any prior turn's unanswered gate: cancel stale pending + // interactions (sparing this turn's own, plus a parked gate this turn answers in-band — + // the resume resolves that one). Best-effort, never blocks the turn. + const answeredTokens = inBandAnswerTokens(request); + void cancelStaleInteractions( + sessionId, + turnId, + answeredTokens, + watchdog.credential, ); - if (turn.attachments.length > 0) { - // A failed claim is accepted as graceful loss: the worst case is that the sweeper - // reclaims the attachment and cold replay renders it as no longer available. - await claimAttachments( - sessionId, - turn.attachments.map((attachment) => attachment.attachmentId), - watchdog.credential, + // Deny-set from THIS run's typed credential material (model connection credentials + + // materialized environment values + MCP connection credentials) and the run credential — + // not process env, which never holds them. A credential value a model echoes back must + // never reach the durable session records unredacted. + const { + emit: persistingEmit, + persist, + flush, + } = buildPersistingEmitter( + sessionId, + watchdog.credential, + liveEmit, + seedForRun(request), + turnId, + request.runContext?.trace?.span_id, + ); + // Record the inbound user turn first so the session record is the full conversation, not just + // agent output. Guard on `tailIsFreshUserMessage`: an approval RESUME's tail is the tool_result + // envelope, so it must not re-persist the ORIGINAL prompt as a duplicate user row. The guard + // writes the prompt only on the turn that first introduced it. + if (tailIsFreshUserMessage(request)) { + persist( + { type: "message", text: turn.text, attachments: turn.attachments }, + "user", ); + if (turn.attachments.length > 0) { + // A failed claim is accepted as graceful loss: the worst case is that the sweeper + // reclaims the attachment and cold replay renders it as no longer available. + await claimAttachments( + sessionId, + turn.attachments.map((attachment) => attachment.attachmentId), + watchdog.credential, + ); + } } + emitFn = (event) => { + if (event.type === "done") terminalRecordEmitted = true; + persistingEmit(event); + }; + flushPersist = flush; + persistError = (message) => persist({ type: "error", message }, "agent"); + persistTerminal = (stopReason) => { + terminalRecordEmitted = true; + persist( + { + type: "done", + ...(stopReason === "cancelled" ? { stopReason } : {}), + }, + "agent", + ); + }; } - emitFn = (event) => { - if (event.type === "done") terminalRecordEmitted = true; - persistingEmit(event); - }; - flushPersist = flush; - persistError = (message) => persist({ type: "error", message }, "agent"); - persistTerminal = (stopReason) => { - terminalRecordEmitted = true; - persist( - { - type: "done", - ...(stopReason === "cancelled" ? { stopReason } : {}), - }, - "agent", - ); - }; + } catch (error) { + if (aliveWatchdog) await aliveWatchdog.release().catch(() => {}); + if (sessionOwned) unregisterExecution(sessionId, turnId); + throw error; } let result: AgentRunResult; @@ -713,6 +754,10 @@ async function runAndStreamWithApiBaseResolved( } } if (aliveWatchdog) await aliveWatchdog.release().catch(() => {}); + // Same `finally` as the watchdog release, so a run that threw still leaves the registry + // clean. Scoped to this turn id, so a turn that finishes after its successor registered + // cannot unregister the successor. + if (sessionOwned) unregisterExecution(sessionId, turnId); } // Streaming delivered the events live, so don't echo them in the terminal record. @@ -779,6 +824,31 @@ function readBodyCapped( }); } +/** `/cancel`'s payload is five short strings. */ +const CANCEL_BODY_MAX_BYTES = 16 * 1024; + +/** A non-empty trimmed string, or null. Used for every id `/cancel` reads. */ +function readRequiredId(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed ? trimmed : null; +} + +/** + * Does the keep-alive pool hold this session parked awaiting an approval? + * + * A Stop against a parked approval has no entry in the execution registry, because no turn is + * running. Without this lookup the runner would answer 404 for exactly the case that has no + * control channel at all today: a parked session stops heartbeating, so the only existing Stop + * signal never reaches it. + */ +function isSessionParked(projectId: string, sessionId: string): boolean { + const key = `${projectId}:${sessionId}`; + return Object.values(keepalivePools).some( + (pool) => pool.get(key)?.state === "awaiting_approval", + ); +} + /** Build the HTTP request listener around a given engine runner (the testable seam). */ export function createRequestListener( run: RunAgent, @@ -848,6 +918,72 @@ export function createRequestListener( return send(res, 200, { ok: true }); } + if (req.method === "POST" && req.url === "/cancel") { + if (!isAuthorized(req)) { + return send(res, 401, { ok: false, error: "Unauthorized" }); + } + // Stop the CURRENT TURN and keep the session warm. This is not `/kill`: the sandbox, + // the native harness session and the keep-alive pool entry all survive, and the next + // message continues the same conversation. + // + // The response is an ACKNOWLEDGEMENT, not an outcome. What happened to the execution + // goes to the API's outcome route, so settlement has one path on every transport. + let cancelBody: { + commandId?: unknown; + projectId?: unknown; + sessionId?: unknown; + targetTurnId?: unknown; + createdAt?: unknown; + }; + try { + const raw = await readBodyCapped(req, CANCEL_BODY_MAX_BYTES); + cancelBody = raw.trim() ? JSON.parse(raw) : {}; + } catch (err) { + if (err instanceof BodyTooLargeError) { + return send(res, 413, { ok: false, error: err.message }); + } + return send(res, 400, { + ok: false, + error: `Invalid JSON: ${err instanceof Error ? err.message : String(err)}`, + }); + } + const commandId = readRequiredId(cancelBody.commandId); + const cancelSessionId = readRequiredId(cancelBody.sessionId); + const cancelProjectId = readRequiredId(cancelBody.projectId); + if (!commandId || !cancelSessionId || !cancelProjectId) { + return send(res, 400, { + ok: false, + error: + "commandId, sessionId and projectId are all required: a pool key is always project-scoped", + }); + } + const command: ControlCommand = { + id: commandId, + projectId: cancelProjectId, + sessionId: cancelSessionId, + kind: "cancel", + target: { + turnId: readRequiredId(cancelBody.targetTurnId), + expectedTurnId: null, + }, + createdAt: + typeof cancelBody.createdAt === "string" + ? cancelBody.createdAt + : "", + }; + if (!holdsSession(cancelProjectId, cancelSessionId, isSessionParked)) { + // 404 is ambiguous on purpose and the API disambiguates it: a `not_held` for a + // session whose row is alive and beating means the call reached the wrong replica. + return send(res, 404, { ok: false, error: "session not held here" }); + } + // Answer before the outcome. The applier reports it separately, and a Stop that takes + // seconds to settle must not hold this request open. + void applyCommand(command, { isParked: isSessionParked }).catch( + () => {}, + ); + return send(res, 202, { ok: true, replicaId: REPLICA_ID }); + } + // POST /stream is the productized name; /run is kept as a back-compat alias // for one release (the SDK still posts /run). Both share the handler. if ( diff --git a/services/runner/src/sessions/applied-commands.ts b/services/runner/src/sessions/applied-commands.ts new file mode 100644 index 00000000000..71e1206b149 --- /dev/null +++ b/services/runner/src/sessions/applied-commands.ts @@ -0,0 +1,91 @@ +/** + * Commands this process has already acted on. + * + * WHY IT MUST OUTLIVE THE DELIVERY PATH. Delivery is at-least-once by design: a lost + * acknowledgement, a retried admission, or a re-armed claim can all bring the same command back. + * Applying a Stop a second time is not harmless — by then the session may be running a NEWER + * turn, and a second abort would kill work the user never asked to stop. + * + * So the set lives at module scope, beside the session pool, not inside a request or a poll + * loop. A loop restart with an empty set would be exactly the bug this prevents. + * + * An already-applied command is a NO-OP THAT STILL ACKNOWLEDGES. It aborts nothing and it + * reports the stored outcome, so a lost acknowledgement is repaired without a second abort. + * + * The entry is written when the command is ACCEPTED, not when the cancel finishes. A duplicate + * that arrives while the first is still cancelling must also be a no-op. + */ + +export interface AppliedCommand { + commandId: string; + /** What this process reported, so a duplicate can repeat the same answer. */ + executionState: string; + executionId: string | null; + result: "applied" | "obsolete"; + appliedAt: number; +} + +/** + * How long an applied command is remembered. Long enough to cover every redelivery path (the + * claim lease is 90 seconds and the sweep runs inside two minutes), short enough that the map + * cannot grow without bound on a long-lived process. + */ +export const APPLIED_COMMAND_TTL_MS = 30 * 60 * 1000; + +/** Hard cap, so a burst cannot grow the map faster than the TTL prunes it. */ +const MAX_APPLIED_COMMANDS = 5000; + +const applied = new Map(); + +function prune(now: number): void { + for (const [id, entry] of applied) { + if (now - entry.appliedAt > APPLIED_COMMAND_TTL_MS) applied.delete(id); + } + while (applied.size > MAX_APPLIED_COMMANDS) { + const oldest = applied.keys().next(); + if (oldest.done) break; + applied.delete(oldest.value); + } +} + +/** What this process already did with `commandId`, if anything. */ +export function recallCommand( + commandId: string, + now: number = Date.now(), +): AppliedCommand | undefined { + const entry = applied.get(commandId); + if (!entry) return undefined; + if (now - entry.appliedAt > APPLIED_COMMAND_TTL_MS) { + applied.delete(commandId); + return undefined; + } + return entry; +} + +/** Record what this process did with a command. Insertion order is the prune order. */ +export function rememberCommand( + entry: Omit, + now: number = Date.now(), +): AppliedCommand { + const stored: AppliedCommand = { ...entry, appliedAt: now }; + applied.delete(entry.commandId); + applied.set(entry.commandId, stored); + prune(now); + return stored; +} + +/** Revise the outcome of a command already accepted, once the cancel settles. */ +export function updateCommandOutcome( + commandId: string, + patch: Pick, +): void { + const entry = applied.get(commandId); + if (!entry) return; + entry.executionState = patch.executionState; + entry.result = patch.result; +} + +/** Test seam. */ +export function resetAppliedCommandsForTest(): void { + applied.clear(); +} diff --git a/services/runner/src/sessions/control-channel.ts b/services/runner/src/sessions/control-channel.ts new file mode 100644 index 00000000000..90349a2dcde --- /dev/null +++ b/services/runner/src/sessions/control-channel.ts @@ -0,0 +1,275 @@ +/** + * Applying a control command, and reporting what it did. + * + * The applier sits ABOVE the transport, not inside it, so every delivery path shares one set of + * guards and one deduplication set. Today there is one path, the direct `POST /cancel` route in + * `server.ts`. A long-poll loop would call the same `applyCommand` and change nothing here. + * + * WHAT THE RUNNER DECIDES AND WHAT IT DOES NOT. It decides whether it holds the named execution + * and whether that execution is old enough to be the one the user meant. It does NOT decide the + * command's fate: it reports an outcome to the API, and the API settles the command and the + * execution together. Settlement has one writer, on every transport. + * + * THE THREE ANSWERS. + * + * stopped — this process held the target execution and aborted it. + * not_running — it holds no execution that can still be stopped. A session + * parked awaiting an approval answers this, and so does a turn + * whose prompt has already settled and is only tearing down. In + * both cases there is nothing to abort, the parked environment + * stays in the pool, and the session stays warm. + * superseded_by_newer_turn — it holds an execution that STARTED AFTER the command was + * created, so the command was meant for a turn that has since + * ended. Nothing is aborted. This check is exact, because it + * compares against this process's own memory of when it started + * the run. + */ + +import { apiBase } from "../apiBase.ts"; +import { REPLICA_ID } from "./alive.ts"; +import { + recallCommand, + rememberCommand, + updateCommandOutcome, +} from "./applied-commands.ts"; +import { findExecution, type LiveExecution } from "./execution-registry.ts"; + +function log(message: string): void { + process.stderr.write(`[control] ${message}\n`); +} + +/** One command as the API delivers it. The same shape arrives on every transport. */ +export interface ControlCommand { + id: string; + projectId: string; + sessionId: string; + kind: "cancel"; + target: { turnId: string | null; expectedTurnId: string | null }; + /** When the API admitted the command. The late-Stop guard compares against this. */ + createdAt: string; +} + +export type ExecutionState = + | "stopped" + | "failed" + | "not_running" + | "superseded_by_newer_turn"; + +export interface ControlOutcome { + /** The command's terminal state, as the runner sees it. */ + result: "applied" | "obsolete"; + execution: { + id: string | null; + state: ExecutionState; + error?: string; + }; +} + +/** How the runner reaches a parked session. Injected so tests need no pool. */ +export interface ParkedLookup { + (projectId: string, sessionId: string): boolean; +} + +export interface ApplyCommandDeps { + /** Overridden in tests. Defaults to the module-level execution registry. */ + findLive?: (projectId: string, sessionId: string) => LiveExecution | undefined; + /** Whether the keep-alive pool holds this session parked awaiting an approval. */ + isParked?: ParkedLookup; + /** Overridden in tests. Defaults to the HTTP report below. */ + report?: (command: ControlCommand, outcome: ControlOutcome) => Promise; + now?: () => number; +} + +/** Does this process hold the session at all? The `/cancel` route answers 404 when it does not. */ +export function holdsSession( + projectId: string, + sessionId: string, + isParked?: ParkedLookup, +): boolean { + if (findExecution(projectId, sessionId)) return true; + return isParked ? isParked(projectId, sessionId) : false; +} + +/** + * Apply one command and report its outcome. Never throws. + * + * Returns the outcome it reported, which is what a duplicate delivery repeats. + */ +export async function applyCommand( + command: ControlCommand, + deps: ApplyCommandDeps = {}, +): Promise { + const findLive = deps.findLive ?? findExecution; + const report = deps.report ?? reportOutcome; + const now = deps.now ?? (() => Date.now()); + + const seen = recallCommand(command.id, now()); + if (seen) { + // A no-op that STILL acknowledges. Aborting a second time could kill a newer turn; not + // acknowledging would leave the command open until the settlement sweep gave up on it. + const outcome: ControlOutcome = { + result: seen.result, + execution: { + id: seen.executionId, + state: seen.executionState as ExecutionState, + }, + }; + log( + `duplicate command=${command.id} session=${command.sessionId} state=${seen.executionState}`, + ); + await report(command, outcome).catch(() => {}); + return outcome; + } + + const createdAtMs = Date.parse(command.createdAt); + const live = findLive(command.projectId, command.sessionId); + const outcome = decideOutcome(command, live, createdAtMs); + + // Remember BEFORE aborting. A duplicate that arrives while the first abort is still settling + // must find the command already taken, not start a second one. + rememberCommand( + { + commandId: command.id, + executionId: outcome.execution.id, + executionState: outcome.execution.state, + result: outcome.result, + }, + now(), + ); + + if (outcome.execution.state === "stopped" && live) { + try { + // The abort is the cancel. It makes the turn end `cancelled`, which is what sends the + // ACP `session/cancel` to the harness and lets the environment be PARKED rather than + // deleted (see `cancel-turn.ts` and `shouldPark`). Stop keeps the session warm. + live.abort(); + log( + `aborted command=${command.id} session=${command.sessionId} turn=${live.turnId}`, + ); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error ?? "abort failed"); + outcome.result = "applied"; + outcome.execution.state = "failed"; + outcome.execution.error = message.slice(0, 2000); + updateCommandOutcome(command.id, { result: "applied", executionState: "failed" }); + log(`abort FAILED command=${command.id} session=${command.sessionId}: ${message}`); + } + } + + // Reported as soon as the abort is issued, not after the harness settles. The command's job + // is to deliver the Stop; the turn's own teardown then writes its transcript and parks the + // sandbox on its own clock, which can take seconds. Waiting for it would make a Stop that + // worked look stuck. + await report(command, outcome).catch((error) => { + log( + `outcome report failed command=${command.id}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }); + return outcome; +} + +function decideOutcome( + command: ControlCommand, + live: LiveExecution | undefined, + createdAtMs: number, +): ControlOutcome { + if (!live) { + // No turn is running here. A parked approval lands here too, and that is the right answer: + // there is nothing to abort, and the parked environment must stay in the pool so the next + // message is warm. Stop ends the work, not the session. + return { + result: "applied", + execution: { id: command.target.turnId, state: "not_running" }, + }; + } + + if (Number.isFinite(createdAtMs) && live.startedAt > createdAtMs) { + // This execution began AFTER the user pressed Stop, so it is not the one they meant. + return { + result: "obsolete", + execution: { id: live.turnId, state: "superseded_by_newer_turn" }, + }; + } + + if (command.target.turnId && command.target.turnId !== live.turnId) { + // A different execution holds the session. The pinned target is gone. + return { + result: "obsolete", + execution: { id: command.target.turnId, state: "not_running" }, + }; + } + + if (live.settled) { + // THE STOP LOST THE RACE BY A MOMENT. The harness prompt already settled and the entry is + // only still here because teardown is running: writing the transcript, exporting the trace, + // parking the environment. There is nothing left to abort. + // + // Doing nothing is not merely tidier, it is the whole fix. `live.abort()` here would abort + // a finished run, and the aborted signal then makes `shouldPark` refuse to park a healthy + // idle environment, so the sandbox is destroyed and the user's next message rebuilds cold. + // The user paid a cold start for pressing Stop as the answer landed. + // + // `obsolete`, not `applied`: the command never stopped anything. `not_running` is the same + // answer a parked approval gets, and it means the same thing here — this process holds no + // execution that can still be stopped. + return { + result: "obsolete", + execution: { id: command.target.turnId ?? live.turnId, state: "not_running" }, + }; + } + + return { + result: "applied", + execution: { id: live.turnId, state: "stopped" }, + }; +} + +/** + * Report a command's outcome to the API. + * + * Authenticates with the shared runner token, not a project credential: the runner holds no + * project credential for a command it was handed, and the command id resolves the project on + * the API side. + */ +export async function reportOutcome( + command: ControlCommand, + outcome: ControlOutcome, +): Promise { + const token = process.env.AGENTA_RUNNER_TOKEN; + if (!token) { + log(`cannot report command=${command.id}: AGENTA_RUNNER_TOKEN is not set`); + return; + } + const url = `${apiBase()}/sessions/control/commands/${encodeURIComponent(command.id)}/outcome`; + const res = await fetch(url, { + method: "POST", + redirect: "error", + headers: { + "content-type": "application/json", + "x-agenta-runner-token": token, + }, + body: JSON.stringify({ + replica_id: REPLICA_ID, + result: outcome.result, + execution: { + id: outcome.execution.id, + state: outcome.execution.state, + ...(outcome.execution.error ? { error: outcome.execution.error } : {}), + }, + }), + }); + if (!res.ok) { + // A 409 means the claim was gone, which is an answer, not a failure to retry: the API has + // already written a terminal outcome for this command. + log( + `outcome HTTP ${res.status} command=${command.id} session=${command.sessionId}`, + ); + return; + } + log( + `outcome reported command=${command.id} session=${command.sessionId} state=${outcome.execution.state}`, + ); +} diff --git a/services/runner/src/sessions/execution-registry.ts b/services/runner/src/sessions/execution-registry.ts new file mode 100644 index 00000000000..0d83a2f3820 --- /dev/null +++ b/services/runner/src/sessions/execution-registry.ts @@ -0,0 +1,133 @@ +/** + * Which executions this runner process is running right now. + * + * WHY IT EXISTS. The abort controller for a session-owned run was a local variable inside the + * request handler in `server.ts`. Nothing outside that closure could reach it, so the only way + * to stop a turn was to take the session's Redis lock away and wait up to 30 seconds for the + * heartbeat to notice. A control command has to reach the running turn directly, and that needs + * a lookup keyed by something the API knows. + * + * THE KEY IS THE SESSION ID, AND THE PROJECT IS CHECKED SEPARATELY. Keying by + * `:` would be tidier, but the project scope is NOT known when a run + * starts: `runContext.project.id` is empty on the live invoke path, and the scope actually used + * for the pool key comes from the signed mount, which the coordinator resolves after the run is + * already in flight (`session-coordinator.ts`, `poolKeyFor(request, signed?.projectId)`). + * Registering under a key that does not exist yet is what made the first version of this + * registry answer "I do not hold that session" for every Stop. + * + * So the entry goes in under the session id at once, and `noteExecutionProject` fills the + * project in as soon as the coordinator knows it. A lookup matches only when the stored project + * agrees, so a Stop from another tenant is REFUSED rather than misrouted. Until the project is + * known the entry matches any project: that window is a few hundred milliseconds at the very + * start of a run, and refusing every Stop in it would reintroduce the bug this comment + * describes. + * + * The limit worth knowing: one entry per session id per process. Two projects running the same + * session id on one runner at the same time keep only the later entry, and the earlier one's + * Stop is then refused with a 404. Refusal is the safe direction, and the keep-alive pool has + * the same shape of key. + * + * `startedAt` is the field that makes a late Stop safe. The API pins the target turn at + * admission and compares its own clock, but the runner's comparison against its OWN memory is + * exact: a command created before an execution started cannot have been meant for it. + * + * Entries are removed in the same `finally` that releases the alive watchdog, so a run that + * threw still leaves the registry clean. + */ + +export interface LiveExecution { + /** Undefined until the coordinator resolves the run's project scope. */ + projectId: string | undefined; + sessionId: string; + /** The execution id, which is the runner's `turn_id`. */ + turnId: string; + /** When this process started the run, in epoch milliseconds. */ + startedAt: number; + /** + * True once the harness prompt has settled, whatever it settled as. + * + * The entry stays registered through teardown, which writes the transcript, exports the + * trace and decides whether to park, and that takes hundreds of milliseconds. A Stop that + * arrives in that window has nothing left to abort, and aborting anyway is actively harmful: + * the abort makes teardown read the run as cancelled-but-unsettled and DESTROY a healthy + * environment that was about to be parked. So the applier reads this flag and does nothing. + */ + settled?: boolean; + /** Stop the run. Aborting is what makes the turn end `cancelled`. */ + abort: () => void; +} + +const executions = new Map(); + +/** + * Register a run as live. A second registration for the same session REPLACES the first, + * because the pool's own supersede path has already torn the previous environment down by the + * time a replacement turn starts. + */ +export function registerExecution(execution: LiveExecution): void { + executions.set(execution.sessionId, execution); +} + +/** + * Fill in the project scope once the coordinator has resolved it. Scoped to the turn id, so a + * late callback from a finished run cannot relabel its successor. + */ +export function noteExecutionProject( + sessionId: string, + turnId: string, + projectId: string, +): void { + const current = executions.get(sessionId); + if (current && current.turnId === turnId) current.projectId = projectId; +} + +/** + * Mark a run's own work as finished, the moment the harness prompt settles and before teardown + * begins. Scoped to the turn id for the same reason `noteExecutionProject` is: a late callback + * from a finished run must not relabel its successor. + * + * Set from inside the turn, not from the request handler that awaits it, because the harmful + * window is exactly the teardown that runs between those two points. + */ +export function noteExecutionSettled(sessionId: string, turnId: string): void { + const current = executions.get(sessionId); + if (current && current.turnId === turnId) current.settled = true; +} + +/** + * Remove a run, but only if it is still the one registered. A turn that finishes after its + * successor registered must not unregister the successor. + */ +export function unregisterExecution(sessionId: string, turnId: string): void { + const current = executions.get(sessionId); + if (current && current.turnId === turnId) executions.delete(sessionId); +} + +/** + * The live execution for a session, when it belongs to the asking project. + * + * A stored project that DISAGREES yields nothing, so a Stop from another tenant is refused. + * A stored project that is not known yet matches, because the run has genuinely not been + * scoped at that point and refusing would drop every Stop in the first moments of a run. + */ +export function findExecution( + projectId: string, + sessionId: string, +): LiveExecution | undefined { + const current = executions.get(sessionId); + if (!current) return undefined; + if (current.projectId !== undefined && current.projectId !== projectId) { + return undefined; + } + return current; +} + +/** Test/inspection snapshot. */ +export function liveExecutions(): LiveExecution[] { + return [...executions.values()]; +} + +/** Test seam: drop everything. Never called by the server. */ +export function resetExecutionsForTest(): void { + executions.clear(); +} diff --git a/services/runner/tests/unit/control-command-apply.test.ts b/services/runner/tests/unit/control-command-apply.test.ts new file mode 100644 index 00000000000..d0349e7d577 --- /dev/null +++ b/services/runner/tests/unit/control-command-apply.test.ts @@ -0,0 +1,463 @@ +/** + * The rules a control command obeys on the runner. + * + * A Stop reaches the runner as a durable command naming one execution. Four rules decide what + * the runner does with it, and this file pins all four: + * + * 1. It aborts the named execution when it holds it, which is what keeps the sandbox warm + * (the abort ends the turn `cancelled`, and only a cancelled turn takes the park path). + * 2. It aborts NOTHING when it holds an execution that started after the command was created. + * That is the late-Stop guard, and it is exact because it reads this process's own memory. + * 3. A session it holds parked awaiting an approval answers `not_running` and stays parked. + * Stop ends the work, not the session. + * 4. The same command delivered twice aborts once and acknowledges twice. + * 5. It aborts NOTHING when the named execution's prompt has already settled and only its + * teardown is still running. That Stop lost the race by a moment, and aborting a finished + * run would destroy the warm environment teardown was about to park. + */ +import assert from "node:assert/strict"; +import { beforeEach, describe, it } from "vitest"; + +import { + applyCommand, + holdsSession, + reportOutcome, + type ControlCommand, + type ControlOutcome, +} from "../../src/sessions/control-channel.ts"; +import { resetAppliedCommandsForTest } from "../../src/sessions/applied-commands.ts"; +import { shouldPark } from "../../src/engines/sandbox_agent/engine.ts"; +import { + isUserStopAbort, + USER_STOP_ABORT_REASON, +} from "../../src/sessions/stop-signal.ts"; +import { + findExecution, + noteExecutionSettled, + registerExecution, + resetExecutionsForTest, + noteExecutionProject, + unregisterExecution, + type LiveExecution, +} from "../../src/sessions/execution-registry.ts"; + +const PROJECT = "11111111-1111-4111-8111-111111111111"; +const SESSION = "sess-42"; +const TURN = "turn-A"; + +/** t=1000 is "now"; a command created at t=1000 is contemporary with a run started at t=900. */ +const COMMAND_CREATED_AT = new Date(1000).toISOString(); + +function command(overrides: Partial = {}): ControlCommand { + return { + id: "cmd-1", + projectId: PROJECT, + sessionId: SESSION, + kind: "cancel", + target: { turnId: TURN, expectedTurnId: null }, + createdAt: COMMAND_CREATED_AT, + ...overrides, + }; +} + +function liveRun( + overrides: Partial = {}, +): { execution: LiveExecution; aborts: number[] } { + const aborts: number[] = []; + const execution: LiveExecution = { + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => aborts.push(Date.now()), + ...overrides, + }; + return { execution, aborts }; +} + +function collector(): { + reported: ControlOutcome[]; + report: (c: ControlCommand, o: ControlOutcome) => Promise; +} { + const reported: ControlOutcome[] = []; + return { + reported, + report: async (_c, o) => { + reported.push(o); + }, + }; +} + +beforeEach(() => { + resetExecutionsForTest(); + resetAppliedCommandsForTest(); +}); + +describe("applyCommand", () => { + it("aborts the live execution the command names and reports it stopped", async () => { + const { execution, aborts } = liveRun(); + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 1); + assert.equal(outcome.result, "applied"); + assert.equal(outcome.execution.state, "stopped"); + assert.equal(outcome.execution.id, TURN); + assert.deepEqual(reported, [outcome]); + }); + + it("aborts nothing when this process holds no execution for the session", async () => { + // The parked-approval case. There is no turn to abort, and the parked environment must + // stay in the pool so the next message is warm. + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => undefined, + report, + }); + + assert.equal(outcome.result, "applied"); + assert.equal(outcome.execution.state, "not_running"); + assert.equal(reported.length, 1); + }); + + it("refuses to abort an execution that started AFTER the command was created", async () => { + const { execution, aborts } = liveRun({ + turnId: "turn-B", + startedAt: 5000, // the command was created at t=1000 + }); + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 0, "a newer turn must never be aborted"); + assert.equal(outcome.result, "obsolete"); + assert.equal(outcome.execution.state, "superseded_by_newer_turn"); + assert.equal(reported.length, 1); + }); + + it("reports not_running when it holds a DIFFERENT, older execution", async () => { + const { execution, aborts } = liveRun({ turnId: "turn-Z", startedAt: 500 }); + const { report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 0); + assert.equal(outcome.result, "obsolete"); + assert.equal(outcome.execution.state, "not_running"); + assert.equal(outcome.execution.id, TURN); + }); + + it("aborts once and acknowledges twice when the same command is delivered twice", async () => { + const { execution, aborts } = liveRun(); + const { reported, report } = collector(); + + await applyCommand(command(), { findLive: () => execution, report }); + await applyCommand(command(), { findLive: () => execution, report }); + + assert.equal(aborts.length, 1, "a second abort could kill a newer turn"); + assert.equal(reported.length, 2, "a lost acknowledgement must be repairable"); + assert.equal(reported[1].execution.state, "stopped"); + }); + + it("remembers the command before aborting, so a duplicate mid-cancel is still a no-op", async () => { + // The abort itself delivers a second copy of the same command, which is what a retried + // admission looks like on the wire. + let nested: ControlOutcome | undefined; + const { report } = collector(); + const aborts: number[] = []; + const execution: LiveExecution = { + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => { + aborts.push(1); + }, + }; + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report: async (c, o) => { + if (nested === undefined) { + nested = await applyCommand(c, { findLive: () => execution, report }); + } + }, + }); + + assert.equal(aborts.length, 1); + assert.equal(outcome.execution.state, "stopped"); + assert.equal(nested?.execution.state, "stopped"); + }); + + it("aborts with the user-stop label, which is what lets the sandbox park", async () => { + // The registry hands the applier whatever abort the transport registered. `shouldPark` + // parks only an abort the runner can prove was a cooperative Stop, so an unlabelled abort + // here would end the turn `cancelled` and then DESTROY the sandbox. This pins the contract + // the applier depends on; `server.ts` is where the label is actually attached. + const controller = new AbortController(); + const execution: LiveExecution = { + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => controller.abort(USER_STOP_ABORT_REASON), + }; + const { report } = collector(); + + await applyCommand(command(), { findLive: () => execution, report }); + + assert.equal(isUserStopAbort(controller.signal), true); + assert.equal( + shouldPark( + { ok: true, stopReason: "cancelled", cancelSettled: true }, + controller.signal, + undefined, + ), + true, + "a Stop delivered as a command must leave the sandbox parkable", + ); + }); + + it("does NOT park when the abort carries no label", () => { + // The regression this guards: the first version of the control route called + // `controller.abort()` with no reason, so every Stop through it destroyed the sandbox. + const controller = new AbortController(); + controller.abort(); + + assert.equal(isUserStopAbort(controller.signal), false); + assert.equal( + shouldPark( + { ok: true, stopReason: "cancelled", cancelSettled: true }, + controller.signal, + undefined, + ), + false, + ); + }); + + it("aborts nothing when the named execution's prompt has already settled", async () => { + // The race the user cannot see: the answer lands, they press Stop a moment later, and the + // entry is still registered because teardown is writing the transcript and parking the + // sandbox. Aborting here stops nothing and makes teardown destroy a healthy environment. + const { execution, aborts } = liveRun({ settled: true }); + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 0, "a finished run must not be aborted"); + assert.equal(outcome.result, "obsolete", "the command stopped nothing"); + assert.equal(outcome.execution.state, "not_running"); + assert.equal(outcome.execution.id, TURN); + assert.deepEqual(reported, [outcome], "and it still acknowledges"); + }); + + it("still aborts an execution whose prompt has NOT settled", async () => { + // The guard must be the flag and not the mere presence of teardown, or every Stop becomes + // a no-op and Stop stops working. + const { execution, aborts } = liveRun({ settled: false }); + const { report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(aborts.length, 1); + assert.equal(outcome.execution.state, "stopped"); + }); + + it("parks the environment of a finished turn that a late Stop did not abort", () => { + // The consequence the fix exists for, stated as the teardown sees it. No abort means no + // aborted signal, so a normally finished turn takes the ordinary park path. + const controller = new AbortController(); + assert.equal( + shouldPark( + { ok: true, stopReason: "end_turn" } as never, + controller.signal, + undefined, + ), + true, + "an un-aborted, cleanly finished turn parks", + ); + // And this is what used to happen instead: the late abort fired, and the same finished + // turn was destroyed rather than parked. + controller.abort(USER_STOP_ABORT_REASON); + assert.equal( + shouldPark( + { ok: true, stopReason: "end_turn" } as never, + controller.signal, + undefined, + ), + false, + "which is why the applier must not abort a settled run", + ); + }); + + it("reports the cancel as failed when the abort itself throws", async () => { + const execution: LiveExecution = { + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => { + throw new Error("controller is gone"); + }, + }; + const { reported, report } = collector(); + + const outcome = await applyCommand(command(), { + findLive: () => execution, + report, + }); + + assert.equal(outcome.execution.state, "failed"); + assert.equal(outcome.execution.error, "controller is gone"); + assert.equal(reported.length, 1); + }); +}); + +describe("the execution registry", () => { + it("refuses a lookup from another project once the scope is known", () => { + const { execution } = liveRun(); + registerExecution(execution); + + assert.equal(findExecution(PROJECT, SESSION)?.turnId, TURN); + assert.equal( + findExecution("22222222-2222-4222-8222-222222222222", SESSION), + undefined, + "two projects may use the same session id; the project is the tenant boundary", + ); + }); + + it("matches any project until the coordinator has resolved the scope", () => { + // `runContext.project.id` is empty on the live invoke path, so a run is registered before + // its project is known. Refusing every Stop in that window is what made the first version + // of this registry answer 404 for every real Stop. + registerExecution(liveRun({ projectId: undefined }).execution); + + assert.equal(findExecution(PROJECT, SESSION)?.turnId, TURN); + + noteExecutionProject(SESSION, TURN, PROJECT); + assert.equal( + findExecution("22222222-2222-4222-8222-222222222222", SESSION), + undefined, + "once the scope is known, another tenant is refused", + ); + }); + + it("does not let a late scope callback relabel a successor turn", () => { + registerExecution(liveRun({ turnId: "turn-2", projectId: undefined }).execution); + + noteExecutionProject(SESSION, "turn-1", "some-other-project"); + + assert.equal(findExecution(PROJECT, SESSION)?.projectId, undefined); + }); + + it("marks only the turn it names as settled", () => { + registerExecution({ + projectId: PROJECT, + sessionId: SESSION, + turnId: TURN, + startedAt: 900, + abort: () => {}, + }); + + // A late callback from a turn that has already been replaced must not mark the successor + // finished, which would make every Stop on the live turn a no-op. + noteExecutionSettled(SESSION, "some-older-turn"); + assert.equal(findExecution(PROJECT, SESSION)?.settled, undefined); + + noteExecutionSettled(SESSION, TURN); + assert.equal(findExecution(PROJECT, SESSION)?.settled, true); + }); + + it("does not let a finished turn unregister its successor", () => { + const first = liveRun({ turnId: "turn-1" }).execution; + const second = liveRun({ turnId: "turn-2" }).execution; + registerExecution(first); + registerExecution(second); + + unregisterExecution(SESSION, "turn-1"); + + assert.equal(findExecution(PROJECT, SESSION)?.turnId, "turn-2"); + }); +}); + +describe("holdsSession", () => { + it("is true for a live execution", () => { + registerExecution(liveRun().execution); + assert.equal(holdsSession(PROJECT, SESSION), true); + }); + + it("is true for a session parked awaiting an approval, which runs no turn", () => { + // This is the case that has no control channel at all today: a parked session stops + // heartbeating, so the existing Stop signal never reaches it. + assert.equal(holdsSession(PROJECT, SESSION), false); + assert.equal( + holdsSession( + PROJECT, + SESSION, + (projectId, sessionId) => + projectId === PROJECT && sessionId === SESSION, + ), + true, + ); + }); + + it("does not match a parked session with the same id in another project", () => { + assert.equal( + holdsSession( + PROJECT, + SESSION, + (projectId, sessionId) => + projectId === "22222222-2222-4222-8222-222222222222" && + sessionId === SESSION, + ), + false, + ); + }); + + it("is false for a session this process does not hold, which is what answers 404", () => { + assert.equal(holdsSession(PROJECT, "other-session", () => false), false); + }); +}); + +describe("reportOutcome", () => { + it("rejects redirects so the runner token cannot be forwarded", async () => { + const previousToken = process.env.AGENTA_RUNNER_TOKEN; + const previousFetch = globalThis.fetch; + let captured: RequestInit | undefined; + process.env.AGENTA_RUNNER_TOKEN = "shared-secret"; + globalThis.fetch = (async (_input, init) => { + captured = init; + return new Response("{}", { status: 200 }); + }) as typeof fetch; + + try { + await reportOutcome(command(), { + result: "applied", + execution: { id: TURN, state: "stopped" }, + }); + } finally { + globalThis.fetch = previousFetch; + if (previousToken === undefined) delete process.env.AGENTA_RUNNER_TOKEN; + else process.env.AGENTA_RUNNER_TOKEN = previousToken; + } + + assert.equal(captured?.redirect, "error"); + }); +}); diff --git a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts index 1018f53e934..a50dc489717 100644 --- a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts +++ b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts @@ -53,6 +53,12 @@ import { flushPromises, type FakeOptions, } from "../utils/sandbox-agent-harness.ts"; +import { + findExecution, + registerExecution, + resetExecutionsForTest, +} from "../../src/sessions/execution-registry.ts"; +import { applyCommand } from "../../src/sessions/control-channel.ts"; // Orchestration cases include Daytona runs: enable it (with a provisioning credential) on top of // the hermetic scrub, then drop the memoized config so the run plan reads the enabled set. @@ -63,6 +69,7 @@ beforeEach(() => { }); afterEach(() => { + resetExecutionsForTest(); vi.unstubAllGlobals(); }); @@ -2600,6 +2607,118 @@ describe("runSandboxAgent default ApprovalResponder wiring", () => { assert.deepEqual(calls.permissionReplies, []); }); + it("marks a paused turn settled after its cancellable teardown window", async () => { + const { deps } = depsWithDefaultResponder(); + const sessionId = "conv-paused-registry"; + const turnId = "turn-paused-registry"; + registerExecution({ + projectId: "11111111-1111-4111-8111-111111111111", + sessionId, + turnId, + startedAt: Date.now(), + abort: () => {}, + }); + + const result = await runSandboxAgent( + { + harness: "claude", + sessionId, + turnId, + permissions: { default: "ask" }, + messages: [{ role: "user", content: "edit the file" }], + }, + undefined, + undefined, + deps, + ); + + assert.equal(result.ok, true); + if (!result.ok) return; + assert.equal(result.stopReason, "paused"); + assert.equal( + findExecution("11111111-1111-4111-8111-111111111111", sessionId)?.settled, + true, + ); + }); + + it("converts a Stop during pause teardown into the turn's cancelled outcome", async () => { + let markPauseTeardownStarted!: () => void; + const pauseTeardownStarted = new Promise((resolve) => { + markPauseTeardownStarted = resolve; + }); + let releasePauseTeardown!: () => void; + const pauseTeardownMayFinish = new Promise((resolve) => { + releasePauseTeardown = resolve; + }); + const { deps } = fakeHarness({ + emitPermission: true, + hangPrompt: true, + afterDestroySession: async () => { + markPauseTeardownStarted(); + await pauseTeardownMayFinish; + }, + }); + delete deps.responderFactory; + const startSandboxAgent = deps.startSandboxAgent!; + deps.startSandboxAgent = async (options) => { + const sandbox = await startSandboxAgent(options); + const cancellable = sandbox as unknown as { + destroySession: (id: string) => Promise; + cancelSession?: (id: string) => Promise; + }; + cancellable.cancelSession = (id) => cancellable.destroySession(id); + return sandbox; + }; + + const projectId = "11111111-1111-4111-8111-111111111111"; + const sessionId = "conv-stop-during-pause-teardown"; + const turnId = "turn-stop-during-pause-teardown"; + const controller = new AbortController(); + registerExecution({ + projectId, + sessionId, + turnId, + startedAt: Date.now(), + abort: () => controller.abort(USER_STOP_ABORT_REASON), + }); + + const turn = runSandboxAgent( + { + harness: "claude", + sessionId, + turnId, + permissions: { default: "ask" }, + messages: [{ role: "user", content: "edit the file" }], + }, + undefined, + controller.signal, + deps, + ); + + await pauseTeardownStarted; + const outcome = await applyCommand( + { + id: "command-stop-during-pause-teardown", + projectId, + sessionId, + kind: "cancel", + target: { turnId, expectedTurnId: turnId }, + createdAt: new Date().toISOString(), + }, + { report: async () => {} }, + ); + assert.equal(outcome.execution.state, "stopped"); + + releasePauseTeardown(); + const result = await turn; + + assert.equal(result.ok, true); + if (!result.ok) return; + assert.equal(result.stopReason, "cancelled"); + assert.equal(result.cancelSettled, true); + assert.equal(findExecution(projectId, sessionId)?.settled, true); + }); + it("effective ask with no decision pauses the tool, no harness reply (F-024)", async () => { const { calls, deps } = depsWithDefaultResponder(); diff --git a/services/runner/tests/unit/server.test.ts b/services/runner/tests/unit/server.test.ts index 8e67d1042a0..45355022561 100644 --- a/services/runner/tests/unit/server.test.ts +++ b/services/runner/tests/unit/server.test.ts @@ -26,6 +26,10 @@ import { import type { SessionEnvironment } from "../../src/engines/sandbox_agent.ts"; import { SessionPool } from "../../src/engines/sandbox_agent/session-pool.ts"; import { HEARTBEAT_INTERVAL_SECONDS } from "../../src/sessions/contract.ts"; +import { + liveExecutions, + resetExecutionsForTest, +} from "../../src/sessions/execution-registry.ts"; const TOKEN_ENV = "AGENTA_RUNNER_TOKEN"; const previousToken = process.env[TOKEN_ENV]; @@ -34,6 +38,7 @@ const LIMIT_ENV = "AGENTA_RUNNER_CONCURRENCY_LIMIT"; const previousLimit = process.env[LIMIT_ENV]; afterEach(() => { + resetExecutionsForTest(); vi.restoreAllMocks(); vi.unstubAllEnvs(); if (previousToken === undefined) delete process.env[TOKEN_ENV]; @@ -796,15 +801,18 @@ describe("createAgentServer", () => { const endings = ingested.filter( (record) => record.record_type === "done", ); - assert.equal(endings.length, 1, "the server must not duplicate runTurn's ending"); + assert.equal( + endings.length, + 1, + "the server must not duplicate runTurn's ending", + ); assert.deepEqual(endings[0].attributes, { type: "done", stopReason: "cancelled", }); assert.equal( records.filter( - (record) => - record.kind === "event" && record.event?.type === "done", + (record) => record.kind === "event" && record.event?.type === "done", ).length, 1, "the normal Stop still streams its one done event", @@ -940,6 +948,7 @@ describe("createAgentServer", () => { records[0].result.error, "A user turn may carry at most 2 attachments.", ); + assert.deepEqual(liveExecutions(), []); } finally { delete process.env.AGENTA_ATTACHMENTS_MAX_PER_TURN; fetchSpy.mockRestore(); diff --git a/services/runner/tests/unit/session-admission.test.ts b/services/runner/tests/unit/session-admission.test.ts index 94420872d6a..d0f0e878d47 100644 --- a/services/runner/tests/unit/session-admission.test.ts +++ b/services/runner/tests/unit/session-admission.test.ts @@ -55,7 +55,9 @@ interface Beat { } /** The fake platform API. `admit` decides what its heartbeat answers for each beat. */ -async function startFakeApi(admit: (beat: Beat) => boolean): Promise<{ +async function startFakeApi( + admit: (beat: Beat) => boolean | Promise, +): Promise<{ url: string; beats: Beat[]; paths: string[]; @@ -66,7 +68,7 @@ async function startFakeApi(admit: (beat: Beat) => boolean): Promise<{ const server = createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c) => chunks.push(c as Buffer)); - req.on("end", () => { + req.on("end", async () => { const path = (req.url ?? "").split("?")[0]; paths.push(path); let body: Record = {}; @@ -87,7 +89,8 @@ async function startFakeApi(admit: (beat: Beat) => boolean): Promise<{ stream: { id: "11111111-1111-1111-1111-111111111111" }, replica_id: body.replica_id ?? null, // A turn-end beat (`is_running: false`) is never an admission question. - is_current_turn: beat.is_running === false ? true : admit(beat), + is_current_turn: + beat.is_running === false ? true : await admit(beat), }), ); return; @@ -352,6 +355,108 @@ describe("runner admission: an admitted turn proceeds", () => { } }); + it("a refused second turn cannot replace the admitted turn's Stop handle", async () => { + let releaseSecondAdmission!: () => void; + const secondAdmissionMayFinish = new Promise((resolve) => { + releaseSecondAdmission = resolve; + }); + let markSecondAdmissionWaiting!: () => void; + const secondAdmissionWaiting = new Promise((resolve) => { + markSecondAdmissionWaiting = resolve; + }); + const api = await startFakeApi(async (beat) => { + if (beat.turn_id !== "turn-B") return true; + markSecondAdmissionWaiting(); + await secondAdmissionMayFinish; + return false; + }); + process.env[INTERNAL_ENV] = api.url; + + let markFirstRunning!: () => void; + const firstRunning = new Promise((resolve) => { + markFirstRunning = resolve; + }); + let markFirstAborted!: () => void; + const firstAborted = new Promise((resolve) => { + markFirstAborted = resolve; + }); + let finishFirstForCleanup!: () => void; + const firstMayFinishForCleanup = new Promise((resolve) => { + finishFirstForCleanup = resolve; + }); + const runCalls: string[] = []; + const runner = await startRunner( + async (request, _emit, signal): Promise => { + runCalls.push(request.turnId ?? "missing"); + assert.equal(request.turnId, "turn-A", "the refused turn never reaches run()"); + markFirstRunning(); + await Promise.race([ + new Promise((resolve) => { + if (signal?.aborted) resolve(); + else signal?.addEventListener("abort", () => resolve(), { once: true }); + }), + firstMayFinishForCleanup, + ]); + if (signal?.aborted) markFirstAborted(); + return { + ok: true, + output: "", + events: [], + ...(signal?.aborted ? { stopReason: "cancelled" as const } : {}), + }; + }, + ); + + const firstRequest = postRun( + runner.url, + sessionRequest({ turnId: "turn-A" }), + ); + let secondRequest: ReturnType | undefined; + try { + await firstRunning; + secondRequest = postRun( + runner.url, + sessionRequest({ turnId: "turn-B" }), + ); + await secondAdmissionWaiting; + + const cancel = await fetch(`${runner.url}/cancel`, { + method: "POST", + headers: { "content-type": "application/json", ...AUTH }, + body: JSON.stringify({ + commandId: "command-stop-A", + projectId: "project-1", + sessionId: "session-admission-1", + targetTurnId: "turn-A", + createdAt: new Date().toISOString(), + }), + }); + + assert.equal(cancel.status, 202, "the runner still holds admitted turn A"); + await firstAborted; + releaseSecondAdmission(); + const [first, second] = await Promise.all([firstRequest, secondRequest]); + assert.equal( + first.records.find((record) => record.kind === "result")?.result?.ok, + true, + ); + assert.equal( + second.records.find((record) => record.kind === "result")?.result?.error, + SESSION_TURN_IN_USE_MESSAGE, + ); + assert.deepEqual(runCalls, ["turn-A"]); + } finally { + releaseSecondAdmission(); + finishFirstForCleanup(); + await Promise.allSettled([ + firstRequest, + ...(secondRequest ? [secondRequest] : []), + ]); + await runner.close(); + await api.close(); + } + }); + it("fails OPEN: an unreachable platform admits the turn rather than refusing it", async () => { // The heartbeat has always failed open, and admission must not change that: a transient API // blip refusing every message would be a worse outage than the bug this slice fixes. The diff --git a/web/mobile/src/features/sessions/useActionableInteractions.ts b/web/mobile/src/features/sessions/useActionableInteractions.ts index 879e983c728..de988c41dd8 100644 --- a/web/mobile/src/features/sessions/useActionableInteractions.ts +++ b/web/mobile/src/features/sessions/useActionableInteractions.ts @@ -1,23 +1,14 @@ -import { - queryInteractions, - type SessionInteraction, - type SessionStream, -} from "@agenta/entities/session" -import {useQuery, useQueryClient} from "@tanstack/react-query" +import {queryInteractions, type SessionInteraction} from "@agenta/entities/session" +import {useQuery} from "@tanstack/react-query" -import {livenessQueryKey} from "./useLivenessPoll" +import {useLivenessPoll} from "./useLivenessPoll" export const actionableInteractionsQueryKey = (projectId: string) => ["mobile", "actionable-interactions", projectId] as const -/** - * Every pending HITL request across the project in ONE query (`session_id` omitted, - * `actionable_only: true`) — the list-badge primitive. Same cadence rules as the liveness poll: - * 15s while anything is pending OR alive (a running turn is what mints new gates), stops when - * idle, re-checks on focus. - */ +/** Poll pending project HITL requests while a gate exists or a turn can create one. */ export const useActionableInteractions = (projectId: string) => { - const queryClient = useQueryClient() + const liveness = useLivenessPoll(projectId) return useQuery({ queryKey: actionableInteractionsQueryKey(projectId), queryFn: ({signal}) => @@ -26,10 +17,8 @@ export const useActionableInteractions = (projectId: string) => { staleTime: 10_000, refetchInterval: (query) => { if ((query.state.data?.length ?? 0) > 0) return 15_000 - const alive = queryClient.getQueryData( - livenessQueryKey(projectId), - ) - return (alive?.length ?? 0) > 0 ? 15_000 : false + // Only running turns can mint new gates. + return (liveness.data ?? []).some((stream) => stream.flags?.is_running) ? 15_000 : false }, refetchOnWindowFocus: true, }) diff --git a/web/mobile/src/features/sessions/useLivenessPoll.ts b/web/mobile/src/features/sessions/useLivenessPoll.ts index bfa5a6b0238..00e9781ae7b 100644 --- a/web/mobile/src/features/sessions/useLivenessPoll.ts +++ b/web/mobile/src/features/sessions/useLivenessPoll.ts @@ -1,15 +1,16 @@ -import {deriveStreamNest, querySessionStreams, type SessionStream} from "@agenta/entities/session" +import { + deriveStreamNest, + livenessPollInterval, + querySessionStreams, + type SessionStream, +} from "@agenta/entities/session" import {useQuery} from "@tanstack/react-query" -/** Shared key so other polls (interactions) can read the alive set from the cache. */ +/** Shared key for the project liveness subscription. */ export const livenessQueryKey = (projectId: string) => ["mobile", "session-liveness", projectId] as const -/** - * Backend liveness for the project's sessions — mirrors the desktop pattern - * (oss AgentChatSlice state/liveness.ts): ONE project-scoped `is_alive=true` query backs every - * badge, low-priority, 15s while anything is alive, stops when idle, re-checks on focus. - */ +/** Poll quickly while work runs, slowly while a session remains warm, and stop when idle. */ export const useLivenessPoll = (projectId: string) => useQuery({ queryKey: livenessQueryKey(projectId), @@ -17,7 +18,7 @@ export const useLivenessPoll = (projectId: string) => querySessionStreams({projectId, isAlive: true, abortSignal: signal, lowPriority: true}), enabled: Boolean(projectId), staleTime: 10_000, - refetchInterval: (query) => ((query.state.data?.length ?? 0) > 0 ? 15_000 : false), + refetchInterval: (query) => livenessPollInterval(query.state.data), refetchOnWindowFocus: true, }) diff --git a/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.test.ts b/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.test.ts new file mode 100644 index 00000000000..8ab106f1b51 --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.test.ts @@ -0,0 +1,141 @@ +import {act, createElement, useCallback} from "react" + +import {latestTurnId} from "@agenta/chat/assets" +import {clearSessionTurnId, getSessionTurnId, setSessionTurnId} from "@agenta/chat/state" +import type {UIMessage} from "ai" +import {createRoot} from "react-dom/client" +import {afterAll, afterEach, beforeAll, describe, expect, it, vi} from "vitest" + +import {stopPinnedExecution} from "./stopWhileResolvingExecution" + +const sessionId = "session-1" + +const deferred = () => { + let resolve!: () => void + const promise = new Promise((done) => { + resolve = done + }) + return {promise, resolve} +} + +beforeAll(() => vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true)) +afterAll(() => vi.unstubAllGlobals()) +afterEach(() => clearSessionTurnId(sessionId)) + +describe("stopPinnedExecution", () => { + it("starts the local abort while cancellation is still pending", async () => { + const held = deferred() + const events: string[] = [] + const stop = vi.fn(() => events.push("stop")) + const cancelExecution = vi.fn(async (executionId: string | undefined) => { + events.push(`cancel:${executionId}`) + await held.promise + }) + + const stopping = stopPinnedExecution({ + stop, + expectedExecutionId: "turn-A", + cancelExecution, + }) + + expect(events).toEqual(["stop", "cancel:turn-A"]) + + held.resolve() + await stopping + expect(cancelExecution).toHaveBeenCalledWith("turn-A") + }) + + it("stops turn B before metadata without restoring turn A's id", async () => { + const stop = vi.fn() + const cancelExecution = vi.fn(async (_executionId: string | undefined) => {}) + setSessionTurnId(sessionId, "turn-A") + + clearSessionTurnId(sessionId) + const messages = [ + {id: "a1", role: "assistant", parts: [], metadata: {turnId: "turn-A"}}, + {id: "u2", role: "user", parts: []}, + ] as UIMessage[] + const turnId = latestTurnId(messages) + if (turnId) setSessionTurnId(sessionId, turnId) + + await stopPinnedExecution({ + stop, + expectedExecutionId: getSessionTurnId(sessionId), + cancelExecution, + }) + + expect(stop).toHaveBeenCalledOnce() + expect(cancelExecution).toHaveBeenCalledWith(undefined) + expect(cancelExecution).not.toHaveBeenCalledWith("turn-A") + }) + + it("keeps turn A pinned when turn B is admitted while cancellation is held", async () => { + const held = deferred() + const cancelled: (string | undefined)[] = [] + setSessionTurnId(sessionId, "turn-A") + + const stopping = stopPinnedExecution({ + stop: vi.fn(), + expectedExecutionId: getSessionTurnId(sessionId), + cancelExecution: async (executionId) => { + await held.promise + cancelled.push(executionId) + }, + }) + + clearSessionTurnId(sessionId) + setSessionTurnId(sessionId, "turn-B") + expect(getSessionTurnId(sessionId)).toBe("turn-B") + + held.resolve() + await stopping + expect(cancelled).toEqual(["turn-A"]) + }) + + it("keeps turn A pinned after the hook remounts and admits turn B", async () => { + const held = deferred() + const cancelled: (string | undefined)[] = [] + const cancelExecution = async (executionId: string | undefined) => { + await held.promise + cancelled.push(executionId) + } + let stopFromMount!: () => Promise + const Harness = () => { + stopFromMount = useCallback( + () => + stopPinnedExecution({ + stop: vi.fn(), + expectedExecutionId: getSessionTurnId(sessionId), + cancelExecution, + }), + [], + ) + return null + } + + const mount = () => { + const host = document.createElement("div") + const root = createRoot(host) + act(() => root.render(createElement(Harness))) + return root + } + + setSessionTurnId(sessionId, "turn-A") + const firstMount = mount() + let stopping!: Promise + act(() => { + stopping = stopFromMount() + }) + act(() => firstMount.unmount()) + + clearSessionTurnId(sessionId) + setSessionTurnId(sessionId, "turn-B") + const secondMount = mount() + expect(getSessionTurnId(sessionId)).toBe("turn-B") + + held.resolve() + await stopping + expect(cancelled).toEqual(["turn-A"]) + act(() => secondMount.unmount()) + }) +}) diff --git a/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.ts b/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.ts new file mode 100644 index 00000000000..18d5b29e915 --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/stopWhileResolvingExecution.ts @@ -0,0 +1,14 @@ +export interface StopPinnedExecutionParams { + stop: () => void + expectedExecutionId: string | undefined + cancelExecution: (executionId: string | undefined) => Promise +} + +export async function stopPinnedExecution({ + stop, + expectedExecutionId, + cancelExecution, +}: StopPinnedExecutionParams): Promise { + stop() + await cancelExecution(expectedExecutionId) +} diff --git a/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts b/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts index f3cceb6bd2e..2aa2864674e 100644 --- a/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts +++ b/web/oss/src/components/AgentChatSlice/hooks/useAgentChatSession.ts @@ -3,6 +3,7 @@ import {useCallback, useEffect, useRef, useState} from "react" import { buildRequestWithinDeadline, getMessageTraceId, + latestTurnId, startupLabelFromDataPart, } from "@agenta/chat/assets" import type {ClientToolOutputHandler} from "@agenta/chat/clientTools" @@ -15,15 +16,18 @@ import { } from "@agenta/chat/state" import {expandedKeysForMessages, pruneExpandedAtom} from "@agenta/chat/state" import { + clearSessionTurnId, + getSessionTurnId, isChatBusy, persistSessionMessagesAtom, sessionMessagesAtom, sessionRecordCountsReadAtom, setSessionStatusAtom, + setSessionTurnId, type SessionChatHooks, } from "@agenta/chat/state" import { - commandSessionStream, + cancelSessionExecution, invalidateSessionListQueries, killSession, recordInteractionAnswerAtom, @@ -52,6 +56,7 @@ import {useAtomValue, useSetAtom, useStore} from "jotai" import {projectIdAtom} from "@/oss/state/project" import {doesAgentChatStopKillSession} from "../assets/constants" +import {stopPinnedExecution} from "../assets/stopWhileResolvingExecution" import {invalidateSessionInspector} from "../components/Inspector/invalidate" import {useChatScopeKey} from "../state/scope" import {openSessionIdsAtomFamily} from "../state/sessions" @@ -130,6 +135,7 @@ export const useAgentChatSession = ({ // instead of sticking to the revision this session first mounted on. const hooks: SessionChatHooks = { prepareRequest: async ({messages, id}) => { + clearSessionTurnId(sessionId) // Bounded: retries while the invocation URL is still loading and rejects if the build // hangs, so a failed send surfaces as an error bubble instead of an eternal spinner // (#6042). The helper owns the not-ready / timed-out errors. @@ -206,10 +212,10 @@ export const useAgentChatSession = ({ const { messages, - sendMessage, + sendMessage: sendChatMessage, status, stop, - regenerate, + regenerate: regenerateChatMessage, setMessages, addToolApprovalResponse, addToolOutput, @@ -230,6 +236,21 @@ export const useAgentChatSession = ({ const busyRef = useRef(busy) busyRef.current = busy + const sendMessage = useCallback( + (...args: Parameters) => { + clearSessionTurnId(sessionId) + return sendChatMessage(...args) + }, + [sendChatMessage, sessionId], + ) + const regenerate = useCallback( + (...args: Parameters) => { + clearSessionTurnId(sessionId) + return regenerateChatMessage(...args) + }, + [regenerateChatMessage, sessionId], + ) + // Mid-stream drive signals: settled write-ish tool calls append file-activity entries (and // throttle-revalidate the drives) as the turn streams, not just at onFinish. useFileActivityDetector({sessionId, messages}) @@ -346,6 +367,11 @@ export const useAgentChatSession = ({ restoredIdsRef.current.has(lastMessage.id) && agentShouldResumeAfterApproval({messages}) + useEffect(() => { + const turnId = latestTurnId(messages) + if (turnId) setSessionTurnId(sessionId, turnId) + }, [messages, sessionId]) + // Surface a stream failure inline: stamp the parsed error onto the failing assistant turn so // it renders as a red error bubble with the real reason (and persists with the session via the // effect below), instead of a transient top banner + a generic "no response". FE-only — it @@ -477,33 +503,50 @@ export const useAgentChatSession = ({ const projectId = useAtomValue(projectIdAtom) + /** Pin the visible execution before client stop unlocks the next send. */ + const stopCurrentExecution = useCallback(async () => { + const expectedExecutionId = getSessionTurnId(sessionId) + if (!projectId || !sessionId) { + stop() + return + } + await stopPinnedExecution({ + stop, + expectedExecutionId, + cancelExecution: (pinnedExecutionId) => + cancelSessionExecution({ + sessionId, + projectId, + expectedExecutionId: pinnedExecutionId, + }), + }) + // Refresh even on conflict because the session state is authoritative. + void invalidateSessionInspector(queryClient, sessionId) + void queryClient.invalidateQueries({queryKey: ["session-liveness"]}) + }, [projectId, sessionId, queryClient, stop]) + const handleStop = useCallback(() => { markStopped() - // A stop voids the pending gate (same rule the queue applies), so the marker must go too — - // otherwise it outlives the abandoned resume and blocks this mount's records adoption. + // Stop clears the pending gate marker before it can block later record adoption. liveGateInteractionRef.current = null - stop() // abort the client stream immediately - if (!projectId || !sessionId) return // Opt-in hard kill (NEXT_PUBLIC_AGENT_CHAT_STOP_KILLS_SESSION): tear the whole session down. if (doesAgentChatStopKillSession()) { + stop() + if (!projectId || !sessionId) return killSession({sessionId, projectId}) .then((ok) => { if (ok) { queryClient.invalidateQueries({queryKey: ["session-liveness"]}) - // Refresh an open Inspector's Runtime lens so its Lifecycle/State reflect the - // kill immediately (mirrors the panel's own Kill button). + // Refresh an open Inspector so it reflects the kill immediately. void invalidateSessionInspector(queryClient, sessionId) } }) .catch(() => {}) return } - // Default Stop: cooperatively cancel the CURRENT TURN. The control-plane `cancel` command - // (no inputs, no force) drops the alive lock; the runner closes the turn as interrupted and - // the session STAYS OPEN so a follow-up prompt resumes it — instead of the old behaviour where - // the client stream aborted but the runner kept running and billing. - commandSessionStream({sessionId, projectId}).catch(() => {}) - }, [markStopped, stop, projectId, sessionId, queryClient]) + // Default Stop cancels the current execution while preserving the warm session. + void stopCurrentExecution() + }, [markStopped, stop, projectId, sessionId, queryClient, stopCurrentExecution]) // ── D9 teardown: `useSessionChat` releases the claim; this tracks what it does not own ── // The startup clock only goes with the session when the session itself is gone — clearing it diff --git a/web/oss/src/components/AgentChatSlice/state/liveness.ts b/web/oss/src/components/AgentChatSlice/state/liveness.ts index 90797abe5f0..0bf9bd1c11e 100644 --- a/web/oss/src/components/AgentChatSlice/state/liveness.ts +++ b/web/oss/src/components/AgentChatSlice/state/liveness.ts @@ -3,6 +3,7 @@ import {sessionLocalSettledAtAtomFamily, sessionStatusAtomFamily} from "@agenta/ import { deriveSessionLifecycle, deriveStreamNest, + livenessPollInterval, querySessionStreams, type SessionLifecycle, type SessionStream, @@ -14,18 +15,7 @@ import {atomWithQuery} from "jotai-tanstack-query" import {projectIdAtom} from "@/oss/state/project" -/** - * Backend liveness for the project's sessions (cross-device truth). The tab dot reads this to - * reflect a session still running on the backend even when THIS browser isn't streaming it (a - * reopened chat, or a run started on another device). - * - * ONE project-scoped query (`is_alive=true`) backs every dot rather than one fetch per session, so - * N idle tabs cost ONE request, not N — important on cold load (see the request-count budget). Only - * alive streams come back, which is exactly what the dot needs (running/alive vs idle); a session - * absent from the result is dormant/cold/dead/new and simply reads as idle. Kept out of the live - * conversation's way: the fetch is LOW-PRIORITY, polls only WHILE something is alive (empty result - * → stop), and re-checks on tab refocus. - */ +/** One low-priority project query supplies cross-device liveness for every tab dot. */ const aliveStreamsQueryAtom = atomWithQuery((get) => { const projectId = get(projectIdAtom) return { @@ -39,7 +29,7 @@ const aliveStreamsQueryAtom = atomWithQuery((get) => { }), enabled: Boolean(projectId), staleTime: 10_000, - refetchInterval: (query) => ((query.state.data?.length ?? 0) > 0 ? 15_000 : false), + refetchInterval: (query) => livenessPollInterval(query.state.data), refetchOnWindowFocus: true, } }) diff --git a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/Client.ts b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/Client.ts index 2155960e439..c9c0b2ee1e3 100644 --- a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/Client.ts +++ b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/Client.ts @@ -2536,4 +2536,82 @@ export class SessionsClient { return handleNonStatusCodeError(_response.error, _response.rawResponse, "POST", "/sessions/unarchive"); } + + /** + * @param {AgentaApi.CancelSessionExecutionRequest} request + * @param {SessionsClient.RequestOptions} requestOptions - Request-specific configuration. + * + * @throws {@link AgentaApi.UnprocessableEntityError} + * + * @example + * await client.sessions.cancelSessionExecution({ + * session_id: "session_id", + * body: {} + * }) + */ + public cancelSessionExecution( + request: AgentaApi.CancelSessionExecutionRequest, + requestOptions?: SessionsClient.RequestOptions, + ): core.HttpResponsePromise { + return core.HttpResponsePromise.fromPromise(this.__cancelSessionExecution(request, requestOptions)); + } + + private async __cancelSessionExecution( + request: AgentaApi.CancelSessionExecutionRequest, + requestOptions?: SessionsClient.RequestOptions, + ): Promise> { + const { session_id: sessionId, body: _body } = request; + const _authRequest: core.AuthRequest = await this._options.authProvider.getAuthRequest(); + const _headers: core.Fetcher.Args["headers"] = mergeHeaders( + _authRequest.headers, + this._options?.headers, + requestOptions?.headers, + ); + const _response = await core.fetcher({ + url: core.url.join( + (await core.Supplier.get(this._options.baseUrl)) ?? + (await core.Supplier.get(this._options.environment)) ?? + environments.AgentaApiEnvironment.Default, + `sessions/${core.url.encodePathParam(sessionId)}/cancel`, + ), + method: "POST", + headers: _headers, + contentType: "application/json", + queryParameters: requestOptions?.queryParams, + requestType: "json", + body: _body, + timeoutMs: (requestOptions?.timeoutInSeconds ?? this._options?.timeoutInSeconds ?? 30) * 1000, + maxRetries: requestOptions?.maxRetries ?? this._options?.maxRetries, + withCredentials: true, + abortSignal: requestOptions?.abortSignal, + fetchFn: this._options?.fetch, + logging: this._options.logging, + }); + if (_response.ok) { + return { data: _response.body, rawResponse: _response.rawResponse }; + } + + if (_response.error.reason === "status-code") { + switch (_response.error.statusCode) { + case 422: + throw new AgentaApi.UnprocessableEntityError( + _response.error.body as AgentaApi.HttpValidationError, + _response.rawResponse, + ); + default: + throw new errors.AgentaApiError({ + statusCode: _response.error.statusCode, + body: _response.error.body, + rawResponse: _response.rawResponse, + }); + } + } + + return handleNonStatusCodeError( + _response.error, + _response.rawResponse, + "POST", + "/sessions/{session_id}/cancel", + ); + } } diff --git a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/CancelSessionExecutionRequest.ts b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/CancelSessionExecutionRequest.ts new file mode 100644 index 00000000000..d3cb6ce31e2 --- /dev/null +++ b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/CancelSessionExecutionRequest.ts @@ -0,0 +1,15 @@ +// This file was auto-generated by Fern from our API Definition. + +import type * as AgentaApi from "../../../../index.js"; + +/** + * @example + * { + * session_id: "session_id", + * body: {} + * } + */ +export interface CancelSessionExecutionRequest { + session_id: string; + body: AgentaApi.SessionCancelRequest | null; +} diff --git a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/SessionStreamCommandRequest.ts b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/SessionStreamCommandRequest.ts index 5d66eef4a2f..a2f69e304b0 100644 --- a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/SessionStreamCommandRequest.ts +++ b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/SessionStreamCommandRequest.ts @@ -13,4 +13,5 @@ export interface SessionStreamCommandRequest { data?: AgentaApi.WorkflowRequestData | null; force?: boolean; detached?: boolean; + expected_execution_id?: string | null; } diff --git a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/index.ts b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/index.ts index 5107e65e7d3..304c3ec79d2 100644 --- a/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/index.ts +++ b/web/packages/agenta-api-client/src/generated/api/resources/sessions/client/requests/index.ts @@ -1,5 +1,6 @@ export type { ArchiveSessionRequest } from "./ArchiveSessionRequest.js"; export type { BodyUploadSessionMountFile } from "./BodyUploadSessionMountFile.js"; +export type { CancelSessionExecutionRequest } from "./CancelSessionExecutionRequest.js"; export type { CreateSessionAttachmentRequest } from "./CreateSessionAttachmentRequest.js"; export type { DeleteSessionRequest } from "./DeleteSessionRequest.js"; export type { DeleteSessionStreamRequest } from "./DeleteSessionStreamRequest.js"; diff --git a/web/packages/agenta-api-client/src/generated/api/types/SessionCancelRequest.ts b/web/packages/agenta-api-client/src/generated/api/types/SessionCancelRequest.ts new file mode 100644 index 00000000000..5010870009e --- /dev/null +++ b/web/packages/agenta-api-client/src/generated/api/types/SessionCancelRequest.ts @@ -0,0 +1,5 @@ +// This file was auto-generated by Fern from our API Definition. + +export interface SessionCancelRequest { + expected_execution_id?: (string | null) | undefined; +} diff --git a/web/packages/agenta-api-client/src/generated/api/types/SessionStream.ts b/web/packages/agenta-api-client/src/generated/api/types/SessionStream.ts index 90177651fb0..6e663895c08 100644 --- a/web/packages/agenta-api-client/src/generated/api/types/SessionStream.ts +++ b/web/packages/agenta-api-client/src/generated/api/types/SessionStream.ts @@ -18,6 +18,8 @@ export interface SessionStream { tags?: (Record | null) | undefined; meta?: (Record | null) | undefined; turn_id?: (string | null) | undefined; + turn_started_at?: (string | null) | undefined; + stopping_turn_id?: (string | null) | undefined; references?: (AgentaApi.SessionReference[] | null) | undefined; archived_at?: (string | null) | undefined; origin?: (AgentaApi.SessionOrigin | null) | undefined; diff --git a/web/packages/agenta-api-client/src/generated/api/types/SessionStreamCommandResponse.ts b/web/packages/agenta-api-client/src/generated/api/types/SessionStreamCommandResponse.ts index 18d1acc5032..2ae62484a8e 100644 --- a/web/packages/agenta-api-client/src/generated/api/types/SessionStreamCommandResponse.ts +++ b/web/packages/agenta-api-client/src/generated/api/types/SessionStreamCommandResponse.ts @@ -8,4 +8,5 @@ export interface SessionStreamCommandResponse { turn_id?: (string | null) | undefined; watcher_id?: (string | null) | undefined; detached?: boolean | undefined; + cancelled_turn_ids?: string[] | undefined; } diff --git a/web/packages/agenta-api-client/src/generated/api/types/index.ts b/web/packages/agenta-api-client/src/generated/api/types/index.ts index 42bca07e3be..6df14bd2fb2 100644 --- a/web/packages/agenta-api-client/src/generated/api/types/index.ts +++ b/web/packages/agenta-api-client/src/generated/api/types/index.ts @@ -378,6 +378,7 @@ export * from "./Selector.js"; export * from "./SessionAttachment.js"; export * from "./SessionAttachmentResponse.js"; export * from "./SessionAttachmentsResponse.js"; +export * from "./SessionCancelRequest.js"; export * from "./SessionDelivery.js"; export * from "./SessionExcludeRequest.js"; export * from "./SessionExpansion.js"; diff --git a/web/packages/agenta-chat/src/assets/agentTurn.ts b/web/packages/agenta-chat/src/assets/agentTurn.ts new file mode 100644 index 00000000000..fb430c5377b --- /dev/null +++ b/web/packages/agenta-chat/src/assets/agentTurn.ts @@ -0,0 +1,18 @@ +import type {UIMessage} from "ai" + +/** Read the runner-minted turn id from merged stream metadata. */ +export const getMessageTurnId = (message: UIMessage | undefined): string | null => { + const turnId = (message?.metadata as {turnId?: unknown} | undefined)?.turnId + return typeof turnId === "string" && turnId.trim() ? turnId : null +} + +/** Read the newest assistant turn id without crossing the latest user-turn boundary. */ +export const latestTurnId = (messages: UIMessage[]): string | null => { + for (let index = messages.length - 1; index >= 0; index--) { + const message = messages[index] + if (message.role === "user") return null + if (message.role !== "assistant") continue + return getMessageTurnId(message) + } + return null +} diff --git a/web/packages/agenta-chat/src/assets/index.ts b/web/packages/agenta-chat/src/assets/index.ts index 3dc5493b5b1..0286cb04d09 100644 --- a/web/packages/agenta-chat/src/assets/index.ts +++ b/web/packages/agenta-chat/src/assets/index.ts @@ -10,3 +10,4 @@ export * from "./conversationLayout" export * from "./jumpToLatest" export * from "./boundedRequest" export {startupLabelFromDataPart} from "./startupPhases" +export {getMessageTurnId, latestTurnId} from "./agentTurn" diff --git a/web/packages/agenta-chat/src/state/sessionEphemera.ts b/web/packages/agenta-chat/src/state/sessionEphemera.ts index ed97d4add0f..8bc8a50959b 100644 --- a/web/packages/agenta-chat/src/state/sessionEphemera.ts +++ b/web/packages/agenta-chat/src/state/sessionEphemera.ts @@ -30,6 +30,20 @@ export const composerDraftBySession = new Map() /** Pending (not yet sent) attachments per session — same lifetime as the drafts. */ export const attachmentsBySession = new Map[]>() +/** In-memory turn guards survive pane remounts but are never restored across page loads. */ +export const turnIdBySession = new Map() + +export const setSessionTurnId = (sessionId: string, turnId: string) => { + turnIdBySession.set(sessionId, turnId) +} + +export const getSessionTurnId = (sessionId: string): string | undefined => + turnIdBySession.get(sessionId) + +export const clearSessionTurnId = (sessionId: string) => { + turnIdBySession.delete(sessionId) +} + // The fresh-session registry moved to @agenta/entities/session — the drive needs the same // predicate, and this package sits ABOVE entity-ui so it cannot be imported from there. export {freshSessionIds} @@ -39,5 +53,6 @@ export {clearSessionFresh, isSessionFresh, markSessionFresh} from "@agenta/entit export const clearSessionEphemera = (sessionId: string) => { composerDraftBySession.delete(sessionId) attachmentsBySession.delete(sessionId) + turnIdBySession.delete(sessionId) freshSessionIds.delete(sessionId) } diff --git a/web/packages/agenta-chat/tests/unit/assets/agentTurn.test.ts b/web/packages/agenta-chat/tests/unit/assets/agentTurn.test.ts new file mode 100644 index 00000000000..231e63d328a --- /dev/null +++ b/web/packages/agenta-chat/tests/unit/assets/agentTurn.test.ts @@ -0,0 +1,80 @@ +import type {UIMessage} from "ai" +import {afterEach, describe, expect, it} from "vitest" + +import {getMessageTurnId, latestTurnId} from "../../../src/assets/agentTurn" +import { + clearSessionEphemera, + clearSessionTurnId, + getSessionTurnId, + setSessionTurnId, +} from "../../../src/state/sessionEphemera" + +const assistant = (id: string, metadata?: unknown): UIMessage => + ({id, role: "assistant", parts: [], metadata}) as UIMessage + +const user = (id: string): UIMessage => ({id, role: "user", parts: []}) as UIMessage + +afterEach(() => { + clearSessionEphemera("s1") + clearSessionEphemera("s2") +}) + +describe("getMessageTurnId", () => { + it("reads the runner-minted id from message metadata", () => { + expect(getMessageTurnId(assistant("a1", {turnId: "turn-1"}))).toBe("turn-1") + }) + + it("rejects missing and malformed ids", () => { + expect(getMessageTurnId(assistant("a1"))).toBeNull() + expect(getMessageTurnId(assistant("a2", {turnId: " "}))).toBeNull() + expect(getMessageTurnId(assistant("a3", {turnId: 7}))).toBeNull() + expect(getMessageTurnId(undefined)).toBeNull() + }) +}) + +describe("latestTurnId", () => { + it("reads only the newest assistant turn", () => { + expect( + latestTurnId([ + user("u1"), + assistant("a1", {turnId: "turn-1"}), + user("u2"), + assistant("a2", {turnId: "turn-2"}), + ]), + ).toBe("turn-2") + }) + + it("does not fall back when the newest assistant has no id", () => { + expect( + latestTurnId([assistant("a1", {turnId: "turn-1"}), assistant("a2")]), + ).toBeNull() + }) + + it("does not cross a trailing user message into an older turn", () => { + expect( + latestTurnId([assistant("a1", {turnId: "turn-A"}), user("u2")]), + ).toBeNull() + }) +}) + +describe("session turn ids", () => { + it("survives a pane remount and is replaced by the next admitted turn", () => { + setSessionTurnId("s1", "turn-A") + expect(getSessionTurnId("s1")).toBe("turn-A") + + setSessionTurnId("s1", "turn-B") + expect(getSessionTurnId("s1")).toBe("turn-B") + }) + + it("is isolated per session and cleared with session ephemera", () => { + setSessionTurnId("s1", "turn-1") + setSessionTurnId("s2", "turn-2") + + clearSessionTurnId("s1") + expect(getSessionTurnId("s1")).toBeUndefined() + expect(getSessionTurnId("s2")).toBe("turn-2") + + clearSessionEphemera("s2") + expect(getSessionTurnId("s2")).toBeUndefined() + }) +}) diff --git a/web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts b/web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts index 742ffc1e5b8..f1c5b1073ba 100644 --- a/web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts +++ b/web/packages/agenta-chat/tests/unit/assets/transcriptToMessages.test.ts @@ -1058,7 +1058,7 @@ describe("transcriptToMessages run-error code", () => { }) describe("transcriptToMessages user-Stop terminal record", () => { - // A cancelled `done` is an ordinary turn terminator during reconstruction. + // A cancelled terminal closes its turn without swallowing the next one. it("closes a stopped turn like a completed one", () => { const messages = transcriptToMessages([ record("r-user", {type: "message", text: "run something long"}, "user"), diff --git a/web/packages/agenta-entities/src/session/api/api.ts b/web/packages/agenta-entities/src/session/api/api.ts index 62862067d05..a30e1aad441 100644 --- a/web/packages/agenta-entities/src/session/api/api.ts +++ b/web/packages/agenta-entities/src/session/api/api.ts @@ -17,6 +17,7 @@ import { sessionInteractionResponseSchema, sessionInteractionsResponseSchema, sessionRecordsQueryResponseSchema, + sessionCancelExecutionResponseSchema, sessionsQueryResponseSchema, sessionStreamCommandResponseSchema, sessionStreamSchema, @@ -43,6 +44,7 @@ import { getLowPrioritySessionsClient, getMountsClient, getSessionsClient, + isAbortError, projectScopedRequest, } from "./client" @@ -937,3 +939,84 @@ export async function readMountFile({ const validated = safeParseWithLogging(mountFileContentResponseSchema, data, "[readMountFile]") return validated?.content ?? null } + +export interface CancelSessionExecutionParams extends SessionScopedParams { + /** Fence Stop to the execution the caller observed. */ + expectedExecutionId?: string + /** Retry identity for this request. Two sends of the same key are one command. */ + idempotencyKey?: string +} + +export interface CancelSessionExecutionResult { + /** The durable command's id and DELIVERY state — never the execution's state. */ + command: {id: string; state: string} + /** What to render: the execution being stopped, or nothing. */ + execution: {id: string | null; state: "stopping" | "idle"} + /** True when the active API path accepted or completed the Stop. */ + accepted: boolean + /** True when the API refused because another execution is running (409). */ + conflict: boolean +} + +/** Cancel current work through Fern while keeping the session warm. */ +export async function cancelSessionExecution({ + sessionId, + projectId, + appId, + abortSignal, + expectedExecutionId, + idempotencyKey, +}: CancelSessionExecutionParams): Promise { + if (!projectId || !sessionId) return null + + try { + const requestOptions = { + ...projectScopedRequest(projectId, appId, abortSignal), + ...(idempotencyKey ? {headers: {"Idempotency-Key": idempotencyKey}} : {}), + } + const {data, rawResponse} = await getSessionsClient() + .cancelSessionExecution( + { + session_id: sessionId, + body: expectedExecutionId ? {expected_execution_id: expectedExecutionId} : null, + }, + requestOptions, + ) + .withRawResponse() + const validated = safeParseWithLogging( + sessionCancelExecutionResponseSchema, + data, + "[cancelSessionExecution]", + ) + if (!validated) return null + if (!("command" in validated)) { + return { + command: {id: "", state: "applied"}, + execution: {id: validated.turn_id ?? null, state: "idle"}, + accepted: true, + conflict: false, + } + } + return { + command: validated.command, + execution: {...validated.execution, id: validated.execution.id ?? null}, + accepted: rawResponse.status === 202, + conflict: false, + } + } catch (error) { + if (isAbortError(error)) throw error + if ((error as {statusCode?: number} | null)?.statusCode === 409) { + return { + command: {id: "", state: "obsolete"}, + execution: {id: null, state: "idle"}, + accepted: false, + conflict: true, + } + } + console.error( + "[cancelSessionExecution] failed:", + error instanceof Error ? error.message : String(error), + ) + return null + } +} diff --git a/web/packages/agenta-entities/src/session/core/liveness.ts b/web/packages/agenta-entities/src/session/core/liveness.ts index 32871468f0e..33a6a539ad0 100644 --- a/web/packages/agenta-entities/src/session/core/liveness.ts +++ b/web/packages/agenta-entities/src/session/core/liveness.ts @@ -88,3 +88,22 @@ export function refineLifecycleWithSandbox( if (sandbox.alive === true) return sandbox.warm ? "warm" : "cold" return lifecycle } + +/** What a liveness-driven `refetchInterval` may return: a period in ms, or `false` to stop. */ +export type LivenessPollInterval = number | false + +/** Fast cadence: something is executing right now, so the view changes on its own. */ +const RUNNING_POLL_MS = 15_000 +/** Slow cadence: nothing runs, but a warm session can be resumed from another device. */ +const RESUMABLE_POLL_MS = 60_000 + +/** Poll `is_running` quickly; `is_alive` alone means warm and uses the slow cadence. */ +export function livenessPollInterval( + rows: readonly (SessionStream | null | undefined)[] | null | undefined, + options?: {idle?: LivenessPollInterval}, +): LivenessPollInterval { + const list = rows ?? [] + if (list.some((row) => row?.flags?.is_running)) return RUNNING_POLL_MS + if (list.some((row) => row?.flags?.is_alive)) return RESUMABLE_POLL_MS + return options?.idle ?? false +} diff --git a/web/packages/agenta-entities/src/session/core/schema.ts b/web/packages/agenta-entities/src/session/core/schema.ts index ca945bb0124..92e64d806f3 100644 --- a/web/packages/agenta-entities/src/session/core/schema.ts +++ b/web/packages/agenta-entities/src/session/core/schema.ts @@ -206,8 +206,20 @@ export const sessionStreamCommandResponseSchema = z.object({ turn_id: z.string().nullish(), watcher_id: z.string().nullish(), detached: z.boolean().nullish(), + cancelled_turn_ids: z.array(z.string()).nullish(), }) +export const sessionCancelExecutionResponseSchema = z.union([ + z.object({ + command: z.object({id: z.string(), state: z.string()}), + execution: z.object({ + id: z.string().nullish(), + state: z.enum(["stopping", "idle"]), + }), + }), + sessionStreamCommandResponseSchema, +]) + export type SessionStream = z.infer export type SessionReference = z.infer export type SessionOrigin = z.infer diff --git a/web/packages/agenta-entities/src/session/index.ts b/web/packages/agenta-entities/src/session/index.ts index ba7fd03a94d..06dd399d4a7 100644 --- a/web/packages/agenta-entities/src/session/index.ts +++ b/web/packages/agenta-entities/src/session/index.ts @@ -18,6 +18,7 @@ export { setSessionHeader, fetchSessionStream, commandSessionStream, + cancelSessionExecution, killSession, deleteSession as deleteSessionRemote, archiveSession as archiveSessionRemote, @@ -38,6 +39,8 @@ export { type RespondInteractionParams, type TransitionInteractionParams, type CommandSessionStreamParams, + type CancelSessionExecutionParams, + type CancelSessionExecutionResult, } from "./api/api" export { getSessionsClient, @@ -80,6 +83,8 @@ export { deriveStreamNest, deriveSessionLifecycle, refineLifecycleWithSandbox, + livenessPollInterval, + type LivenessPollInterval, type SessionLifecycle, type SessionStreamNest, type SandboxLiveness, diff --git a/web/packages/agenta-entities/tests/unit/session-cancel-api.test.ts b/web/packages/agenta-entities/tests/unit/session-cancel-api.test.ts new file mode 100644 index 00000000000..f96d2d6230f --- /dev/null +++ b/web/packages/agenta-entities/tests/unit/session-cancel-api.test.ts @@ -0,0 +1,113 @@ +import {beforeEach, describe, expect, it, vi} from "vitest" + +const fernCancelSessionExecution = vi.fn() + +vi.mock("@agenta/sdk/resources", () => ({ + getSessionsClient: () => ({cancelSessionExecution: fernCancelSessionExecution}), + getLowPrioritySessionsClient: vi.fn(), + getMountsClient: vi.fn(), + getLowPriorityMountsClient: vi.fn(), +})) + +import {cancelSessionExecution} from "../../src/session/api/api" + +const response = { + command: {id: "command-1", state: "pending"}, + execution: {id: "turn-1", state: "stopping"}, +} + +beforeEach(() => { + fernCancelSessionExecution.mockReset() + fernCancelSessionExecution.mockReturnValue({ + withRawResponse: () => Promise.resolve({data: response, rawResponse: {status: 202}}), + }) +}) + +describe("cancelSessionExecution", () => { + it("uses the Fern route with project query scope and the observed turn", async () => { + const result = await cancelSessionExecution({ + projectId: "project-1", + appId: "app-1", + sessionId: "session-1", + expectedExecutionId: "turn-1", + idempotencyKey: "stop-1", + }) + + expect(fernCancelSessionExecution).toHaveBeenCalledWith( + { + session_id: "session-1", + body: {expected_execution_id: "turn-1"}, + }, + { + queryParams: {project_id: "project-1", application_id: "app-1"}, + abortSignal: undefined, + headers: {"Idempotency-Key": "stop-1"}, + }, + ) + expect(result).toEqual({...response, accepted: true, conflict: false}) + }) + + it("maps Fern 409 to the stale-execution conflict result", async () => { + fernCancelSessionExecution.mockReturnValue({ + withRawResponse: () => Promise.reject({statusCode: 409}), + }) + + const result = await cancelSessionExecution({ + projectId: "project-1", + sessionId: "session-1", + }) + + expect(result).toEqual({ + command: {id: "", state: "obsolete"}, + execution: {id: null, state: "idle"}, + accepted: false, + conflict: true, + }) + }) + + it("accepts and normalizes the API flag-off legacy cancel payload", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) + fernCancelSessionExecution.mockReturnValue({ + withRawResponse: () => + Promise.resolve({ + data: { + mode: "cancel", + session_id: "session-1", + turn_id: "turn-1", + watcher_id: null, + detached: true, + cancelled_turn_ids: [], + }, + rawResponse: {status: 200}, + }), + }) + + const result = await cancelSessionExecution({ + projectId: "project-1", + sessionId: "session-1", + expectedExecutionId: "turn-1", + }) + + expect(result).toEqual({ + command: {id: "", state: "applied"}, + execution: {id: "turn-1", state: "idle"}, + accepted: true, + conflict: false, + }) + expect(consoleError).not.toHaveBeenCalled() + consoleError.mockRestore() + }) + + it("rejects malformed successful payloads at the Zod boundary", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) + fernCancelSessionExecution.mockReturnValue({ + withRawResponse: () => + Promise.resolve({data: {command: null}, rawResponse: {status: 202}}), + }) + + await expect( + cancelSessionExecution({projectId: "project-1", sessionId: "session-1"}), + ).resolves.toBeNull() + expect(consoleError).toHaveBeenCalled() + }) +}) diff --git a/web/packages/agenta-entities/tests/unit/session-liveness.test.ts b/web/packages/agenta-entities/tests/unit/session-liveness.test.ts index e5db3f25ea3..f1d943a563c 100644 --- a/web/packages/agenta-entities/tests/unit/session-liveness.test.ts +++ b/web/packages/agenta-entities/tests/unit/session-liveness.test.ts @@ -9,6 +9,7 @@ import {describe, expect, it} from "vitest" import { deriveSessionLifecycle, deriveStreamNest, + livenessPollInterval, refineLifecycleWithSandbox, } from "../../src/session/core/liveness" import type {SessionStream} from "../../src/session/core/schema" @@ -92,3 +93,31 @@ describe("refineLifecycleWithSandbox", () => { expect(refineLifecycleWithSandbox("cold", {alive: true, warm: false})).toBe("cold") }) }) + +// Every liveness poll shares the running-versus-warm cadence rule. +describe("livenessPollInterval", () => { + const rows = (...flags: Partial>[]) => flags.map(streamWith) + + it("polls fast while any row is running", () => { + expect(livenessPollInterval(rows({is_alive: true, is_running: true}))).toBe(15_000) + expect(livenessPollInterval(rows({is_alive: true}, {is_running: true}))).toBe(15_000) + }) + + // A stopped session stays alive for warm resume but no longer polls quickly. + it("drops to the slow cadence for a session that is alive but not running", () => { + expect(livenessPollInterval(rows({is_alive: true}))).toBe(60_000) + }) + + it("stops by default when nothing is alive", () => { + expect(livenessPollInterval(rows({}))).toBe(false) + expect(livenessPollInterval([])).toBe(false) + expect(livenessPollInterval(null)).toBe(false) + expect(livenessPollInterval(undefined)).toBe(false) + }) + + // The rail must still DISCOVER a run it did not start, so it names a floor instead of false. + it("uses the caller's idle floor when one is given", () => { + expect(livenessPollInterval([], {idle: 60_000})).toBe(60_000) + expect(livenessPollInterval(rows({}), {idle: 60_000})).toBe(60_000) + }) +}) diff --git a/web/packages/agenta-navigation/src/dynamic/sessionsSource.ts b/web/packages/agenta-navigation/src/dynamic/sessionsSource.ts index b0359f489aa..dcf61ca86ac 100644 --- a/web/packages/agenta-navigation/src/dynamic/sessionsSource.ts +++ b/web/packages/agenta-navigation/src/dynamic/sessionsSource.ts @@ -1,4 +1,9 @@ -import {queryInteractions, querySessions, type SessionStream} from "@agenta/entities/session" +import { + livenessPollInterval, + queryInteractions, + querySessions, + type SessionStream, +} from "@agenta/entities/session" import { agentWorkflowsListQueryStateAtom, appWorkflowsListQueryAtom, @@ -105,28 +110,12 @@ const requestFilters = (filters: SidebarSessionFilters) => { } } -/** Fast enough that a dot clears about when the stream does. */ -const LIVE_POLL_MS = 15_000 - /** Slow enough to be background noise, quick enough to notice a run you did not start. */ const IDLE_POLL_MS = 60_000 -/** - * Poll fast while something can still change, slowly the rest of the time. - * - * A row's dot is driven by `is_alive`/`is_running`, which the server flips when the stream ends — - * with no request, the dot stays filled until you reload. The BASELINE matters just as much: a - * turn started under another agent (a trigger, another browser) is invisible to this client, so a - * rail that stopped polling when it looked quiet could never discover it, and only the session you - * were driving yourself ever appeared to run. - * - * Both intervals are gated: the source only subscribes while the Sessions group is open and the - * rail is expanded, and React Query holds the timer while the window is unfocused. - */ +/** Poll fast for running work and keep a slow baseline for cross-client discovery. */ export const livePollInterval = (rows: SessionStream[] | null | undefined) => - (rows ?? []).some((row) => row.flags?.is_alive || row.flags?.is_running) - ? LIVE_POLL_MS - : IDLE_POLL_MS + livenessPollInterval(rows, {idle: IDLE_POLL_MS}) /** * One request per selected agent, merged — see `requestFilters` on why they cannot be one. diff --git a/web/packages/agenta-navigation/tests/unit/sidebarChildren.test.ts b/web/packages/agenta-navigation/tests/unit/sidebarChildren.test.ts index a84fc93c5cb..bacdad4bf10 100644 --- a/web/packages/agenta-navigation/tests/unit/sidebarChildren.test.ts +++ b/web/packages/agenta-navigation/tests/unit/sidebarChildren.test.ts @@ -470,17 +470,21 @@ describe("localSessionRefsMatching", () => { }) }) -// The baseline is the half that is easy to lose: without it the rail can only ever show the run -// you started yourself, because a turn under another agent reaches this client through the poll. +// The baseline discovers runs started by another client. describe("livePollInterval", () => { // Only `flags` is read; the rest of a SessionStream is irrelevant here. const rows = (...flags: {is_alive?: boolean; is_running?: boolean}[]) => flags.map((f) => ({session_id: "s1", flags: f})) as Parameters[0] & object[] - it("polls fast while a session is alive or running", () => { - expect(livePollInterval(rows({is_alive: true}))).toBe(15_000) - expect(livePollInterval(rows({is_running: true}))).toBe(15_000) + it("polls fast only while a session is RUNNING", () => { + expect(livePollInterval(rows({is_alive: true, is_running: true}))).toBe(15_000) + expect(livePollInterval(rows({}, {is_running: true}))).toBe(15_000) + }) + + // Warm but idle sessions use the slow cadence. + it("drops to the slow baseline for a session that is alive but not running", () => { + expect(livePollInterval(rows({is_alive: true}))).toBe(60_000) }) it("keeps a slow baseline when every row looks idle", () => {