From 2d9f2321fde304a494e6b576eed8fceb3556205a Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 15:34:18 -0700 Subject: [PATCH 01/15] F: publish a finished task's pull request in production (#103) TaskPublishing (runner/publishing.ts) publishes the task's PR with the real publisher: GhAlreadyFixedGateway, GhPullRequestGateway and a GitBranchPusher that pushes from the runner-owned repository to the configured repository on GH_HOST (never a configured URL), overwriting only ledger-owned commits. - Mode by status: a running task whose plan items have all run gets a ready PR; a task in needs human gets a draft PR with what sent it there. - When: after a start/resume run ends; when a refused start/resume moves the task to needs human; on the new `publish` runner action; once at startup, after the CLI re-verifies the lock, if a publish is owed. - Schema v10 `publish_outcomes`: the last outcome, shown as `publish` in GET /api/runner and replayed by the publish action. Tasks already running or in needs human at the upgrade are recorded as `not published`. - Shutdown: publishes are aborted in step 1 and awaited before the Store write gate closes. Start, resume and a second publish are refused while a publish runs. Demos never publish. - github.baseBranch is required with a runner block, checked before the lock. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- docs/architecture.md | 2 +- docs/implementation/pull-request-opening.md | 16 +- docs/implementation/runner-lifecycle.md | 1 + github/merge.ts | 2 + runner/branch-push.ts | 2 +- runner/production.ts | 35 +- runner/publishing.ts | 163 +++++++++ runner/store.ts | 48 ++- test/cli.test.ts | 12 +- test/publish.test.ts | 2 +- test/runner-lifecycle-store.test.ts | 30 +- test/runner-production.test.ts | 2 +- test/runner-publishing.test.ts | 363 ++++++++++++++++++++ web/cli.ts | 13 +- web/server.ts | 60 +++- 16 files changed, 728 insertions(+), 27 deletions(-) create mode 100644 runner/publishing.ts create mode 100644 test/runner-publishing.test.ts diff --git a/README.md b/README.md index e0697d5a..41d32414 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The issue must match the stored plan. The authenticated `gh` account must be abl ## Runner (opt-in) -A configuration with a `github` block may also add a `runner` block. The runner carries out plan items with Claude Code in the agent container and commits each item in its own repository, never in yours: +A configuration with a `github` block may also add a `runner` block. The runner carries out plan items with Claude Code in the agent container and commits each item in its own repository, never in yours. It then publishes the task's pull request, so the `github` block needs `"baseBranch": "main"` (the branch the PR targets): ```json { @@ -51,7 +51,7 @@ A configuration with a `github` block may also add a `runner` block. The runner - `diagnosticsCapBytes`: the size retention trims that directory back to. It is a target, not a hard limit: the file just saved is always kept, even when it alone passes the cap. The default is 256 MiB. - `limits`: task storage limits (`workBytes`, `workInodes`, `metadataBytes`, `metadataInodes`). -The runner needs Docker and `CLAUDE_CODE_OAUTH_TOKEN`. At start, before the server opens, codeboost recovers what an earlier run left and builds the agent image; a recovery it cannot finish safely stops startup with what to do. `POST /api/runner` with `start` or `resume` runs the plan; the review screen has no buttons for them yet. +The runner needs Docker and `CLAUDE_CODE_OAUTH_TOKEN`. At start, before the server opens, codeboost recovers what an earlier run left and builds the agent image; a recovery it cannot finish safely stops startup with what to do. `POST /api/runner` with `start` or `resume` runs the plan; the review screen has no buttons for them yet. When a run has completed every item, codeboost pushes the task head to a `codeboost/…` branch with your `gh` credentials and opens a ready pull request into `baseBranch`; a task that needs a person gets a draft pull request with its problems. A publish that was refused (for example, the branch holds a commit codeboost did not make) is retried with the `publish` action, and `GET /api/runner` shows the last outcome under `publish`. Demos never publish. ## Library diff --git a/docs/architecture.md b/docs/architecture.md index 1f6fb7ab..e1bb6293 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -167,7 +167,7 @@ Dependencies point downward only. `core/` imports nothing that does input or out | Merge coordinator | `runner/merge.ts` | Runs the full merge gate twice, re-reads the local generation, then merges the exact reviewed head. Tracks merge-queue attempts. | Yes | | Runner coordinator | `runner/coordinator.ts`, `runner/lifecycle.ts` | Attempt admission, concurrency slots, compare-and-swap on results, retries, shutdown order. | Not connected | | Execution | `runner/execution.ts` | Runs plan items in order: fresh workspace, prompt, agent, post-run audit, then the runner's own commit and ledger entry. Pauses in needs amendment on an out-of-scope edit; moves to needs human on a safety violation. | Not connected; uses a test workspace until #66 part 2 and #87 | -| Publishing | `runner/publish.ts` | Runs the already-fixed check, pushes the task head, then opens or reuses the task's pull request (a draft when the task needs a person). Records each opening before calling GitHub, so a lost outcome can be found again by a marker. | Not connected; the push step is injected until the runner's commit bundle (PR #90) exists | +| Publishing | `runner/publish.ts`, `runner/publishing.ts`, `runner/branch-push.ts` | Runs the already-fixed check, pushes the task head, then opens or reuses the task's pull request (a draft when the task needs a person). Records each opening before calling GitHub, so a lost outcome can be found again by a marker. `TaskPublishing` (#103) publishes when a run ends, on the `publish` runner action, and once at startup, and records the last outcome. | With a runner block (needs `github.baseBranch`); never in a demo | | Lock and startup recovery | `runner/recovery.ts` | One runner per database, held as an OS lock on the database file's device and inode. Startup recovery finalizes interrupted attempts and removes leftover containers and storage. | Lock in use; recovery not connected | | Store | `runner/store.ts` | The only database handle and all SQL. Revision numbers, snapshots, ledger, approvals, choices, notes, suggestion requests, tasks, attempts, user actions, feedback events, merge attempts, pull request openings, already-fixed results. | In use | | Plan logic | `core/plan.ts`, `core/parse-v1.ts`, `schema/` | Schema validation, YAML and JSON import, meaning checks (IDs, dependencies, paths, projected file operations), literal command parsing, suggestion edits. | Yes | diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index b7adf3d2..12c30ccd 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -85,7 +85,7 @@ The adapter refuses any answer for a PR that is not open. A PR closed between th **Deadlines.** Each PR operation (open, lookup, refresh, draft change) has one deadline for all of its commands and poll waits together: 60 seconds by default (`operationMs`), combined with the caller's signal. Stopping `gh` at the deadline can take up to 6 seconds more (see Transport), so one operation settles within 66 seconds. A whole publish has no deadline of its own. Its caller bounds it with the signal. A caller that runs publish inside an HTTP request must give it a deadline below the request budget. -**Shutdown.** The publisher reads the coordinator's `closing` flag (the `closing` dependency) and its own flag right before each push, opening, description update and ready change, with no await in between (runner-lifecycle.md, "Irreversible actions"). Either flag refuses with `ShuttingDownError`. An update refused this way stays in flight for the next publish to settle. A draft change on its own is not refused: it only makes a PR safer, and `close()` stops it through the abort. The draft change inside an update comes after `beforeReady`, so it is refused with the update. `close()` sets the publisher's flag, aborts every publish in progress, and awaits their settlement. The server must await `close()` before the Store's write gate closes (shutdown step 3), so a publish's last records still land. Nothing calls the publisher yet; the change that wires it in adds `close()` to the shutdown order. +**Shutdown.** The publisher reads the coordinator's `closing` flag (the `closing` dependency) and its own flag right before each push, opening, description update and ready change, with no await in between (runner-lifecycle.md, "Irreversible actions"). Either flag refuses with `ShuttingDownError`. An update refused this way stays in flight for the next publish to settle. A draft change on its own is not refused: it only makes a PR safer, and `close()` stops it through the abort. The draft change inside an update comes after `beforeReady`, so it is refused with the update. `close()` sets the publisher's flag, aborts every publish in progress, and awaits their settlement. The server must await `close()` before the Store's write gate closes (shutdown step 3), so a publish's last records still land. `TaskPublishing.close()` (#103) starts this in shutdown step 1, in the same turn the coordinator rejects admission, and the server awaits it before step 3. **Repository.** The publisher refuses a check or PR gateway configured for another repository than its own, because the Store records each PR under the publisher's repository. @@ -122,13 +122,23 @@ Some repositories do not support draft PRs (for example private repositories on - **Repository renames and transfers.** Every PR record is kept under the configured repository name. After a rename or transfer, publish fails closed (the task's PRs are refused or reported as unsettled) until the records are moved; nothing here moves them. -- **Push.** `BranchPusher` is injected. The real push needs the runner's commits in a runner-owned host repository, fetched from the bundles `commitTaskChanges` returns (#66, #87). - **Continue from possibly already fixed.** The Continue and Cancel actions are user actions for a later slice. - **Close the draft on cancel.** The design closes the draft PR when a person cancels a needs-human task. The PR record is kept for that. +## When the server publishes (#103) + +`TaskPublishing` (`runner/publishing.ts`) owns publishing in production. `setUpRunner` builds the publisher with the real gateways (`GhAlreadyFixedGateway`, `GhPullRequestGateway`) and a `GitBranchPusher` that pushes from the runner-owned repository to `pushUrl(github.repository)` on `GH_HOST`, never to a configured URL. The pusher may overwrite only commits the task's ledger records as `owned`. Before #22 adds rebasing, nothing in production writes a `foreign` ledger entry (only `recordRebase` does), so codeboost pushes no head the ledger calls foreign. + +- **Base branch.** `github.baseBranch` is required with a runner block. The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. +- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: "time budget ran out" when the budget has passed, else its current attempt's safety finding, else that attempt's diagnostic. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. +- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. +- **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. +- **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. +- **Demo.** A demo never publishes, even with a `github` block or an injected publisher. + ## Tests -`test/already-fixed.test.ts` and `test/publish.test.ts` cover each row above. Each guard was also broken on purpose, and a test failed each time: own-PR and own-commit exclusion, repository-qualified exclusion, every bound, the ancestor check, the fail-closed catch, the issue-number boundary, the state-version and head re-read, the status check before the no-changes shortcut, the version-owned status change, the open-state check, the settle time, the branch hash, the environment allowlist, reuse of the earlier PR, lost-opening recovery, adoption on the main path and counting that PR as the task's own, drafting a PR that a task which cannot publish still has ready, adopting an abandoned opening's PR that appears after the task stopped, the abort checks on the no-changes path and in that draft step (a lookup that fails, one that answers, a draft change that fails and one that lands), recording a late PR by adoption alone, an abort during the head-mismatch draft change, the `closing` checks before a push and before a ready change, `close()` aborting a publish in progress, the repository check, the definite 422 refusal, a retargeted own PR and another base's PR from the branch, refusals matched on stderr and GitHub's `errors[].message` only, the configured base in recovery and the head settle, the draft step finding the task's PR in any base, closing-only cross-references, both sides of a manual link, the query text, the fence length, and the description bounds. +`test/runner-publishing.test.ts` covers the server wiring (#103) with a local bare repository as the remote and GitHub faked: a completed run opens one ready PR with the branch at the task head, a needs-human task gets a draft with its problems, a branch holding someone else's commit is refused and the `publish` action retries it (and its replay reports the outcome), a refused resume that moves the task to needs human publishes a draft while one on a task already there publishes nothing, start, resume and a second publish are refused while a publish runs, shutdown during the push or during the opening closes the Store only after the publish settled, a PR whose reply arrives just after the abort is still recorded before the write gate closes, and the next start publishes once or recovers the opening by its marker. Removing the startup publish, the after-run publish, the publish after a refused resume, the status-change check, the busy guard, the replay update or the shutdown wait, or moving that wait after the write gate, each fails a test. `test/already-fixed.test.ts` and `test/publish.test.ts` cover each row above. Each guard was also broken on purpose, and a test failed each time: own-PR and own-commit exclusion, repository-qualified exclusion, every bound, the ancestor check, the fail-closed catch, the issue-number boundary, the state-version and head re-read, the status check before the no-changes shortcut, the version-owned status change, the open-state check, the settle time, the branch hash, the environment allowlist, reuse of the earlier PR, lost-opening recovery, adoption on the main path and counting that PR as the task's own, drafting a PR that a task which cannot publish still has ready, adopting an abandoned opening's PR that appears after the task stopped, the abort checks on the no-changes path and in that draft step (a lookup that fails, one that answers, a draft change that fails and one that lands), recording a late PR by adoption alone, an abort during the head-mismatch draft change, the `closing` checks before a push and before a ready change, `close()` aborting a publish in progress, the repository check, the definite 422 refusal, a retargeted own PR and another base's PR from the branch, refusals matched on stderr and GitHub's `errors[].message` only, the configured base in recovery and the head settle, the draft step finding the task's PR in any base, closing-only cross-references, both sides of a manual link, the query text, the fence length, and the description bounds. ## Test this document with a reader diff --git a/docs/implementation/runner-lifecycle.md b/docs/implementation/runner-lifecycle.md index 9813e299..98aa54da 100644 --- a/docs/implementation/runner-lifecycle.md +++ b/docs/implementation/runner-lifecycle.md @@ -253,6 +253,7 @@ codeboost never writes the user's repository. The commits it makes live in a **r |---|---|---| | 1 | Set `stopping` on the server and make the runner coordinator reject admission, in the same synchronous turn, before any active-work list is copied. New API requests get HTTP 503, and new runner work throws `ShuttingDownError`. **Admitted requests are not rejected:** AGENTS.md says to drain already-admitted HTTP requests, and to recheck the flag only at irreversible boundaries. So after reading a body, the server rechecks `stopping` only for `merge`. The question coordinator also stops admission in this turn (`questions.stopAdmission()`): a question admitted before shutdown still saves its note, but starts no agent and gets a retryable "Server stopped" answer (`markStopped`). **Changed while implementing F1c:** an earlier revision of this row rejected every admitted write and closed the Store write gate here. That broke the drain rule and the existing browser regression "drains an in-flight question request before closing its agent manager", so the gate now closes in step 3. | server flag, merge recheck and question admission stop: yes. Runner barrier: new. | | 2 | Stop accepting connections, and wait for admitted requests up to the drain limit (at most 14.5 s, below the 15 s request timeout). | yes | +| 2a | Abort every publish in progress (`TaskPublishing.close()`, #103) in the same turn as step 1, and await it here, at the end of the drain, before the write gate closes: the publisher already refuses every later push and opening through the coordinator's `closing` flag, and a publish's last records (an abandoned opening, the outcome) must still land. A publish stopped this way is owed, so the next start publishes it again. | new (#103) | | 3 | After the drain limit, abort the signals of the remaining requests, destroy requests that are still reading a body, then **close the `Store` write gate**, then await request-owned work (the merge coordinator). From here on, every `Store` write throws `ShuttingDownError` unless the caller passes the shutdown capability. The server hands that capability at construction only to coordinators' settlement and close code (the runner, questions' `finishAnswer`, merge settlement after an irreversible command, and E3's `settleSuggestion` when it is wired), and to startup recovery; HTTP handlers and `ReviewService` never receive it. So a request-path write still pending after the abort (an aborted `/api/review` whose `load()` reaches `recordHistory`, or merge reconciliation and queue polling after a GitHub await) fails. The server maps `ShuttingDownError` to HTTP 503 "The review server is shutting down.", never to the 409 used for review errors, and there is no partial view. **Today the merge coordinator catches every error in `displayStatus()` and `#pollQueue()`**, which would swallow the gate error and answer 200; F1 makes each of those catch blocks rethrow `ShuttingDownError` first. The next startup checks GitHub again, so a skipped merge observation is recovered. The server also closes idle keep-alive connections once shutdown starts and after each request finishes, so a finished request's socket cannot hold `server.close()` open until its keep-alive timeout. | abort: yes. Gate, capability, rethrow and idle-connection close: new. | | 4 | For every `pending` and `running` attempt that has an in-memory job, record the first reason `shutdown` **only if no first reason is set yet**. A job already marked `cancelled`, `stale` or `time-limit` keeps its reason. For `running`, call `cancel('shutdown')` and await `settled`. For `pending`, await its preparation promise (the D start call itself returns at once, so it is never in progress), remove only its host-side preparation files (task storage waits for the terminal write, as in step 5), and cancel any handle that start returned, then await its `settled` (the "Launch" table). F does not abandon a job after a timer (decision 4). **This ends:** D's supervisor escalates to a forced kill, and retries unfinished container, network or setup cleanup every second for at most 60 seconds after the first failure (`CLEANUP_RETRY_WINDOW_MS`, `agents/adapters/supervisor.ts`). Retries after the first get only what is left of the window, and every cleanup subprocess is killed at its deadline, so settlement ends about 90 seconds after cleanup starts at worst (up to 30 seconds for the first, failed attempt, then the 60-second window). Then `settled` resolves with a `stopReason` and an `unreleased` list, even if Docker is unreachable or hangs. F records `unreleased` durably before it frees the slot. See the prerequisite below. | new | | 5 | For each attempt from step 4, in the order set by the task-storage holder row: the partial-output export (writable attempts), then the terminal state from its first reason (a `shutdown` reason gives `cancelled` "Stopped by shutdown") with its `diagnostic_ref`, then `removeTaskFilesystems`, and only then free its slot. A failed write leaves the row non-terminal, and a failed removal keeps the slot held. Startup recovery handles both. | new | diff --git a/github/merge.ts b/github/merge.ts index 7b19d8cc..47b5c92e 100644 --- a/github/merge.ts +++ b/github/merge.ts @@ -69,6 +69,8 @@ export interface GhMergeConfig { pullRequest: number; issue: number; method?: 'merge' | 'squash' | 'rebase'; + /** The branch a task's pull request targets (#103). Required when the review has a runner block. */ + baseBranch?: string; } type BranchRules = Pick; diff --git a/runner/branch-push.ts b/runner/branch-push.ts index 7561ba0a..b813bb75 100644 --- a/runner/branch-push.ts +++ b/runner/branch-push.ts @@ -37,7 +37,7 @@ const BRANCH = /^codeboost\/[a-z0-9]+(?:-[a-z0-9]+)*$/; const REMOTE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; const HOST = /^[A-Za-z0-9.-]+(?::[0-9]+)?$/; // The variables whose values `gh` may authenticate with; their exact values are removed from error text. -const TOKEN_VARIABLES = ['GH_TOKEN', 'GITHUB_TOKEN', 'GH_ENTERPRISE_TOKEN', 'GITHUB_ENTERPRISE_TOKEN'] as const; +export const TOKEN_VARIABLES = ['GH_TOKEN', 'GITHUB_TOKEN', 'GH_ENTERPRISE_TOKEN', 'GITHUB_ENTERPRISE_TOKEN'] as const; // GitHub token shapes, removed from any text that leaves this module in case a server or proxy echoed one back. No word // boundaries: a token glued to other text (a URL, a path) is still removed. const TOKEN = /(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})/g; diff --git a/runner/production.ts b/runner/production.ts index 7b2ecc4e..e8a6e850 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -5,13 +5,17 @@ import { buildAgentImage } from '../agents/container/image.ts'; import { exportTaskDiff, removeTaskFilesystemsAsync, type TaskStorageLimits } from '../agents/container/storage.ts'; import { recoverLeftovers } from '../agents/recovery.ts'; import { startClaudeInvocation } from '../agents/adapters/claude.ts'; +import { GhAlreadyFixedGateway } from '../github/already-fixed.ts'; import { GhIssueGateway, type IssueText } from '../github/issues.ts'; +import { GhPullRequestGateway } from '../github/pull-requests.ts'; import { identityKey } from '../core/identity.ts'; import type { RunnerDeps } from './coordinator.ts'; import { DEFAULT_DIAGNOSTICS_CAP_BYTES } from './diagnostics.ts'; import { executionDeps, SafetyFindings, type AgentLauncher, type ExecutionSources } from './execution.ts'; import { isUuidV4, quoteForTerminal, type ShutdownCapability } from './lifecycle.ts'; import { recoverStartup, removalCommand, type RecoveryDeps, type RecoveryReport, type RunnerLock } from './recovery.ts'; +import { GitBranchPusher, pushUrl } from './branch-push.ts'; +import { PullRequestPublisher } from './publish.ts'; import type { ReviewService } from './review.ts'; import { openRunnerRepository, ownerOnlyDirectory } from './runner-repository.ts'; import { createTaskWorkspace, workspaceFilesystems } from './workspace.ts'; @@ -132,8 +136,26 @@ export function recoveryWarnings(report: Pick `Unknown entry in the runner's attempt directory, left in place: ${quoteForTerminal(entry)}`)]; } +/** + * The branch a task's PR targets: `github.baseBranch`, required with a runner block (#103). Checked before anything is + * created; Git's own refusal would come only after a run, at the push or the opening. + */ +export function baseBranch(github: { baseBranch?: unknown } | undefined): string { + const name = github?.baseBranch; + // A short branch name, as GitHub's pull request API takes it: not a full ref (`refs/heads/main`) and not HEAD. + if (typeof name !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._/-]{0,199}$/.test(name) || /\.\.|\/\/|\/\.|@\{|\.lock(?:\/|$)|[./]$/.test(name) + || name === 'HEAD' || name.startsWith('refs/')) + throw new Error('The runner publishes pull requests: add github.baseBranch, the branch they target (for example "main"), to the review configuration.'); + return name; +} + export interface RunnerAssembly { readonly deps: RunnerDeps; + /** + * Builds the task's PR publisher (#103) once the coordinator exists: `closing` is its admission flag, read before each + * irreversible GitHub change. Absent: nothing is published (tests that only run items). + */ + readonly publisher?: (closing: () => boolean) => PullRequestPublisher; readonly sources: ExecutionSources; readonly findings: SafetyFindings; readonly recovery: RecoveryReport; @@ -151,6 +173,9 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd const { service, config } = o, review = service.config; if (review.demo) throw new Error('Demos never run the runner.'); if (!review.github) throw new Error('The runner reads the plan\'s issue from GitHub: add a github block to the review configuration.'); + const base = baseBranch(review.github); + // The push URL too (the repository's shape, GH_HOST), before recovery and the image build rather than after them. + pushUrl(review.github.repository, o.env as NodeJS.ProcessEnv); const token = o.env.CLAUDE_CODE_OAUTH_TOKEN; if (!token) throw new Error(RUNNER_CREDENTIAL_MISSING); // Decision 1, step 4: the database path still names the locked file, now that the Store has opened it. @@ -201,5 +226,13 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd const findings = new SafetyFindings(service.store, o.capability); const deps = executionDeps(service.store, workspace, claudeLauncher({ imageId, runnerRoot: config.root, runnerOwner, token }), sources, runnerOwner, findings, { diagnostics: { directory: diagnosticsDir, capBytes: config.diagnosticsCapBytes ?? DEFAULT_DIAGNOSTICS_CAP_BYTES } }); - return { deps, sources, findings, recovery }; + const github = review.github; + // Never a `url` here (#101 review, finding 6): the push goes to the configured repository on GH_HOST, from the runner's + // own repository. Only commits the ledger records as codeboost's may be overwritten; before #22 adds rebasing, the + // ledger has no other kind of commit codeboost pushes (only recordRebase writes foreign entries). + const pusher = new GitBranchPusher({ repository, repositoryId: identity.repositoryId, remote: github.repository, env: o.env as NodeJS.ProcessEnv, + ownedCommits: requested => service.store.getLedger(requested).filter(entry => entry.origin === 'owned').map(entry => entry.sha) }); + const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: new GhAlreadyFixedGateway({ repository: github.repository }), + pulls: new GhPullRequestGateway({ repository: github.repository }), pusher, closing }, { repository: github.repository, baseBranch: base }); + return { deps, sources, findings, recovery, publisher }; } diff --git a/runner/publishing.ts b/runner/publishing.ts new file mode 100644 index 00000000..fd2fffff --- /dev/null +++ b/runner/publishing.ts @@ -0,0 +1,163 @@ +import { identityKey, type PlanIdentity } from '../core/identity.ts'; +import { PullRequestMisplaced, PullRequestRefused } from '../github/pull-requests.ts'; +import { BranchPushRefused, redact, TOKEN_VARIABLES } from './branch-push.ts'; +import type { RunnerCoordinator } from './coordinator.ts'; +import type { ItemExecutor } from './execution.ts'; +import { GuardRefusal, ShuttingDownError, settleWith, type ShutdownCapability } from './lifecycle.ts'; +import { OpeningUnsettled, type PublishOutcome, type PullRequestPublisher } from './publish.ts'; +import type { PublishRecord, Store } from './store.ts'; + +/** Outcomes after which nothing is owed until the task changes again. Every other record (a refusal, a failure, a stop) leaves the publish owed. */ +const SETTLED = ['opened', 'possibly already fixed', 'draft skipped', 'draft unsupported', 'no changes', + // Recorded by the v10 upgrade for a task that reached its status before publishing existed: only the action publishes it. + 'not published']; +/** Refusals a person acts on; anything else that fails is reported as `failed`. */ +const REFUSALS = [GuardRefusal, BranchPushRefused, OpeningUnsettled, PullRequestMisplaced, PullRequestRefused]; + +/** + * Publishes a task's pull request in production (#103): when a run ends, on the `publish` action, and once at startup. + * The task's status decides the mode: a running task whose every plan item has run gets a ready PR, a task in needs human + * a draft PR with its problems. One publish per task at a time, in the background, tracked until it settles; its outcome + * is recorded in the Store, where GET /api/runner reads it. Shutdown aborts every publish and awaits it before the Store's + * write gate closes, so a publish's last records still land. + */ +export class TaskPublishing { + #store: Store; #publisher: PullRequestPublisher; #runner: RunnerCoordinator; #executor: ItemExecutor; + #write: (fn: () => T) => T; + #closing = false; + /** Publishes in progress, by task, from scheduling until the outcome is recorded. */ + #running = new Map>(); + constructor(store: Store, publisher: PullRequestPublisher, runner: RunnerCoordinator, executor: ItemExecutor, capability?: ShutdownCapability) { + this.#store = store; this.#publisher = publisher; this.#runner = runner; this.#executor = executor; this.#write = settleWith(capability); + } + + /** Whether a publish of this task is in progress. */ + busy(identity: PlanIdentity): boolean { return this.#running.has(identityKey(identity)); } + lastOutcome(identity: PlanIdentity): PublishRecord | null { return this.#store.lastPublish(identity); } + /** Resolves once the task's publish in progress, if any, has recorded its outcome. */ + async settled(identity: PlanIdentity): Promise { await this.#running.get(identityKey(identity)); } + + /** + * What a publish would do now: a ready or a draft PR. Throws the refusal otherwise, with what a person can act on. + * Writes nothing, so the view asks it too and never offers what the action would refuse. + */ + mode(identity: PlanIdentity, progress?: { next: string | null }): { draft: boolean } { + if (this.#closing || this.#runner.closing) throw new ShuttingDownError(); + if (this.busy(identity)) throw new GuardRefusal('A pull request is already being published for this task.'); + // Publishing reads the task head, so it waits for the run (and the attempt) to end: assertPublishable refuses an active attempt. + if (this.#runner.isActive(identity) || this.#executor.busy(identity)) throw new GuardRefusal('A run of this task is still in progress; publish once it has ended.'); + const task = this.#store.getTask(identity); + if (task.status === 'running') { + const { next } = progress ?? this.#executor.progress(identity); + if (next !== null) throw new GuardRefusal(`${next} has not run yet; publish once every plan item has run.`); + this.#store.assertPublishableNow(identity, false); + return { draft: false }; + } + if (task.status === 'needs human') { + this.#store.assertPublishableNow(identity, true); + return { draft: true }; + } + throw new GuardRefusal(`The task is ${task.status}; a pull request is published for a running task whose plan items have all run, or a task that needs a person.`); + } + + /** + * The publish owed, if any: the task can be published, and no publish has settled since the task last changed. Null + * when nothing is owed. + */ + owed(identity: PlanIdentity): { draft: boolean } | null { + let mode; + try { mode = this.mode(identity); } + catch (error) { if (error instanceof GuardRefusal || error instanceof ShuttingDownError) return null; throw error; } + const last = this.#store.lastPublish(identity); + return !last || !SETTLED.includes(last.outcome) || last.stateVersion !== this.#store.getTask(identity).stateVersion ? mode : null; + } + + /** + * The `publish` action: start a publish now, or throw its refusal. Runs after the caller's transaction commits; its + * outcome replaces the action's saved response, so a replay reports it. + */ + request(identity: PlanIdentity, actionId: string): { draft: boolean } { + const mode = this.mode(identity); + this.#schedule(identity, mode.draft, actionId); + return mode; + } + + /** A run of the task ended, or the server started: publish if one is owed. Never throws. */ + publishIfOwed(identity: PlanIdentity): void { + try { const owed = this.owed(identity); if (owed) this.#schedule(identity, owed.draft); } + catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error))}`); } + } + + /** + * Shutdown: refuse new publishes, abort those in progress (the publisher refuses every later push, opening and ready + * change) and await their recorded outcomes. + */ + async close(): Promise { + this.#closing = true; + await this.#publisher.close(); + while (this.#running.size) await Promise.all([...this.#running.values()]); + } + + #schedule(identity: PlanIdentity, draft: boolean, actionId?: string): void { + const key = identityKey(identity); + // Reserved in this turn, so a second request in the same transaction is refused. + const reserved = Promise.withResolvers(); + this.#running.set(key, reserved.promise); + const run = async () => { + let record: Omit; + try { + // The problems are read when the publish starts: the task is in needs human, and its last attempt says why. + const outcome = await this.#publisher.publish(identity, draft ? { problems: this.#problems(identity) } : {}); + record = describe(outcome, draft); + } catch (error) { + const stopped = error instanceof ShuttingDownError || (this.#closing && (error as Error)?.name === 'AbortError'); + record = { outcome: stopped ? 'stopped' : REFUSALS.some(type => error instanceof type) ? 'refused' : 'failed', draft, message: message(error) }; + } + try { this.#write(() => this.#store.recordPublish(identity, record, actionId)); } + catch (error) { console.error(`Could not record the publish outcome: ${JSON.stringify(message(error))}`); } + }; + // Starts once the caller's transaction (the action's userAction) has committed; a rollback starts nothing. + this.#store.afterCommit(() => { + void run().finally(() => { this.#running.delete(key); reserved.resolve(); }); + }, () => { this.#running.delete(key); reserved.resolve(); }); + } + + /** + * Why a needs-human task needs a person, for its draft PR, from what sent it there now: its spent budget, or its current + * attempt's safety finding or diagnostic. An earlier attempt's finding, which a person may have dealt with, is not used. + */ + #problems(identity: PlanIdentity): string[] { + const task = this.#store.getTask(identity); + if (task.budgetDeadline !== null && task.budgetDeadline <= Date.now()) return ['The task\'s time budget ran out before its plan finished.']; + const current = task.currentAttemptId === null ? undefined : this.#store.getAttempt(identity, task.currentAttemptId); + if (current?.safetyFinding) return [current.safetyFinding]; + if (current?.diagnostic) return [current.diagnostic]; + return ['The task needs a person.']; + } +} + +function describe(outcome: PublishOutcome, draft: boolean): Omit { + const ready = 'leftReady' in outcome && outcome.leftReady !== undefined ? ` Pull request #${outcome.leftReady} could not be made a draft and may still be ready for review.` : ''; + switch (outcome.kind) { + case 'opened': return { outcome: outcome.kind, draft, number: outcome.number, url: outcome.url, + message: `Pull request #${outcome.number} is open${outcome.draft ? ' as a draft' : ''}; the task is ${outcome.status}.${ready}` }; + case 'possibly already fixed': return { outcome: outcome.kind, draft, message: `The issue may already be fixed (${checkSummary(outcome.result)}), so no pull request was opened.${ready}` }; + case 'draft skipped': return { outcome: outcome.kind, draft, message: `The issue may already be fixed (${checkSummary(outcome.result)}), so no draft pull request was opened.${ready}` }; + case 'draft unsupported': return { outcome: outcome.kind, draft, ...(outcome.number === null ? {} : { number: outcome.number }), + message: 'This repository does not support draft pull requests, so the task, which needs a person, has none.' }; + case 'no changes': return { outcome: outcome.kind, draft, message: `The task head is its base: there is nothing to publish.${ready}` }; + } +} +function checkSummary(result: Extract['result']): string { + if (result.outcome === 'unknown') return `the check could not finish: ${result.reason}`; + // Only codeboost-shaped fields: a commit subject is someone else's text. + if (result.outcome === 'found') return result.matches.map(match => match.kind === 'pull request' ? `pull request ${match.repository}#${match.number}` + : match.kind === 'commit' ? `commit ${match.sha.slice(0, 12)}` : 'the issue was closed').join(', '); + return 'clear'; +} +/** + * An error's text for the record and the runner view. A `gh` failure carries its output, which a server or proxy may have + * echoed a token into: every configured token value and every GitHub token shape is removed, as for the push. + */ +const message = (error: unknown) => redact(error instanceof Error ? error.message : String(error), + TOKEN_VARIABLES.flatMap(name => process.env[name] ?? [])).slice(0, 2000); diff --git a/runner/store.ts b/runner/store.ts index bc06ef66..8f8fe7ed 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -50,6 +50,19 @@ export interface MergeAttempt { export function mergeActionResponse(attempt: MergeAttempt) { return { attemptId: attempt.id, state: attempt.state, reason: attempt.reason, url: attempt.url }; } +/** + * The last publish of a task (#103), as GET /api/runner shows it. `outcome` is the publisher's outcome kind, or `refused` + * (a guard or GitHub refusal a person acts on), `failed` (anything else) or `stopped` (shutdown). `stateVersion` is the + * task's state version when it was recorded: a publish owed since then has not run yet. + */ +export interface PublishRecord { + outcome: string; draft: boolean; message: string; stateVersion: number; at: string; number?: number; url?: string; +} +/** What the publish action replays once its publish has settled: the outcome, as `publish.last` shows it. */ +export function publishActionResponse(record: PublishRecord) { + return { outcome: record.outcome, draft: record.draft, message: record.message, ...(record.number === undefined ? {} : { number: record.number }), + ...(record.url === undefined ? {} : { url: record.url }) }; +} export interface TaskRecord { planKey: string; status: TaskStatus; stateVersion: number; contextGeneration: number; assignmentId: string; referencedCodeHash: string; currentAttemptId: string | null; requeuePending: boolean; cancelRequested: string | null; rebaseInProgress: unknown; budgetDeadline: number | null; @@ -102,8 +115,8 @@ export class Store { this.#db.exec('PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;'); this.#transaction(() => { const version = this.#get('PRAGMA user_version')!.user_version as number; - if (![0, 1, 2, 3, 4, 5, 6, 7, 8, 9].includes(version)) throw new Error('Unsupported store schema version.'); - if (version === 9) return; + if (![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10].includes(version)) throw new Error('Unsupported store schema version.'); + if (version === 10) return; if (version === 0) this.#db.exec(` CREATE TABLE plans (key TEXT PRIMARY KEY, issue INTEGER NOT NULL, revision INTEGER NOT NULL, snapshot_id TEXT); CREATE TABLE revisions (key TEXT NOT NULL REFERENCES plans(key), revision INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(key,revision)); @@ -148,6 +161,17 @@ export class Store { if (!columns.includes('storage_base')) this.#db.exec('ALTER TABLE attempts ADD COLUMN storage_base TEXT'); this.#db.exec('PRAGMA user_version=9'); } + // The last publish of each task (#103): what GET /api/runner shows, kept across restarts so a refusal stays visible. + // A task already running or in needs human reached that status before codeboost published anything: it is recorded as + // not published, at its current state version, so the first start after the upgrade opens no PR a person did not ask + // for. The publish action publishes it. + if (version < 10) { + this.#db.exec('CREATE TABLE IF NOT EXISTS publish_outcomes (plan_key TEXT PRIMARY KEY REFERENCES tasks(plan_key), data TEXT NOT NULL)'); + this.#run(`INSERT OR IGNORE INTO publish_outcomes (plan_key,data) SELECT plan_key, json_object('outcome','not published','draft',json('false'), + 'message','This task reached its status before codeboost published pull requests. Use the publish action to publish it.', + 'stateVersion',state_version,'at',?) FROM tasks WHERE status IN ('running','needs human')`, new Date().toISOString()); + this.#db.exec('PRAGMA user_version=10'); + } }); } catch (error) { this.#db.close(); throw error; } } @@ -1281,6 +1305,26 @@ export class Store { return this.#task(key).state_version as number; }); } + /** + * Record a publish's outcome. An observation, not a task change: the task's state version does not move, so a refused + * publish leaves the task exactly as it was. Settlement writes go through the shutdown capability. + */ + recordPublish(identity: PlanIdentity, record: Omit, actionId?: string): PublishRecord { + const key = identityKey(identity); + return this.#transaction(() => { + const saved: PublishRecord = { ...record, message: record.message.slice(0, 2000), stateVersion: this.#task(key).state_version as number, at: new Date().toISOString() }; + this.#run('INSERT INTO publish_outcomes (plan_key,data) VALUES (?,?) ON CONFLICT(plan_key) DO UPDATE SET data=excluded.data', key, encode(saved)); + // The same transaction refreshes the publish action's replay, as for a merge, so a resent click reports this outcome. + if (actionId !== undefined) this.#run(`UPDATE user_actions SET response=? WHERE plan_key=? AND action_id=? AND kind='publish' AND json_extract(response,'$.ok')=1`, + encode({ ok: true, value: publishActionResponse(saved) }), key, actionId); + return saved; + }); + } + lastPublish(identity: PlanIdentity): PublishRecord | null { + const key = identityKey(identity); this.#task(key); + const row = this.#get('SELECT data FROM publish_outcomes WHERE plan_key=?', key); + return row ? decode(row.data) : null; + } /** Whether the task can be published in this mode right now (status, no attempt, merge, requeue or rebase). */ canPublish(identity: PlanIdentity, draft: boolean): boolean { const key = identityKey(identity), task = this.#task(key); diff --git a/test/cli.test.ts b/test/cli.test.ts index 9a1832c9..d752d4db 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -16,12 +16,12 @@ const cli = fileURLToPath(new URL('../web/cli.ts', import.meta.url)); const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); /** A non-demo review with a runner block, and a HOME of its own so the lock directory is the test's. */ -function review() { +function review(github: Record = { baseBranch: 'main' }) { const root = mkdtempSync(join(tmpdir(), 'codeboost-cli-')); roots.push(root); const demo = createDemo(join(root, 'demo')), store = new Store(demo.database), issue = store.getPlan(demo.identity).issue; store.close(); mkdirSync(join(root, 'runner'), { mode: 0o700 }); mkdirSync(join(root, 'home'), { mode: 0o700 }); const config = join(root, 'review.json'); - writeFileSync(config, JSON.stringify({ ...demo, demo: false, github: { repository: 'owner/repo', pullRequest: 1, issue }, + writeFileSync(config, JSON.stringify({ ...demo, demo: false, github: { repository: 'owner/repo', pullRequest: 1, issue, ...github }, runner: { root: join(root, 'runner'), committer: { name: 'codeboost', email: 'runner@codeboost.invalid' } } })); /** * A `docker` that answers every call with nothing (no leftovers, an image that matches the pinned profile) once the hold @@ -64,6 +64,14 @@ it('refuses a runner startup without a token with its message, exit 1 and the lo expect(result.stderr.trim()).toBe(RUNNER_CREDENTIAL_MISSING); } }); +it('refuses a runner without github.baseBranch before the lock and before the token check (#103)', () => { + for (const github of [{}, { baseBranch: 'refs/heads/main' }]) { + const result = review(github).run('--port', '0'); + expect(result.status).toBe(1); + // Not the missing-token refusal that the runner's setup would give after the lock: the CLI checked the block first. + expect(result.stderr.trim()).toMatch(/^The runner publishes pull requests: add github\.baseBranch/); + } +}); it('refuses a bad port and an unknown preparation with a message, never a stack', () => { const { run } = review(); const port = run('--port', 'x'); diff --git a/test/publish.test.ts b/test/publish.test.ts index 9e32ea54..efd15bc8 100644 --- a/test/publish.test.ts +++ b/test/publish.test.ts @@ -324,7 +324,7 @@ describe('schema v7', () => { expect(upgraded.taskPullRequests(identity)).toEqual([]); expect(upgraded.latestAlreadyFixed(identity)).toBeNull(); const db = new DatabaseSync(path, { readOnly: true }); - expect(db.prepare('PRAGMA user_version').get()).toEqual({ user_version: 9 }); + expect(db.prepare('PRAGMA user_version').get()).toEqual({ user_version: 10 }); const names = db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table','index') AND (name LIKE '%pull_requests%' OR name LIKE 'already_fixed_checks%') AND name NOT LIKE 'sqlite_autoindex%' ORDER BY name").all().map(row => row.name); expect(names).toEqual(['already_fixed_checks', 'already_fixed_checks_task', 'task_pull_requests', 'task_pull_requests_number', 'task_pull_requests_opening', 'task_pull_requests_task']); db.close(); diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index 7e2cf2d3..c7669cb0 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -662,7 +662,7 @@ describe('durable safety findings (#87 item 3)', () => { db.exec('ALTER TABLE attempts DROP COLUMN safety_finding; PRAGMA user_version=7;'); db.close(); const reopened = open(path); expect(reopened.getAttempt(identity, attempt.id).safetyFinding).toBeNull(); - expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 9 }); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 10 }); }); }); @@ -686,6 +686,32 @@ describe('allocation baseline (#91)', () => { db.exec('ALTER TABLE attempts DROP COLUMN metadata_baseline; ALTER TABLE attempts DROP COLUMN storage_base; PRAGMA user_version=8;'); db.close(); const reopened = open(path); expect(reopened.interruptedAttempts()).toEqual([expect.objectContaining({ id: attempt.id, metadataBaseline: null, storageBase: null })]); - expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 9 }); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 10 }); + }); +}); +describe('publish outcomes (#103)', () => { + it('adds the outcome table to a version 9 database, and records an outcome without moving the task', () => { + const { path, store } = queued(); + store.close(); stores.splice(stores.indexOf(store), 1); + const db = new DatabaseSync(path); + db.exec('DROP TABLE publish_outcomes; PRAGMA user_version=9;'); db.close(); + const reopened = open(path), before = reopened.getTask(identity); + expect(reopened.lastPublish(identity)).toBeNull(); + expect(reopened.recordPublish(identity, { outcome: 'refused', draft: false, message: 'x'.repeat(3000) })) + .toMatchObject({ outcome: 'refused', stateVersion: before.stateVersion, message: 'x'.repeat(2000) }); + reopened.recordPublish(identity, { outcome: 'opened', draft: false, message: 'Pull request #1 is open.', number: 1, url: 'https://github.com/o/r/pull/1' }); + expect(reopened.lastPublish(identity)).toMatchObject({ outcome: 'opened', number: 1 }); + expect(reopened.getTask(identity)).toEqual(before); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 10 }); + }); + it('records a task already running at the upgrade as not published, so the first start publishes nothing for it', () => { + const { path, store } = queued(); admit(store); + expect(store.getTask(identity).status).toBe('running'); + const version = store.getTask(identity).stateVersion; + store.close(); stores.splice(stores.indexOf(store), 1); + const db = new DatabaseSync(path); + db.exec('DROP TABLE publish_outcomes; PRAGMA user_version=9;'); db.close(); + expect(open(path).lastPublish(identity)).toMatchObject({ outcome: 'not published', draft: false, stateVersion: version, + message: expect.stringMatching(/Use the publish action/) }); }); }); diff --git a/test/runner-production.test.ts b/test/runner-production.test.ts index 692b7ca2..6b2baaa6 100644 --- a/test/runner-production.test.ts +++ b/test/runner-production.test.ts @@ -41,7 +41,7 @@ function fixture() { const service = new ReviewService(demo); const issue = service.store.getPlan(demo.identity).issue; // A real review, not a demo: the runner needs a github block for the issue text. - const config = { ...demo, demo: false, github: { repository: 'owner/repo', pullRequest: 1, issue } }; + const config = { ...demo, demo: false, github: { repository: 'owner/repo', pullRequest: 1, issue, baseBranch: 'main' } }; service.config = config; cleanups.push(() => { try { service.close(); } catch { /* closed by the test */ } }); return { root, service, config, demo }; diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts new file mode 100644 index 00000000..2fa27156 --- /dev/null +++ b/test/runner-publishing.test.ts @@ -0,0 +1,363 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { fixtureGit as git } from './fixtures/git.ts'; +import { createDemo } from '../scripts/demo.ts'; +import { startServer } from '../web/server.ts'; +import { Store } from '../runner/store.ts'; +import type { ReviewService } from '../runner/review.ts'; +import type { PreparedAttempt, RunnerDeps } from '../runner/coordinator.ts'; +import { SafetyFindings, type ExecutionSources } from '../runner/execution.ts'; +import { GitBranchPusher } from '../runner/branch-push.ts'; +import { PullRequestPublisher } from '../runner/publish.ts'; +import { ensureCommit, openRunnerRepository } from '../runner/runner-repository.ts'; +import { baseBranch } from '../runner/production.ts'; +import { PullRequestMisplaced, type OpenPullRequestInput, type PullRequestGateway } from '../github/pull-requests.ts'; +import type { AlreadyFixedGateway } from '../github/already-fixed.ts'; +import type { PlanIdentity } from '../core/identity.ts'; + +vi.setConfig({ testTimeout: 30_000 }); +const roots: string[] = [], cleanups: (() => Promise | void)[] = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +const OWNER = 'c'.repeat(32), REPO = 'acme/app'; +type App = Awaited>; + +interface Pr { number: number; url: string; draft: boolean; open: boolean; base: string; headBranch: string; marker: string; body: string } +/** + * GitHub, as far as publishing sees it: pull requests in memory, and branches in a real local bare repository, so a PR's + * head is whatever the push left on its branch. `onOpen` runs after GitHub created the PR and before the reply arrives. + */ +class FakeGitHub { + prs: Pr[] = []; + calls: string[] = []; + onOpen?: (signal?: AbortSignal) => Promise; + readonly remote: string; + constructor(remote: string) { this.remote = remote; } + head(branch: string): string | null { + try { return git(this.remote, 'rev-parse', '--verify', '-q', `refs/heads/${branch}`); } catch { return null; } + } + #view(pr: Pr) { return { number: pr.number, url: pr.url, draft: pr.draft, headSha: this.head(pr.headBranch) ?? '' }; } + checks: AlreadyFixedGateway = { repository: REPO, check: async () => { this.calls.push('check'); return { outcome: 'clear', baseHead: 'b'.repeat(40) }; } }; + pulls: PullRequestGateway = { + repository: REPO, + open: async (input: OpenPullRequestInput, signal?: AbortSignal) => { + this.calls.push(`open ${input.draft ? 'draft' : 'ready'}`); + if (this.head(input.headBranch) === null) throw new Error('No such branch on GitHub.'); + const pr: Pr = { number: 100 + this.prs.length, url: `https://github.com/${REPO}/pull/${100 + this.prs.length}`, draft: input.draft, open: true, + base: input.base, headBranch: input.headBranch, marker: input.marker, body: input.body }; + this.prs.push(pr); + await this.onOpen?.(signal); + return this.#view(pr); + }, + findOpened: async input => { + const open = this.prs.filter(pr => pr.open && pr.headBranch === input.headBranch); + const own = open.filter(pr => input.markers.includes(pr.marker)); + if (own.length > 1 || (own[0] && own[0].base !== input.base)) throw new PullRequestMisplaced('The task\'s pull requests are not where it publishes.'); + if (open.some(pr => pr.base === input.base && !input.markers.includes(pr.marker))) throw new Error('An open pull request exists that codeboost did not open.'); + return own[0] ? { ...this.#view(own[0]), marker: own[0].marker } : null; + }, + findOwned: async input => this.prs.filter(pr => pr.open && pr.headBranch === input.headBranch && input.markers.includes(pr.marker)) + .map(pr => ({ ...this.#view(pr), marker: pr.marker, base: pr.base })), + readPull: async number => { const pr = this.prs.find(candidate => candidate.number === number)!; return { open: pr.open, headBranch: pr.headBranch, base: pr.base, marker: pr.marker }; }, + markDraft: async number => { const pr = this.prs.find(candidate => candidate.number === number)!; pr.draft = true; return this.#view(pr); }, + refresh: async (number, input) => { + this.calls.push(`refresh ${input.draft ? 'draft' : 'ready'}`); + const pr = this.prs.find(candidate => candidate.number === number)!; + input.beforeReady?.(); + pr.body = input.body; pr.draft = input.draft; + return this.#view(pr); + }, + }; +} + +/** Every plan item completed, changing nothing: the task is running and its plan has run. */ +function completeAll(service: ReviewService) { + const s = service.store, id = service.config.identity, head = s.getSnapshot(id).head; + s.transitionTask(id, s.getTask(id).stateVersion, 'queued'); + for (const item of s.getPlan(id).items) { + const attempt = s.admitAttempt(id, { expectedStateVersion: s.getTask(id).stateVersion, kind: 'execute', item: item.id, expectedContext: s.currentContext(id), deadline: Date.now() + 60_000 }); + s.markRunning(id, attempt.id); + s.settleAttempt(id, attempt.id, { firstReason: null, exitCode: 0, valid: true, result: { head, unchanged: true, inScope: [], outOfScope: [] } }); + } + expect(s.getTask(id).status).toBe('running'); +} +/** The first item failed with a 1 ms task budget, which has passed: the task is running, and its next admission is refused. */ +function budgetSpent(service: ReviewService) { + const s = service.store, id = service.config.identity; + s.transitionTask(id, s.getTask(id).stateVersion, 'queued'); + const admit = (budgetMs?: number) => s.admitAttempt(id, { expectedStateVersion: s.getTask(id).stateVersion, kind: 'execute', item: s.getPlan(id).items[0]!.id, + expectedContext: s.currentContext(id), deadline: Date.now() + 60_000, ...(budgetMs ? { budgetMs } : {}) }); + const attempt = admit(1); + s.markRunning(id, attempt.id); + s.settleAttempt(id, attempt.id, { firstReason: null, exitCode: 1, valid: false, detail: 'The tests failed in retry.ts.' }); + const until = Date.now() + 5; while (Date.now() < until) { /* let the budget pass */ } + return admit; +} +/** As production gets there: an admission after the budget passed moves the task to needs human, and is refused. */ +function needsHuman(service: ReviewService) { + const admit = budgetSpent(service); + expect(() => admit()).toThrow(/time budget/); + expect(service.store.getTask(service.config.identity).status).toBe('needs human'); +} +const BUDGET = 'The task\'s time budget ran out before its plan finished.'; +/** Wait until the startup publish (or any in progress) has settled, then return the GitHub calls made so far. */ +async function quiet(app: App, w: World, identity: PlanIdentity) { await app.publishing!.settled(identity); return [...w.github.calls]; } + +interface World { root: string; demo: ReturnType; remote: string; github: FakeGitHub } +function world(): World { + const root = mkdtempSync(join(tmpdir(), 'codeboost-publishing-')); roots.push(root); + const demo = createDemo(join(root, 'demo')), remote = join(root, 'remote.git'); + git(root, 'init', '-q', '--bare', remote); + return { root, demo, remote, github: new FakeGitHub(remote) }; +} + +/** + * A server whose runner completes every item it admits without an agent (each changes nothing), and publishes through the + * real publisher and the real pusher into `w.remote`, with GitHub faked. `before` shapes the Store before the runner + * exists, as an earlier process would have left it. + */ +async function serve(w: World, options: { before?: (service: ReviewService) => void; onPushSpawn?: (n: number, app: () => App, close: () => Promise) => void; demo?: boolean; startup?: boolean } = {}) { + let app: App | undefined, spawns = 0, closing: Promise | undefined; + const close = () => closing ??= app!.close(); + let branchOf: (identity: PlanIdentity) => string = () => ''; + const config = { ...w.demo, demo: options.demo ?? false }; + app = await startServer(config, 0, undefined, undefined, 2_000, undefined, undefined, undefined, async service => { + options.before?.(service); + const repository = await openRunnerRepository({ runnerRoot: join(w.root, 'runner'), runnerOwner: OWNER, repositoryId: service.config.identity.repositoryId, source: service.config.repository }); + await ensureCommit(repository, service.store.getSnapshot(service.config.identity).head); + const prepared: PreparedAttempt = { clone: { id: 'clone', taskId: 'task', directory: '/tmp/x', head: 'f'.repeat(40) }, vendor: 'claude', approvedArgv: [] }; + const deps: RunnerDeps = { runnerOwner: OWNER, kinds: ['execute'], prepare: async () => prepared, cleanupPreparation: async () => undefined, + start: input => ({ attemptId: input.attemptId, cancel: () => undefined, + settled: Promise.resolve({ attemptId: input.attemptId, context: input.context, exitCode: 0, signal: null, stdout: '', stderr: '' }) }), + validate: () => ({ head: service.store.getSnapshot(service.config.identity).head, unchanged: true, inScope: [], outOfScope: [] }) }; + const sources: ExecutionSources = { planContext: () => service.planContext(), issue: () => ({ number: 3, title: '', body: '', comments: [] }), lessons: () => [], vendor: () => 'claude' }; + const pusher = new GitBranchPusher({ repository, repositoryId: service.config.identity.repositoryId, remote: REPO, url: w.remote, + ownedCommits: identity => service.store.getLedger(identity).filter(entry => entry.origin === 'owned').map(entry => entry.sha), + onProcessGroup: () => options.onPushSpawn?.(++spawns, () => app!, close) }); + const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: w.github.checks, pulls: w.github.pulls, pusher, closing }, { repository: REPO, baseBranch: 'main' }); + branchOf = identity => publisher(() => false).branch(identity); + return { deps, sources, findings: new SafetyFindings(service.store), publisher, + recovery: { finalized: [], requeue: [], removedDirectories: [], unknownEntries: [], unmatchedStorage: [], repairedMerges: [] } }; + }); + cleanups.push(close); + // As the CLI does once it has verified the lock. + if (options.startup !== false) app.publishOwed(); + const identity = w.demo.identity; + return { app, close, identity, store: app.service.store, branch: branchOf(identity), head: app.service.store.getSnapshot(identity).head }; +} +const view = async (app: App) => (await fetch(`${new URL(app.url).origin}/api/runner`, { headers: { 'x-codeboost-token': app.token } })).json() as Promise>; +async function act(app: App, action: string, actionId = randomUUID(), expectedStateVersion?: number) { + const { stateVersion } = await view(app); + const response = await fetch(`${new URL(app.url).origin}/api/runner`, { method: 'POST', headers: { 'x-codeboost-token': app.token, 'content-type': 'application/json' }, + body: JSON.stringify({ action, expectedStateVersion: expectedStateVersion ?? stateVersion, actionId }) }); + return { status: response.status, body: await response.json() as Record }; +} +const publishSettled = (app: App, identity: PlanIdentity) => vi.waitFor(async () => { + expect(app.publishing!.busy(identity)).toBe(false); + expect(app.service.store.lastPublish(identity)).not.toBeNull(); +}, { timeout: 20_000, interval: 20 }); + +describe('publishing a finished task (#103)', () => { + it('opens one ready pull request when a run completes the plan, with the branch at the task head', async () => { + const w = world(); + const { app, close, identity, store, branch, head } = await serve(w); + expect((await view(app)).publish).toMatchObject({ available: true, publishable: false, last: null }); + expect((await act(app, 'start')).body.result).toMatchObject({ outcome: 'started' }); + await publishSettled(app, identity); + expect(w.github.prs).toHaveLength(1); + expect(w.github.prs[0]).toMatchObject({ draft: false, base: 'main', headBranch: branch, open: true }); + expect(w.github.head(branch)).toBe(head); + expect(store.getTask(identity).status).toBe('in review'); + expect((await view(app)).publish).toMatchObject({ active: false, publishable: false, + last: { outcome: 'opened', draft: false, number: 100, url: `https://github.com/${REPO}/pull/100` } }); + // Nothing is owed any more: a restart publishes nothing (no check, no open, no refresh). + await close(); + const calls = [...w.github.calls], again = await serve(w); + expect(again.app.publishing!.busy(identity)).toBe(false); + expect(await quiet(again.app, w, identity)).toEqual(calls); + }); + + it('opens a draft pull request with its problems for a task that needs a person', async () => { + const w = world(); + const { app, identity, store, branch, head } = await serve(w, { before: needsHuman }); + await publishSettled(app, identity); + expect(w.github.prs).toHaveLength(1); + expect(w.github.prs[0]).toMatchObject({ draft: true, headBranch: branch }); + expect(w.github.prs[0]!.body).toContain(BUDGET); + expect(w.github.head(branch)).toBe(head); + expect(store.getTask(identity).status).toBe('needs human'); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true, number: 100 }); + }); + + it('leaves a task whose branch holds someone else\'s commit running, and the publish action retries once a person fixed it', async () => { + const v = world(); + // A person pushed the base commit (not codeboost's) to the task's branch before the publish. + const s2 = await serve(v, { before: service => { + completeAll(service); + const name = new PullRequestPublisher(service.store, { checks: v.github.checks, pulls: v.github.pulls, pusher: { push: async () => undefined } }, { repository: REPO, baseBranch: 'main' }).branch(service.config.identity); + git(v.demo.repository, '-c', 'protocol.file.allow=always', 'push', '-q', v.remote, `${service.store.getSnapshot(service.config.identity).base}:refs/heads/${name}`); + } }); + await publishSettled(s2.app, s2.identity); + expect(s2.store.lastPublish(s2.identity)).toMatchObject({ outcome: 'refused', draft: false, message: expect.stringMatching(/which codeboost did not make\. Nothing was pushed\./) }); + expect(s2.store.getTask(s2.identity).status).toBe('running'); + expect(v.github.prs).toHaveLength(0); + expect((await view(s2.app)).publish).toMatchObject({ active: false, publishable: true, last: { outcome: 'refused' } }); + // A person deletes the branch; the publish action runs again. + git(v.remote, 'update-ref', '-d', `refs/heads/${s2.branch}`); + const actionId = randomUUID(), version = s2.store.getTask(s2.identity).stateVersion; + expect(await act(s2.app, 'publish', actionId, version)).toMatchObject({ status: 200, body: { result: { outcome: 'publishing', draft: false } } }); + expect((await view(s2.app)).publish).toMatchObject({ active: true }); + await s2.app.publishing!.settled(s2.identity); + expect(s2.store.lastPublish(s2.identity)).toMatchObject({ outcome: 'opened', draft: false }); + expect(v.github.head(s2.branch)).toBe(s2.head); + expect(s2.store.getTask(s2.identity).status).toBe('in review'); + // The same action ID replays the publish's outcome, not "publishing", and publishes nothing again. + const calls = [...v.github.calls]; + expect((await act(s2.app, 'publish', actionId, version)).body.result).toMatchObject({ outcome: 'opened', draft: false, number: 100 }); + expect(s2.app.publishing!.busy(s2.identity)).toBe(false); + expect(v.github.calls).toEqual(calls); + }); + + it('refuses the publish action while nothing can be published, and records the refusal', async () => { + const w = world(); + const { app } = await serve(w); + const refused = await act(app, 'publish'); + expect(refused).toMatchObject({ status: 409, body: { error: expect.stringMatching(/The task is in review/) } }); + expect(w.github.calls).toEqual([]); + }); + + it('closes the Store only after a publish stopped during its push has settled, and the next start publishes once', async () => { + const w = world(); + let closedWhileBusy: boolean | undefined; + const first = await serve(w, { before: completeAll, onPushSpawn: (n, app, close) => { + // The third Git call is the push (after the commit check and the read of the remote). + if (n !== 3) return; + const service = app().service, closeStore = service.close.bind(service); + service.close = () => { closedWhileBusy = app().publishing!.busy(w.demo.identity); closeStore(); }; + void close(); + } }); + await vi.waitFor(() => expect(closedWhileBusy).toBe(false), { timeout: 20_000 }); + const store = new Store(w.demo.database); + try { expect(store.lastPublish(first.identity)).toMatchObject({ outcome: 'stopped', draft: false }); } finally { store.close(); } + expect(w.github.prs).toHaveLength(0); + const second = await serve(w); + await publishSettled(second.app, second.identity); + expect(second.store.lastPublish(second.identity)).toMatchObject({ outcome: 'opened' }); + expect(w.github.prs).toHaveLength(1); + expect(w.github.head(second.branch)).toBe(second.head); + }); + + it('keeps an opening whose reply was lost to shutdown, and the next start recovers it by its marker', async () => { + const w = world(); + let closedWhileBusy: boolean | undefined; + let appRef: App | undefined, closeApp: (() => Promise) | undefined; + // GitHub created the PR; shutdown aborts the call before its reply arrives. + w.github.onOpen = signal => new Promise((_, reject) => { + const service = appRef!.service, closeStore = service.close.bind(service); + service.close = () => { closedWhileBusy = appRef!.publishing!.busy(w.demo.identity); closeStore(); }; + signal!.addEventListener('abort', () => reject(signal!.reason), { once: true }); + void closeApp!(); + }); + const first = await serve(w, { before: completeAll, onPushSpawn: (_, app, close) => { appRef = app(); closeApp = close; } }); + await vi.waitFor(() => expect(closedWhileBusy).toBe(false), { timeout: 20_000 }); + const store = new Store(w.demo.database); + try { + expect(store.lastPublish(first.identity)).toMatchObject({ outcome: 'stopped' }); + expect(store.taskPullRequests(first.identity)).toMatchObject([{ state: 'opening' }]); + } finally { store.close(); } + expect(w.github.prs).toHaveLength(1); + w.github.onOpen = undefined; + const second = await serve(w); + await publishSettled(second.app, second.identity); + // Recovered, not opened again. + expect(w.github.prs).toHaveLength(1); + expect(second.store.taskPullRequests(second.identity)).toMatchObject([{ state: 'opened', number: 100 }]); + expect(second.store.getTask(second.identity).status).toBe('in review'); + expect(second.store.lastPublish(second.identity)).toMatchObject({ outcome: 'opened', number: 100 }); + }); + + it('publishes a draft when a resume is refused because the budget ran out, which moves the task to needs human', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: service => { budgetSpent(service); } }); + expect(store.getTask(identity).status).toBe('running'); + expect(await act(app, 'resume')).toMatchObject({ status: 409, body: { error: expect.stringMatching(/time budget/) } }); + expect(store.getTask(identity).status).toBe('needs human'); + await publishSettled(app, identity); + expect(w.github.prs).toEqual([expect.objectContaining({ draft: true, body: expect.stringContaining(BUDGET) })]); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }); + }); + + it('publishes nothing when resume is refused for a task already in needs human, though its last publish was refused', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { startup: false, before: service => { + needsHuman(service); + service.store.recordPublish(service.config.identity, { outcome: 'refused', draft: true, message: 'The branch moved.' }); + } }); + expect(await act(app, 'resume')).toMatchObject({ status: 409, body: { error: expect.stringMatching(/needs human/) } }); + expect(app.publishing!.busy(identity)).toBe(false); + expect(w.github.calls).toEqual([]); + // The publish action is how a person retries it. + expect(await act(app, 'publish')).toMatchObject({ status: 200, body: { result: { outcome: 'publishing', draft: true } } }); + await app.publishing!.settled(identity); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }); + }); + + it('refuses start, resume and a second publish while a publish runs, and records the publish refusal', async () => { + const w = world(); + const opening = Promise.withResolvers(), release = Promise.withResolvers(); + w.github.onOpen = async () => { opening.resolve(); await release.promise; }; + const { app, identity, store } = await serve(w, { before: completeAll }); + await opening.promise; + expect((await view(app)).publish).toMatchObject({ active: true, publishable: false }); + for (const action of ['start', 'resume']) + expect(await act(app, action)).toMatchObject({ status: 409, body: { error: 'A pull request is being published for this task; try again when it has finished.' } }); + const actionId = randomUUID(), version = store.getTask(identity).stateVersion; + const second = await act(app, 'publish', actionId, version); + expect(second).toMatchObject({ status: 409, body: { error: 'A pull request is already being published for this task.' } }); + release.resolve(); + await app.publishing!.settled(identity); + expect(store.getAttempts(identity)).toHaveLength(3); + // The refusal was recorded: the same action ID replays it, and starts nothing. + expect(await act(app, 'publish', actionId, version)).toMatchObject({ status: 409, body: second.body }); + expect(w.github.prs).toHaveLength(1); + }); + + it('awaits a publish stopped by shutdown before the write gate closes, so a PR GitHub opened as the abort came is recorded', async () => { + const w = world(); + let appRef: App | undefined, closeApp: (() => Promise) | undefined; + // GitHub's reply arrives a moment after shutdown aborted the call: the publisher records the PR it opened. + w.github.onOpen = signal => new Promise(resolve => { + signal!.addEventListener('abort', () => setTimeout(resolve, 300), { once: true }); + void closeApp!(); + }); + const first = await serve(w, { before: completeAll, onPushSpawn: (_, app, close) => { appRef = app(); closeApp = close; } }); + await vi.waitFor(() => expect(appRef?.publishing?.busy(first.identity)).toBe(false), { timeout: 20_000 }); + await closeApp!(); + const store = new Store(w.demo.database); + try { expect(store.taskPullRequests(first.identity)).toMatchObject([{ state: 'opened', number: 100 }]); } finally { store.close(); } + }); + + it('never publishes in a demo, even with a publisher', async () => { + const w = world(); + const { app } = await serve(w, { demo: true, before: completeAll }); + expect(app.publishing).toBeNull(); + expect((await view(app)).publish).toMatchObject({ available: false }); + expect(await act(app, 'publish')).toMatchObject({ status: 409 }); + expect(w.github.calls).toEqual([]); + }); +}); + +describe('github.baseBranch', () => { + it('is required, and refused when Git would refuse it as a branch name', () => { + expect(baseBranch({ baseBranch: 'main' })).toBe('main'); + expect(baseBranch({ baseBranch: 'release/2.x' })).toBe('release/2.x'); + for (const bad of [undefined, '', '-main', 'a..b', 'a//b', 'a/', 'a.', 'a.lock', 'a/.b', 'a@{b', 'a b', 'a\nb']) + expect(() => baseBranch({ baseBranch: bad })).toThrow(/github\.baseBranch/); + expect(() => baseBranch(undefined)).toThrow(/github\.baseBranch/); + }); +}); diff --git a/web/cli.ts b/web/cli.ts index 135421f2..9b98ec14 100644 --- a/web/cli.ts +++ b/web/cli.ts @@ -5,14 +5,14 @@ import { startServer, type RunnerSetup } from './server.ts'; import { createDemo } from '../scripts/demo.ts'; import { Store, requireSupportedNode } from '../runner/store.ts'; import { acquireRunnerLock, releasePreparation } from '../runner/recovery.ts'; -import { parseRunnerConfig, recoveryWarnings, setUpRunner } from '../runner/production.ts'; +import { baseBranch, parseRunnerConfig, recoveryWarnings, setUpRunner } from '../runner/production.ts'; requireSupportedNode(); /** A refusal the person acts on (bad input, a held lock, a blocked recovery): its message and exit 1, not a stack. */ function refuse(error: unknown): never { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } const checked = (fn: () => T): T => { try { return fn(); } catch (error) { return refuse(error); } }; const { values } = parseArgs({ options: { demo: { type:'boolean' }, directory:{type:'string'}, config:{type:'string'}, port:{type:'string'}, help:{type:'boolean'}, 'release-preparation':{type:'string'} } }); if (values.help || (!values.demo && !values.config && values['release-preparation'] === undefined)) { - console.log('codeboost local review\n\nDemo: npm run demo\nExisting store: npm start -- --config /absolute/path/review.json\nOptions: --port 4318 --directory /path/to/demo\n\nThe configuration binds a trusted repository, database, plan identity, and known path identity. Configure the question agent in Settings; Ask runs it in a Docker container (Claude Code needs CLAUDE_CODE_OAUTH_TOKEN; Codex cannot answer questions yet). A github block enables the guarded merge gate; demos never merge. A runner block (root, committer, optional diagnosticsDir and limits) turns on the runner: it needs Docker, the github block and CLAUDE_CODE_OAUTH_TOKEN, and runs startup recovery before the server opens.\n\n--release-preparation : after startup recovery reports a preparation whose process was never recorded, and you have stopped that process, remove its attempt directory (refused while any process still uses it).'); + console.log('codeboost local review\n\nDemo: npm run demo\nExisting store: npm start -- --config /absolute/path/review.json\nOptions: --port 4318 --directory /path/to/demo\n\nThe configuration binds a trusted repository, database, plan identity, and known path identity. Configure the question agent in Settings; Ask runs it in a Docker container (Claude Code needs CLAUDE_CODE_OAUTH_TOKEN; Codex cannot answer questions yet). A github block enables the guarded merge gate; demos never merge. A runner block (root, committer, optional diagnosticsDir and limits) turns on the runner: it needs Docker, the github block with a baseBranch for its pull requests, and CLAUDE_CODE_OAUTH_TOKEN, and runs startup recovery before the server opens.\n\n--release-preparation : after startup recovery reports a preparation whose process was never recorded, and you have stopped that process, remove its attempt directory (refused while any process still uses it).'); } else if (values['release-preparation'] !== undefined) { const attemptId = values['release-preparation']; checked(() => { @@ -36,7 +36,12 @@ if (values.help || (!values.demo && !values.config && values['release-preparatio if (!Number.isInteger(port) || port < 0 || port > 65535) refuse(new Error('Invalid port.')); const config = checked(() => values.demo ? createDemo(values.directory ?? '.codeboost-local/demo') : JSON.parse(readFileSync(resolve(values.config!), 'utf8'))); // Checked before the lock, so a malformed block changes nothing. Demos never run the runner, however they were opened. - const runnerConfig = checked(() => config.demo !== true && config.runner !== undefined ? parseRunnerConfig(config.runner) : null); + const runnerConfig = checked(() => { + if (config.demo === true || config.runner === undefined) return null; + // The PR's base branch too (#103); a missing github block is refused by the runner's setup, with its own message. + if (config.github) baseBranch(config.github); + return parseRunnerConfig(config.runner); + }); // Decision 1: one runner per database, held as an OS lock keyed by the database file's device and inode. const lock = checked(() => acquireRunnerLock(config.database)); const runnerSetup: RunnerSetup | undefined = runnerConfig ? async (service, capability) => { @@ -73,6 +78,8 @@ if (values.help || (!values.demo && !values.config && values['release-preparatio // The database path changed: a refusal the person acts on, so its message, not a stack. refuse(error); } + // Only now, under a lock verified to name the database: a publish an earlier process owed (#103). + if (!stopping) app.publishOwed(); const stop = () => void app.close().then(() => { lock.release(); process.exit(0); }, error => { lock.release(); console.error(error instanceof Error ? error.message : error); process.exit(1); }); // A second Ctrl+C does not skip shutdown (runner-lifecycle.md): agents are still being stopped and awaited. diff --git a/web/server.ts b/web/server.ts index d52f907a..fa14b5fd 100644 --- a/web/server.ts +++ b/web/server.ts @@ -8,6 +8,7 @@ import { GhMergeGateway, type MergeGateway } from '../github/merge.ts'; import { MERGE_OPERATION_TIMEOUT_MS, MergeCoordinator, MergeNotApplied, MergeOutcomeUnknown } from '../runner/merge.ts'; import { RunnerCoordinator, type RunnerDeps } from '../runner/coordinator.ts'; import { ItemExecutor } from '../runner/execution.ts'; +import { TaskPublishing } from '../runner/publishing.ts'; import type { RunnerAssembly } from '../runner/production.ts'; import type { ShutdownCapability } from '../runner/lifecycle.ts'; import { BadRequest, GuardRefusal, ShuttingDownError, assertUuidV4, isUuidV4, sameContext } from '../runner/lifecycle.ts'; @@ -39,6 +40,8 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge let questions: Questions, merges: MergeCoordinator | null, issues: IssueBoard, runner: RunnerCoordinator | null, suggestions: SuggestionCoordinator | null; /** Runs a task's plan items; one per Store, like the coordinator. Only the production runner has one. */ let executor: ItemExecutor | null = null; + /** Publishes the task's pull request (#103); only a production runner whose setup built a publisher has one. */ + let publishing: TaskPublishing | null = null; // Only coordinators' settlement and close code receive this; HTTP handlers never do. const capability = service.store.shutdownCapability(); try { @@ -68,6 +71,8 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge const assembly = await runnerSetup(service, capability); runner = new RunnerCoordinator(service.store, assembly.deps, undefined, capability); executor = new ItemExecutor(service.store, runner, assembly.sources, assembly.findings, { capability }); + // A demo never publishes, whatever its github block or an injected setup provides (setUpRunner refuses demos too). + if (assembly.publisher && !config.demo) { const coordinator = runner; publishing = new TaskPublishing(service.store, assembly.publisher(() => coordinator.closing), runner, executor, capability); } } catch (error) { service.close(); throw error; } } const loadReview=()=>{const view=service.load();return {...view,notes:view.notes.map(note=>({...note,answerActive:questions.isRunning(note.id)}))};}; @@ -96,6 +101,8 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge if (runner!.isActive(identity)) throw new GuardRefusal('An attempt is already active for this task.'); // Between two items no attempt is active, but the run that admits the next one is still going. if (executor.busy(identity)) throw new GuardRefusal('An earlier run of this task is still finishing; try again when that run has ended.'); + // A publish reads the task head and pushes it; a new item must not move it meanwhile. + if (publishing?.busy(identity)) throw new GuardRefusal('A pull request is being published for this task; try again when it has finished.'); const merge = service.store.getMergeAttempt(identity); if (merge && (merge.state === 'submitting' || merge.state === 'queued')) throw new GuardRefusal('A merge is in progress; wait for its outcome.'); // Only the view stops here: the action lets admission refuse, because its refusal also moves the idle task to needs @@ -145,15 +152,34 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge // One progress read per poll, shared by both flags. const progress = free && executor ? executor.progress(identity) : undefined; return { available: !!runner, task, attempts, startable: !!progress && offered('start', progress), resumable: !!progress && offered('resume', progress), - stateVersion: task.stateVersion, retryable, stopRequested: status.stopRequested, unresolved: status.unresolved }; + stateVersion: task.stateVersion, retryable, stopRequested: status.stopRequested, unresolved: status.unresolved, publish: publishView(progress) }; }; + /** The task's publishing (#103): in progress, offered (what the publish action would run), and the last outcome. */ + const publishView = (progress?: ReturnType) => { + if (!publishing) return { available: false, active: false, publishable: false, draft: false, last: null }; + let mode: { draft: boolean } | null = null; + try { mode = publishing.mode(identity, progress); } catch (error) { if (!(error instanceof GuardRefusal) && !(error instanceof ShuttingDownError)) throw error; } + return { available: true, active: publishing.busy(identity), publishable: mode !== null, draft: mode?.draft ?? false, last: publishing.lastOutcome(identity) }; + }; + /** A run's outcome is in the task and attempt rows; once it ends, the task's status decides whether a publish is owed. */ + const afterRun = (outcome: Promise) => void outcome + .catch(error => console.error(`Runner run failed: ${JSON.stringify(error instanceof Error ? error.message : String(error))}`)) + .finally(() => publishing?.publishIfOwed(identity)); const runnerAction = (input: Record) => { const { action, attemptId, expectedStateVersion, actionId } = input; // Malformed requests are refused before userAction, so nothing is recorded under their action ID (HTTP 400). - if (!['cancel-attempt', 'retry', 'cancel-task', 'start', 'resume'].includes(action as string)) throw new BadRequest('Unsupported runner action.'); + if (!['cancel-attempt', 'retry', 'cancel-task', 'start', 'resume', 'publish'].includes(action as string)) throw new BadRequest('Unsupported runner action.'); if (!Number.isSafeInteger(expectedStateVersion)) throw new BadRequest('expectedStateVersion must be an integer.'); if (action === 'cancel-attempt' || action === 'retry') assertUuidV4(attemptId, 'Attempt ID'); - return service.store.userAction(identity, { actionId: actionId as string, kind: action as string, request: { attemptId, expectedStateVersion } }, () => { + // A refused start or resume can still move the task to needs human (an expired budget is committed with the refusal), + // and a task that moves there is owed a draft PR (#103). Only that move publishes: a person who pressed resume on a + // task already in needs human asked for no publish. + if (action === 'start' || action === 'resume') { + const before = service.store.getTask(identity).status; + try { return act(); } finally { if (service.store.getTask(identity).status !== before) publishing?.publishIfOwed(identity); } + } + return act(); + function act() { return service.store.userAction(identity, { actionId: actionId as string, kind: action as string, request: { attemptId, expectedStateVersion } }, () => { if (service.store.getTask(identity).stateVersion !== expectedStateVersion) throw new GuardRefusal('Stale task state. Reload before writing.'); if (action === 'cancel-task') return { outcome: runner ? runner.cancelTask(identity, expectedStateVersion as number, actionId as string) : service.store.cancelTask(identity, expectedStateVersion as number, actionId as string) }; if (!runner) throw new GuardRefusal(config.demo ? RUNNER_NOT_IN_DEMO : RUNNER_NOT_CONFIGURED); @@ -163,10 +189,17 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge if (choice.queue) service.store.transitionTask(identity, expectedStateVersion as number, 'queued'); const begun = executor!.begin(identity, { fromItem: choice.fromItem, claimRequeue: choice.claimRequeue }); // The run goes on after this request; its outcome is in the task and attempt rows. An error the run throws (storage, - // a missing snapshot) is only logged, and the next start or resume derives what is owed again. - void begun.outcome.catch(error => console.error(`Runner run failed: ${JSON.stringify(error instanceof Error ? error.message : String(error))}`)); + // a missing snapshot) is only logged, and the next start or resume derives what is owed again. Once it ends, the + // task's PR is published if the task is finished or needs a person (#103). + afterRun(begun.outcome); return begun.attemptId ? { outcome: 'started', attemptId: begun.attemptId, item: choice.fromItem } : { outcome: 'settled' }; } + if (action === 'publish') { + // Retries a publish that failed or was refused (#103); it runs after this action commits, in the background. + if (!publishing) throw new GuardRefusal(config.demo ? 'Demos never publish pull requests.' : 'This runner does not publish pull requests.'); + // Replaced by the publish's outcome once it settles (recordPublish), so a replay reports that. + return { outcome: 'publishing', draft: publishing.request(identity, actionId as string).draft }; + } if (action === 'cancel-attempt') { if (!runner.stop(identity, attemptId as string, 'cancelled')) throw new GuardRefusal('That attempt is not running.'); return { outcome: 'stopping' }; @@ -177,7 +210,7 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge const retry = runner.retry(identity, attemptId as string, { expectedStateVersion: expectedStateVersion as number, kind: last.kind, item: last.item, expectedContext: service.store.currentContext(identity), deadline: Date.now() + 10 * 60_000 }); return { outcome: 'started', attemptId: retry.id }; - }).response; + }).response; } }; /** Handles of suggestion requests started by this process, removed once their outcome settles. */ const suggestionHandles = new Map(); @@ -331,12 +364,22 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge server.requestTimeout = 15000; await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', () => { server.removeListener('error', reject); resolve(); }); }).catch(error => { service.close(); throw error; }); const address = server.address(); if (!address || typeof address === 'string') throw new Error('Cannot determine local address.'); - return { server, service, token, runner, executor, url: `http://127.0.0.1:${address.port}/#${token}`, close: async () => { + return { server, service, token, runner, executor, publishing, + /** + * Startup (#103): a publish an earlier process owed (a lost opening, a run that ended before its publish completed) + * runs once, in the background; recovery finds a lost opening by its marker. The caller runs it once it has checked + * that the runner lock still names the database, since a publish pushes. + */ + publishOwed: () => publishing?.publishIfOwed(identity), + url: `http://127.0.0.1:${address.port}/#${token}`, close: async () => { // Step 1, one synchronous turn: reject new API requests and new runner work. Admitted requests drain (step 2). stopping = true; // Same turn as the admission flag: a request already reading its body must not start a new Ask worker. questions.stopAdmission(); runner?.rejectAdmission(); + // Publishes are aborted at once (the publisher already refuses every later push and opening, through the + // coordinator's flag) and awaited before the write gate closes, so their last records still land (#103). + const publishingClosed = publishing?.close().then(() => undefined, (error: unknown) => error); server.closeIdleConnections(); const closing = new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); let timer: ReturnType | undefined; @@ -347,6 +390,7 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge active.abort.abort(reason); if(active.readingBody)active.request.destroy(reason); } + const publishingFailure = await publishingClosed; // Step 3: after the drain, close the Store write gate. A request-path write still pending after the abort // (for example merge reconciliation after a GitHub await) now fails with 503; settling coordinators keep the capability. service.store.closeWrites(); @@ -355,7 +399,7 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge const issuesClosed=issues.close(); // Every step runs even when an earlier one fails: agents are still stopped, plan runs awaited and the Store closed // last. The first failure is reported; a later one never hides it. - const failures: unknown[] = []; + const failures: unknown[] = publishingFailure === undefined ? [] : [publishingFailure]; const step = async (run: () => Promise | unknown) => { try { await run(); } catch (error) { failures.push(error); } }; await step(() => merges?.close()); await step(() => issuesClosed); From 28de408e6814695b7630b286485ae829e38c53d8 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 19:19:38 -0700 Subject: [PATCH 02/15] F: test and document that a published head carries the branch's own commits (#113) A task's head is the user's branch (base..HEAD, recorded by the review before the runner's first commit) plus the runner's commits on top, so the published PR carries commits the ledger has no entry for. That is the design: they count as foreign, review shows their changes in the Unplanned row, and the merge gate blocks until each is assigned or accepted. A test publishes the demo task, whose last commit a person made, and checks the branch holds it, review shows its change as Unplanned, and the merge gate reports the unplanned blocker. The docs say what is published and that the gate still inspects github.pullRequest, not the published PR. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 3 ++- test/runner-publishing.test.ts | 22 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index 12c30ccd..f2119ff5 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -129,6 +129,7 @@ Some repositories do not support draft PRs (for example private repositories on `TaskPublishing` (`runner/publishing.ts`) owns publishing in production. `setUpRunner` builds the publisher with the real gateways (`GhAlreadyFixedGateway`, `GhPullRequestGateway`) and a `GitBranchPusher` that pushes from the runner-owned repository to `pushUrl(github.repository)` on `GH_HOST`, never to a configured URL. The pusher may overwrite only commits the task's ledger records as `owned`. Before #22 adds rebasing, nothing in production writes a `foreign` ledger entry (only `recordRebase` does), so codeboost pushes no head the ledger calls foreign. +- **What is published.** The whole task head, `base..head` (#113). Before the runner's first commit, the review records the user's checkout, so a task usually starts from the branch's own commits: a person made them, and the ledger has no entry for them, so they count as foreign (the design: a commit not in the ledger is foreign). The runner's commits come on top. All of them reach the `codeboost/…` branch and the PR, as the design intends. Review shows a foreign commit's changes in the Unplanned row, and the merge gate blocks (`unplanned`) until a person assigns or accepts each one. The gate inspects and merges the PR named in `github.pullRequest`, not yet the task's published PR, so this protects the published PR only when the two are the same; pointing the gate at the published PR is a follow-up. The push may overwrite only an owned commit already on the branch. Mapping commits through a rebase is #22's. - **Base branch.** `github.baseBranch` is required with a runner block. The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. - **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: "time budget ran out" when the budget has passed, else its current attempt's safety finding, else that attempt's diagnostic. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. - **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. @@ -138,7 +139,7 @@ Some repositories do not support draft PRs (for example private repositories on ## Tests -`test/runner-publishing.test.ts` covers the server wiring (#103) with a local bare repository as the remote and GitHub faked: a completed run opens one ready PR with the branch at the task head, a needs-human task gets a draft with its problems, a branch holding someone else's commit is refused and the `publish` action retries it (and its replay reports the outcome), a refused resume that moves the task to needs human publishes a draft while one on a task already there publishes nothing, start, resume and a second publish are refused while a publish runs, shutdown during the push or during the opening closes the Store only after the publish settled, a PR whose reply arrives just after the abort is still recorded before the write gate closes, and the next start publishes once or recovers the opening by its marker. Removing the startup publish, the after-run publish, the publish after a refused resume, the status-change check, the busy guard, the replay update or the shutdown wait, or moving that wait after the write gate, each fails a test. `test/already-fixed.test.ts` and `test/publish.test.ts` cover each row above. Each guard was also broken on purpose, and a test failed each time: own-PR and own-commit exclusion, repository-qualified exclusion, every bound, the ancestor check, the fail-closed catch, the issue-number boundary, the state-version and head re-read, the status check before the no-changes shortcut, the version-owned status change, the open-state check, the settle time, the branch hash, the environment allowlist, reuse of the earlier PR, lost-opening recovery, adoption on the main path and counting that PR as the task's own, drafting a PR that a task which cannot publish still has ready, adopting an abandoned opening's PR that appears after the task stopped, the abort checks on the no-changes path and in that draft step (a lookup that fails, one that answers, a draft change that fails and one that lands), recording a late PR by adoption alone, an abort during the head-mismatch draft change, the `closing` checks before a push and before a ready change, `close()` aborting a publish in progress, the repository check, the definite 422 refusal, a retargeted own PR and another base's PR from the branch, refusals matched on stderr and GitHub's `errors[].message` only, the configured base in recovery and the head settle, the draft step finding the task's PR in any base, closing-only cross-references, both sides of a manual link, the query text, the fence length, and the description bounds. +`test/runner-publishing.test.ts` covers the server wiring (#103) with a local bare repository as the remote and GitHub faked: a completed run opens one ready PR with the branch at the task head, a needs-human task gets a draft with its problems, a branch holding someone else's commit is refused and the `publish` action retries it (and its replay reports the outcome), a refused resume that moves the task to needs human publishes a draft while one on a task already there publishes nothing, start, resume and a second publish are refused while a publish runs, shutdown during the push or during the opening closes the Store only after the publish settled, a PR whose reply arrives just after the abort is still recorded before the write gate closes, and the next start publishes once or recovers the opening by its marker. Removing the startup publish, the after-run publish, the publish after a refused resume, the status-change check, the busy guard, the replay update or the shutdown wait, or moving that wait after the write gate, each fails a test. It also publishes a head that holds the branch's own (foreign) commit, and checks that review shows it as Unplanned and the merge gate blocks on it (#113); removing that blocker fails the test. `test/already-fixed.test.ts` and `test/publish.test.ts` cover each row above. Each guard was also broken on purpose, and a test failed each time: own-PR and own-commit exclusion, repository-qualified exclusion, every bound, the ancestor check, the fail-closed catch, the issue-number boundary, the state-version and head re-read, the status check before the no-changes shortcut, the version-owned status change, the open-state check, the settle time, the branch hash, the environment allowlist, reuse of the earlier PR, lost-opening recovery, adoption on the main path and counting that PR as the task's own, drafting a PR that a task which cannot publish still has ready, adopting an abandoned opening's PR that appears after the task stopped, the abort checks on the no-changes path and in that draft step (a lookup that fails, one that answers, a draft change that fails and one that lands), recording a late PR by adoption alone, an abort during the head-mismatch draft change, the `closing` checks before a push and before a ready change, `close()` aborting a publish in progress, the repository check, the definite 422 refusal, a retargeted own PR and another base's PR from the branch, refusals matched on stderr and GitHub's `errors[].message` only, the configured base in recovery and the head settle, the draft step finding the task's PR in any base, closing-only cross-references, both sides of a manual link, the query text, the fence length, and the description bounds. ## Test this document with a reader diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 2fa27156..21c71d67 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -14,6 +14,8 @@ import { GitBranchPusher } from '../runner/branch-push.ts'; import { PullRequestPublisher } from '../runner/publish.ts'; import { ensureCommit, openRunnerRepository } from '../runner/runner-repository.ts'; import { baseBranch } from '../runner/production.ts'; +import { MergeCoordinator } from '../runner/merge.ts'; +import type { MergeGateway } from '../github/merge.ts'; import { PullRequestMisplaced, type OpenPullRequestInput, type PullRequestGateway } from '../github/pull-requests.ts'; import type { AlreadyFixedGateway } from '../github/already-fixed.ts'; import type { PlanIdentity } from '../core/identity.ts'; @@ -342,6 +344,26 @@ describe('publishing a finished task (#103)', () => { try { expect(store.taskPullRequests(first.identity)).toMatchObject([{ state: 'opened', number: 100 }]); } finally { store.close(); } }); + it('publishes the whole task head, the branch\'s own commits too, which review shows as Unplanned and the merge gate blocks (#113)', async () => { + const w = world(); + const { app, identity, store, branch, head } = await serve(w, { before: completeAll }); + await publishSettled(app, identity); + // The demo branch's last commit was made by a person, not by codeboost: the ledger does not record it. + const owned = store.getLedger(identity).filter(entry => entry.origin === 'owned').map(entry => entry.sha); + expect(owned).not.toContain(head); + // It is published as it is: the PR's branch is the whole head. + expect(w.github.head(branch)).toBe(head); + expect(w.github.prs).toEqual([expect.objectContaining({ open: true, draft: false })]); + // Review shows its change in the Unplanned row, and the merge gate blocks until a person assigns or accepts it. + const view = app.service.load(); + expect(view.segments.filter(segment => segment.row === 'Unplanned').map(segment => segment.path)).toContain('debug.log'); + const gateway: MergeGateway = { inspect: async () => ({ base: view.snapshot.base, head: view.snapshot.head, pullRequestState: 'OPEN', mergeable: 'MERGEABLE', + rulesKnown: true, atomicBaseGuard: true, mergeQueue: false, requiredChecks: [], alreadyFixed: 'clear' }), merge: async () => { throw new Error('not merged here'); } }; + const status = await new MergeCoordinator(app.service, gateway).status(view); + expect(status).toMatchObject({ ready: false }); + expect(status.blockers).toContainEqual(expect.objectContaining({ code: 'unplanned' })); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); From cfb38a9d8a078a5329b79ffeb37cbdc99412e594 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 19:47:58 -0700 Subject: [PATCH 03/15] F: review fixes for publishing (#103): finding before budget, one gh env, CLI test - A draft PR's problems name the current attempt's safety finding before a spent budget, so a budget that ran out later cannot hide the violation. - The already-fixed and PR gateways take the setup's env, the same as the push, so GH_HOST and credentials agree; recorded errors are redacted with that env's token values (RunnerAssembly.env, TaskPublishing). - A CLI test shows the startup publish runs after the lock is verified. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 4 +-- github/already-fixed.ts | 8 ++++-- github/pull-requests.ts | 5 ++-- runner/production.ts | 10 +++++-- runner/publishing.ts | 22 +++++++++------ test/cli.test.ts | 28 ++++++++++++++++++- test/publish.test.ts | 12 +++++++- test/runner-publishing.test.ts | 31 +++++++++++++++++++-- web/server.ts | 2 +- 9 files changed, 99 insertions(+), 23 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index 12c30ccd..b8dee59f 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -129,8 +129,8 @@ Some repositories do not support draft PRs (for example private repositories on `TaskPublishing` (`runner/publishing.ts`) owns publishing in production. `setUpRunner` builds the publisher with the real gateways (`GhAlreadyFixedGateway`, `GhPullRequestGateway`) and a `GitBranchPusher` that pushes from the runner-owned repository to `pushUrl(github.repository)` on `GH_HOST`, never to a configured URL. The pusher may overwrite only commits the task's ledger records as `owned`. Before #22 adds rebasing, nothing in production writes a `foreign` ledger entry (only `recordRebase` does), so codeboost pushes no head the ledger calls foreign. -- **Base branch.** `github.baseBranch` is required with a runner block. The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. -- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: "time budget ran out" when the budget has passed, else its current attempt's safety finding, else that attempt's diagnostic. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. +- **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. +- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. - **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. - **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. diff --git a/github/already-fixed.ts b/github/already-fixed.ts index 35ec68b6..ee882239 100644 --- a/github/already-fixed.ts +++ b/github/already-fixed.ts @@ -100,7 +100,11 @@ export function mentionsIssue(message: string, repository: string, issue: number || new RegExp(`(? runWithInput('gh', args, { timeout: 30_000, maxBuffer: 8 * 1024 * 1024, signal: options?.signal, env: ghEnvironment(), killGraceMs: CHECK_KILL_GRACE_MS, pipeGraceMs: CHECK_PIPE_GRACE_MS })); + this.run = run ?? ((args, options) => runWithInput('gh', args, { timeout: 30_000, maxBuffer: 8 * 1024 * 1024, signal: options?.signal, env: ghEnvironment(config.env), killGraceMs: CHECK_KILL_GRACE_MS, pipeGraceMs: CHECK_PIPE_GRACE_MS })); } async #json(args: readonly string[], signal?: AbortSignal): Promise { diff --git a/github/pull-requests.ts b/github/pull-requests.ts index 20a38ec0..264e87ff 100644 --- a/github/pull-requests.ts +++ b/github/pull-requests.ts @@ -110,12 +110,13 @@ export class GhPullRequestGateway implements PullRequestGateway { readonly run: RunGhWithInput; /** One deadline for a whole operation (every command and poll wait in it), not a fresh allowance per command. */ readonly operationMs: number; - constructor(config: { repository: string; operationMs?: number }, run?: RunGhWithInput) { + /** `env`: the environment `gh` reads its allowlisted variables (GH_HOST, tokens) from. Default `process.env`. */ + constructor(config: { repository: string; operationMs?: number; env?: NodeJS.ProcessEnv }, run?: RunGhWithInput) { if (!REPOSITORY.test(config.repository)) throw new Error('A GitHub repository is required to open pull requests.'); if (config.operationMs !== undefined && (!Number.isSafeInteger(config.operationMs) || config.operationMs < 1)) throw new Error('Invalid operation deadline.'); this.repository = config.repository; this.operationMs = config.operationMs ?? PR_OPERATION_DEADLINE_MS; - this.run = run ?? ((args, options) => runWithInput('gh', args, { input: options?.input, timeout: 30_000, maxBuffer: 8 * 1024 * 1024, signal: options?.signal, env: ghEnvironment() })); + this.run = run ?? ((args, options) => runWithInput('gh', args, { input: options?.input, timeout: 30_000, maxBuffer: 8 * 1024 * 1024, signal: options?.signal, env: ghEnvironment(config.env) })); } /** The caller's signal combined with this operation's single deadline; every command and wait in it uses the result. */ diff --git a/runner/production.ts b/runner/production.ts index e8a6e850..88c7049b 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -156,6 +156,8 @@ export interface RunnerAssembly { * irreversible GitHub change. Absent: nothing is published (tests that only run items). */ readonly publisher?: (closing: () => boolean) => PullRequestPublisher; + /** The environment the publisher's gh calls and push run with; its token values are removed from recorded errors. */ + readonly env?: NodeJS.ProcessEnv; readonly sources: ExecutionSources; readonly findings: SafetyFindings; readonly recovery: RecoveryReport; @@ -232,7 +234,9 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd // ledger has no other kind of commit codeboost pushes (only recordRebase writes foreign entries). const pusher = new GitBranchPusher({ repository, repositoryId: identity.repositoryId, remote: github.repository, env: o.env as NodeJS.ProcessEnv, ownedCommits: requested => service.store.getLedger(requested).filter(entry => entry.origin === 'owned').map(entry => entry.sha) }); - const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: new GhAlreadyFixedGateway({ repository: github.repository }), - pulls: new GhPullRequestGateway({ repository: github.repository }), pusher, closing }, { repository: github.repository, baseBranch: base }); - return { deps, sources, findings, recovery, publisher }; + // The same environment as the push, so the PR calls go to the same GH_HOST with the same credentials. + const env = o.env as NodeJS.ProcessEnv; + const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: new GhAlreadyFixedGateway({ repository: github.repository, env }), + pulls: new GhPullRequestGateway({ repository: github.repository, env }), pusher, closing }, { repository: github.repository, baseBranch: base }); + return { deps, sources, findings, recovery, publisher, env }; } diff --git a/runner/publishing.ts b/runner/publishing.ts index fd2fffff..dec8fdf0 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -24,11 +24,15 @@ const REFUSALS = [GuardRefusal, BranchPushRefused, OpeningUnsettled, PullRequest export class TaskPublishing { #store: Store; #publisher: PullRequestPublisher; #runner: RunnerCoordinator; #executor: ItemExecutor; #write: (fn: () => T) => T; + /** Token values to remove from recorded error text: those of the environment the gh calls and the push run with. */ + #secrets: string[]; #closing = false; /** Publishes in progress, by task, from scheduling until the outcome is recorded. */ #running = new Map>(); - constructor(store: Store, publisher: PullRequestPublisher, runner: RunnerCoordinator, executor: ItemExecutor, capability?: ShutdownCapability) { + constructor(store: Store, publisher: PullRequestPublisher, runner: RunnerCoordinator, executor: ItemExecutor, capability?: ShutdownCapability, + env: NodeJS.ProcessEnv = process.env) { this.#store = store; this.#publisher = publisher; this.#runner = runner; this.#executor = executor; this.#write = settleWith(capability); + this.#secrets = TOKEN_VARIABLES.flatMap(name => env[name] ?? []); } /** Whether a publish of this task is in progress. */ @@ -85,7 +89,7 @@ export class TaskPublishing { /** A run of the task ended, or the server started: publish if one is owed. Never throws. */ publishIfOwed(identity: PlanIdentity): void { try { const owed = this.owed(identity); if (owed) this.#schedule(identity, owed.draft); } - catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error))}`); } + catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } } /** @@ -111,10 +115,10 @@ export class TaskPublishing { record = describe(outcome, draft); } catch (error) { const stopped = error instanceof ShuttingDownError || (this.#closing && (error as Error)?.name === 'AbortError'); - record = { outcome: stopped ? 'stopped' : REFUSALS.some(type => error instanceof type) ? 'refused' : 'failed', draft, message: message(error) }; + record = { outcome: stopped ? 'stopped' : REFUSALS.some(type => error instanceof type) ? 'refused' : 'failed', draft, message: message(error, this.#secrets) }; } try { this.#write(() => this.#store.recordPublish(identity, record, actionId)); } - catch (error) { console.error(`Could not record the publish outcome: ${JSON.stringify(message(error))}`); } + catch (error) { console.error(`Could not record the publish outcome: ${JSON.stringify(message(error, this.#secrets))}`); } }; // Starts once the caller's transaction (the action's userAction) has committed; a rollback starts nothing. this.#store.afterCommit(() => { @@ -123,14 +127,15 @@ export class TaskPublishing { } /** - * Why a needs-human task needs a person, for its draft PR, from what sent it there now: its spent budget, or its current - * attempt's safety finding or diagnostic. An earlier attempt's finding, which a person may have dealt with, is not used. + * Why a needs-human task needs a person, for its draft PR, from what sent it there now: its current attempt's safety + * finding first (a budget that also ran out since must not hide it), then its spent budget, then that attempt's + * diagnostic. An earlier attempt's finding, which a person may have dealt with, is not used. */ #problems(identity: PlanIdentity): string[] { const task = this.#store.getTask(identity); - if (task.budgetDeadline !== null && task.budgetDeadline <= Date.now()) return ['The task\'s time budget ran out before its plan finished.']; const current = task.currentAttemptId === null ? undefined : this.#store.getAttempt(identity, task.currentAttemptId); if (current?.safetyFinding) return [current.safetyFinding]; + if (task.budgetDeadline !== null && task.budgetDeadline <= Date.now()) return ['The task\'s time budget ran out before its plan finished.']; if (current?.diagnostic) return [current.diagnostic]; return ['The task needs a person.']; } @@ -159,5 +164,4 @@ function checkSummary(result: Extract * An error's text for the record and the runner view. A `gh` failure carries its output, which a server or proxy may have * echoed a token into: every configured token value and every GitHub token shape is removed, as for the push. */ -const message = (error: unknown) => redact(error instanceof Error ? error.message : String(error), - TOKEN_VARIABLES.flatMap(name => process.env[name] ?? [])).slice(0, 2000); +const message = (error: unknown, secrets: readonly string[]) => redact(error instanceof Error ? error.message : String(error), secrets).slice(0, 2000); diff --git a/test/cli.test.ts b/test/cli.test.ts index d752d4db..78137fb5 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -52,7 +52,7 @@ function review(github: Record = { baseBranch: 'main' }) { const { CLAUDE_CODE_OAUTH_TOKEN: _token, ...env } = process.env; return spawnSync(process.execPath, [cli, '--config', config, ...args], { encoding: 'utf8', env: { ...env, HOME: join(root, 'home') }, timeout: 20_000 }); }; - return { run, fakeDocker, start, database: demo.database }; + return { run, fakeDocker, start, database: demo.database, identity: demo.identity }; } it('refuses a runner startup without a token with its message, exit 1 and the lock released', () => { @@ -134,6 +134,32 @@ it('ignores a runner block in a demo configuration opened with --config, and ser } finally { child.kill('SIGKILL'); } }); +it('starts a publish the database owes once startup has verified the lock (#103)', async () => { + const { fakeDocker, start, database, identity } = review(), docker = fakeDocker(); + // A task an earlier process left in needs human with no draft PR yet: a draft publish is owed. + const store = new Store(database); + try { + store.transitionTask(identity, store.getTask(identity).stateVersion, 'queued'); + const attempt = store.admitAttempt(identity, { expectedStateVersion: store.getTask(identity).stateVersion, kind: 'execute', item: store.getPlan(identity).items[0]!.id, + expectedContext: store.currentContext(identity), deadline: Date.now() + 60_000 }); + store.markRunning(identity, attempt.id); + store.settleAttempt(identity, attempt.id, { firstReason: null, exitCode: 1, valid: false }); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'needs human'); + } finally { store.close(); } + // A gh that records each call and fails, so nothing reaches GitHub. + const bin = docker.path.split(':')[0]!, calls = join(bin, 'gh-calls'); + writeFileSync(join(bin, 'gh'), `#!/bin/sh\necho "$@" >> '${calls}'\nexit 1\n`); chmodSync(join(bin, 'gh'), 0o755); + docker.release(); + const cli = start(docker); + try { + await cli.until(() => cli.out().includes('Review ready'), 'the server to open'); + await cli.until(() => existsSync(calls), 'the startup publish to call gh'); + cli.child.kill('SIGINT'); + expect(await cli.exited).toBe(0); + const reopened = new Store(database); + try { expect(reopened.lastPublish(identity)).toMatchObject({ outcome: 'failed', draft: true }); } finally { reopened.close(); } + } finally { cli.child.kill('SIGKILL'); } +}); it('refuses with a message, not a stack, when the database path changes during a runner startup', async () => { const { fakeDocker, start, database } = review(), docker = fakeDocker(); const cli = start(docker); diff --git a/test/publish.test.ts b/test/publish.test.ts index efd15bc8..177ac8e7 100644 --- a/test/publish.test.ts +++ b/test/publish.test.ts @@ -1,4 +1,6 @@ -import { existsSync } from 'node:fs'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { Store } from '../runner/store.ts'; import { GuardRefusal, ShuttingDownError } from '../runner/lifecycle.ts'; @@ -1351,6 +1353,14 @@ describe('GitHub PR adapter', () => { expect(await new GhPullRequestGateway({ repository: 'owner/repo' }, async () => JSON.stringify([response({ body: `${marker}\nplan quoting ${other}` })])) .findOpened({ ...input, markers: [other, marker] })).toMatchObject({ marker }); }); + it('runs gh with the environment it was given, so its GH_HOST matches the push\'s', async () => { + const dir = mkdtempSync(join(tmpdir(), 'codeboost-gh-')); + try { + writeFileSync(join(dir, 'gh'), '#!/bin/sh\nprintf \'{"number":7,"state":"open","body":"","head":{"ref":"x"},"base":{"ref":"%s"}}\' "$GH_HOST"\n', { mode: 0o755 }); + const gh = new GhPullRequestGateway({ repository: 'owner/repo', env: { PATH: `${dir}:${process.env.PATH}`, GH_HOST: 'ghe.example.com' } }); + expect((await gh.readPull(7)).base).toBe('ghe.example.com'); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); it('fails a draft opening that GitHub created as ready, so the opening stays owned for recovery', async () => { const gh = new GhPullRequestGateway({ repository: 'owner/repo' }, async () => JSON.stringify(response({ draft: false }))); await expect(gh.open({ ...input, draft: true })).rejects.toThrow(/as ready, not as a draft/); diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 2fa27156..5473fa4f 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -121,7 +121,7 @@ function world(): World { * real publisher and the real pusher into `w.remote`, with GitHub faked. `before` shapes the Store before the runner * exists, as an earlier process would have left it. */ -async function serve(w: World, options: { before?: (service: ReviewService) => void; onPushSpawn?: (n: number, app: () => App, close: () => Promise) => void; demo?: boolean; startup?: boolean } = {}) { +async function serve(w: World, options: { before?: (service: ReviewService) => void; onPushSpawn?: (n: number, app: () => App, close: () => Promise) => void; demo?: boolean; startup?: boolean; env?: NodeJS.ProcessEnv } = {}) { let app: App | undefined, spawns = 0, closing: Promise | undefined; const close = () => closing ??= app!.close(); let branchOf: (identity: PlanIdentity) => string = () => ''; @@ -141,7 +141,7 @@ async function serve(w: World, options: { before?: (service: ReviewService) => v onProcessGroup: () => options.onPushSpawn?.(++spawns, () => app!, close) }); const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: w.github.checks, pulls: w.github.pulls, pusher, closing }, { repository: REPO, baseBranch: 'main' }); branchOf = identity => publisher(() => false).branch(identity); - return { deps, sources, findings: new SafetyFindings(service.store), publisher, + return { deps, sources, findings: new SafetyFindings(service.store), publisher, ...(options.env ? { env: options.env } : {}), recovery: { finalized: [], requeue: [], removedDirectories: [], unknownEntries: [], unmatchedStorage: [], repairedMerges: [] } }; }); cleanups.push(close); @@ -281,6 +281,25 @@ describe('publishing a finished task (#103)', () => { expect(second.store.lastPublish(second.identity)).toMatchObject({ outcome: 'opened', number: 100 }); }); + it('names the current attempt\'s safety finding in the draft, even when the budget has also run out', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: service => { + const s = service.store, id = service.config.identity; + s.transitionTask(id, s.getTask(id).stateVersion, 'queued'); + const attempt = s.admitAttempt(id, { expectedStateVersion: s.getTask(id).stateVersion, kind: 'execute', item: s.getPlan(id).items[0]!.id, + expectedContext: s.currentContext(id), deadline: Date.now() + 60_000, budgetMs: 1 }); + s.markRunning(id, attempt.id); + s.recordSafetyFinding(id, attempt.id, 'P1 wrote to a path outside its workspace.'); + s.settleAttempt(id, attempt.id, { firstReason: null, exitCode: 0, valid: false }); + const until = Date.now() + 5; while (Date.now() < until) { /* let the budget pass */ } + } }); + expect(store.getTask(identity).status).toBe('needs human'); + await publishSettled(app, identity); + expect(w.github.prs).toHaveLength(1); + expect(w.github.prs[0]!.body).toContain('P1 wrote to a path outside its workspace.'); + expect(w.github.prs[0]!.body).not.toContain(BUDGET); + }); + it('publishes a draft when a resume is refused because the budget ran out, which moves the task to needs human', async () => { const w = world(); const { app, identity, store } = await serve(w, { before: service => { budgetSpent(service); } }); @@ -342,6 +361,14 @@ describe('publishing a finished task (#103)', () => { try { expect(store.taskPullRequests(first.identity)).toMatchObject([{ state: 'opened', number: 100 }]); } finally { store.close(); } }); + it('removes the publishing environment\'s token values from a recorded failure', async () => { + const w = world(), token = 'not-a-github-shaped-secret-4711'; + w.github.onOpen = async () => { throw new Error(`gh failed: Authorization: token ${token}`); }; + const { app, identity, store } = await serve(w, { before: completeAll, env: { GH_ENTERPRISE_TOKEN: token } }); + await publishSettled(app, identity); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed', message: 'gh failed: Authorization: token [token]' }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); diff --git a/web/server.ts b/web/server.ts index fa14b5fd..89a52918 100644 --- a/web/server.ts +++ b/web/server.ts @@ -72,7 +72,7 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge runner = new RunnerCoordinator(service.store, assembly.deps, undefined, capability); executor = new ItemExecutor(service.store, runner, assembly.sources, assembly.findings, { capability }); // A demo never publishes, whatever its github block or an injected setup provides (setUpRunner refuses demos too). - if (assembly.publisher && !config.demo) { const coordinator = runner; publishing = new TaskPublishing(service.store, assembly.publisher(() => coordinator.closing), runner, executor, capability); } + if (assembly.publisher && !config.demo) { const coordinator = runner; publishing = new TaskPublishing(service.store, assembly.publisher(() => coordinator.closing), runner, executor, capability, assembly.env); } } catch (error) { service.close(); throw error; } } const loadReview=()=>{const view=service.load();return {...view,notes:view.notes.map(note=>({...note,answerActive:questions.isRunning(note.id)}))};}; From b2fe79628c439270014b55584f1cec18e5d827fb Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 22:26:53 -0700 Subject: [PATCH 04/15] F: settle a publish action whose process stopped before its outcome (#103) The publish action's ID lived only in memory, so a crash after its 'publishing' reply committed and before the publish recorded an outcome left the replay saying 'publishing' for ever. A reply that still says 'publishing' now counts as unsettled: any later outcome of the task settles it (one publish runs per task), and startup settles it as 'stopped' (interrupted) when no publish is owed. Copilot review on PR #110. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 2 +- runner/publishing.ts | 15 +++++++++++ runner/store.ts | 12 +++++++-- test/runner-publishing.test.ts | 30 +++++++++++++++++++++ web/server.ts | 2 +- 5 files changed, 57 insertions(+), 4 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index b8dee59f..762b2ed3 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -132,7 +132,7 @@ Some repositories do not support draft PRs (for example private repositories on - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. - **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. - **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. -- **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. +- **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. - **Demo.** A demo never publishes, even with a `github` block or an injected publisher. diff --git a/runner/publishing.ts b/runner/publishing.ts index dec8fdf0..97f44779 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -92,6 +92,21 @@ export class TaskPublishing { catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } } + /** + * Startup, once the runner lock is verified: publish if one is owed. Otherwise a publish action whose process stopped + * before its publish recorded an outcome (a crash) is settled now, so its replay stops saying `publishing`; the task's + * records (a PR it opened, recovered by its marker on the next publish) show what that publish did. Never throws. + */ + startup(identity: PlanIdentity): void { + try { + const owed = this.owed(identity); + if (owed) { this.#schedule(identity, owed.draft); return; } + if (this.#store.hasUnsettledPublishAction(identity)) + this.#write(() => this.#store.recordPublish(identity, { outcome: 'stopped', draft: false, + message: 'The publish was interrupted before it recorded an outcome (codeboost stopped). Its pull request, if it opened one, is in the task\'s records; the publish action runs it again.' })); + } catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } + } + /** * Shutdown: refuse new publishes, abort those in progress (the publisher refuses every later push, opening and ready * change) and await their recorded outcomes. diff --git a/runner/store.ts b/runner/store.ts index 13b06147..41894525 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -1315,11 +1315,19 @@ export class Store { const saved: PublishRecord = { ...record, message: record.message.slice(0, 2000), stateVersion: this.#task(key).state_version as number, at: new Date().toISOString() }; this.#run('INSERT INTO publish_outcomes (plan_key,data) VALUES (?,?) ON CONFLICT(plan_key) DO UPDATE SET data=excluded.data', key, encode(saved)); // The same transaction refreshes the publish action's replay, as for a merge, so a resent click reports this outcome. - if (actionId !== undefined) this.#run(`UPDATE user_actions SET response=? WHERE plan_key=? AND action_id=? AND kind='publish' AND json_extract(response,'$.ok')=1`, - encode({ ok: true, value: publishActionResponse(saved) }), key, actionId); + // A reply still saying `publishing` is an action whose publish never recorded an outcome: its process stopped (a + // crash) before it could. One publish runs per task, so this outcome is that action's continuation and settles it + // too; otherwise its replay would say `publishing` for ever. + this.#run(`UPDATE user_actions SET response=? WHERE plan_key=? AND kind='publish' AND json_extract(response,'$.ok')=1 + AND (action_id=? OR json_extract(response,'$.value.outcome')='publishing')`, encode({ ok: true, value: publishActionResponse(saved) }), key, actionId ?? null); return saved; }); } + /** Whether a publish action's saved reply still says `publishing`: its publish has not recorded an outcome yet. */ + hasUnsettledPublishAction(identity: PlanIdentity): boolean { + return !!this.#get(`SELECT 1 FROM user_actions WHERE plan_key=? AND kind='publish' AND json_extract(response,'$.ok')=1 + AND json_extract(response,'$.value.outcome')='publishing' LIMIT 1`, identityKey(identity)); + } lastPublish(identity: PlanIdentity): PublishRecord | null { const key = identityKey(identity); this.#task(key); const row = this.#get('SELECT data FROM publish_outcomes WHERE plan_key=?', key); diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 5473fa4f..ba88249b 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -369,6 +369,36 @@ describe('publishing a finished task (#103)', () => { expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed', message: 'gh failed: Authorization: token [token]' }); }); + describe('a publish action whose process stopped before its outcome was recorded', () => { + /** As the crashed process left it: the action committed its `publishing` reply, and its publish recorded nothing. */ + const plant = (service: ReviewService, actionId: string) => { + const s = service.store, id = service.config.identity, expectedStateVersion = s.getTask(id).stateVersion; + s.userAction(id, { actionId, kind: 'publish', request: { attemptId: undefined, expectedStateVersion } }, () => ({ outcome: 'publishing', draft: false })); + return expectedStateVersion; + }; + it('is settled by the startup publish when one is owed, so its replay reports that outcome', async () => { + const w = world(), actionId = randomUUID(); + let version = -1; + const { app, identity, store } = await serve(w, { before: service => { completeAll(service); version = plant(service, actionId); } }); + await publishSettled(app, identity); + expect(store.hasUnsettledPublishAction(identity)).toBe(false); + expect((await act(app, 'publish', actionId, version)).body.result).toMatchObject({ outcome: 'opened', draft: false, number: 100 }); + expect(w.github.prs).toHaveLength(1); + }); + it('is settled as interrupted at startup when nothing is owed, instead of replaying `publishing` for ever', async () => { + const w = world(), actionId = randomUUID(); + let version = -1; + const { app, identity, store } = await serve(w, { before: service => { + completeAll(service); version = plant(service, actionId); + // Meanwhile the task left what a publish serves (a person cancelled it), so nothing is owed at startup. + service.store.cancelTask(service.config.identity, service.store.getTask(service.config.identity).stateVersion, randomUUID()); + } }); + expect(store.hasUnsettledPublishAction(identity)).toBe(false); + expect((await act(app, 'publish', actionId, version)).body.result).toMatchObject({ outcome: 'stopped', message: expect.stringMatching(/interrupted before it recorded an outcome/) }); + expect(w.github.calls).toEqual([]); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); diff --git a/web/server.ts b/web/server.ts index e410e556..fee4e7a2 100644 --- a/web/server.ts +++ b/web/server.ts @@ -406,7 +406,7 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge * runs once, in the background; recovery finds a lost opening by its marker. The caller runs it once it has checked * that the runner lock still names the database, since a publish pushes. */ - publishOwed: () => publishing?.publishIfOwed(identity), + publishOwed: () => publishing?.startup(identity), url: `http://127.0.0.1:${address.port}/#${token}`, close: async () => { // Step 1, one synchronous turn: reject new API requests and new runner work. Admitted requests drain (step 2). stopping = true; From b35a2a03642fd9f603c28df94b0d0c266c7d684c Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 22:51:13 -0700 Subject: [PATCH 05/15] F: record every publish as in flight before it starts; settle a leftover record at startup (#103) An automatic publish (after a run, at startup) had no durable in-flight marker: a crash after its PR opened and moved the task to in review, but before its outcome was written, left publish.last empty or stale for good, since startup then saw nothing owed and no unsettled action. Every publish now records 'publishing' before its first external call (and does not run if that write fails); startup publishes again when one is owed and otherwise settles a leftover 'publishing' record as 'stopped'. Copilot review on PR #110 (previously missed finding). Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 2 +- runner/publishing.ts | 15 +++++--- test/runner-publishing.test.ts | 38 +++++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index 762b2ed3..23d066d2 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -132,7 +132,7 @@ Some repositories do not support draft PRs (for example private repositories on - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. - **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. - **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. -- **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. +- **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. - **Demo.** A demo never publishes, even with a `github` block or an injected publisher. diff --git a/runner/publishing.ts b/runner/publishing.ts index 97f44779..1772044b 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -11,6 +11,8 @@ import type { PublishRecord, Store } from './store.ts'; const SETTLED = ['opened', 'possibly already fixed', 'draft skipped', 'draft unsupported', 'no changes', // Recorded by the v10 upgrade for a task that reached its status before publishing existed: only the action publishes it. 'not published']; +/** The record a publish writes before it starts; one left at startup is a publish whose process stopped (#103). */ +const IN_FLIGHT = 'publishing'; /** Refusals a person acts on; anything else that fails is reported as `failed`. */ const REFUSALS = [GuardRefusal, BranchPushRefused, OpeningUnsettled, PullRequestMisplaced, PullRequestRefused]; @@ -93,15 +95,16 @@ export class TaskPublishing { } /** - * Startup, once the runner lock is verified: publish if one is owed. Otherwise a publish action whose process stopped - * before its publish recorded an outcome (a crash) is settled now, so its replay stops saying `publishing`; the task's - * records (a PR it opened, recovered by its marker on the next publish) show what that publish did. Never throws. + * Startup, once the runner lock is verified: publish if one is owed. Otherwise a publish whose process stopped before + * it recorded an outcome (a crash) is settled now: its in-flight record, and any action reply still saying + * `publishing`, become `stopped`; the task's records (a PR it opened, recovered by its marker on the next publish) + * show what that publish did. Never throws. */ startup(identity: PlanIdentity): void { try { const owed = this.owed(identity); if (owed) { this.#schedule(identity, owed.draft); return; } - if (this.#store.hasUnsettledPublishAction(identity)) + if (this.#store.hasUnsettledPublishAction(identity) || this.#store.lastPublish(identity)?.outcome === IN_FLIGHT) this.#write(() => this.#store.recordPublish(identity, { outcome: 'stopped', draft: false, message: 'The publish was interrupted before it recorded an outcome (codeboost stopped). Its pull request, if it opened one, is in the task\'s records; the publish action runs it again.' })); } catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } @@ -123,6 +126,10 @@ export class TaskPublishing { const reserved = Promise.withResolvers(); this.#running.set(key, reserved.promise); const run = async () => { + // Durable in-flight ownership before the first external write (AGENTS.md): if this process stops before the outcome + // is recorded, startup finds the marker and publishes again or settles it as interrupted. No marker, no publish. + try { this.#write(() => this.#store.recordPublish(identity, { outcome: IN_FLIGHT, draft, message: 'A pull request is being published.' })); } + catch (error) { console.error(`Could not record the publish as started, so it did not run: ${JSON.stringify(message(error, this.#secrets))}`); return; } let record: Omit; try { // The problems are read when the publish starts: the task is in needs human, and its last attempt says why. diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index ba88249b..172e8d4e 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -369,6 +369,44 @@ describe('publishing a finished task (#103)', () => { expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed', message: 'gh failed: Authorization: token [token]' }); }); + describe('a publish\'s in-flight record', () => { + it('is written before any GitHub call', async () => { + const w = world(); + let seen: unknown; + const opening = Promise.withResolvers(); + w.github.checks.check = async () => { opening.resolve(); return { outcome: 'clear', baseHead: 'b'.repeat(40) }; }; + const { app, identity, store } = await serve(w, { before: completeAll, startup: false }); + app.publishOwed(); + seen = store.lastPublish(identity)?.outcome; + await opening.promise; + expect(seen).toBe('publishing'); + await publishSettled(app, identity); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened' }); + }); + it('left by a process that stopped after the PR opened is settled as interrupted at startup', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: service => { + completeAll(service); + const s = service.store, id = service.config.identity; + // The crashed process recorded its publish as started; its outcome never landed, and nothing is owed now. + s.recordPublish(id, { outcome: 'publishing', draft: false, message: 'A pull request is being published.' }); + s.cancelTask(id, s.getTask(id).stateVersion, randomUUID()); + } }); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'stopped', message: expect.stringMatching(/interrupted before it recorded an outcome/) }); + expect(app.publishing!.busy(identity)).toBe(false); + expect(w.github.calls).toEqual([]); + }); + it('starts no publish when it cannot be written', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: completeAll, startup: false }); + vi.spyOn(store, 'recordPublish').mockImplementationOnce(() => { throw new Error('disk full'); }); + app.publishOwed(); + await app.publishing!.settled(identity); + expect(w.github.calls).toEqual([]); + expect(store.lastPublish(identity)).toBeNull(); + }); + }); + describe('a publish action whose process stopped before its outcome was recorded', () => { /** As the crashed process left it: the action committed its `publishing` reply, and its publish recorded nothing. */ const plant = (service: ReviewService, actionId: string) => { From 1439ccf02be270e4ce0fefed7d38763becc898c5 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 23:08:28 -0700 Subject: [PATCH 06/15] F: pay what an earlier run owes before publishing; check the github block before the lock (#103) - A task whose earlier run owes a safety escalation or a scope pause (its write failed, or the process stopped after the item settled) is not finished work, but every item has run, so no later run would pay it and a ready PR would show unreviewed out-of-scope changes. Publishing refuses it (mode), and a run's end, startup and a refused publish action have the executor pay it first (ItemExecutor.owes/payOwed). - The CLI refuses a runner with no github block before taking the lock, so the missing block is reported as itself, not as a lock conflict. Copilot review on PR #110. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 2 +- runner/execution.ts | 22 ++++++++++++++ runner/production.ts | 3 +- runner/publishing.ts | 13 +++++++- test/cli.test.ts | 17 ++++++++--- test/runner-publishing.test.ts | 33 +++++++++++++++++++++ web/cli.ts | 8 +++-- web/server.ts | 5 ++++ 8 files changed, 93 insertions(+), 10 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index 23d066d2..6068e93b 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -130,7 +130,7 @@ Some repositories do not support draft PRs (for example private repositories on `TaskPublishing` (`runner/publishing.ts`) owns publishing in production. `setUpRunner` builds the publisher with the real gateways (`GhAlreadyFixedGateway`, `GhPullRequestGateway`) and a `GitBranchPusher` that pushes from the runner-owned repository to `pushUrl(github.repository)` on `GH_HOST`, never to a configured URL. The pusher may overwrite only commits the task's ledger records as `owned`. Before #22 adds rebasing, nothing in production writes a `foreign` ledger entry (only `recordRebase` does), so codeboost pushes no head the ledger calls foreign. - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. -- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. +- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. - **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. - **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. diff --git a/runner/execution.ts b/runner/execution.ts index 49684a56..f6cadd5e 100644 --- a/runner/execution.ts +++ b/runner/execution.ts @@ -327,6 +327,28 @@ export class ItemExecutor { } /** Whether a run of this task is still in progress here (its last write may still be to come). */ busy(identity: PlanIdentity): boolean { return this.#inFlight.has(identityKey(identity)); } + /** + * Whether an earlier run left work owed that the next run pays before any item: a safety finding not yet acted on, or + * a scope pause never recorded (a failed write, the write gate, a crash). Reads only. + */ + owes(identity: PlanIdentity): boolean { + return this.#store.getAttempts(identity).some(earlier => this.#findings.get(earlier.id) !== undefined) || this.#unpausedScopeFinding(identity) !== null; + } + /** + * Pay what an earlier run owes, outside a run, as a run would before its first item (#103: a task that owes a pause or + * an escalation must not be published as finished). Null when nothing is owed, or a run of the task is in progress + * (it pays its own); otherwise the outcome: needs human, needs amendment, or stopped when it could not be written. + */ + payOwed(identity: PlanIdentity): ExecutionOutcome | null { + if (this.#runner.closing || this.#runner.isActive(identity) || this.busy(identity)) return null; + const stopped = (item: string, state: string, reason: string | null): ExecutionOutcome => ({ kind: 'stopped', item, state, reason, completed: [] }); + for (const earlier of this.#store.getAttempts(identity)) { + const finding = this.#findings.get(earlier.id); + if (finding) return this.#escalate(identity, earlier, finding, stopped, [], true); + } + const owed = this.#unpausedScopeFinding(identity); + return owed ? this.#pause(identity, owed.row, owed.result, stopped, [], true) : null; + } /** Held in #inFlight from its start to its return, so close() can await it. */ #track(key: string, run: Promise): Promise { this.#inFlight.set(key, run.catch(() => undefined)); diff --git a/runner/production.ts b/runner/production.ts index 88c7049b..c717a1fc 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -45,6 +45,7 @@ export const EXECUTE_STORAGE: TaskStorageLimits = Object.freeze({ const EXECUTE_SCHEMA = '{"$schema":"https://json-schema.org/draft/2020-12/schema","title":"codeboost execute summary","type":"string"}\n'; /** How long one read of the issue serves later plan items: the length of a short multi-item run. */ export const ISSUE_REUSE_MS = 5 * 60_000; +export const RUNNER_NEEDS_GITHUB = 'The runner reads the plan\'s issue from GitHub: add a github block to the review configuration.'; export const RUNNER_CREDENTIAL_MISSING = 'The runner runs Claude Code, which needs CLAUDE_CODE_OAUTH_TOKEN. Create one with `claude setup-token`, set it, and restart codeboost.'; /** Check the `runner` block of a review configuration before anything is created. */ @@ -174,7 +175,7 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd onSlowStart?: () => void }): Promise { const { service, config } = o, review = service.config; if (review.demo) throw new Error('Demos never run the runner.'); - if (!review.github) throw new Error('The runner reads the plan\'s issue from GitHub: add a github block to the review configuration.'); + if (!review.github) throw new Error(RUNNER_NEEDS_GITHUB); const base = baseBranch(review.github); // The push URL too (the repository's shape, GH_HOST), before recovery and the image build rather than after them. pushUrl(review.github.repository, o.env as NodeJS.ProcessEnv); diff --git a/runner/publishing.ts b/runner/publishing.ts index 1772044b..9ad0c472 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -52,6 +52,9 @@ export class TaskPublishing { if (this.busy(identity)) throw new GuardRefusal('A pull request is already being published for this task.'); // Publishing reads the task head, so it waits for the run (and the attempt) to end: assertPublishable refuses an active attempt. if (this.#runner.isActive(identity) || this.#executor.busy(identity)) throw new GuardRefusal('A run of this task is still in progress; publish once it has ended.'); + // An earlier run owes a safety escalation or a scope pause (its write failed, or the process stopped): the task is not + // finished work until that is settled, so a ready PR must not show its unreviewed changes. publishIfOwed pays it. + if (this.#executor.owes(identity)) throw new GuardRefusal('An earlier run left a safety finding or a scope pause that has not been acted on yet; it is settled before anything is published.'); const task = this.#store.getTask(identity); if (task.status === 'running') { const { next } = progress ?? this.#executor.progress(identity); @@ -90,7 +93,7 @@ export class TaskPublishing { /** A run of the task ended, or the server started: publish if one is owed. Never throws. */ publishIfOwed(identity: PlanIdentity): void { - try { const owed = this.owed(identity); if (owed) this.#schedule(identity, owed.draft); } + try { this.#payOwed(identity); const owed = this.owed(identity); if (owed) this.#schedule(identity, owed.draft); } catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } } @@ -102,6 +105,7 @@ export class TaskPublishing { */ startup(identity: PlanIdentity): void { try { + this.#payOwed(identity); const owed = this.owed(identity); if (owed) { this.#schedule(identity, owed.draft); return; } if (this.#store.hasUnsettledPublishAction(identity) || this.#store.lastPublish(identity)?.outcome === IN_FLIGHT) @@ -120,6 +124,13 @@ export class TaskPublishing { while (this.#running.size) await Promise.all([...this.#running.values()]); } + /** Settle what an earlier run owes before deciding what to publish (see mode); a task it moves is then not finished work. */ + #payOwed(identity: PlanIdentity): void { + if (this.#closing || this.#runner.closing) return; + const paid = this.#executor.payOwed(identity); + if (paid?.kind === 'stopped') console.error(`Could not settle what an earlier run owes: ${JSON.stringify(paid.reason ?? '')}`); + } + #schedule(identity: PlanIdentity, draft: boolean, actionId?: string): void { const key = identityKey(identity); // Reserved in this turn, so a second request in the same transaction is refused. diff --git a/test/cli.test.ts b/test/cli.test.ts index 78137fb5..ef0cccb4 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -1,6 +1,6 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { spawn, spawnSync } from 'node:child_process'; import { chmodSync, existsSync, renameSync, unlinkSync } from 'node:fs'; import { BASE_IMAGE, CLAUDE_VERSION, CODEX_VERSION } from '../agents/container/image.ts'; @@ -9,19 +9,20 @@ import { fileURLToPath } from 'node:url'; import { afterEach, expect, it, vi } from 'vitest'; import { createDemo } from '../scripts/demo.ts'; import { Store } from '../runner/store.ts'; -import { RUNNER_CREDENTIAL_MISSING } from '../runner/production.ts'; +import { acquireRunnerLock } from '../runner/recovery.ts'; +import { RUNNER_CREDENTIAL_MISSING, RUNNER_NEEDS_GITHUB } from '../runner/production.ts'; vi.setConfig({ testTimeout: 30_000 }); const cli = fileURLToPath(new URL('../web/cli.ts', import.meta.url)); const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); /** A non-demo review with a runner block, and a HOME of its own so the lock directory is the test's. */ -function review(github: Record = { baseBranch: 'main' }) { +function review(github: Record | null = { baseBranch: 'main' }) { const root = mkdtempSync(join(tmpdir(), 'codeboost-cli-')); roots.push(root); const demo = createDemo(join(root, 'demo')), store = new Store(demo.database), issue = store.getPlan(demo.identity).issue; store.close(); mkdirSync(join(root, 'runner'), { mode: 0o700 }); mkdirSync(join(root, 'home'), { mode: 0o700 }); const config = join(root, 'review.json'); - writeFileSync(config, JSON.stringify({ ...demo, demo: false, github: { repository: 'owner/repo', pullRequest: 1, issue, ...github }, + writeFileSync(config, JSON.stringify({ ...demo, demo: false, ...(github ? { github: { repository: 'owner/repo', pullRequest: 1, issue, ...github } } : {}), runner: { root: join(root, 'runner'), committer: { name: 'codeboost', email: 'runner@codeboost.invalid' } } })); /** * A `docker` that answers every call with nothing (no leftovers, an image that matches the pinned profile) once the hold @@ -71,6 +72,14 @@ it('refuses a runner without github.baseBranch before the lock and before the to // Not the missing-token refusal that the runner's setup would give after the lock: the CLI checked the block first. expect(result.stderr.trim()).toMatch(/^The runner publishes pull requests: add github\.baseBranch/); } + // No github block at all: refused as itself, before the lock, too. Another runner holds the lock meanwhile, so a check + // made after the lock would report the lock instead. + const { run, database } = review(null); + const held = acquireRunnerLock(database, { lockRoot: join(dirname(dirname(database)), 'home', '.codeboost', 'locks') }); + try { + const missing = run('--port', '0'); + expect([missing.status, missing.stderr.trim()]).toEqual([1, RUNNER_NEEDS_GITHUB]); + } finally { held.release(); } }); it('refuses a bad port and an unknown preparation with a message, never a stack', () => { const { run } = review(); diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 172e8d4e..7e41aa9a 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -86,6 +86,21 @@ function completeAll(service: ReviewService) { } expect(s.getTask(id).status).toBe('running'); } +/** + * Every plan item completed, but the last changed a file outside its plan item, and its pause for amendment was never + * recorded (the write failed, or the process stopped): the next run owes that pause. + */ +function completeAllOwingPause(service: ReviewService) { + const s = service.store, id = service.config.identity, head = s.getSnapshot(id).head, items = s.getPlan(id).items; + s.transitionTask(id, s.getTask(id).stateVersion, 'queued'); + for (const [index, item] of items.entries()) { + const attempt = s.admitAttempt(id, { expectedStateVersion: s.getTask(id).stateVersion, kind: 'execute', item: item.id, expectedContext: s.currentContext(id), deadline: Date.now() + 60_000 }); + s.markRunning(id, attempt.id); + s.settleAttempt(id, attempt.id, { firstReason: null, exitCode: 0, valid: true, + result: { head, unchanged: true, inScope: [], outOfScope: index === items.length - 1 ? ['run.sh'] : [] } }); + } + expect(s.getTask(id).status).toBe('running'); +} /** The first item failed with a 1 ms task budget, which has passed: the task is running, and its next admission is refused. */ function budgetSpent(service: ReviewService) { const s = service.store, id = service.config.identity; @@ -437,6 +452,24 @@ describe('publishing a finished task (#103)', () => { }); }); + it('pays a scope pause an earlier run owes instead of publishing its unreviewed changes as ready, at startup', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: completeAllOwingPause }); + await app.publishing!.settled(identity); + expect(store.getTask(identity).status).toBe('needs amendment'); + expect(w.github.calls).toEqual([]); + expect(store.lastPublish(identity)).toBeNull(); + }); + it('refuses the publish action while a scope pause is owed, and pays it', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: completeAllOwingPause, startup: false }); + expect((await view(app)).publish).toMatchObject({ publishable: false }); + expect(await act(app, 'publish')).toMatchObject({ status: 409, body: { error: expect.stringMatching(/scope pause that has not been acted on/) } }); + expect(store.getTask(identity).status).toBe('needs amendment'); + await app.publishing!.settled(identity); + expect(w.github.calls).toEqual([]); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); diff --git a/web/cli.ts b/web/cli.ts index 189b9f6e..086b25e9 100644 --- a/web/cli.ts +++ b/web/cli.ts @@ -6,7 +6,7 @@ import { productionPlanning } from './planning.ts'; import { createDemo } from '../scripts/demo.ts'; import { Store, requireSupportedNode } from '../runner/store.ts'; import { acquireRunnerLock, releasePreparation } from '../runner/recovery.ts'; -import { baseBranch, parseRunnerConfig, recoveryWarnings, setUpRunner } from '../runner/production.ts'; +import { RUNNER_NEEDS_GITHUB, baseBranch, parseRunnerConfig, recoveryWarnings, setUpRunner } from '../runner/production.ts'; requireSupportedNode(); /** A refusal the person acts on (bad input, a held lock, a blocked recovery): its message and exit 1, not a stack. */ function refuse(error: unknown): never { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } @@ -39,8 +39,10 @@ if (values.help || (!values.demo && !values.config && values['release-preparatio // Checked before the lock, so a malformed block changes nothing. Demos never run the runner, however they were opened. const runnerConfig = checked(() => { if (config.demo === true || config.runner === undefined) return null; - // The PR's base branch too (#103); a missing github block is refused by the runner's setup, with its own message. - if (config.github) baseBranch(config.github); + // The github block and the PR's base branch too (#103), before the lock: a missing block or base branch must be reported + // as itself, not as a lock conflict. + if (!config.github) throw new Error(RUNNER_NEEDS_GITHUB); + baseBranch(config.github); return parseRunnerConfig(config.runner); }); // Decision 1: one runner per database, held as an OS lock keyed by the database file's device and inode. diff --git a/web/server.ts b/web/server.ts index fee4e7a2..0246c087 100644 --- a/web/server.ts +++ b/web/server.ts @@ -194,6 +194,11 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge const before = service.store.getTask(identity).status; try { return act(); } finally { if (service.store.getTask(identity).status !== before) publishing?.publishIfOwed(identity); } } + // A publish refused because an earlier run owes a pause or an escalation pays it here, outside the refused transaction + // (AGENTS.md: a refusal's durable change is committed outside it); a task it sends to needs human then gets its draft. + if (action === 'publish') { + try { return act(); } catch (error) { if (error instanceof GuardRefusal) publishing?.publishIfOwed(identity); throw error; } + } return act(); function act() { return service.store.userAction(identity, { actionId: actionId as string, kind: action as string, request: { attemptId, expectedStateVersion } }, () => { if (service.store.getTask(identity).stateVersion !== expectedStateVersion) throw new GuardRefusal('Stale task state. Reload before writing.'); From f60ca06f4c82434cd39ee483be8418d26a85968b Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 23:26:19 -0700 Subject: [PATCH 07/15] F: fixes from an independent crash-point review of publishing (#103) - A ready publish needs every item of the current plan revision completed at that revision, checked in the Store's publish guard before every guarded write: a revision that adds an item during a publish's first GitHub lookups no longer gets a ready PR. - Only a publish action refused because of owed work pays it; a refusal for a stale view, a busy task or a replay starts nothing. - A task recorded as 'not published' at the v10 upgrade stays unpublished until a person asks, whatever changes since. - A failure while paying owed work no longer stops startup from settling an in-flight record or a stuck reply. - A publish refused while a lost opening settles is retried at that opening's deadline (settleRemaining). - Startup settles a reply that is the only thing stuck with the outcome on record, leaving that record (and its version stamp) as it is. - The owed-work scan runs only for a status that could publish, not on every poll. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 6 +- runner/publish.ts | 10 +++ runner/publishing.ts | 45 ++++++++++-- runner/store.ts | 15 ++++ test/publish.test.ts | 23 ++++++ test/runner-publishing.test.ts | 77 +++++++++++++++++++-- web/server.ts | 4 +- 7 files changed, 165 insertions(+), 15 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index 6068e93b..e7ba90de 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -130,9 +130,9 @@ Some repositories do not support draft PRs (for example private repositories on `TaskPublishing` (`runner/publishing.ts`) owns publishing in production. `setUpRunner` builds the publisher with the real gateways (`GhAlreadyFixedGateway`, `GhPullRequestGateway`) and a `GitBranchPusher` that pushes from the runner-owned repository to `pushUrl(github.repository)` on `GH_HOST`, never to a configured URL. The pusher may overwrite only commits the task's ledger records as `owned`. Before #22 adds rebasing, nothing in production writes a `foreign` ledger entry (only `recordRebase` does), so codeboost pushes no head the ledger calls foreign. - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. -- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. -- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. -- **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. +- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. +- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, whatever changes since. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish refused because a lost opening is still settling (`OpeningUnsettled`) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). +- **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`. When only an action reply is stuck (its publish never recorded itself as started), the reply reports the outcome on record, and that record is left as it is. A failure while paying owed work does not stop this; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. - **Demo.** A demo never publishes, even with a `github` block or an injected publisher. diff --git a/runner/publish.ts b/runner/publish.ts index 4b7319ff..e8d4700c 100644 --- a/runner/publish.ts +++ b/runner/publish.ts @@ -111,6 +111,16 @@ export class PullRequestPublisher { finally { inflight.delete(key); this.#running.delete(publish); } } + /** + * How long until the task's lost opening may be abandoned (0 when it may be now, null when there is none): its own + * deadline, its creation plus the settle time, on the clock recovery uses. + */ + settleRemaining(identity: PlanIdentity): number | null { + const lost = this.#store.taskPullRequests(identity).find(pr => pr.state === 'opening'); + if (!lost) return null; + return Math.max(0, Date.parse(lost.createdAt) + (this.#config.settleMs ?? DEFAULT_SETTLE_MS) - (this.#config.now ?? Date.now)()); + } + async #publish(identity: PlanIdentity, input: { problems?: readonly string[] }, signal?: AbortSignal): Promise { const draft = input.problems !== undefined; const recovered = await this.#recover(identity, draft, signal); diff --git a/runner/publishing.ts b/runner/publishing.ts index 9ad0c472..aa84311f 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -29,6 +29,8 @@ export class TaskPublishing { /** Token values to remove from recorded error text: those of the environment the gh calls and the push run with. */ #secrets: string[]; #closing = false; + /** Retries of a publish refused while a lost opening settles (OpeningUnsettled), cleared at shutdown. */ + #retries = new Set>(); /** Publishes in progress, by task, from scheduling until the outcome is recorded. */ #running = new Map>(); constructor(store: Store, publisher: PullRequestPublisher, runner: RunnerCoordinator, executor: ItemExecutor, capability?: ShutdownCapability, @@ -52,23 +54,31 @@ export class TaskPublishing { if (this.busy(identity)) throw new GuardRefusal('A pull request is already being published for this task.'); // Publishing reads the task head, so it waits for the run (and the attempt) to end: assertPublishable refuses an active attempt. if (this.#runner.isActive(identity) || this.#executor.busy(identity)) throw new GuardRefusal('A run of this task is still in progress; publish once it has ended.'); - // An earlier run owes a safety escalation or a scope pause (its write failed, or the process stopped): the task is not - // finished work until that is settled, so a ready PR must not show its unreviewed changes. publishIfOwed pays it. - if (this.#executor.owes(identity)) throw new GuardRefusal('An earlier run left a safety finding or a scope pause that has not been acted on yet; it is settled before anything is published.'); const task = this.#store.getTask(identity); if (task.status === 'running') { const { next } = progress ?? this.#executor.progress(identity); if (next !== null) throw new GuardRefusal(`${next} has not run yet; publish once every plan item has run.`); + this.#assertNothingOwed(identity); this.#store.assertPublishableNow(identity, false); return { draft: false }; } if (task.status === 'needs human') { + this.#assertNothingOwed(identity); this.#store.assertPublishableNow(identity, true); return { draft: true }; } throw new GuardRefusal(`The task is ${task.status}; a pull request is published for a running task whose plan items have all run, or a task that needs a person.`); } + /** + * An earlier run owes a safety escalation or a scope pause (its write failed, or the process stopped): the task is not + * finished work until that is settled, so a ready PR must not show its unreviewed changes. publishIfOwed pays it. Read + * only for a status that could publish, so a poll of any other task does not scan its attempts. + */ + #assertNothingOwed(identity: PlanIdentity): void { + if (this.#executor.owes(identity)) throw new GuardRefusal('An earlier run left a safety finding or a scope pause that has not been acted on yet; it is settled before anything is published.'); + } + /** * The publish owed, if any: the task can be published, and no publish has settled since the task last changed. Null * when nothing is owed. @@ -78,6 +88,9 @@ export class TaskPublishing { try { mode = this.mode(identity); } catch (error) { if (error instanceof GuardRefusal || error instanceof ShuttingDownError) return null; throw error; } const last = this.#store.lastPublish(identity); + // A task that reached its status before publishing existed is published only when a person asks (the v10 upgrade's + // record), whatever changes since. + if (last?.outcome === 'not published') return null; return !last || !SETTLED.includes(last.outcome) || last.stateVersion !== this.#store.getTask(identity).stateVersion ? mode : null; } @@ -108,9 +121,13 @@ export class TaskPublishing { this.#payOwed(identity); const owed = this.owed(identity); if (owed) { this.#schedule(identity, owed.draft); return; } - if (this.#store.hasUnsettledPublishAction(identity) || this.#store.lastPublish(identity)?.outcome === IN_FLIGHT) - this.#write(() => this.#store.recordPublish(identity, { outcome: 'stopped', draft: false, + const last = this.#store.lastPublish(identity); + if (!last || last.outcome === IN_FLIGHT) { + if (last || this.#store.hasUnsettledPublishAction(identity)) this.#write(() => this.#store.recordPublish(identity, { outcome: 'stopped', draft: false, message: 'The publish was interrupted before it recorded an outcome (codeboost stopped). Its pull request, if it opened one, is in the task\'s records; the publish action runs it again.' })); + // Only a reply is stuck (its publish never recorded itself as started): it reports the outcome on record, which is + // left as it is, stamp included, so what is owed does not change. + } else if (this.#store.hasUnsettledPublishAction(identity)) this.#write(() => this.#store.settleUnsettledPublishReplies(identity, last)); } catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } } @@ -120,6 +137,8 @@ export class TaskPublishing { */ async close(): Promise { this.#closing = true; + for (const timer of this.#retries) clearTimeout(timer); + this.#retries.clear(); await this.#publisher.close(); while (this.#running.size) await Promise.all([...this.#running.values()]); } @@ -127,8 +146,12 @@ export class TaskPublishing { /** Settle what an earlier run owes before deciding what to publish (see mode); a task it moves is then not finished work. */ #payOwed(identity: PlanIdentity): void { if (this.#closing || this.#runner.closing) return; - const paid = this.#executor.payOwed(identity); - if (paid?.kind === 'stopped') console.error(`Could not settle what an earlier run owes: ${JSON.stringify(paid.reason ?? '')}`); + // Its failure must not stop the caller from settling what it can (an in-flight record, a stuck reply): the task still + // owes the work, and mode() refuses to publish until it is paid. + try { + const paid = this.#executor.payOwed(identity); + if (paid?.kind === 'stopped') console.error(`Could not settle what an earlier run owes: ${JSON.stringify(paid.reason ?? '')}`); + } catch (error) { console.error(`Could not settle what an earlier run owes: ${JSON.stringify(message(error, this.#secrets))}`); } } #schedule(identity: PlanIdentity, draft: boolean, actionId?: string): void { @@ -147,6 +170,14 @@ export class TaskPublishing { const outcome = await this.#publisher.publish(identity, draft ? { problems: this.#problems(identity) } : {}); record = describe(outcome, draft); } catch (error) { + // Refused while a lost opening settles: nothing else would try again (no run ends, startup has passed), so it is + // tried again at that opening's own deadline, plus a second. + const remaining = error instanceof OpeningUnsettled && !this.#closing ? this.#publisher.settleRemaining(identity) : null; + if (remaining !== null) { + const timer = setTimeout(() => { this.#retries.delete(timer); this.publishIfOwed(identity); }, remaining + 1_000); + timer.unref?.(); + this.#retries.add(timer); + } const stopped = error instanceof ShuttingDownError || (this.#closing && (error as Error)?.name === 'AbortError'); record = { outcome: stopped ? 'stopped' : REFUSALS.some(type => error instanceof type) ? 'refused' : 'failed', draft, message: message(error, this.#secrets) }; } diff --git a/runner/store.ts b/runner/store.ts index 41894525..bc1c5a30 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -1141,6 +1141,16 @@ export class Store { // Interrupted work waiting to be requeued, or a rebase in progress, is not finished work to publish. if (task.requeue_pending === 1) throw new GuardRefusal('The task has interrupted work waiting to be requeued.'); if (task.rebase_in_progress !== null) throw new GuardRefusal('A rebase is in progress for this task.'); + // A ready PR is finished work: every item of the current plan revision has a completed execute attempt. Checked here, + // at every guarded publish write, so a plan revision that adds an item while a publish awaits GitHub refuses it. + if (!draft) { + const current = this.#current(key), revision = current.revision as number; + const items = decode(this.#get('SELECT data FROM revisions WHERE key=? AND revision=?', key, revision)!.data).items; + const finished = new Set(this.#db.prepare(`SELECT DISTINCT item FROM attempts WHERE plan_key=? AND kind='execute' AND item IS NOT NULL + AND state='completed' AND json_extract(context,'$.planRevision') = ?`).all(key, revision).map(entry => entry.item as string)); + const unrun = items.find(item => !finished.has(item.id)); + if (unrun) throw new GuardRefusal(`${unrun.id} has not run yet; publish once every plan item has run.`); + } } /** * Records a pre-PR check. A match, or a check that could not be completed, moves a running task to possibly already @@ -1323,6 +1333,11 @@ export class Store { return saved; }); } + /** Settle every publish action reply still saying `publishing` with `record`, the outcome on record, unchanged. */ + settleUnsettledPublishReplies(identity: PlanIdentity, record: PublishRecord): void { + this.#run(`UPDATE user_actions SET response=? WHERE plan_key=? AND kind='publish' AND json_extract(response,'$.ok')=1 + AND json_extract(response,'$.value.outcome')='publishing'`, encode({ ok: true, value: publishActionResponse(record) }), identityKey(identity)); + } /** Whether a publish action's saved reply still says `publishing`: its publish has not recorded an outcome yet. */ hasUnsettledPublishAction(identity: PlanIdentity): boolean { return !!this.#get(`SELECT 1 FROM user_actions WHERE plan_key=? AND kind='publish' AND json_extract(response,'$.ok')=1 diff --git a/test/publish.test.ts b/test/publish.test.ts index 177ac8e7..95268c1d 100644 --- a/test/publish.test.ts +++ b/test/publish.test.ts @@ -1232,6 +1232,29 @@ describe('gh subprocess environment', () => { }); }); +describe('a plan revision during a publish (#103 review)', () => { + it('refuses a ready publish when a revision adds an item while its first GitHub lookups run', async () => { + const store = runningTask(), live = new Map(), hidden = new Set(); + // A lost opening, so the next publish awaits GitHub (its recovery) before its guarded read. + await expect(harness(store, { live, openTimesOut: true }).publisher.publish(identity)).rejects.toThrow('timeout'); + for (const marker of live.keys()) hidden.add(marker); + let revised = false; + const { publisher, log } = harness(store, { live, hidden, config: { settleMs: 0 }, onFind: () => { + if (revised) return; + revised = true; + // A reviewer saves a revision that adds P2, which never ran. + const current = store.getPlan(identity); + store.importRevision(JSON.stringify({ ...current, revision: current.revision + 1, items: [...current.items, + { id: 'P2', title: 'Log it', intent: 'Log rejected input', files: [{ path: 'a.ts', kind: 'edit', renamed_from: null, change: 'Log' }], acceptance: [{ type: 'check', text: 'Logged' }], depends_on: [] }] }), + 'json', context, current.revision); + } }); + // The revision's items have not run at that revision (P1 ran at the old one), so the ready publish is refused. + await expect(publisher.publish(identity)).rejects.toThrow(/^P1 has not run yet; publish once every plan item has run\.$/); + expect(log.some(line => line.startsWith('push') || line.startsWith('open'))).toBe(false); + expect(store.getTask(identity).status).toBe('running'); + }); +}); + describe('GitHub PR adapter', () => { const marker = ''; const response = (over: Record = {}) => ({ number: 7, html_url: 'https://github.com/owner/repo/pull/7', state: 'open', draft: true, body: `${marker}\nplan`, diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 7e41aa9a..dc445e60 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -36,6 +36,8 @@ class FakeGitHub { prs: Pr[] = []; calls: string[] = []; onOpen?: (signal?: AbortSignal) => Promise; + /** PRs GitHub's list does not show yet (it lags behind them). */ + hidden = new Set(); readonly remote: string; constructor(remote: string) { this.remote = remote; } head(branch: string): string | null { @@ -55,13 +57,13 @@ class FakeGitHub { return this.#view(pr); }, findOpened: async input => { - const open = this.prs.filter(pr => pr.open && pr.headBranch === input.headBranch); + const open = this.prs.filter(pr => pr.open && !this.hidden.has(pr.number) && pr.headBranch === input.headBranch); const own = open.filter(pr => input.markers.includes(pr.marker)); if (own.length > 1 || (own[0] && own[0].base !== input.base)) throw new PullRequestMisplaced('The task\'s pull requests are not where it publishes.'); if (open.some(pr => pr.base === input.base && !input.markers.includes(pr.marker))) throw new Error('An open pull request exists that codeboost did not open.'); return own[0] ? { ...this.#view(own[0]), marker: own[0].marker } : null; }, - findOwned: async input => this.prs.filter(pr => pr.open && pr.headBranch === input.headBranch && input.markers.includes(pr.marker)) + findOwned: async input => this.prs.filter(pr => pr.open && !this.hidden.has(pr.number) && pr.headBranch === input.headBranch && input.markers.includes(pr.marker)) .map(pr => ({ ...this.#view(pr), marker: pr.marker, base: pr.base })), readPull: async number => { const pr = this.prs.find(candidate => candidate.number === number)!; return { open: pr.open, headBranch: pr.headBranch, base: pr.base, marker: pr.marker }; }, markDraft: async number => { const pr = this.prs.find(candidate => candidate.number === number)!; pr.draft = true; return this.#view(pr); }, @@ -136,7 +138,7 @@ function world(): World { * real publisher and the real pusher into `w.remote`, with GitHub faked. `before` shapes the Store before the runner * exists, as an earlier process would have left it. */ -async function serve(w: World, options: { before?: (service: ReviewService) => void; onPushSpawn?: (n: number, app: () => App, close: () => Promise) => void; demo?: boolean; startup?: boolean; env?: NodeJS.ProcessEnv } = {}) { +async function serve(w: World, options: { before?: (service: ReviewService) => void; onPushSpawn?: (n: number, app: () => App, close: () => Promise) => void; demo?: boolean; startup?: boolean; env?: NodeJS.ProcessEnv; settleMs?: number } = {}) { let app: App | undefined, spawns = 0, closing: Promise | undefined; const close = () => closing ??= app!.close(); let branchOf: (identity: PlanIdentity) => string = () => ''; @@ -154,7 +156,7 @@ async function serve(w: World, options: { before?: (service: ReviewService) => v const pusher = new GitBranchPusher({ repository, repositoryId: service.config.identity.repositoryId, remote: REPO, url: w.remote, ownedCommits: identity => service.store.getLedger(identity).filter(entry => entry.origin === 'owned').map(entry => entry.sha), onProcessGroup: () => options.onPushSpawn?.(++spawns, () => app!, close) }); - const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: w.github.checks, pulls: w.github.pulls, pusher, closing }, { repository: REPO, baseBranch: 'main' }); + const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: w.github.checks, pulls: w.github.pulls, pusher, closing }, { repository: REPO, baseBranch: 'main', ...(options.settleMs ? { settleMs: options.settleMs } : {}) }); branchOf = identity => publisher(() => false).branch(identity); return { deps, sources, findings: new SafetyFindings(service.store), publisher, ...(options.env ? { env: options.env } : {}), recovery: { finalized: [], requeue: [], removedDirectories: [], unknownEntries: [], unmatchedStorage: [], repairedMerges: [] } }; @@ -470,6 +472,73 @@ describe('publishing a finished task (#103)', () => { expect(w.github.calls).toEqual([]); }); + describe('round 1 of the independent review', () => { + it('starts nothing when the publish action is refused for a stale view, though a publish is owed', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: service => { + completeAll(service); + service.store.recordPublish(service.config.identity, { outcome: 'refused', draft: false, message: 'The branch moved.' }); + }, startup: false }); + const stale = store.getTask(identity).stateVersion - 1; + expect(await act(app, 'publish', randomUUID(), stale)).toMatchObject({ status: 409, body: { error: 'Stale task state. Reload before writing.' } }); + expect(app.publishing!.busy(identity)).toBe(false); + expect(w.github.calls).toEqual([]); + }); + it('keeps a task recorded as not published at the upgrade unpublished at startup, after unrelated changes', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: service => { + completeAll(service); + const s = service.store, id = service.config.identity; + s.recordPublish(id, { outcome: 'not published', draft: false, message: 'Before publishing existed.' }); + // An unrelated change since moves the task's state version. + s.transitionTask(id, s.getTask(id).stateVersion, 'needs human'); + } }); + await app.publishing!.settled(identity); + expect(w.github.calls).toEqual([]); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'not published' }); + }); + it('still settles a leftover in-flight record at startup when paying owed work fails', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { startup: false, before: service => { + const s = service.store, id = service.config.identity; + s.recordPublish(id, { outcome: 'publishing', draft: false, message: 'A pull request is being published.' }); + } }); + vi.spyOn(app.executor!, 'payOwed').mockImplementation(() => { throw new Error('The snapshot of P1\'s commit is missing.'); }); + app.publishOwed(); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'stopped' }); + }); + it('settles only a stuck reply with the outcome on record, leaving that record as it is', async () => { + const w = world(), actionId = randomUUID(); + const first = await serve(w, { before: completeAll }); + await publishSettled(first.app, first.identity); + const opened = first.store.lastPublish(first.identity)!; + expect(opened).toMatchObject({ outcome: 'opened', number: 100 }); + // A publish action committed its reply, and the process stopped before that publish recorded itself as started. + const version = first.store.getTask(first.identity).stateVersion; + first.store.userAction(first.identity, { actionId, kind: 'publish', request: { attemptId: undefined, expectedStateVersion: version } }, () => ({ outcome: 'publishing', draft: false })); + await first.close(); + const second = await serve(w); + expect(second.store.lastPublish(second.identity)).toEqual(opened); + expect((await act(second.app, 'publish', actionId, version)).body.result).toMatchObject({ outcome: 'opened', number: 100 }); + }); + it('retries a publish refused while a lost opening settles, at that opening\'s deadline', async () => { + const w = world(); + // The first publish's opening reply is lost: GitHub created the PR, its list does not show it yet. + w.github.onOpen = async () => { w.github.hidden.add(w.github.prs.at(-1)!.number); throw new Error('timed out'); }; + const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 2_500 }); + await publishSettled(app, identity); + expect(store.taskPullRequests(identity)).toMatchObject([{ state: 'opening' }]); + w.github.onOpen = undefined; + // Asked again at once: the opening is still settling, so this publish is refused. + await act(app, 'publish'); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'refused' }), { timeout: 5_000 }); + w.github.hidden.clear(); + // Retried by itself at the deadline: the lost opening's PR is recovered by its marker, not opened again. + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', number: 100 }), { timeout: 8_000, interval: 50 }); + expect(w.github.prs).toHaveLength(1); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); diff --git a/web/server.ts b/web/server.ts index 0246c087..4c48e628 100644 --- a/web/server.ts +++ b/web/server.ts @@ -196,8 +196,10 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge } // A publish refused because an earlier run owes a pause or an escalation pays it here, outside the refused transaction // (AGENTS.md: a refusal's durable change is committed outside it); a task it sends to needs human then gets its draft. + // Only a refusal that came from owed work: one for a stale view, a busy task or a replayed refusal starts nothing. if (action === 'publish') { - try { return act(); } catch (error) { if (error instanceof GuardRefusal) publishing?.publishIfOwed(identity); throw error; } + const owing = !!publishing && !!executor?.owes(identity); + try { return act(); } catch (error) { if (owing && error instanceof GuardRefusal) publishing?.publishIfOwed(identity); throw error; } } return act(); function act() { return service.store.userAction(identity, { actionId: actionId as string, kind: action as string, request: { attemptId, expectedStateVersion } }, () => { From 111b1402aa336865325854b3e113496cd58ec67d Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 23:38:15 -0700 Subject: [PATCH 08/15] F: fixes from round 2 of the independent review of publishing (#103) - A publish that ends with an opening still in flight (its own reply was lost, not only a refusal while one settles) is retried at the opening's deadline; a list-lag OpeningUnsettled is retried after 30 s. - 'not published' (v10) holds only until a run starts after the upgrade. - Only the publish action's own owed-work refusal pays that work; a stale request or any replay applies nothing. - A draft waits only for an owed safety finding, not an owed scope pause (which a needs-human task cannot record). - startup() leaves a publish of this process alone; the stopped record it writes keeps the interrupted publish's draft flag; the retry decision cannot throw out of run(). - owes() reads only owed findings in memory and completed out-of-scope attempts (Store.scopeFindingHeads), not every attempt, on a poll. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 4 +- runner/execution.ts | 11 +++- runner/publishing.ts | 50 +++++++++++------ runner/store.ts | 5 ++ test/runner-publishing.test.ts | 59 +++++++++++++++++++++ web/server.ts | 12 +++-- 6 files changed, 117 insertions(+), 24 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index e7ba90de..b0187b16 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -130,8 +130,8 @@ Some repositories do not support draft PRs (for example private repositories on `TaskPublishing` (`runner/publishing.ts`) owns publishing in production. `setUpRunner` builds the publisher with the real gateways (`GhAlreadyFixedGateway`, `GhPullRequestGateway`) and a `GitBranchPusher` that pushes from the runner-owned repository to `pushUrl(github.repository)` on `GH_HOST`, never to a configured URL. The pusher may overwrite only commits the task's ledger records as `owned`. Before #22 adds rebasing, nothing in production writes a `foreign` ledger entry (only `recordRebase` does), so codeboost pushes no head the ledger calls foreign. - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. -- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. -- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, whatever changes since. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish refused because a lost opening is still settling (`OpeningUnsettled`) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). +- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A draft waits only for an owed safety finding, not an owed scope pause: a draft invites no review, and a task in needs human cannot record a pause. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. +- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide). Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). One refused because GitHub's PR list lags behind a recorded PR is tried again after 30 seconds. - **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`. When only an action reply is stuck (its publish never recorded itself as started), the reply reports the outcome on record, and that record is left as it is. A failure while paying owed work does not stop this; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. - **Demo.** A demo never publishes, even with a `github` block or an injected publisher. diff --git a/runner/execution.ts b/runner/execution.ts index f6cadd5e..6126742e 100644 --- a/runner/execution.ts +++ b/runner/execution.ts @@ -95,6 +95,8 @@ export class SafetyFindings { /** Whether the finding is only in memory: the Store knows nothing of it, so the executor must act on it itself. */ unsaved(attemptId: string): boolean { return this.#unsaved.has(attemptId); } settle(attemptId: string): void { this.#unsaved.delete(attemptId); } + /** The attempts whose finding is owed (only in memory). Usually none, so a check over them costs nothing. */ + owedAttempts(): string[] { return [...this.#unsaved.keys()]; } } export interface ExecutionResult { head: string; unchanged: boolean; inScope: string[]; outOfScope: string[] } interface Private { workspace: WorkspaceRef; prompt: string; baseHead: string; linkSnapshot: DeclaredLinkSnapshot | undefined; @@ -331,8 +333,13 @@ export class ItemExecutor { * Whether an earlier run left work owed that the next run pays before any item: a safety finding not yet acted on, or * a scope pause never recorded (a failed write, the write gate, a crash). Reads only. */ - owes(identity: PlanIdentity): boolean { - return this.#store.getAttempts(identity).some(earlier => this.#findings.get(earlier.id) !== undefined) || this.#unpausedScopeFinding(identity) !== null; + owes(identity: PlanIdentity, options: { scope?: boolean } = {}): boolean { + // Cheap enough for a poll: owed findings are in memory (usually none), and only completed attempts that changed files + // outside their items are read, not every attempt with its result. + const key = identityKey(identity); + if (this.#findings.owedAttempts().some(id => this.#store.attemptOwner(id) === key)) return true; + if (options.scope === false) return false; + return this.#store.scopeFindingHeads(identity).some(head => !this.#store.checkpointAtHead(identity, head)); } /** * Pay what an earlier run owes, outside a run, as a run would before its first item (#103: a task that owes a pause or diff --git a/runner/publishing.ts b/runner/publishing.ts index aa84311f..f68bbcab 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -13,6 +13,10 @@ const SETTLED = ['opened', 'possibly already fixed', 'draft skipped', 'draft uns 'not published']; /** The record a publish writes before it starts; one left at startup is a publish whose process stopped (#103). */ const IN_FLIGHT = 'publishing'; +/** The refusal while an earlier run owes work; the publish action that gets it pays that work (web/server.ts). */ +export const OWED_REFUSAL = 'An earlier run left a safety finding or a scope pause that has not been acted on yet; it is settled before anything is published.'; +/** How long a publish refused because GitHub's PR list lags behind a PR waits before it is tried again. */ +export const LIST_LAG_RETRY_MS = 30_000; /** Refusals a person acts on; anything else that fails is reported as `failed`. */ const REFUSALS = [GuardRefusal, BranchPushRefused, OpeningUnsettled, PullRequestMisplaced, PullRequestRefused]; @@ -63,7 +67,9 @@ export class TaskPublishing { return { draft: false }; } if (task.status === 'needs human') { - this.#assertNothingOwed(identity); + // A draft invites no review, and a scope pause cannot be recorded for a task already in needs human, so only an owed + // safety finding (its escalation records it) holds a draft back. + this.#assertNothingOwed(identity, false); this.#store.assertPublishableNow(identity, true); return { draft: true }; } @@ -75,8 +81,8 @@ export class TaskPublishing { * finished work until that is settled, so a ready PR must not show its unreviewed changes. publishIfOwed pays it. Read * only for a status that could publish, so a poll of any other task does not scan its attempts. */ - #assertNothingOwed(identity: PlanIdentity): void { - if (this.#executor.owes(identity)) throw new GuardRefusal('An earlier run left a safety finding or a scope pause that has not been acted on yet; it is settled before anything is published.'); + #assertNothingOwed(identity: PlanIdentity, scope = true): void { + if (this.#executor.owes(identity, { scope })) throw new GuardRefusal(OWED_REFUSAL); } /** @@ -88,9 +94,9 @@ export class TaskPublishing { try { mode = this.mode(identity); } catch (error) { if (error instanceof GuardRefusal || error instanceof ShuttingDownError) return null; throw error; } const last = this.#store.lastPublish(identity); - // A task that reached its status before publishing existed is published only when a person asks (the v10 upgrade's - // record), whatever changes since. - if (last?.outcome === 'not published') return null; + // A task that reached its status before publishing existed (the v10 upgrade's record) is published only when a person + // asks, until a run starts after the upgrade: that run's end is the first state the upgrade did not decide. + if (last?.outcome === 'not published' && !this.#store.getAttempts(identity).some(attempt => attempt.createdAt > last.at)) return null; return !last || !SETTLED.includes(last.outcome) || last.stateVersion !== this.#store.getTask(identity).stateVersion ? mode : null; } @@ -117,13 +123,15 @@ export class TaskPublishing { * show what that publish did. Never throws. */ startup(identity: PlanIdentity): void { + // A publish of this process is running: its record is its own, not one a crash left. + if (this.busy(identity)) return; try { this.#payOwed(identity); const owed = this.owed(identity); if (owed) { this.#schedule(identity, owed.draft); return; } const last = this.#store.lastPublish(identity); if (!last || last.outcome === IN_FLIGHT) { - if (last || this.#store.hasUnsettledPublishAction(identity)) this.#write(() => this.#store.recordPublish(identity, { outcome: 'stopped', draft: false, + if (last || this.#store.hasUnsettledPublishAction(identity)) this.#write(() => this.#store.recordPublish(identity, { outcome: 'stopped', draft: last?.draft ?? false, message: 'The publish was interrupted before it recorded an outcome (codeboost stopped). Its pull request, if it opened one, is in the task\'s records; the publish action runs it again.' })); // Only a reply is stuck (its publish never recorded itself as started): it reports the outcome on record, which is // left as it is, stamp included, so what is owed does not change. @@ -154,6 +162,22 @@ export class TaskPublishing { } catch (error) { console.error(`Could not settle what an earlier run owes: ${JSON.stringify(message(error, this.#secrets))}`); } } + /** + * A publish that ended with an opening still in flight (its reply was lost, or it was refused while one settles) or + * refused because GitHub's list lags behind a PR is tried again by itself: nothing else would (no run ends, startup has + * passed). It waits for the opening's own deadline, or a short while for the list, plus a second. Never throws. + */ + #retryIfUnsettled(identity: PlanIdentity, unsettled: boolean): void { + try { + if (this.#closing) return; + const remaining = this.#publisher.settleRemaining(identity) ?? (unsettled ? LIST_LAG_RETRY_MS : null); + if (remaining === null) return; + const timer = setTimeout(() => { this.#retries.delete(timer); this.publishIfOwed(identity); }, remaining + 1_000); + timer.unref?.(); + this.#retries.add(timer); + } catch (error) { console.error(`Could not arrange a retry of the publish: ${JSON.stringify(message(error, this.#secrets))}`); } + } + #schedule(identity: PlanIdentity, draft: boolean, actionId?: string): void { const key = identityKey(identity); // Reserved in this turn, so a second request in the same transaction is refused. @@ -164,25 +188,19 @@ export class TaskPublishing { // is recorded, startup finds the marker and publishes again or settles it as interrupted. No marker, no publish. try { this.#write(() => this.#store.recordPublish(identity, { outcome: IN_FLIGHT, draft, message: 'A pull request is being published.' })); } catch (error) { console.error(`Could not record the publish as started, so it did not run: ${JSON.stringify(message(error, this.#secrets))}`); return; } - let record: Omit; + let record: Omit, unsettled = false; try { // The problems are read when the publish starts: the task is in needs human, and its last attempt says why. const outcome = await this.#publisher.publish(identity, draft ? { problems: this.#problems(identity) } : {}); record = describe(outcome, draft); } catch (error) { - // Refused while a lost opening settles: nothing else would try again (no run ends, startup has passed), so it is - // tried again at that opening's own deadline, plus a second. - const remaining = error instanceof OpeningUnsettled && !this.#closing ? this.#publisher.settleRemaining(identity) : null; - if (remaining !== null) { - const timer = setTimeout(() => { this.#retries.delete(timer); this.publishIfOwed(identity); }, remaining + 1_000); - timer.unref?.(); - this.#retries.add(timer); - } + unsettled = error instanceof OpeningUnsettled; const stopped = error instanceof ShuttingDownError || (this.#closing && (error as Error)?.name === 'AbortError'); record = { outcome: stopped ? 'stopped' : REFUSALS.some(type => error instanceof type) ? 'refused' : 'failed', draft, message: message(error, this.#secrets) }; } try { this.#write(() => this.#store.recordPublish(identity, record, actionId)); } catch (error) { console.error(`Could not record the publish outcome: ${JSON.stringify(message(error, this.#secrets))}`); } + this.#retryIfUnsettled(identity, unsettled); }; // Starts once the caller's transaction (the action's userAction) has committed; a rollback starts nothing. this.#store.afterCommit(() => { diff --git a/runner/store.ts b/runner/store.ts index bc1c5a30..bc39f55f 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -829,6 +829,11 @@ export class Store { AND state='completed' AND ${rev} = ?`).all(key, revision).map(entry => entry.item as string); return { started: row.started === 1, begun: row.begun === 1, earlierCommits: !!earlier, finished }; } + /** The heads of completed execute attempts that changed files outside their plan item, oldest first. */ + scopeFindingHeads(identity: PlanIdentity): string[] { + return this.#db.prepare(`SELECT json_extract(result,'$.head') AS head FROM attempts WHERE plan_key=? AND kind='execute' AND state='completed' + AND json_valid(result) AND json_array_length(json_extract(result,'$.outOfScope')) > 0 ORDER BY rowid`).all(identityKey(identity)).map(row => row.head as string); + } getAttempts(identity: PlanIdentity): AttemptRecord[] { return this.#db.prepare('SELECT * FROM attempts WHERE plan_key=? ORDER BY rowid').all(identityKey(identity)).map(row => this.#attemptRecord(row)); } diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index dc445e60..ba6be3d5 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -14,6 +14,7 @@ import { GitBranchPusher } from '../runner/branch-push.ts'; import { PullRequestPublisher } from '../runner/publish.ts'; import { ensureCommit, openRunnerRepository } from '../runner/runner-repository.ts'; import { baseBranch } from '../runner/production.ts'; +import { OWED_REFUSAL } from '../runner/publishing.ts'; import { PullRequestMisplaced, type OpenPullRequestInput, type PullRequestGateway } from '../github/pull-requests.ts'; import type { AlreadyFixedGateway } from '../github/already-fixed.ts'; import type { PlanIdentity } from '../core/identity.ts'; @@ -539,6 +540,64 @@ describe('publishing a finished task (#103)', () => { }); }); + describe('round 2 of the independent review', () => { + it('retries by itself a publish whose own opening reply was lost', async () => { + const w = world(); + // GitHub creates the PR, but its list does not show it yet, and the reply is lost. + w.github.onOpen = async () => { w.github.hidden.add(w.github.prs.at(-1)!.number); w.github.onOpen = undefined; throw new Error('timed out'); }; + const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 2_000 }); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed' }), { timeout: 5_000 }); + w.github.hidden.clear(); + // No action and no restart: the retry at the opening's deadline recovers it by its marker. + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', number: 100 }), { timeout: 8_000, interval: 50 }); + expect(w.github.prs).toHaveLength(1); + }); + it('publishes a task recorded as not published at the upgrade once a run has started after it', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: service => { + service.store.recordPublish(service.config.identity, { outcome: 'not published', draft: false, message: 'Before publishing existed.' }); + // A person resumed it after the upgrade, and that run completed the plan. + completeAll(service); + } }); + await publishSettled(app, identity); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened' }); + }); + it('pays owed work only for its own refusal: a stale or replayed publish action starts nothing', async () => { + const w = world(), actionId = randomUUID(); + const { app, identity, store } = await serve(w, { before: completeAllOwingPause, startup: false }); + const stale = store.getTask(identity).stateVersion - 1; + expect(await act(app, 'publish', actionId, stale)).toMatchObject({ status: 409, body: { error: 'Stale task state. Reload before writing.' } }); + expect(store.getTask(identity).status).toBe('running'); + // The replay of that saved refusal applies nothing either. + expect(await act(app, 'publish', actionId, stale)).toMatchObject({ status: 409 }); + expect(store.getTask(identity).status).toBe('running'); + expect(app.executor!.owes(identity)).toBe(true); + // Its own refusal for the owed pause pays it. + expect(await act(app, 'publish')).toMatchObject({ status: 409, body: { error: OWED_REFUSAL } }); + expect(store.getTask(identity).status).toBe('needs amendment'); + }); + it('publishes the draft of a needs-human task although a scope pause is owed', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: service => { + completeAllOwingPause(service); + const s = service.store, id = service.config.identity; + s.transitionTask(id, s.getTask(id).stateVersion, 'needs human'); + } }); + await publishSettled(app, identity); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }); + expect(w.github.prs).toEqual([expect.objectContaining({ draft: true })]); + }); + it('keeps an interrupted draft publish recorded as a draft when startup settles it', async () => { + const w = world(); + const { identity, store } = await serve(w, { before: service => { + const s = service.store, id = service.config.identity; + s.recordPublish(id, { outcome: 'publishing', draft: true, message: 'A pull request is being published.' }); + s.cancelTask(id, s.getTask(id).stateVersion, randomUUID()); + } }); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'stopped', draft: true }); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); diff --git a/web/server.ts b/web/server.ts index 4c48e628..9f53170d 100644 --- a/web/server.ts +++ b/web/server.ts @@ -8,7 +8,7 @@ import { GhMergeGateway, type MergeGateway } from '../github/merge.ts'; import { MERGE_OPERATION_TIMEOUT_MS, MergeCoordinator, MergeNotApplied, MergeOutcomeUnknown } from '../runner/merge.ts'; import { RunnerCoordinator, type RunnerDeps } from '../runner/coordinator.ts'; import { ItemExecutor } from '../runner/execution.ts'; -import { TaskPublishing } from '../runner/publishing.ts'; +import { OWED_REFUSAL, TaskPublishing } from '../runner/publishing.ts'; import type { RunnerAssembly } from '../runner/production.ts'; import type { ShutdownCapability } from '../runner/lifecycle.ts'; import { BadRequest, GuardRefusal, ShuttingDownError, assertUuidV4, isUuidV4, sameContext } from '../runner/lifecycle.ts'; @@ -196,10 +196,14 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge } // A publish refused because an earlier run owes a pause or an escalation pays it here, outside the refused transaction // (AGENTS.md: a refusal's durable change is committed outside it); a task it sends to needs human then gets its draft. - // Only a refusal that came from owed work: one for a stale view, a busy task or a replayed refusal starts nothing. + // Only this action's own refusal for owed work: one for a stale view or a busy task, and any replay (which applies + // nothing, AGENTS.md), starts nothing. if (action === 'publish') { - const owing = !!publishing && !!executor?.owes(identity); - try { return act(); } catch (error) { if (owing && error instanceof GuardRefusal) publishing?.publishIfOwed(identity); throw error; } + let replay: boolean; + try { replay = !!service.store.savedAction(identity, { actionId: actionId as string, kind: 'publish', request: { attemptId, expectedStateVersion } }); } + catch { replay = true; } + try { return act(); } + catch (error) { if (!replay && error instanceof GuardRefusal && error.message === OWED_REFUSAL) publishing?.publishIfOwed(identity); throw error; } } return act(); function act() { return service.store.userAction(identity, { actionId: actionId as string, kind: action as string, request: { attemptId, expectedStateVersion } }, () => { From 699a35182394bed1ca66ae55a917d15cec27bd96 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sat, 3 Oct 2026 23:49:07 -0700 Subject: [PATCH 09/15] F: fixes from round 3 of the independent review of publishing (#103) - Bound the automatic retry (AGENTS.md: back off recurring external polling): one timer per task; a lost opening is waited for only while its deadline is ahead, so one still stuck after it (two PRs carry its marker, the repository changed, GitHub keeps failing) no longer re-publishes every second; the list-lag retry runs once per chain. - A person's start or resume that moves an upgraded task (v10 'not published') ends that hold, so a refused resume that sends it to needs human gets its draft. - Owed work is paid only where a run would (running, queued) or a draft waits on it (needs human). - The 'not published' tests space their timestamps, which are compared. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 2 +- runner/publishing.ts | 44 ++++++++++++++------- test/runner-publishing.test.ts | 38 +++++++++++++++++- web/server.ts | 2 +- 4 files changed, 68 insertions(+), 18 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index b0187b16..4da07371 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -131,7 +131,7 @@ Some repositories do not support draft PRs (for example private repositories on - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. - **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A draft waits only for an owed safety finding, not an owed scope pause: a draft invites no review, and a task in needs human cannot record a pause. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. -- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide). Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). One refused because GitHub's PR list lags behind a recorded PR is tried again after 30 seconds. +- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide), or a person's `start` or `resume` moves the task. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). One refused because GitHub's PR list lags behind a recorded PR is tried again once, after 30 seconds. Retries are bounded: one timer per task, and an opening is waited for only while its deadline is ahead. One still in place after its deadline is stuck for a reason a person must fix (two PRs carry its marker, the repository changed, GitHub keeps failing), so the next publish waits for a run's end, a restart or the `publish` action. - **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`. When only an action reply is stuck (its publish never recorded itself as started), the reply reports the outcome on record, and that record is left as it is. A failure while paying owed work does not stop this; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. - **Demo.** A demo never publishes, even with a `github` block or an injected publisher. diff --git a/runner/publishing.ts b/runner/publishing.ts index f68bbcab..58ef665b 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -34,7 +34,9 @@ export class TaskPublishing { #secrets: string[]; #closing = false; /** Retries of a publish refused while a lost opening settles (OpeningUnsettled), cleared at shutdown. */ - #retries = new Set>(); + #retries = new Map>(); + /** Tasks whose list-lag retry has run since their last settled publish: it runs once, not every 30 s for ever. */ + #listLagRetried = new Set(); /** Publishes in progress, by task, from scheduling until the outcome is recorded. */ #running = new Map>(); constructor(store: Store, publisher: PullRequestPublisher, runner: RunnerCoordinator, executor: ItemExecutor, capability?: ShutdownCapability, @@ -89,14 +91,14 @@ export class TaskPublishing { * The publish owed, if any: the task can be published, and no publish has settled since the task last changed. Null * when nothing is owed. */ - owed(identity: PlanIdentity): { draft: boolean } | null { + owed(identity: PlanIdentity, options: { personAsked?: boolean } = {}): { draft: boolean } | null { let mode; try { mode = this.mode(identity); } catch (error) { if (error instanceof GuardRefusal || error instanceof ShuttingDownError) return null; throw error; } const last = this.#store.lastPublish(identity); // A task that reached its status before publishing existed (the v10 upgrade's record) is published only when a person // asks, until a run starts after the upgrade: that run's end is the first state the upgrade did not decide. - if (last?.outcome === 'not published' && !this.#store.getAttempts(identity).some(attempt => attempt.createdAt > last.at)) return null; + if (last?.outcome === 'not published' && !options.personAsked && !this.#store.getAttempts(identity).some(attempt => attempt.createdAt > last.at)) return null; return !last || !SETTLED.includes(last.outcome) || last.stateVersion !== this.#store.getTask(identity).stateVersion ? mode : null; } @@ -110,9 +112,12 @@ export class TaskPublishing { return mode; } - /** A run of the task ended, or the server started: publish if one is owed. Never throws. */ - publishIfOwed(identity: PlanIdentity): void { - try { this.#payOwed(identity); const owed = this.owed(identity); if (owed) this.#schedule(identity, owed.draft); } + /** + * A run of the task ended, the server started, or a person's start or resume moved the task (`personAsked`: it also ends + * the v10 upgrade's `not published` hold, as a run starting would): publish if one is owed. Never throws. + */ + publishIfOwed(identity: PlanIdentity, options: { personAsked?: boolean } = {}): void { + try { this.#payOwed(identity); const owed = this.owed(identity, options); if (owed) this.#schedule(identity, owed.draft); } catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } } @@ -145,7 +150,7 @@ export class TaskPublishing { */ async close(): Promise { this.#closing = true; - for (const timer of this.#retries) clearTimeout(timer); + for (const timer of this.#retries.values()) clearTimeout(timer); this.#retries.clear(); await this.#publisher.close(); while (this.#running.size) await Promise.all([...this.#running.values()]); @@ -154,6 +159,8 @@ export class TaskPublishing { /** Settle what an earlier run owes before deciding what to publish (see mode); a task it moves is then not finished work. */ #payOwed(identity: PlanIdentity): void { if (this.#closing || this.#runner.closing) return; + // Only where a run would pay it before its first item (a running or queued task), or where a draft waits on it. + if (!['running', 'queued', 'needs human'].includes(this.#store.getTask(identity).status)) return; // Its failure must not stop the caller from settling what it can (an in-flight record, a stuck reply): the task still // owes the work, and mode() refuses to publish until it is paid. try { @@ -165,16 +172,23 @@ export class TaskPublishing { /** * A publish that ended with an opening still in flight (its reply was lost, or it was refused while one settles) or * refused because GitHub's list lags behind a PR is tried again by itself: nothing else would (no run ends, startup has - * passed). It waits for the opening's own deadline, or a short while for the list, plus a second. Never throws. + * passed). Bounded (AGENTS.md: back off recurring external polling): one timer per task; an opening is waited for only + * while its deadline is ahead (at the deadline the next publish abandons or adopts it, so one that is still there + * after it is stuck for a reason a person must fix, not a timing one); the list is waited for once per chain, 30 s. + * Never throws. */ - #retryIfUnsettled(identity: PlanIdentity, unsettled: boolean): void { + #retryIfUnsettled(identity: PlanIdentity, record: Omit, unsettled: boolean): void { try { - if (this.#closing) return; - const remaining = this.#publisher.settleRemaining(identity) ?? (unsettled ? LIST_LAG_RETRY_MS : null); - if (remaining === null) return; - const timer = setTimeout(() => { this.#retries.delete(timer); this.publishIfOwed(identity); }, remaining + 1_000); + const key = identityKey(identity); + if (SETTLED.includes(record.outcome)) this.#listLagRetried.delete(key); + if (this.#closing || this.#retries.has(key)) return; + const remaining = this.#publisher.settleRemaining(identity); + let wait: number | null = remaining !== null && remaining > 0 ? remaining : null; + if (wait === null && remaining === null && unsettled && !this.#listLagRetried.has(key)) { this.#listLagRetried.add(key); wait = LIST_LAG_RETRY_MS; } + if (wait === null) return; + const timer = setTimeout(() => { this.#retries.delete(key); this.publishIfOwed(identity); }, wait + 1_000); timer.unref?.(); - this.#retries.add(timer); + this.#retries.set(key, timer); } catch (error) { console.error(`Could not arrange a retry of the publish: ${JSON.stringify(message(error, this.#secrets))}`); } } @@ -200,7 +214,7 @@ export class TaskPublishing { } try { this.#write(() => this.#store.recordPublish(identity, record, actionId)); } catch (error) { console.error(`Could not record the publish outcome: ${JSON.stringify(message(error, this.#secrets))}`); } - this.#retryIfUnsettled(identity, unsettled); + this.#retryIfUnsettled(identity, record, unsettled); }; // Starts once the caller's transaction (the action's userAction) has committed; a rollback starts nothing. this.#store.afterCommit(() => { diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index ba6be3d5..1157b19c 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -490,6 +490,8 @@ describe('publishing a finished task (#103)', () => { const { app, identity, store } = await serve(w, { before: service => { completeAll(service); const s = service.store, id = service.config.identity; + // The upgrade comes after those runs (a later millisecond: timestamps are compared). + const until = Date.now() + 3; while (Date.now() < until) { /* let the clock move */ } s.recordPublish(id, { outcome: 'not published', draft: false, message: 'Before publishing existed.' }); // An unrelated change since moves the task's state version. s.transitionTask(id, s.getTask(id).stateVersion, 'needs human'); @@ -556,7 +558,8 @@ describe('publishing a finished task (#103)', () => { const w = world(); const { app, identity, store } = await serve(w, { before: service => { service.store.recordPublish(service.config.identity, { outcome: 'not published', draft: false, message: 'Before publishing existed.' }); - // A person resumed it after the upgrade, and that run completed the plan. + // A person resumed it after the upgrade (a later millisecond: timestamps are compared), and that run completed the plan. + const until = Date.now() + 3; while (Date.now() < until) { /* let the clock move */ } completeAll(service); } }); await publishSettled(app, identity); @@ -598,6 +601,39 @@ describe('publishing a finished task (#103)', () => { }); }); + describe('round 3 of the independent review', () => { + it('does not retry for ever when a lost opening stays stuck past its deadline', async () => { + const w = world(); + // The opening's reply is lost (GitHub created the PR; its list does not show it yet). + w.github.onOpen = async () => { w.github.hidden.add(w.github.prs.at(-1)!.number); w.github.onOpen = undefined; throw new Error('timed out'); }; + const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 1_000, startup: false }); + const starts = vi.spyOn(store, 'recordPublish'); + app.publishOwed(); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed' }), { timeout: 5_000 }); + // Someone copied its description onto another PR from the branch: recovery cannot tell which is the task's, so the + // opening stays, and every publish of it is refused for a reason only a person can fix. + const pr = w.github.prs[0]!; + w.github.prs.push({ ...pr, number: 999, url: `https://github.com/${REPO}/pull/999`, base: 'release' }); + w.github.hidden.clear(); + await new Promise(resolve => setTimeout(resolve, 6_000)); + const publishes = starts.mock.calls.filter(call => call[1].outcome === 'publishing').length; + // The first publish and one retry at the opening's deadline; not one more every second. + expect(publishes).toBeLessThanOrEqual(2); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'refused' }); + }); + it('publishes the draft of an upgraded task when a person\'s refused resume moves it to needs human', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { startup: false, before: service => { + budgetSpent(service); + service.store.recordPublish(service.config.identity, { outcome: 'not published', draft: false, message: 'Before publishing existed.' }); + } }); + expect(await act(app, 'resume')).toMatchObject({ status: 409, body: { error: expect.stringMatching(/time budget/) } }); + expect(store.getTask(identity).status).toBe('needs human'); + await publishSettled(app, identity); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); diff --git a/web/server.ts b/web/server.ts index 9f53170d..fecc0c8a 100644 --- a/web/server.ts +++ b/web/server.ts @@ -192,7 +192,7 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge // task already in needs human asked for no publish. if (action === 'start' || action === 'resume') { const before = service.store.getTask(identity).status; - try { return act(); } finally { if (service.store.getTask(identity).status !== before) publishing?.publishIfOwed(identity); } + try { return act(); } finally { if (service.store.getTask(identity).status !== before) publishing?.publishIfOwed(identity, { personAsked: true }); } } // A publish refused because an earlier run owes a pause or an escalation pays it here, outside the refused transaction // (AGENTS.md: a refusal's durable change is committed outside it); a task it sends to needs human then gets its draft. From 4d692d80eacabff7195209161541248718464669 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sun, 4 Oct 2026 00:00:49 -0700 Subject: [PATCH 10/15] F: fixes from round 4 of the independent review of publishing (#103) - Retry budget per chain (until a publish settles): at most one retry at a lost opening's deadline and one short retry, so a GitHub outage that fails every open no longer republishes every settle time for ever. - An 'opened' outcome that left the PR or the task where the publish did not mean them (GitHub showed another head, so the PR was drafted and the task stayed running; a draft change failed) is recorded with `reconcile`: still owed, and given the short retry. - A refusal just as an opening's deadline passes gets the short retry. - The short retry's delay is a test seam (RunnerAssembly.shortRetryMs); tests now cover the list-lag retry, the outage bound and reconciling. - Tests that proved less than they claimed: the owed-work failure test now reaches the failure; the CLI baseBranch half holds the lock; the CLI startup test stops only after the outcome is recorded. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 2 +- runner/production.ts | 2 + runner/publishing.ts | 50 +++++++++++------ runner/store.ts | 2 + test/cli.test.ts | 15 +++-- test/runner-publishing.test.ts | 61 +++++++++++++++++++-- web/server.ts | 2 +- 7 files changed, 108 insertions(+), 26 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index 4da07371..a8a4a84d 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -131,7 +131,7 @@ Some repositories do not support draft PRs (for example private repositories on - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. - **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A draft waits only for an owed safety finding, not an owed scope pause: a draft invites no review, and a task in needs human cannot record a pause. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. -- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide), or a person's `start` or `resume` moves the task. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). One refused because GitHub's PR list lags behind a recorded PR is tried again once, after 30 seconds. Retries are bounded: one timer per task, and an opening is waited for only while its deadline is ahead. One still in place after its deadline is stuck for a reason a person must fix (two PRs carry its marker, the repository changed, GitHub keeps failing), so the next publish waits for a run's end, a restart or the `publish` action. +- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide), or a person's `start` or `resume` moves the task. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). A short retry, after 30 seconds, follows a publish refused because GitHub's PR list lags behind a recorded PR, one refused just as an opening's deadline passed, and one whose open or update left work behind. That last case covers GitHub showing another head than the one pushed (the PR is made a draft and the task stays running), and a draft change that failed. Such an `opened` outcome is recorded with `reconcile`, which keeps the publish owed. Retries are bounded: one timer per task, and per chain (until a publish settles) at most one retry at an opening's deadline and one short retry. Whatever is still unsettled after them is stuck for a reason a person must fix (two PRs carry a marker, the repository changed, GitHub keeps failing every open), so the next publish waits for a run's end, a restart or the `publish` action. - **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`. When only an action reply is stuck (its publish never recorded itself as started), the reply reports the outcome on record, and that record is left as it is. A failure while paying owed work does not stop this; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. - **Demo.** A demo never publishes, even with a `github` block or an injected publisher. diff --git a/runner/production.ts b/runner/production.ts index c717a1fc..fb67a1eb 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -159,6 +159,8 @@ export interface RunnerAssembly { readonly publisher?: (closing: () => boolean) => PullRequestPublisher; /** The environment the publisher's gh calls and push run with; its token values are removed from recorded errors. */ readonly env?: NodeJS.ProcessEnv; + /** Tests only: the delay of the publisher's short retry (SHORT_RETRY_MS, 30 s). */ + readonly shortRetryMs?: number; readonly sources: ExecutionSources; readonly findings: SafetyFindings; readonly recovery: RecoveryReport; diff --git a/runner/publishing.ts b/runner/publishing.ts index 58ef665b..144f72f1 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -15,8 +15,11 @@ const SETTLED = ['opened', 'possibly already fixed', 'draft skipped', 'draft uns const IN_FLIGHT = 'publishing'; /** The refusal while an earlier run owes work; the publish action that gets it pays that work (web/server.ts). */ export const OWED_REFUSAL = 'An earlier run left a safety finding or a scope pause that has not been acted on yet; it is settled before anything is published.'; -/** How long a publish refused because GitHub's PR list lags behind a PR waits before it is tried again. */ -export const LIST_LAG_RETRY_MS = 30_000; +/** + * How long a publish waits before its short retry: refused because GitHub's PR list lags behind a PR, refused just as + * an opening's deadline passed, or opened while GitHub still showed another head (`reconcile`). + */ +export const SHORT_RETRY_MS = 30_000; /** Refusals a person acts on; anything else that fails is reported as `failed`. */ const REFUSALS = [GuardRefusal, BranchPushRefused, OpeningUnsettled, PullRequestMisplaced, PullRequestRefused]; @@ -33,15 +36,20 @@ export class TaskPublishing { /** Token values to remove from recorded error text: those of the environment the gh calls and the push run with. */ #secrets: string[]; #closing = false; - /** Retries of a publish refused while a lost opening settles (OpeningUnsettled), cleared at shutdown. */ + /** The automatic retry armed for each task (at most one), cleared at shutdown. */ #retries = new Map>(); - /** Tasks whose list-lag retry has run since their last settled publish: it runs once, not every 30 s for ever. */ - #listLagRetried = new Set(); + /** + * The automatic retries each task has had since its last settled publish: at most one at an opening's deadline and one + * short one per chain, so a GitHub outage that fails every open cannot republish every settle time for ever. + */ + #retried = new Map(); + #shortRetryMs: number; /** Publishes in progress, by task, from scheduling until the outcome is recorded. */ #running = new Map>(); constructor(store: Store, publisher: PullRequestPublisher, runner: RunnerCoordinator, executor: ItemExecutor, capability?: ShutdownCapability, - env: NodeJS.ProcessEnv = process.env) { + env: NodeJS.ProcessEnv = process.env, options: { shortRetryMs?: number } = {}) { this.#store = store; this.#publisher = publisher; this.#runner = runner; this.#executor = executor; this.#write = settleWith(capability); + this.#shortRetryMs = options.shortRetryMs ?? SHORT_RETRY_MS; this.#secrets = TOKEN_VARIABLES.flatMap(name => env[name] ?? []); } @@ -99,7 +107,9 @@ export class TaskPublishing { // A task that reached its status before publishing existed (the v10 upgrade's record) is published only when a person // asks, until a run starts after the upgrade: that run's end is the first state the upgrade did not decide. if (last?.outcome === 'not published' && !options.personAsked && !this.#store.getAttempts(identity).some(attempt => attempt.createdAt > last.at)) return null; - return !last || !SETTLED.includes(last.outcome) || last.stateVersion !== this.#store.getTask(identity).stateVersion ? mode : null; + // `reconcile`: an open or update that left the PR or the task not where the publish meant them (GitHub showed another + // head, or a draft change failed); the next publish reconciles it, so it is still owed. + return !last || !SETTLED.includes(last.outcome) || last.reconcile || last.stateVersion !== this.#store.getTask(identity).stateVersion ? mode : null; } /** @@ -170,22 +180,27 @@ export class TaskPublishing { } /** - * A publish that ended with an opening still in flight (its reply was lost, or it was refused while one settles) or - * refused because GitHub's list lags behind a PR is tried again by itself: nothing else would (no run ends, startup has - * passed). Bounded (AGENTS.md: back off recurring external polling): one timer per task; an opening is waited for only - * while its deadline is ahead (at the deadline the next publish abandons or adopts it, so one that is still there - * after it is stuck for a reason a person must fix, not a timing one); the list is waited for once per chain, 30 s. - * Never throws. + * A publish that ended with an opening still in flight (its reply was lost, or it was refused while one settles), + * refused because GitHub's list lags behind a PR, or opened without settling (`reconcile`) is tried again by itself: + * nothing else would (no run ends, startup has passed). Bounded (AGENTS.md: back off recurring external polling): one + * timer per task, and per chain (until a publish settles) at most one retry at an opening's deadline, while it is + * still ahead, and one short retry. What is still unsettled after them is stuck for a reason a person must fix (two + * PRs carry a marker, the repository changed, GitHub keeps failing): the next publish waits for a run's end, a restart + * or the action. Never throws. */ #retryIfUnsettled(identity: PlanIdentity, record: Omit, unsettled: boolean): void { try { const key = identityKey(identity); - if (SETTLED.includes(record.outcome)) this.#listLagRetried.delete(key); + if (SETTLED.includes(record.outcome) && !record.reconcile) { this.#retried.delete(key); return; } if (this.#closing || this.#retries.has(key)) return; + const retried = this.#retried.get(key) ?? { deadline: 0, short: 0 }; const remaining = this.#publisher.settleRemaining(identity); - let wait: number | null = remaining !== null && remaining > 0 ? remaining : null; - if (wait === null && remaining === null && unsettled && !this.#listLagRetried.has(key)) { this.#listLagRetried.add(key); wait = LIST_LAG_RETRY_MS; } + let wait: number | null = null; + if (remaining !== null && remaining > 0 && retried.deadline < 1) { retried.deadline++; wait = remaining; } + // Refused for the list's lag or just as an opening's deadline passed, or opened without settling: one short retry. + else if ((unsettled || record.reconcile) && retried.short < 1) { retried.short++; wait = this.#shortRetryMs; } if (wait === null) return; + this.#retried.set(key, retried); const timer = setTimeout(() => { this.#retries.delete(key); this.publishIfOwed(identity); }, wait + 1_000); timer.unref?.(); this.#retries.set(key, timer); @@ -240,7 +255,10 @@ export class TaskPublishing { function describe(outcome: PublishOutcome, draft: boolean): Omit { const ready = 'leftReady' in outcome && outcome.leftReady !== undefined ? ` Pull request #${outcome.leftReady} could not be made a draft and may still be ready for review.` : ''; switch (outcome.kind) { + // A ready publish whose task did not reach in review (GitHub showed another head, so the PR was drafted), or any + // publish that left a PR ready it meant to draft, is not done: the next publish reconciles it. case 'opened': return { outcome: outcome.kind, draft, number: outcome.number, url: outcome.url, + ...((!draft && outcome.status !== 'in review') || outcome.leftReady !== undefined ? { reconcile: true } : {}), message: `Pull request #${outcome.number} is open${outcome.draft ? ' as a draft' : ''}; the task is ${outcome.status}.${ready}` }; case 'possibly already fixed': return { outcome: outcome.kind, draft, message: `The issue may already be fixed (${checkSummary(outcome.result)}), so no pull request was opened.${ready}` }; case 'draft skipped': return { outcome: outcome.kind, draft, message: `The issue may already be fixed (${checkSummary(outcome.result)}), so no draft pull request was opened.${ready}` }; diff --git a/runner/store.ts b/runner/store.ts index bc39f55f..28efaa45 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -57,6 +57,8 @@ export function mergeActionResponse(attempt: MergeAttempt) { */ export interface PublishRecord { outcome: string; draft: boolean; message: string; stateVersion: number; at: string; number?: number; url?: string; + /** An `opened` outcome that left the PR or the task not where the publish meant them: still owed (#103). */ + reconcile?: boolean; } /** What the publish action replays once its publish has settled: the outcome, as `publish.last` shows it. */ export function publishActionResponse(record: PublishRecord) { diff --git a/test/cli.test.ts b/test/cli.test.ts index ef0cccb4..81aa978b 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -67,10 +67,15 @@ it('refuses a runner startup without a token with its message, exit 1 and the lo }); it('refuses a runner without github.baseBranch before the lock and before the token check (#103)', () => { for (const github of [{}, { baseBranch: 'refs/heads/main' }]) { - const result = review(github).run('--port', '0'); - expect(result.status).toBe(1); - // Not the missing-token refusal that the runner's setup would give after the lock: the CLI checked the block first. - expect(result.stderr.trim()).toMatch(/^The runner publishes pull requests: add github\.baseBranch/); + // Another runner holds the lock, so a check made after the lock (as the runner's setup also makes it) would report + // the lock instead. + const { run, database } = review(github); + const held = acquireRunnerLock(database, { lockRoot: join(dirname(dirname(database)), 'home', '.codeboost', 'locks') }); + try { + const result = run('--port', '0'); + expect(result.status).toBe(1); + expect(result.stderr.trim()).toMatch(/^The runner publishes pull requests: add github\.baseBranch/); + } finally { held.release(); } } // No github block at all: refused as itself, before the lock, too. Another runner holds the lock meanwhile, so a check // made after the lock would report the lock instead. @@ -163,6 +168,8 @@ it('starts a publish the database owes once startup has verified the lock (#103) try { await cli.until(() => cli.out().includes('Review ready'), 'the server to open'); await cli.until(() => existsSync(calls), 'the startup publish to call gh'); + // Stopped only once that publish has recorded its outcome: a stop while gh runs would record `stopped` instead. + await cli.until(() => { const store = new Store(database); try { return store.lastPublish(identity)?.outcome === 'failed'; } finally { store.close(); } }, 'the publish outcome'); cli.child.kill('SIGINT'); expect(await cli.exited).toBe(0); const reopened = new Store(database); diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 1157b19c..8c21dd51 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -44,7 +44,12 @@ class FakeGitHub { head(branch: string): string | null { try { return git(this.remote, 'rev-parse', '--verify', '-q', `refs/heads/${branch}`); } catch { return null; } } - #view(pr: Pr) { return { number: pr.number, url: pr.url, draft: pr.draft, headSha: this.head(pr.headBranch) ?? '' }; } + /** The next answer about a PR shows this head instead of the branch's (GitHub lags behind a push), once. */ + staleHeadOnce?: string; + #view(pr: Pr) { + const stale = this.staleHeadOnce; this.staleHeadOnce = undefined; + return { number: pr.number, url: pr.url, draft: pr.draft, headSha: stale ?? this.head(pr.headBranch) ?? '' }; + } checks: AlreadyFixedGateway = { repository: REPO, check: async () => { this.calls.push('check'); return { outcome: 'clear', baseHead: 'b'.repeat(40) }; } }; pulls: PullRequestGateway = { repository: REPO, @@ -139,7 +144,7 @@ function world(): World { * real publisher and the real pusher into `w.remote`, with GitHub faked. `before` shapes the Store before the runner * exists, as an earlier process would have left it. */ -async function serve(w: World, options: { before?: (service: ReviewService) => void; onPushSpawn?: (n: number, app: () => App, close: () => Promise) => void; demo?: boolean; startup?: boolean; env?: NodeJS.ProcessEnv; settleMs?: number } = {}) { +async function serve(w: World, options: { before?: (service: ReviewService) => void; onPushSpawn?: (n: number, app: () => App, close: () => Promise) => void; demo?: boolean; startup?: boolean; env?: NodeJS.ProcessEnv; settleMs?: number; shortRetryMs?: number } = {}) { let app: App | undefined, spawns = 0, closing: Promise | undefined; const close = () => closing ??= app!.close(); let branchOf: (identity: PlanIdentity) => string = () => ''; @@ -159,7 +164,7 @@ async function serve(w: World, options: { before?: (service: ReviewService) => v onProcessGroup: () => options.onPushSpawn?.(++spawns, () => app!, close) }); const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: w.github.checks, pulls: w.github.pulls, pusher, closing }, { repository: REPO, baseBranch: 'main', ...(options.settleMs ? { settleMs: options.settleMs } : {}) }); branchOf = identity => publisher(() => false).branch(identity); - return { deps, sources, findings: new SafetyFindings(service.store), publisher, ...(options.env ? { env: options.env } : {}), + return { deps, sources, findings: new SafetyFindings(service.store), publisher, ...(options.env ? { env: options.env } : {}), ...(options.shortRetryMs ? { shortRetryMs: options.shortRetryMs } : {}), recovery: { finalized: [], requeue: [], removedDirectories: [], unknownEntries: [], unmatchedStorage: [], repairedMerges: [] } }; }); cleanups.push(close); @@ -504,10 +509,17 @@ describe('publishing a finished task (#103)', () => { const w = world(); const { app, identity, store } = await serve(w, { startup: false, before: service => { const s = service.store, id = service.config.identity; + // Running, with P2 and P3 still to run: owed work would be paid, and nothing can be published. + s.transitionTask(id, s.getTask(id).stateVersion, 'queued'); + const attempt = s.admitAttempt(id, { expectedStateVersion: s.getTask(id).stateVersion, kind: 'execute', item: s.getPlan(id).items[0]!.id, expectedContext: s.currentContext(id), deadline: Date.now() + 60_000 }); + s.markRunning(id, attempt.id); + s.settleAttempt(id, attempt.id, { firstReason: null, exitCode: 0, valid: true, result: { head: s.getSnapshot(id).head, unchanged: true, inScope: [], outOfScope: [] } }); s.recordPublish(id, { outcome: 'publishing', draft: false, message: 'A pull request is being published.' }); } }); - vi.spyOn(app.executor!, 'payOwed').mockImplementation(() => { throw new Error('The snapshot of P1\'s commit is missing.'); }); + expect(store.getTask(identity).status).toBe('running'); + const pay = vi.spyOn(app.executor!, 'payOwed').mockImplementation(() => { throw new Error('The snapshot of P1\'s commit is missing.'); }); app.publishOwed(); + expect(pay).toHaveBeenCalled(); expect(store.lastPublish(identity)).toMatchObject({ outcome: 'stopped' }); }); it('settles only a stuck reply with the outcome on record, leaving that record as it is', async () => { @@ -634,6 +646,47 @@ describe('publishing a finished task (#103)', () => { }); }); + describe('round 4 of the independent review', () => { + it('stops retrying when GitHub keeps failing every open: one retry at the deadline, not one every settle time', async () => { + const w = world(); + // GitHub fails every open, creating nothing. + w.github.onOpen = async () => { w.github.prs.pop(); throw new Error('HTTP 502'); }; + const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 800, startup: false }); + const starts = vi.spyOn(store, 'recordPublish'); + app.publishOwed(); + await new Promise(resolve => setTimeout(resolve, 6_000)); + expect(starts.mock.calls.filter(call => call[1].outcome === 'publishing').length).toBe(2); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed' }); + }); + it('reconciles a ready PR that GitHub showed at another head: still owed, and published again by itself', async () => { + const w = world(); + w.github.staleHeadOnce = 'a'.repeat(40); + const { app, identity, store } = await serve(w, { before: completeAll, shortRetryMs: 300, startup: false }); + app.publishOwed(); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', reconcile: true }), { timeout: 10_000 }); + expect(store.getTask(identity).status).toBe('running'); + expect(w.github.prs[0]!.draft).toBe(true); + // The short retry publishes again, and this time GitHub shows the pushed head. + await vi.waitFor(() => expect(store.getTask(identity).status).toBe('in review'), { timeout: 10_000, interval: 50 }); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened' }); + expect(store.lastPublish(identity)!.reconcile).toBeUndefined(); + expect(w.github.prs).toEqual([expect.objectContaining({ draft: false })]); + }); + it('retries once, after a short wait, a publish refused because GitHub\'s list lags behind its PR', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: needsHuman, shortRetryMs: 300 }); + await publishSettled(app, identity); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }); + // Owed again (an earlier attempt failed), and GitHub's list does not show the PR for now. + store.recordPublish(identity, { outcome: 'refused', draft: true, message: 'The branch moved.' }); + w.github.hidden.add(100); + app.publishOwed(); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'refused', message: expect.stringMatching(/list does not show it yet/) }), { timeout: 10_000 }); + w.github.hidden.clear(); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }), { timeout: 10_000, interval: 50 }); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); diff --git a/web/server.ts b/web/server.ts index fecc0c8a..0bcf4e49 100644 --- a/web/server.ts +++ b/web/server.ts @@ -88,7 +88,7 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge runner = new RunnerCoordinator(service.store, assembly.deps, undefined, capability); executor = new ItemExecutor(service.store, runner, assembly.sources, assembly.findings, { capability }); // A demo never publishes, whatever its github block or an injected setup provides (setUpRunner refuses demos too). - if (assembly.publisher && !config.demo) { const coordinator = runner; publishing = new TaskPublishing(service.store, assembly.publisher(() => coordinator.closing), runner, executor, capability, assembly.env); } + if (assembly.publisher && !config.demo) { const coordinator = runner; publishing = new TaskPublishing(service.store, assembly.publisher(() => coordinator.closing), runner, executor, capability, assembly.env, { ...(assembly.shortRetryMs !== undefined ? { shortRetryMs: assembly.shortRetryMs } : {}) }); } } catch (error) { service.close(); throw error; } } const loadReview=()=>{const view=service.load();return {...view,notes:view.notes.map(note=>({...note,answerActive:questions.isRunning(note.id)}))};}; From 87161732e514d7dd3fa08cbcd9b8578fbbc3f9a8 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sun, 4 Oct 2026 00:13:13 -0700 Subject: [PATCH 11/15] F: fixes from round 5 of the independent review of publishing (#103) - A run's end, a person's action or a restart starts a new retry chain with its own budget (AGENTS.md: reset on a lifecycle change or user action); a retry timer's own publish does not, so retries stay bounded. - No short retry while a lost opening's deadline is still ahead (it would only be refused again). - `reconcile` only while the task can still be published; the action reply carries it; the settleHead comment no longer overstates. - The owed-work path of a refused publish action carries the person's intent (personAsked), like start/resume. - publishOwed runs the lock check itself, first: no order of the CLI's lines can start a startup publish before the lock is verified. - Tests: the one-short-retry bound, the new chain on a person's action, and the lock check running first. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 2 +- runner/publish.ts | 2 +- runner/publishing.ts | 23 ++++++-- runner/store.ts | 2 +- test/runner-publishing.test.ts | 60 ++++++++++++++++++--- web/cli.ts | 6 +-- web/server.ts | 8 +-- 7 files changed, 80 insertions(+), 23 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index a8a4a84d..f03cfdf2 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -131,7 +131,7 @@ Some repositories do not support draft PRs (for example private repositories on - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. - **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A draft waits only for an owed safety finding, not an owed scope pause: a draft invites no review, and a task in needs human cannot record a pause. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. -- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it (`publishOwed`) after it has verified that the runner lock still names the database, because a publish pushes. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide), or a person's `start` or `resume` moves the task. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). A short retry, after 30 seconds, follows a publish refused because GitHub's PR list lags behind a recorded PR, one refused just as an opening's deadline passed, and one whose open or update left work behind. That last case covers GitHub showing another head than the one pushed (the PR is made a draft and the task stays running), and a draft change that failed. Such an `opened` outcome is recorded with `reconcile`, which keeps the publish owed. Retries are bounded: one timer per task, and per chain (until a publish settles) at most one retry at an opening's deadline and one short retry. Whatever is still unsettled after them is stuck for a reason a person must fix (two PRs carry a marker, the repository changed, GitHub keeps failing every open), so the next publish waits for a run's end, a restart or the `publish` action. +- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it through `publishOwed`, which first runs the check that the runner lock still names the database, because a publish pushes; if the check fails, nothing starts. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide), or a person's `start` or `resume` moves the task. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). A short retry, after 30 seconds, follows a publish refused because GitHub's PR list lags behind a recorded PR, one refused just as an opening's deadline passed, and one whose open or update left work behind. That last case covers GitHub showing another head than the one pushed (the PR is made a draft and the task stays running), and a draft change that failed. Such an `opened` outcome is recorded with `reconcile`, which keeps the publish owed. Retries are bounded: one timer per task, and per chain at most one retry at an opening's deadline (while it is still ahead) and one short retry. A chain ends when a publish settles. A new one starts with a run's end, a person's action or a restart, never with a retry's own publish. Whatever is still unsettled after them is stuck for a reason a person must fix (two PRs carry a marker, the repository changed, GitHub keeps failing every open), so the next publish waits for a run's end, a restart or the `publish` action. - **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`. When only an action reply is stuck (its publish never recorded itself as started), the reply reports the outcome on record, and that record is left as it is. A failure while paying owed work does not stop this; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. - **Demo.** A demo never publishes, even with a `github` block or an injected publisher. diff --git a/runner/publish.ts b/runner/publish.ts index e8d4700c..f6d61836 100644 --- a/runner/publish.ts +++ b/runner/publish.ts @@ -320,7 +320,7 @@ export class PullRequestPublisher { // a draft, whatever GitHub returned. if (pr.draft || (!draft && status === 'in review' && pr.headSha === head)) return opened; let drafted; - // Only the GitHub call's failure becomes leftReady (the PR stays ready; the next publish reconciles it). A Store + // Only the GitHub call's failure becomes leftReady (the PR stays ready; the next publish of a task that can still be published reconciles it). A Store // failure after a draft change that landed propagates, so it is not misreported as a ready PR. try { drafted = await this.#pulls.markDraft(pr.number, { base, headBranch: branch, marker: marker(openingId) }, signal); } catch (error) { if (signal?.aborted) throw error; return { ...opened, leftReady: pr.number }; } diff --git a/runner/publishing.ts b/runner/publishing.ts index 144f72f1..b895daa2 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -118,6 +118,8 @@ export class TaskPublishing { */ request(identity: PlanIdentity, actionId: string): { draft: boolean } { const mode = this.mode(identity); + // A person's action starts a new chain, with its own automatic retries (AGENTS.md: reset on an explicit user action). + this.#retried.delete(identityKey(identity)); this.#schedule(identity, mode.draft, actionId); return mode; } @@ -126,8 +128,16 @@ export class TaskPublishing { * A run of the task ended, the server started, or a person's start or resume moved the task (`personAsked`: it also ends * the v10 upgrade's `not published` hold, as a run starting would): publish if one is owed. Never throws. */ - publishIfOwed(identity: PlanIdentity, options: { personAsked?: boolean } = {}): void { - try { this.#payOwed(identity); const owed = this.owed(identity, options); if (owed) this.#schedule(identity, owed.draft); } + publishIfOwed(identity: PlanIdentity, options: { personAsked?: boolean; retry?: boolean } = {}): void { + try { + this.#payOwed(identity); + const owed = this.owed(identity, options); + if (!owed) return; + // A run's end, a person's action or a restart is a meaningful lifecycle change: a new chain, with its own automatic + // retries. A retry timer's own publish is not, or the retries would be unbounded again. + if (!options.retry) this.#retried.delete(identityKey(identity)); + this.#schedule(identity, owed.draft); + } catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } } @@ -198,10 +208,11 @@ export class TaskPublishing { let wait: number | null = null; if (remaining !== null && remaining > 0 && retried.deadline < 1) { retried.deadline++; wait = remaining; } // Refused for the list's lag or just as an opening's deadline passed, or opened without settling: one short retry. - else if ((unsettled || record.reconcile) && retried.short < 1) { retried.short++; wait = this.#shortRetryMs; } + // Not while an opening's deadline is still ahead: a publish before it would only be refused again. + else if ((unsettled || record.reconcile) && !(remaining !== null && remaining > 0) && retried.short < 1) { retried.short++; wait = this.#shortRetryMs; } if (wait === null) return; this.#retried.set(key, retried); - const timer = setTimeout(() => { this.#retries.delete(key); this.publishIfOwed(identity); }, wait + 1_000); + const timer = setTimeout(() => { this.#retries.delete(key); this.publishIfOwed(identity, { retry: true }); }, wait + 1_000); timer.unref?.(); this.#retries.set(key, timer); } catch (error) { console.error(`Could not arrange a retry of the publish: ${JSON.stringify(message(error, this.#secrets))}`); } @@ -258,7 +269,9 @@ function describe(outcome: PublishOutcome, draft: boolean): Omit v }); cleanups.push(close); // As the CLI does once it has verified the lock. - if (options.startup !== false) app.publishOwed(); + if (options.startup !== false) app.publishOwed(() => undefined); const identity = w.demo.identity; return { app, close, identity, store: app.service.store, branch: branchOf(identity), head: app.service.store.getSnapshot(identity).head }; } @@ -399,7 +399,7 @@ describe('publishing a finished task (#103)', () => { const opening = Promise.withResolvers(); w.github.checks.check = async () => { opening.resolve(); return { outcome: 'clear', baseHead: 'b'.repeat(40) }; }; const { app, identity, store } = await serve(w, { before: completeAll, startup: false }); - app.publishOwed(); + app.publishOwed(() => undefined); seen = store.lastPublish(identity)?.outcome; await opening.promise; expect(seen).toBe('publishing'); @@ -423,7 +423,7 @@ describe('publishing a finished task (#103)', () => { const w = world(); const { app, identity, store } = await serve(w, { before: completeAll, startup: false }); vi.spyOn(store, 'recordPublish').mockImplementationOnce(() => { throw new Error('disk full'); }); - app.publishOwed(); + app.publishOwed(() => undefined); await app.publishing!.settled(identity); expect(w.github.calls).toEqual([]); expect(store.lastPublish(identity)).toBeNull(); @@ -518,7 +518,7 @@ describe('publishing a finished task (#103)', () => { } }); expect(store.getTask(identity).status).toBe('running'); const pay = vi.spyOn(app.executor!, 'payOwed').mockImplementation(() => { throw new Error('The snapshot of P1\'s commit is missing.'); }); - app.publishOwed(); + app.publishOwed(() => undefined); expect(pay).toHaveBeenCalled(); expect(store.lastPublish(identity)).toMatchObject({ outcome: 'stopped' }); }); @@ -620,7 +620,7 @@ describe('publishing a finished task (#103)', () => { w.github.onOpen = async () => { w.github.hidden.add(w.github.prs.at(-1)!.number); w.github.onOpen = undefined; throw new Error('timed out'); }; const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 1_000, startup: false }); const starts = vi.spyOn(store, 'recordPublish'); - app.publishOwed(); + app.publishOwed(() => undefined); await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed' }), { timeout: 5_000 }); // Someone copied its description onto another PR from the branch: recovery cannot tell which is the task's, so the // opening stays, and every publish of it is refused for a reason only a person can fix. @@ -653,7 +653,7 @@ describe('publishing a finished task (#103)', () => { w.github.onOpen = async () => { w.github.prs.pop(); throw new Error('HTTP 502'); }; const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 800, startup: false }); const starts = vi.spyOn(store, 'recordPublish'); - app.publishOwed(); + app.publishOwed(() => undefined); await new Promise(resolve => setTimeout(resolve, 6_000)); expect(starts.mock.calls.filter(call => call[1].outcome === 'publishing').length).toBe(2); expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed' }); @@ -662,7 +662,7 @@ describe('publishing a finished task (#103)', () => { const w = world(); w.github.staleHeadOnce = 'a'.repeat(40); const { app, identity, store } = await serve(w, { before: completeAll, shortRetryMs: 300, startup: false }); - app.publishOwed(); + app.publishOwed(() => undefined); await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', reconcile: true }), { timeout: 10_000 }); expect(store.getTask(identity).status).toBe('running'); expect(w.github.prs[0]!.draft).toBe(true); @@ -680,13 +680,57 @@ describe('publishing a finished task (#103)', () => { // Owed again (an earlier attempt failed), and GitHub's list does not show the PR for now. store.recordPublish(identity, { outcome: 'refused', draft: true, message: 'The branch moved.' }); w.github.hidden.add(100); - app.publishOwed(); + app.publishOwed(() => undefined); await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'refused', message: expect.stringMatching(/list does not show it yet/) }), { timeout: 10_000 }); w.github.hidden.clear(); await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }), { timeout: 10_000, interval: 50 }); }); }); + describe('round 5 of the independent review', () => { + it('gives a persistent list lag one short retry, not one every period', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: needsHuman, shortRetryMs: 200 }); + await publishSettled(app, identity); + store.recordPublish(identity, { outcome: 'refused', draft: true, message: 'The branch moved.' }); + // GitHub's list keeps leaving the PR out. + w.github.hidden.add(100); + const starts = vi.spyOn(store, 'recordPublish'); + app.publishOwed(() => undefined); + await new Promise(resolve => setTimeout(resolve, 5_000)); + expect(starts.mock.calls.filter(call => call[1].outcome === 'publishing').length).toBe(2); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'refused', message: expect.stringMatching(/list does not show it yet/) }); + }); + it('starts no startup publish unless the lock check passes, which runs first', async () => { + const w = world(); + const { app, identity } = await serve(w, { before: completeAll, startup: false }); + const order: string[] = []; + expect(() => app.publishOwed(() => { order.push('verify'); throw new Error('The database path changed while opening. Refusing to start.'); })) + .toThrow('The database path changed'); + expect(app.publishing!.busy(identity)).toBe(false); + expect(w.github.calls).toEqual([]); + app.publishOwed(() => { order.push('verify'); expect(app.publishing!.busy(identity)).toBe(false); }); + expect(order).toEqual(['verify', 'verify']); + expect(app.publishing!.busy(identity)).toBe(true); + await publishSettled(app, identity); + }); + it('starts a new chain with its own retries when a person asks again', async () => { + const w = world(); + w.github.onOpen = async () => { w.github.prs.pop(); throw new Error('HTTP 502'); }; + const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 800, startup: false }); + const starts = vi.spyOn(store, 'recordPublish'); + const publishes = () => starts.mock.calls.filter(call => call[1].outcome === 'publishing').length; + app.publishOwed(() => undefined); + // The first chain: its publish and one retry at the opening's deadline, then nothing. + await new Promise(resolve => setTimeout(resolve, 4_000)); + expect(publishes()).toBe(2); + // A person asks again: a new chain, which gets its own retry at its opening's deadline. + await act(app, 'publish'); + await new Promise(resolve => setTimeout(resolve, 4_000)); + expect(publishes()).toBe(4); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); diff --git a/web/cli.ts b/web/cli.ts index 086b25e9..3ff42d13 100644 --- a/web/cli.ts +++ b/web/cli.ts @@ -76,14 +76,14 @@ if (values.help || (!values.demo && !values.config && values['release-preparatio console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } for (const signal of ['SIGINT', 'SIGTERM'] as const) process.removeListener(signal, duringStartup); - try { lock.verify(); } + // A publish an earlier process owed (#103) starts only under a lock verified to name the database: publishOwed runs the + // check itself, first, so no order of these lines can start one before it. + try { if (stopping) lock.verify(); else app.publishOwed(() => lock.verify()); } catch (error) { await app.close(); lock.release(); // The database path changed: a refusal the person acts on, so its message, not a stack. refuse(error); } - // Only now, under a lock verified to name the database: a publish an earlier process owed (#103). - if (!stopping) app.publishOwed(); const stop = () => void app.close().then(() => { lock.release(); process.exit(0); }, error => { lock.release(); console.error(error instanceof Error ? error.message : error); process.exit(1); }); // A second Ctrl+C does not skip shutdown (runner-lifecycle.md): agents are still being stopped and awaited. diff --git a/web/server.ts b/web/server.ts index 0bcf4e49..d4376db5 100644 --- a/web/server.ts +++ b/web/server.ts @@ -203,7 +203,7 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge try { replay = !!service.store.savedAction(identity, { actionId: actionId as string, kind: 'publish', request: { attemptId, expectedStateVersion } }); } catch { replay = true; } try { return act(); } - catch (error) { if (!replay && error instanceof GuardRefusal && error.message === OWED_REFUSAL) publishing?.publishIfOwed(identity); throw error; } + catch (error) { if (!replay && error instanceof GuardRefusal && error.message === OWED_REFUSAL) publishing?.publishIfOwed(identity, { personAsked: true }); throw error; } } return act(); function act() { return service.store.userAction(identity, { actionId: actionId as string, kind: action as string, request: { attemptId, expectedStateVersion } }, () => { @@ -414,10 +414,10 @@ export async function startServer(config: ReviewConfig, port = 4318, questionAge return { server, service, token, runner, executor, publishing, /** * Startup (#103): a publish an earlier process owed (a lost opening, a run that ended before its publish completed) - * runs once, in the background; recovery finds a lost opening by its marker. The caller runs it once it has checked - * that the runner lock still names the database, since a publish pushes. + * runs once, in the background; recovery finds a lost opening by its marker. A publish pushes, so `verifyLock` + * (the runner lock still names the database) runs first, here: if it throws, nothing starts and its error is thrown. */ - publishOwed: () => publishing?.startup(identity), + publishOwed: (verifyLock: () => void) => { verifyLock(); publishing?.startup(identity); }, url: `http://127.0.0.1:${address.port}/#${token}`, close: async () => { // Step 1, one synchronous turn: reject new API requests and new runner work. Admitted requests drain (step 2). stopping = true; From 2f8934e14f25053f8f2676ce438aa82d730a2269 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sun, 4 Oct 2026 00:22:18 -0700 Subject: [PATCH 12/15] F: fixes from round 6 of the independent review of publishing (#103) - A person's publish action drops a retry the old chain armed, so it cannot fire later on the new chain's budget (at most two automatic publishes per chain, not three). - Tests for two round-5 fixes that no test covered: no reconcile for a task that can no longer be published, and the action reply carrying reconcile. Co-Authored-By: Claude Opus 5.5 --- runner/publishing.ts | 6 +++++- test/runner-publishing.test.ts | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/runner/publishing.ts b/runner/publishing.ts index b895daa2..cd4e33e4 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -119,7 +119,11 @@ export class TaskPublishing { request(identity: PlanIdentity, actionId: string): { draft: boolean } { const mode = this.mode(identity); // A person's action starts a new chain, with its own automatic retries (AGENTS.md: reset on an explicit user action). - this.#retried.delete(identityKey(identity)); + // A retry the old chain armed is dropped with it, so it cannot fire later on the new chain's budget. + const key = identityKey(identity); + this.#retried.delete(key); + const armed = this.#retries.get(key); + if (armed) { clearTimeout(armed); this.#retries.delete(key); } this.#schedule(identity, mode.draft, actionId); return mode; } diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 5db9f451..3438be0f 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -731,6 +731,44 @@ describe('publishing a finished task (#103)', () => { }); }); + describe('round 6 of the independent review', () => { + it('drops the old chain\'s armed retry when a person asks again', async () => { + const w = world(); + w.github.onOpen = async () => { w.github.prs.pop(); throw new Error('HTTP 502'); }; + const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 1_500, startup: false }); + const starts = vi.spyOn(store, 'recordPublish'); + const publishes = () => starts.mock.calls.filter(call => call[1].outcome === 'publishing').length; + app.publishOwed(() => undefined); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed' }), { timeout: 5_000 }); + // The first chain's retry at its opening's deadline is armed; a person asks again before it fires. + await act(app, 'publish'); + await new Promise(resolve => setTimeout(resolve, 6_000)); + // The first publish, the person's, and that chain's one retry: the old chain's timer never fires on the new budget. + expect(publishes()).toBe(3); + }); + it('records no reconcile for a ready PR whose task can no longer be published, and replays reconcile when it is set', async () => { + const w = world(); + // The task is cancelled while its PR opens: the PR is recorded, and nothing would ever reconcile it. + let cancel: (() => void) | undefined; + w.github.onOpen = async () => { cancel?.(); }; + const first = await serve(w, { before: completeAll, startup: false }); + cancel = () => first.store.cancelTask(first.identity, first.store.getTask(first.identity).stateVersion, randomUUID()); + first.app.publishOwed(() => undefined); + await publishSettled(first.app, first.identity); + expect(first.store.getTask(first.identity).status).toBe('cancelled'); + expect(first.store.lastPublish(first.identity)).toMatchObject({ outcome: 'opened' }); + expect(first.store.lastPublish(first.identity)!.reconcile).toBeUndefined(); + // A publish action whose PR GitHub shows at another head: its saved reply, replayed, says reconcile too. + const v = world(); + v.github.staleHeadOnce = 'a'.repeat(40); + const second = await serve(v, { before: completeAll, startup: false }); + const actionId = randomUUID(), version = second.store.getTask(second.identity).stateVersion; + await act(second.app, 'publish', actionId, version); + await second.app.publishing!.settled(second.identity); + expect((await act(second.app, 'publish', actionId, version)).body.result).toMatchObject({ outcome: 'opened', reconcile: true }); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); From bb4c79185eb06d3de7aeb75c1b6c1909ac57cc41 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sun, 4 Oct 2026 00:40:48 -0700 Subject: [PATCH 13/15] F: fixes from round 7 of the independent review of publishing (#103) - Tests for the owed-safety-finding gate: a ready publish is refused while a finding is owed, the action pays it and the draft follows; a needs-human task's owed finding is paid at startup before its draft; a replay of that refusal pays nothing. - The draft of a task whose owed finding was paid names that finding: the executor settles it in memory only, so TaskPublishing keeps the reason payOwed returned (it was lost before). - A run's end starting a new chain drops the old chain's armed retry, as a person's action does (#newChain): one timer per task, tested. Co-Authored-By: Claude Opus 5.5 --- runner/publishing.ts | 28 +++++++++++++---- test/runner-publishing.test.ts | 56 ++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/runner/publishing.ts b/runner/publishing.ts index cd4e33e4..a3331b4d 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -44,6 +44,11 @@ export class TaskPublishing { */ #retried = new Map(); #shortRetryMs: number; + /** + * The reason an owed escalation was paid with (a safety finding whose save had failed): the executor settles it in + * memory only, so the task's attempt rows do not carry it, and the draft PR would otherwise not say why (#103). + */ + #escalations = new Map(); /** Publishes in progress, by task, from scheduling until the outcome is recorded. */ #running = new Map>(); constructor(store: Store, publisher: PullRequestPublisher, runner: RunnerCoordinator, executor: ItemExecutor, capability?: ShutdownCapability, @@ -119,11 +124,7 @@ export class TaskPublishing { request(identity: PlanIdentity, actionId: string): { draft: boolean } { const mode = this.mode(identity); // A person's action starts a new chain, with its own automatic retries (AGENTS.md: reset on an explicit user action). - // A retry the old chain armed is dropped with it, so it cannot fire later on the new chain's budget. - const key = identityKey(identity); - this.#retried.delete(key); - const armed = this.#retries.get(key); - if (armed) { clearTimeout(armed); this.#retries.delete(key); } + this.#newChain(identity); this.#schedule(identity, mode.draft, actionId); return mode; } @@ -139,7 +140,7 @@ export class TaskPublishing { if (!owed) return; // A run's end, a person's action or a restart is a meaningful lifecycle change: a new chain, with its own automatic // retries. A retry timer's own publish is not, or the retries would be unbounded again. - if (!options.retry) this.#retried.delete(identityKey(identity)); + if (!options.retry) this.#newChain(identity); this.#schedule(identity, owed.draft); } catch (error) { console.error(`Could not start publishing: ${JSON.stringify(message(error, this.#secrets))}`); } @@ -189,10 +190,22 @@ export class TaskPublishing { // owes the work, and mode() refuses to publish until it is paid. try { const paid = this.#executor.payOwed(identity); + if (paid?.kind === 'needs human') this.#escalations.set(identityKey(identity), paid.reason); if (paid?.kind === 'stopped') console.error(`Could not settle what an earlier run owes: ${JSON.stringify(paid.reason ?? '')}`); } catch (error) { console.error(`Could not settle what an earlier run owes: ${JSON.stringify(message(error, this.#secrets))}`); } } + /** + * A new chain (a run's end, a person's action, a restart): a fresh retry budget, and a retry the old chain armed is + * dropped with it, so it cannot fire later on the new chain's budget. One timer per task at all times. + */ + #newChain(identity: PlanIdentity): void { + const key = identityKey(identity); + this.#retried.delete(key); + const armed = this.#retries.get(key); + if (armed) { clearTimeout(armed); this.#retries.delete(key); } + } + /** * A publish that ended with an opening still in flight (its reply was lost, or it was refused while one settles), * refused because GitHub's list lags behind a PR, or opened without settling (`reconcile`) is tried again by itself: @@ -259,6 +272,9 @@ export class TaskPublishing { */ #problems(identity: PlanIdentity): string[] { const task = this.#store.getTask(identity); + // An owed finding paid here first: only this process knows its text. + const escalated = this.#escalations.get(identityKey(identity)); + if (escalated) return [escalated]; const current = task.currentAttemptId === null ? undefined : this.#store.getAttempt(identity, task.currentAttemptId); if (current?.safetyFinding) return [current.safetyFinding]; if (task.budgetDeadline !== null && task.budgetDeadline <= Date.now()) return ['The task\'s time budget ran out before its plan finished.']; diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 3438be0f..7844100d 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -148,6 +148,7 @@ async function serve(w: World, options: { before?: (service: ReviewService) => v let app: App | undefined, spawns = 0, closing: Promise | undefined; const close = () => closing ??= app!.close(); let branchOf: (identity: PlanIdentity) => string = () => ''; + let findings: SafetyFindings | undefined; const config = { ...w.demo, demo: options.demo ?? false }; app = await startServer(config, 0, undefined, undefined, 2_000, undefined, undefined, undefined, async service => { options.before?.(service); @@ -164,14 +165,15 @@ async function serve(w: World, options: { before?: (service: ReviewService) => v onProcessGroup: () => options.onPushSpawn?.(++spawns, () => app!, close) }); const publisher = (closing: () => boolean) => new PullRequestPublisher(service.store, { checks: w.github.checks, pulls: w.github.pulls, pusher, closing }, { repository: REPO, baseBranch: 'main', ...(options.settleMs ? { settleMs: options.settleMs } : {}) }); branchOf = identity => publisher(() => false).branch(identity); - return { deps, sources, findings: new SafetyFindings(service.store), publisher, ...(options.env ? { env: options.env } : {}), ...(options.shortRetryMs ? { shortRetryMs: options.shortRetryMs } : {}), + findings = new SafetyFindings(service.store); + return { deps, sources, findings, publisher, ...(options.env ? { env: options.env } : {}), ...(options.shortRetryMs ? { shortRetryMs: options.shortRetryMs } : {}), recovery: { finalized: [], requeue: [], removedDirectories: [], unknownEntries: [], unmatchedStorage: [], repairedMerges: [] } }; }); cleanups.push(close); // As the CLI does once it has verified the lock. if (options.startup !== false) app.publishOwed(() => undefined); const identity = w.demo.identity; - return { app, close, identity, store: app.service.store, branch: branchOf(identity), head: app.service.store.getSnapshot(identity).head }; + return { app, close, identity, store: app.service.store, branch: branchOf(identity), head: app.service.store.getSnapshot(identity).head, findings: findings! }; } const view = async (app: App) => (await fetch(`${new URL(app.url).origin}/api/runner`, { headers: { 'x-codeboost-token': app.token } })).json() as Promise>; async function act(app: App, action: string, actionId = randomUUID(), expectedStateVersion?: number) { @@ -769,6 +771,56 @@ describe('publishing a finished task (#103)', () => { }); }); + describe('round 7 of the independent review', () => { + /** A safety finding the audit made on the last attempt, whose save failed: the executor holds it, owed, in memory. */ + const owedFinding = (store: Store, findings: SafetyFindings, identity: PlanIdentity) => { + const attempt = store.getAttempts(identity).at(-1)!; + vi.spyOn(store, 'recordSafetyFinding').mockImplementationOnce(() => { throw new Error('database is locked'); }); + findings.record(attempt.id, 'The last item wrote outside its workspace.'); + expect(findings.owedAttempts()).toEqual([attempt.id]); + }; + it('refuses a ready publish while a safety finding is owed, pays it, and publishes the draft instead', async () => { + const w = world(), actionId = randomUUID(); + const { app, identity, store, findings } = await serve(w, { before: completeAll, startup: false }); + owedFinding(store, findings, identity); + expect((await view(app)).publish).toMatchObject({ publishable: false }); + const version = store.getTask(identity).stateVersion; + expect(await act(app, 'publish', actionId, version)).toMatchObject({ status: 409, body: { error: OWED_REFUSAL } }); + // Paid: the task went to a person, and its draft is published with the finding as its problem. + expect(store.getTask(identity).status).toBe('needs human'); + await publishSettled(app, identity); + expect(w.github.prs).toEqual([expect.objectContaining({ draft: true, body: expect.stringContaining('The last item wrote outside its workspace.') })]); + // A replay of that refusal applies nothing: no payment, no publish. + const pay = vi.spyOn(app.executor!, 'payOwed'), publishIfOwed = vi.spyOn(app.publishing!, 'publishIfOwed'); + expect(await act(app, 'publish', actionId, version)).toMatchObject({ status: 409, body: { error: OWED_REFUSAL } }); + expect(pay).not.toHaveBeenCalled(); + expect(publishIfOwed).not.toHaveBeenCalled(); + }); + it('pays a safety finding owed on a needs-human task before its draft, at startup', async () => { + const w = world(); + const { app, identity, store, findings } = await serve(w, { before: needsHuman, startup: false }); + owedFinding(store, findings, identity); + app.publishOwed(() => undefined); + await publishSettled(app, identity); + expect(findings.owedAttempts()).toEqual([]); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }); + }); + it('keeps one retry timer per task when a run\'s end starts a new chain while one is armed', async () => { + const w = world(); + w.github.onOpen = async () => { w.github.prs.pop(); throw new Error('HTTP 502'); }; + const { app, identity, store } = await serve(w, { before: completeAll, settleMs: 1_500, startup: false }); + const starts = vi.spyOn(store, 'recordPublish'); + const publishes = () => starts.mock.calls.filter(call => call[1].outcome === 'publishing').length; + app.publishOwed(() => undefined); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'failed' }), { timeout: 5_000 }); + // A run's end starts a new chain while the first chain's retry is armed. + app.publishing!.publishIfOwed(identity); + await new Promise(resolve => setTimeout(resolve, 6_000)); + // The first publish, the new chain's, and that chain's one retry; the first chain's timer never fires. + expect(publishes()).toBe(3); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll }); From 58fdcf414d5cac3e6ebed1b5275e774ff5b6d43d Mon Sep 17 00:00:00 2001 From: mchwang Date: Sun, 4 Oct 2026 00:52:38 -0700 Subject: [PATCH 14/15] F: fixes from round 8 of the independent review of publishing (#103) A paid safety finding names a draft only while the task stays in needs human on the same current attempt; a later return has its own reason. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 2 +- runner/publishing.ts | 13 +++++++----- test/runner-publishing.test.ts | 23 +++++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index f03cfdf2..57f3b2e0 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -130,7 +130,7 @@ Some repositories do not support draft PRs (for example private repositories on `TaskPublishing` (`runner/publishing.ts`) owns publishing in production. `setUpRunner` builds the publisher with the real gateways (`GhAlreadyFixedGateway`, `GhPullRequestGateway`) and a `GitBranchPusher` that pushes from the runner-owned repository to `pushUrl(github.repository)` on `GH_HOST`, never to a configured URL. The pusher may overwrite only commits the task's ledger records as `owned`. Before #22 adds rebasing, nothing in production writes a `foreign` ledger entry (only `recordRebase` does), so codeboost pushes no head the ledger calls foreign. - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. -- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A draft waits only for an owed safety finding, not an owed scope pause: a draft invites no review, and a task in needs human cannot record a pause. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. +- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A draft waits only for an owed safety finding, not an owed scope pause: a draft invites no review, and a task in needs human cannot record a pause. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. That draft names the paid finding, which only this process holds. It does so only while the task stays in needs human on the same current attempt. A later run that returns the task to needs human has its own reason. - **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it through `publishOwed`, which first runs the check that the runner lock still names the database, because a publish pushes; if the check fails, nothing starts. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide), or a person's `start` or `resume` moves the task. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). A short retry, after 30 seconds, follows a publish refused because GitHub's PR list lags behind a recorded PR, one refused just as an opening's deadline passed, and one whose open or update left work behind. That last case covers GitHub showing another head than the one pushed (the PR is made a draft and the task stays running), and a draft change that failed. Such an `opened` outcome is recorded with `reconcile`, which keeps the publish owed. Retries are bounded: one timer per task, and per chain at most one retry at an opening's deadline (while it is still ahead) and one short retry. A chain ends when a publish settles. A new one starts with a run's end, a person's action or a restart, never with a retry's own publish. Whatever is still unsettled after them is stuck for a reason a person must fix (two PRs carry a marker, the repository changed, GitHub keeps failing every open), so the next publish waits for a run's end, a restart or the `publish` action. - **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`. When only an action reply is stuck (its publish never recorded itself as started), the reply reports the outcome on record, and that record is left as it is. A failure while paying owed work does not stop this; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. diff --git a/runner/publishing.ts b/runner/publishing.ts index a3331b4d..35b8b0d8 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -48,7 +48,7 @@ export class TaskPublishing { * The reason an owed escalation was paid with (a safety finding whose save had failed): the executor settles it in * memory only, so the task's attempt rows do not carry it, and the draft PR would otherwise not say why (#103). */ - #escalations = new Map(); + #escalations = new Map(); /** Publishes in progress, by task, from scheduling until the outcome is recorded. */ #running = new Map>(); constructor(store: Store, publisher: PullRequestPublisher, runner: RunnerCoordinator, executor: ItemExecutor, capability?: ShutdownCapability, @@ -190,7 +190,8 @@ export class TaskPublishing { // owes the work, and mode() refuses to publish until it is paid. try { const paid = this.#executor.payOwed(identity); - if (paid?.kind === 'needs human') this.#escalations.set(identityKey(identity), paid.reason); + // Tied to the attempt current once paid: a later run (a new attempt) that ends in needs human again has its own reason. + if (paid?.kind === 'needs human') this.#escalations.set(identityKey(identity), { reason: paid.reason, attemptId: this.#store.getTask(identity).currentAttemptId }); if (paid?.kind === 'stopped') console.error(`Could not settle what an earlier run owes: ${JSON.stringify(paid.reason ?? '')}`); } catch (error) { console.error(`Could not settle what an earlier run owes: ${JSON.stringify(message(error, this.#secrets))}`); } } @@ -272,9 +273,11 @@ export class TaskPublishing { */ #problems(identity: PlanIdentity): string[] { const task = this.#store.getTask(identity); - // An owed finding paid here first: only this process knows its text. - const escalated = this.#escalations.get(identityKey(identity)); - if (escalated) return [escalated]; + // An owed finding paid here first: only this process knows its text. Only while the task is still in needs human from + // that payment (the same current attempt); once a later run has started, the finding is an earlier attempt's. + const key = identityKey(identity), escalated = this.#escalations.get(key); + if (escalated && task.status === 'needs human' && escalated.attemptId === task.currentAttemptId) return [escalated.reason]; + this.#escalations.delete(key); const current = task.currentAttemptId === null ? undefined : this.#store.getAttempt(identity, task.currentAttemptId); if (current?.safetyFinding) return [current.safetyFinding]; if (task.budgetDeadline !== null && task.budgetDeadline <= Date.now()) return ['The task\'s time budget ran out before its plan finished.']; diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 7844100d..0c1820fb 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -804,6 +804,29 @@ describe('publishing a finished task (#103)', () => { await publishSettled(app, identity); expect(findings.owedAttempts()).toEqual([]); expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }); + expect(w.github.prs).toEqual([expect.objectContaining({ draft: true, body: expect.stringContaining('The last item wrote outside its workspace.') })]); + }); + it('names a later return to needs human by its own reason, not a finding paid before', async () => { + const w = world(); + const { app, identity, store, findings } = await serve(w, { before: completeAll, startup: false }); + owedFinding(store, findings, identity); + app.publishOwed(() => undefined); + await publishSettled(app, identity); + expect(w.github.prs).toEqual([expect.objectContaining({ body: expect.stringContaining('The last item wrote outside its workspace.') })]); + // A person dealt with the finding and the task ran again: its new attempt failed, and the next admission came after + // the task's budget, so it is back in needs human for that reason. + store.transitionTask(identity, store.getTask(identity).stateVersion, 'queued'); + const admit = (now?: number) => store.admitAttempt(identity, { expectedStateVersion: store.getTask(identity).stateVersion, kind: 'execute', + item: store.getPlan(identity).items[0]!.id, expectedContext: store.currentContext(identity), deadline: (now ?? Date.now()) + 60_000, ...(now ? { now } : {}) }); + const attempt = admit(); + store.markRunning(identity, attempt.id); + store.settleAttempt(identity, attempt.id, { firstReason: null, exitCode: 1, valid: false, detail: 'The tests failed in retry.ts.' }); + expect(() => admit(store.getTask(identity).budgetDeadline! + 1)).toThrow(/time budget/); + expect(store.getTask(identity).status).toBe('needs human'); + app.publishing!.publishIfOwed(identity); + await publishSettled(app, identity); + expect(w.github.prs).toEqual([expect.objectContaining({ draft: true, body: expect.stringContaining('The tests failed in retry.ts.') })]); + expect(w.github.prs[0]!.body).not.toContain('The last item wrote outside its workspace.'); }); it('keeps one retry timer per task when a run\'s end starts a new chain while one is armed', async () => { const w = world(); From d94948eb1091edc2002a91fdfee479faf6496684 Mon Sep 17 00:00:00 2001 From: mchwang Date: Sun, 4 Oct 2026 01:06:36 -0700 Subject: [PATCH 15/15] F: fixes from round 9 of the independent review of publishing (#103) A publish refused by a guard (a race) gets the one short retry; a draft names a finding whose save failed even after the budget ran out; a needs-human task pays only an owed finding, not a scope pause. Co-Authored-By: Claude Opus 5.5 --- docs/implementation/pull-request-opening.md | 4 +- runner/execution.ts | 4 +- runner/publishing.ts | 11 +++-- test/runner-publishing.test.ts | 53 +++++++++++++++++++++ 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/docs/implementation/pull-request-opening.md b/docs/implementation/pull-request-opening.md index 57f3b2e0..3edf2cad 100644 --- a/docs/implementation/pull-request-opening.md +++ b/docs/implementation/pull-request-opening.md @@ -130,8 +130,8 @@ Some repositories do not support draft PRs (for example private repositories on `TaskPublishing` (`runner/publishing.ts`) owns publishing in production. `setUpRunner` builds the publisher with the real gateways (`GhAlreadyFixedGateway`, `GhPullRequestGateway`) and a `GitBranchPusher` that pushes from the runner-owned repository to `pushUrl(github.repository)` on `GH_HOST`, never to a configured URL. The pusher may overwrite only commits the task's ledger records as `owned`. Before #22 adds rebasing, nothing in production writes a `foreign` ledger entry (only `recordRebase` does), so codeboost pushes no head the ledger calls foreign. - **Base branch.** `github.baseBranch` is required with a runner block. The `gh` calls run with the same environment as the push (its `GH_HOST` and credentials). The CLI checks it before it takes the lock, and `setUpRunner` checks it again before anything is created. -- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding, else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A draft waits only for an owed safety finding, not an owed scope pause: a draft invites no review, and a task in needs human cannot record a pause. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. That draft names the paid finding, which only this process holds. It does so only while the task stays in needs human on the same current attempt. A later run that returns the task to needs human has its own reason. -- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it through `publishOwed`, which first runs the check that the runner lock still names the database, because a publish pushes; if the check fails, nothing starts. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide), or a person's `start` or `resume` moves the task. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). A short retry, after 30 seconds, follows a publish refused because GitHub's PR list lags behind a recorded PR, one refused just as an opening's deadline passed, and one whose open or update left work behind. That last case covers GitHub showing another head than the one pushed (the PR is made a draft and the task stays running), and a draft change that failed. Such an `opened` outcome is recorded with `reconcile`, which keeps the publish owed. Retries are bounded: one timer per task, and per chain at most one retry at an opening's deadline (while it is still ahead) and one short retry. A chain ends when a publish settles. A new one starts with a run's end, a person's action or a restart, never with a retry's own publish. Whatever is still unsettled after them is stuck for a reason a person must fix (two PRs carry a marker, the repository changed, GitHub keeps failing every open), so the next publish waits for a run's end, a restart or the `publish` action. +- **Mode.** The task's status decides it. A `running` task whose every plan item has completed at the current revision gets a ready PR. A task in `needs human`, by any path (a safety finding, the time budget), gets a draft PR. Its problems say what sent it there now: its current attempt's safety finding (or, when that finding's save failed, the attempt's diagnostic that carries it), else "time budget ran out" when the budget has passed, else that attempt's diagnostic. A budget that ran out after a safety finding does not hide the finding. An earlier attempt's finding, which a person may have dealt with, is not used. Any other status, an active attempt, a run still in progress, or a publish already in progress refuses. A draft waits only for an owed safety finding, not an owed scope pause: a draft invites no review, and a task in needs human cannot record a pause. A ready publish also needs every item of the current plan revision to have completed at that revision. The Store's publish guard checks this before every guarded write, so a revision that adds an item while a publish awaits GitHub refuses it. So does work an earlier run still owes: a safety finding not yet acted on, or a scope pause never recorded (its write failed, or the process stopped). Every plan item has run, so no later run would pay it, and a ready PR would show unreviewed out-of-scope changes. A run's end, startup and a refused `publish` action therefore have the executor pay it first (`ItemExecutor.payOwed`, as a run does before its first item). The task then moves to needs amendment or needs human, and a draft follows for the latter. For a task already in needs human, only an owed safety finding is paid. That draft names the paid finding, which only this process holds. It does so only while the task stays in needs human on the same current attempt. A later run that returns the task to needs human has its own reason. +- **When.** Once a run started by `start` or `resume` has ended, with no attempt active (`assertPublishable` refuses one). On the `publish` runner action (`expectedStateVersion` and an action ID, like the other runner actions), which retries a refused or failed publish. When a refused `start` or `resume` moves the task to needs human (an expired budget is committed with the refusal); a refused action that leaves the status as it was publishes nothing, so pressing resume on a task already in needs human never pushes. Once at startup, if a publish is owed: the CLI starts it through `publishOwed`, which first runs the check that the runner lock still names the database, because a publish pushes; if the check fails, nothing starts. A task recorded as `not published` by the v10 upgrade stays unpublished until a person uses the `publish` action, or until a run starts after the upgrade (its end is the first state the upgrade did not decide), or a person's `start` or `resume` moves the task. Otherwise a publish is owed when the task can be published and no publish has settled (`opened`, `possibly already fixed`, `draft skipped`, `draft unsupported`, `no changes`) at the task's current state version. So a refusal, a failure or a stop leaves it owed, and the next start recovers a lost opening by its marker. A publish that ends with an opening still in flight (its own reply was lost, or it was refused while one settles) is tried again at that opening's own deadline, plus a second (`settleRemaining`; a timer, cleared at shutdown). A short retry, after 30 seconds, follows a publish refused because GitHub's PR list lags behind a recorded PR, one refused by a guard (mostly a race: the task, its review or its head changed during the publish; a refusal that persists is in the mode too, so the retry finds nothing owed), one refused just as an opening's deadline passed, and one whose open or update left work behind. That last case covers GitHub showing another head than the one pushed (the PR is made a draft and the task stays running), and a draft change that failed. Such an `opened` outcome is recorded with `reconcile`, which keeps the publish owed. Retries are bounded: one timer per task, and per chain at most one retry at an opening's deadline (while it is still ahead) and one short retry. A chain ends when a publish settles. A new one starts with a run's end, a person's action or a restart, never with a retry's own publish. Whatever is still unsettled after them is stuck for a reason a person must fix (two PRs carry a marker, the repository changed, GitHub keeps failing every open), so the next publish waits for a run's end, a restart or the `publish` action. - **Outcome.** Each publish runs in the background, one per task, and records its outcome in `publish_outcomes` (schema v10): the outcome kind, or `refused` (a guard, `BranchPushRefused`, `OpeningUnsettled`, `PullRequestMisplaced`, `PullRequestRefused`), `failed` or `stopped` (shutdown), with its message (any token value or token shape removed, as for the push) and the task's state version. The record does not move the task's state version. The same transaction replaces the `publish` action's saved response with this outcome, so resending its action ID reports the outcome, not `publishing`. Before it starts, every publish (after a run, at startup or on the action) records itself as `publishing`, its durable in-flight ownership. If that write fails, the publish does not run. A `publishing` record left at startup is a publish whose process stopped before it recorded an outcome. Startup publishes again if one is owed, and otherwise settles the record as `stopped`. When only an action reply is stuck (its publish never recorded itself as started), the reply reports the outcome on record, and that record is left as it is. A failure while paying owed work does not stop this; the task's PR records show what that publish did. A reply that still says `publishing` belongs to an action whose process stopped (a crash) before its publish recorded an outcome. Any later outcome of the task settles it too, since one publish runs per task. At startup, if no publish is owed, it is settled as `stopped` ("interrupted before it recorded an outcome"), so no replay says `publishing` for ever. The v10 upgrade records every task already `running` or in `needs human` as `not published`, a settled outcome: the first start after the upgrade opens no PR a person did not ask for, and the `publish` action publishes it. `GET /api/runner` shows it as `publish.last`, with `publish.active` and `publish.publishable`. - **Start and resume** refuse while a publish of the task is in progress, so a new item cannot move the head being pushed. A second `publish` is refused too, and the refusal is saved under its action ID. Cancel task is not refused: a draft PR that GitHub opens after a cancel is recorded, and closing it is the later "close the draft on cancel" slice. - **Demo.** A demo never publishes, even with a `github` block or an injected publisher. diff --git a/runner/execution.ts b/runner/execution.ts index 6126742e..2c8b5d31 100644 --- a/runner/execution.ts +++ b/runner/execution.ts @@ -345,14 +345,16 @@ export class ItemExecutor { * Pay what an earlier run owes, outside a run, as a run would before its first item (#103: a task that owes a pause or * an escalation must not be published as finished). Null when nothing is owed, or a run of the task is in progress * (it pays its own); otherwise the outcome: needs human, needs amendment, or stopped when it could not be written. + * `scope: false` pays only an owed safety finding (a task in needs human cannot record a pause). */ - payOwed(identity: PlanIdentity): ExecutionOutcome | null { + payOwed(identity: PlanIdentity, options: { scope?: boolean } = {}): ExecutionOutcome | null { if (this.#runner.closing || this.#runner.isActive(identity) || this.busy(identity)) return null; const stopped = (item: string, state: string, reason: string | null): ExecutionOutcome => ({ kind: 'stopped', item, state, reason, completed: [] }); for (const earlier of this.#store.getAttempts(identity)) { const finding = this.#findings.get(earlier.id); if (finding) return this.#escalate(identity, earlier, finding, stopped, [], true); } + if (options.scope === false) return null; const owed = this.#unpausedScopeFinding(identity); return owed ? this.#pause(identity, owed.row, owed.result, stopped, [], true) : null; } diff --git a/runner/publishing.ts b/runner/publishing.ts index 35b8b0d8..3d378a63 100644 --- a/runner/publishing.ts +++ b/runner/publishing.ts @@ -2,7 +2,7 @@ import { identityKey, type PlanIdentity } from '../core/identity.ts'; import { PullRequestMisplaced, PullRequestRefused } from '../github/pull-requests.ts'; import { BranchPushRefused, redact, TOKEN_VARIABLES } from './branch-push.ts'; import type { RunnerCoordinator } from './coordinator.ts'; -import type { ItemExecutor } from './execution.ts'; +import { SAFETY_VIOLATION, type ItemExecutor } from './execution.ts'; import { GuardRefusal, ShuttingDownError, settleWith, type ShutdownCapability } from './lifecycle.ts'; import { OpeningUnsettled, type PublishOutcome, type PullRequestPublisher } from './publish.ts'; import type { PublishRecord, Store } from './store.ts'; @@ -189,7 +189,8 @@ export class TaskPublishing { // Its failure must not stop the caller from settling what it can (an in-flight record, a stuck reply): the task still // owes the work, and mode() refuses to publish until it is paid. try { - const paid = this.#executor.payOwed(identity); + // A task in needs human cannot record a scope pause, and its draft does not wait for one (mode): only a finding. + const paid = this.#executor.payOwed(identity, { scope: this.#store.getTask(identity).status !== 'needs human' }); // Tied to the attempt current once paid: a later run (a new attempt) that ends in needs human again has its own reason. if (paid?.kind === 'needs human') this.#escalations.set(identityKey(identity), { reason: paid.reason, attemptId: this.#store.getTask(identity).currentAttemptId }); if (paid?.kind === 'stopped') console.error(`Could not settle what an earlier run owes: ${JSON.stringify(paid.reason ?? '')}`); @@ -252,7 +253,9 @@ export class TaskPublishing { const outcome = await this.#publisher.publish(identity, draft ? { problems: this.#problems(identity) } : {}); record = describe(outcome, draft); } catch (error) { - unsettled = error instanceof OpeningUnsettled; + // A guard refusal is mostly a race (the task, its review or its head changed during the publish): one short retry + // publishes the new state. One that persists is in mode() too, so the retry finds nothing owed. + unsettled = error instanceof OpeningUnsettled || error instanceof GuardRefusal; const stopped = error instanceof ShuttingDownError || (this.#closing && (error as Error)?.name === 'AbortError'); record = { outcome: stopped ? 'stopped' : REFUSALS.some(type => error instanceof type) ? 'refused' : 'failed', draft, message: message(error, this.#secrets) }; } @@ -280,6 +283,8 @@ export class TaskPublishing { this.#escalations.delete(key); const current = task.currentAttemptId === null ? undefined : this.#store.getAttempt(identity, task.currentAttemptId); if (current?.safetyFinding) return [current.safetyFinding]; + // A finding whose save failed while the run still moved the task: the attempt's diagnostic is the finding's text. + if (current?.diagnostic?.startsWith(SAFETY_VIOLATION)) return [current.diagnostic]; if (task.budgetDeadline !== null && task.budgetDeadline <= Date.now()) return ['The task\'s time budget ran out before its plan finished.']; if (current?.diagnostic) return [current.diagnostic]; return ['The task needs a person.']; diff --git a/test/runner-publishing.test.ts b/test/runner-publishing.test.ts index 0c1820fb..5dabde22 100644 --- a/test/runner-publishing.test.ts +++ b/test/runner-publishing.test.ts @@ -7,6 +7,7 @@ import { fixtureGit as git } from './fixtures/git.ts'; import { createDemo } from '../scripts/demo.ts'; import { startServer } from '../web/server.ts'; import { Store } from '../runner/store.ts'; +import { GuardRefusal } from '../runner/lifecycle.ts'; import type { ReviewService } from '../runner/review.ts'; import type { PreparedAttempt, RunnerDeps } from '../runner/coordinator.ts'; import { SafetyFindings, type ExecutionSources } from '../runner/execution.ts'; @@ -844,6 +845,58 @@ describe('publishing a finished task (#103)', () => { }); }); + describe('round 9 of the independent review', () => { + it('retries once, after a short wait, a publish refused because the task changed during it', async () => { + const w = world(); + const { app, identity, store } = await serve(w, { before: completeAll, shortRetryMs: 200, startup: false }); + const check = store.recordAlreadyFixed.bind(store); + vi.spyOn(store, 'recordAlreadyFixed').mockImplementationOnce(() => { throw new GuardRefusal('The review changed during the check. Reload before writing.'); }) + .mockImplementation(check); + app.publishOwed(() => undefined); + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'refused', message: expect.stringMatching(/review changed/) }), { timeout: 10_000 }); + // Nothing a person must fix: the short retry publishes the task as it is now. + await vi.waitFor(() => expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: false }), { timeout: 10_000, interval: 50 }); + expect(w.github.prs).toEqual([expect.objectContaining({ draft: false })]); + }); + it('names a safety finding whose save failed in the draft, even after the budget ran out', async () => { + const w = world(); + const finding = 'Safety violation: The item wrote outside its workspace.'; + const before = (service: ReviewService) => { + const s = service.store, id = service.config.identity; + s.transitionTask(id, s.getTask(id).stateVersion, 'queued'); + const admit = (now?: number) => s.admitAttempt(id, { expectedStateVersion: s.getTask(id).stateVersion, kind: 'execute', item: s.getPlan(id).items[0]!.id, + expectedContext: s.currentContext(id), deadline: (now ?? Date.now()) + 60_000, ...(now ? { now } : {}) }); + const attempt = admit(); + s.markRunning(id, attempt.id); + // The finding's own save failed, so the attempt row carries it only as its diagnostic; the run still escalated. + s.settleAttempt(id, attempt.id, { firstReason: null, exitCode: 1, valid: false, detail: finding }); + s.transitionTask(id, s.getTask(id).stateVersion, 'needs human'); + }; + const { app, identity, store } = await serve(w, { before, startup: false }); + expect(store.getAttempts(identity).at(-1)).toMatchObject({ safetyFinding: null, diagnostic: expect.stringContaining(finding) }); + vi.spyOn(Date, 'now').mockReturnValue(store.getTask(identity).budgetDeadline! + 1); + app.publishOwed(() => undefined); + await publishSettled(app, identity); + vi.mocked(Date.now).mockRestore(); + expect(w.github.prs).toEqual([expect.objectContaining({ draft: true, body: expect.stringContaining(finding) })]); + expect(w.github.prs[0]!.body).not.toContain(BUDGET); + }); + it('pays no scope pause for a needs-human task, so nothing is logged as unsettled', async () => { + const w = world(); + const before = (service: ReviewService) => { + completeAllOwingPause(service); + const s = service.store, id = service.config.identity; + s.transitionTask(id, s.getTask(id).stateVersion, 'needs human'); + }; + const { app, identity, store } = await serve(w, { before, startup: false }); + const errors = vi.spyOn(console, 'error'); + app.publishOwed(() => undefined); + await publishSettled(app, identity); + expect(errors.mock.calls.filter(call => String(call[0]).includes('Could not settle what an earlier run owes'))).toEqual([]); + expect(store.lastPublish(identity)).toMatchObject({ outcome: 'opened', draft: true }); + }); + }); + it('never publishes in a demo, even with a publisher', async () => { const w = world(); const { app } = await serve(w, { demo: true, before: completeAll });