diff --git a/docs/api/agent.md b/docs/api/agent.md index 745c55c7..5e682aec 100644 --- a/docs/api/agent.md +++ b/docs/api/agent.md @@ -865,7 +865,7 @@ Required for `mode === 'open-pr'` — the GH owner/repo (`tangle-network/tax-age ##### allowCreateForKinds? -> `optional` **allowCreateForKinds?**: readonly (`"knowledge.wiki"` \| `"knowledge.claim"` \| `"knowledge.raw"` \| `"knowledge.stale"` \| `"system-prompt"` \| `"tool-doc"` \| `"new-tool"` \| `"rag"` \| `"memory"` \| `"scaffolding"` \| `"output-schema"` \| `"websearch.outdated"` \| `"prior-run-summary"` \| `"cluster"`)[] +> `optional` **allowCreateForKinds?**: readonly (`"memory"` \| `"knowledge.wiki"` \| `"knowledge.claim"` \| `"knowledge.raw"` \| `"knowledge.stale"` \| `"system-prompt"` \| `"tool-doc"` \| `"new-tool"` \| `"rag"` \| `"scaffolding"` \| `"output-schema"` \| `"websearch.outdated"` \| `"prior-run-summary"` \| `"cluster"`)[] Defined in: [agent/improvement-adapter.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/agent/improvement-adapter.ts#L100) diff --git a/docs/api/index.md b/docs/api/index.md index a4aa92a6..86ca200b 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -8,6 +8,352 @@ ## Classes +### FileAgentCandidateExecutionClaimStore + +Defined in: [candidate-execution/claim-file-store.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L74) + +Cross-process lifecycle implemented as fsynced, create-if-absent records. + +#### Implements + +- [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) + +#### Constructors + +##### Constructor + +> **new FileAgentCandidateExecutionClaimStore**(`options`): [`FileAgentCandidateExecutionClaimStore`](#fileagentcandidateexecutionclaimstore) + +Defined in: [candidate-execution/claim-file-store.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L78) + +###### Parameters + +###### options + +[`FileAgentCandidateExecutionClaimStoreOptions`](#fileagentcandidateexecutionclaimstoreoptions) + +###### Returns + +[`FileAgentCandidateExecutionClaimStore`](#fileagentcandidateexecutionclaimstore) + +#### Methods + +##### tryClaim() + +> **tryClaim**(`requested`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L86) + +###### Parameters + +###### requested + +[`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`tryClaim`](#tryclaim-1) + +##### getAttempt() + +> **getAttempt**(`requestedAttempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +Defined in: [candidate-execution/claim-file-store.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L115) + +###### Parameters + +###### requestedAttempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`getAttempt`](#getattempt-1) + +##### markCandidateMayRun() + +> **markCandidateMayRun**(`requestedLease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L121) + +Persist the point after which candidate code may have run. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`markCandidateMayRun`](#markcandidatemayrun-1) + +##### stageTerminal() + +> **stageTerminal**(`requestedLease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L158) + +Fsync the complete terminal record into the durable outbox. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### result + +[`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`stageTerminal`](#stageterminal-1) + +##### finish() + +> **finish**(`requestedLease`, `requestedTerminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L191) + +Publish exactly the staged terminal identified by `terminalDigest`. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### requestedTerminalDigest + +`` `sha256:${string}` `` + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`finish`](#finish-1) + +##### recoverExpired() + +> **recoverExpired**(`requestedAttempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim-file-store.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L228) + +Write a failed terminal only after the lease expired and a trusted worker +independently proved process death plus model and memory closure. + +###### Parameters + +###### requestedAttempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### evidence + +[`AgentCandidateExecutionRecoveryEvidence`](#agentcandidateexecutionrecoveryevidence) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`recoverExpired`](#recoverexpired-1) + +*** + +### InMemoryAgentCandidateExecutionClaimStore + +Defined in: [candidate-execution/claim.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L290) + +Single-process lifecycle implementation. + +#### Implements + +- [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) + +#### Constructors + +##### Constructor + +> **new InMemoryAgentCandidateExecutionClaimStore**(`options?`): [`InMemoryAgentCandidateExecutionClaimStore`](#inmemoryagentcandidateexecutionclaimstore) + +Defined in: [candidate-execution/claim.ts:296](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L296) + +###### Parameters + +###### options? + +`InMemoryAgentCandidateExecutionClaimStoreOptions` = `{}` + +###### Returns + +[`InMemoryAgentCandidateExecutionClaimStore`](#inmemoryagentcandidateexecutionclaimstore) + +#### Methods + +##### tryClaim() + +> **tryClaim**(`requested`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +Defined in: [candidate-execution/claim.ts:300](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L300) + +###### Parameters + +###### requested + +[`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`tryClaim`](#tryclaim-1) + +##### getAttempt() + +> **getAttempt**(`requestedAttempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +Defined in: [candidate-execution/claim.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L323) + +###### Parameters + +###### requestedAttempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`getAttempt`](#getattempt-1) + +##### markCandidateMayRun() + +> **markCandidateMayRun**(`requestedLease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +Defined in: [candidate-execution/claim.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L333) + +Persist the point after which candidate code may have run. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`markCandidateMayRun`](#markcandidatemayrun-1) + +##### stageTerminal() + +> **stageTerminal**(`requestedLease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +Defined in: [candidate-execution/claim.ts:350](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L350) + +Fsync the complete terminal record into the durable outbox. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### result + +[`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`stageTerminal`](#stageterminal-1) + +##### finish() + +> **finish**(`requestedLease`, `requestedTerminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim.ts:365](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L365) + +Publish exactly the staged terminal identified by `terminalDigest`. + +###### Parameters + +###### requestedLease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### requestedTerminalDigest + +`` `sha256:${string}` `` + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`finish`](#finish-1) + +##### recoverExpired() + +> **recoverExpired**(`requestedAttempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L383) + +Write a failed terminal only after the lease expired and a trusted worker +independently proved process death plus model and memory closure. + +###### Parameters + +###### requestedAttempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### evidence + +[`AgentCandidateExecutionRecoveryEvidence`](#agentcandidateexecutionrecoveryevidence) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +###### Implementation of + +[`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore).[`recoverExpired`](#recoverexpired-1) + +*** + ### CircuitOpenError Defined in: [conversation/call-policy.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/conversation/call-policy.ts#L43) @@ -831,71 +1177,2537 @@ Defined in: [sessions.ts:45](https://github.com/tangle-network/agent-runtime/blo [`RuntimeSessionStore`](#runtimesessionstore).[`get`](#get-1) -##### put() +##### put() + +> **put**(`session`): `void` + +Defined in: [sessions.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/sessions.ts#L49) + +###### Parameters + +###### session + +`RuntimeSession` + +###### Returns + +`void` + +###### Implementation of + +[`RuntimeSessionStore`](#runtimesessionstore).[`put`](#put-2) + +##### appendEvent() + +> **appendEvent**(`sessionId`, `event`): `void` + +Defined in: [sessions.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/sessions.ts#L53) + +###### Parameters + +###### sessionId + +`string` + +###### event + +[`RuntimeStreamEvent`](#runtimestreamevent) + +###### Returns + +`void` + +###### Implementation of + +[`RuntimeSessionStore`](#runtimesessionstore).[`appendEvent`](#appendevent-1) + +##### listEvents() + +> **listEvents**(`sessionId`): [`RuntimeStreamEvent`](#runtimestreamevent)[] + +Defined in: [sessions.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/sessions.ts#L59) + +###### Parameters + +###### sessionId + +`string` + +###### Returns + +[`RuntimeStreamEvent`](#runtimestreamevent)[] + +###### Implementation of + +[`RuntimeSessionStore`](#runtimesessionstore).[`listEvents`](#listevents-1) + +## Interfaces + +### FileAgentCandidateExecutionClaimStoreOptions + +Defined in: [candidate-execution/claim-file-store.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L66) + +#### Properties + +##### directory + +> **directory**: `string` + +Defined in: [candidate-execution/claim-file-store.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L68) + +Evaluator-owned directory shared by every process allowed to execute candidates. + +##### now? + +> `optional` **now?**: () => `number` + +Defined in: [candidate-execution/claim-file-store.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-file-store.ts#L70) + +Testable evaluator clock; defaults to `Date.now`. + +###### Returns + +`number` + +*** + +### AgentCandidateExecutionCleanupHandles + +Defined in: [candidate-execution/claim.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L39) + +Non-secret identities a trusted recovery worker needs to close an abandoned attempt. + +#### Properties + +##### preparationId + +> `readonly` **preparationId**: `string` + +Defined in: [candidate-execution/claim.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L40) + +##### modelGrantDigest + +> `readonly` **modelGrantDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/claim.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L41) + +##### resolvedModel + +> `readonly` **resolvedModel**: `AgentCandidateResolvedModel` + +Defined in: [candidate-execution/claim.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L42) + +##### traceRunId + +> `readonly` **traceRunId**: `string` + +Defined in: [candidate-execution/claim.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L43) + +##### cleanupTimeoutMs + +> `readonly` **cleanupTimeoutMs**: `number` + +Defined in: [candidate-execution/claim.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L44) + +##### memory? + +> `readonly` `optional` **memory?**: `object` + +Defined in: [candidate-execution/claim.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L45) + +###### accessDigest + +> `readonly` **accessDigest**: `` `sha256:${string}` `` + +###### effectiveNamespace + +> `readonly` **effectiveNamespace**: `string` + +*** + +### AgentCandidateExecutionClaim + +Defined in: [candidate-execution/claim.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L52) + +Immutable signed identity stored for one execution attempt. + +#### Properties + +##### executionId + +> `readonly` **executionId**: `string` + +Defined in: [candidate-execution/claim.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L53) + +##### attempt + +> `readonly` **attempt**: `number` + +Defined in: [candidate-execution/claim.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L54) + +##### maxAttempts + +> `readonly` **maxAttempts**: `number` + +Defined in: [candidate-execution/claim.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L55) + +##### retryPolicy + +> `readonly` **retryPolicy**: `"none"` \| `"pre-model-infrastructure-only"` + +Defined in: [candidate-execution/claim.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L56) + +##### bundleDigest + +> `readonly` **bundleDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/claim.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L57) + +##### executionPlanDigest + +> `readonly` **executionPlanDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/claim.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L58) + +##### retryLineageDigest + +> `readonly` **retryLineageDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/claim.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L60) + +Frozen plan identity with only attempt number and per-attempt grant identity normalized. + +##### leaseExpiresAtMs + +> `readonly` **leaseExpiresAtMs**: `number` + +Defined in: [candidate-execution/claim.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L62) + +The winning lease stops authorizing a new terminal write at this instant. + +##### resultTimeoutMs + +> `readonly` **resultTimeoutMs**: `number` + +Defined in: [candidate-execution/claim.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L64) + +Frozen budget for task verification, executable grading, and receipt construction. + +##### cleanup + +> `readonly` **cleanup**: [`AgentCandidateExecutionCleanupHandles`](#agentcandidateexecutioncleanuphandles) + +Defined in: [candidate-execution/claim.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L66) + +Non-secret handles retained so an expired attempt can be closed and reconciled. + +*** + +### AgentCandidateExecutionLease + +Defined in: [candidate-execution/claim.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L70) + +Secret capability required to finish the acquired attempt. + +#### Properties + +##### executionId + +> `readonly` **executionId**: `string` + +Defined in: [candidate-execution/claim.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L71) + +##### attempt + +> `readonly` **attempt**: `number` + +Defined in: [candidate-execution/claim.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L72) + +##### token + +> `readonly` **token**: `string` + +Defined in: [candidate-execution/claim.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L73) + +##### expiresAtMs + +> `readonly` **expiresAtMs**: `number` + +Defined in: [candidate-execution/claim.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L74) + +*** + +### AgentCandidateExecutionUsage + +Defined in: [candidate-execution/claim.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L85) + +Exact fixed-point usage proven by the closed evaluator model ledger. + +#### Properties + +##### costUsdNanos + +> `readonly` **costUsdNanos**: `number` + +Defined in: [candidate-execution/claim.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L86) + +##### inputTokens + +> `readonly` **inputTokens**: `number` + +Defined in: [candidate-execution/claim.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L87) + +##### outputTokens + +> `readonly` **outputTokens**: `number` + +Defined in: [candidate-execution/claim.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L88) + +##### cachedInputTokens + +> `readonly` **cachedInputTokens**: `number` + +Defined in: [candidate-execution/claim.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L89) + +##### reasoningTokens + +> `readonly` **reasoningTokens**: `number` + +Defined in: [candidate-execution/claim.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L90) + +##### modelCalls + +> `readonly` **modelCalls**: `number` + +Defined in: [candidate-execution/claim.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L91) + +*** + +### AgentCandidateExecutionRecoveryEvidence + +Defined in: [candidate-execution/claim.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L128) + +Trusted, independently observed closure facts for one expired winning lease. + +#### Properties + +##### failureClass + +> `readonly` **failureClass**: [`AgentCandidateExecutionFailureClass`](#agentcandidateexecutionfailureclass) + +Defined in: [candidate-execution/claim.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L129) + +##### usage + +> `readonly` **usage**: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage) + +Defined in: [candidate-execution/claim.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L130) + +##### modelSettlement + +> `readonly` **modelSettlement**: `AgentCandidateArtifactRef` + +Defined in: [candidate-execution/claim.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L131) + +##### failureEvidence? + +> `readonly` `optional` **failureEvidence?**: `AgentCandidateArtifactRef` + +Defined in: [candidate-execution/claim.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L132) + +##### process + +> `readonly` **process**: `object` + +Defined in: [candidate-execution/claim.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L133) + +###### stopped + +> `readonly` **stopped**: `true` + +###### executionPlanDigest + +> `readonly` **executionPlanDigest**: `` `sha256:${string}` `` + +##### model + +> `readonly` **model**: `object` + +Defined in: [candidate-execution/claim.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L137) + +###### closed + +> `readonly` **closed**: `true` + +###### preparationId + +> `readonly` **preparationId**: `string` + +###### grantDigest + +> `readonly` **grantDigest**: `` `sha256:${string}` `` + +##### memory? + +> `readonly` `optional` **memory?**: `object` + +Defined in: [candidate-execution/claim.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L142) + +###### closed + +> `readonly` **closed**: `true` + +###### preparationId + +> `readonly` **preparationId**: `string` + +###### accessDigest + +> `readonly` **accessDigest**: `` `sha256:${string}` `` + +###### effectiveNamespace + +> `readonly` **effectiveNamespace**: `string` + +*** + +### AgentCandidateExecutionAttemptRef + +Defined in: [candidate-execution/claim.ts:150](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L150) + +#### Properties + +##### executionId + +> `readonly` **executionId**: `string` + +Defined in: [candidate-execution/claim.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L151) + +##### attempt + +> `readonly` **attempt**: `number` + +Defined in: [candidate-execution/claim.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L152) + +*** + +### AgentCandidateExecutionAttemptRecord + +Defined in: [candidate-execution/claim.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L156) + +Persisted state available to a fresh trusted recovery worker after a crash. + +#### Properties + +##### claim + +> `readonly` **claim**: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) + +Defined in: [candidate-execution/claim.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L157) + +##### phase + +> `readonly` **phase**: [`AgentCandidateExecutionPhase`](#agentcandidateexecutionphase) + +Defined in: [candidate-execution/claim.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L158) + +##### staged? + +> `readonly` `optional` **staged?**: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord) + +Defined in: [candidate-execution/claim.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L160) + +Durable outbox content written before the terminal compare-and-set. + +##### terminal? + +> `readonly` `optional` **terminal?**: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord) + +Defined in: [candidate-execution/claim.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L161) + +*** + +### AgentCandidateExecutionClaimStore + +Defined in: [candidate-execution/claim.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L233) + +Atomic one-shot store for candidate execution attempts. + +Implementations must linearize both methods across every process sharing the +store. Terminal publication is deliberately two-step: `stageTerminal` +fsyncs the complete immutable outbox record, then `finish` publishes exactly +those staged bytes by digest. A crash between the two leaves recoverable +evidence rather than an ambiguous completed run. + +#### Methods + +##### tryClaim() + +> **tryClaim**(`claim`): `Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +Defined in: [candidate-execution/claim.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L234) + +###### Parameters + +###### claim + +[`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionClaimResult`](#agentcandidateexecutionclaimresult)\> + +##### getAttempt() + +> **getAttempt**(`attempt`): `Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +Defined in: [candidate-execution/claim.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L235) + +###### Parameters + +###### attempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionAttemptRecord`](#agentcandidateexecutionattemptrecord) \| `undefined`\> + +##### markCandidateMayRun() + +> **markCandidateMayRun**(`lease`): `Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +Defined in: [candidate-execution/claim.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L239) + +Persist the point after which candidate code may have run. + +###### Parameters + +###### lease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionPhaseResult`](#agentcandidateexecutionphaseresult)\> + +##### stageTerminal() + +> **stageTerminal**(`lease`, `result`): `Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +Defined in: [candidate-execution/claim.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L243) + +Fsync the complete terminal record into the durable outbox. + +###### Parameters + +###### lease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### result + +[`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionStageResult`](#agentcandidateexecutionstageresult)\> + +##### finish() + +> **finish**(`lease`, `terminalDigest`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L248) + +Publish exactly the staged terminal identified by `terminalDigest`. + +###### Parameters + +###### lease + +[`AgentCandidateExecutionLease`](#agentcandidateexecutionlease) + +###### terminalDigest + +`` `sha256:${string}` `` + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +##### recoverExpired() + +> **recoverExpired**(`attempt`, `evidence`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/claim.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L256) + +Write a failed terminal only after the lease expired and a trusted worker +independently proved process death plus model and memory closure. + +###### Parameters + +###### attempt + +[`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +###### evidence + +[`AgentCandidateExecutionRecoveryEvidence`](#agentcandidateexecutionrecoveryevidence) + +###### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +*** + +### DisposePreparedAgentCandidateOptions + +Defined in: [candidate-execution/dispose.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/dispose.ts#L10) + +#### Properties + +##### cleanupTimeoutMs? + +> `optional` **cleanupTimeoutMs?**: `number` + +Defined in: [candidate-execution/dispose.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/dispose.ts#L11) + +*** + +### ExecutePreparedAgentCandidateOptions + +Defined in: [candidate-execution/execute.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L60) + +#### Properties + +##### executor + +> **executor**: [`AgentCandidateExecutorPort`](#agentcandidateexecutorport) + +Defined in: [candidate-execution/execute.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L61) + +##### grader + +> **grader**: [`AgentCandidateBenchmarkGraderPort`](#agentcandidatebenchmarkgraderport) + +Defined in: [candidate-execution/execute.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L62) + +##### outputArtifacts + +> **outputArtifacts**: [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) + +Defined in: [candidate-execution/execute.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L63) + +##### traceStore + +> **traceStore**: `TraceStore` + +Defined in: [candidate-execution/execute.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L64) + +##### claimStore + +> **claimStore**: [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) + +Defined in: [candidate-execution/execute.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L66) + +Long-lived evaluator-owned store shared by every process that can run this benchmark. + +##### cleanupTimeoutMs? + +> `optional` **cleanupTimeoutMs?**: `number` + +Defined in: [candidate-execution/execute.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L68) + +Maximum time to prove process death and revoke protected access after a run ends. + +##### resultTimeoutMs? + +> `optional` **resultTimeoutMs?**: `number` + +Defined in: [candidate-execution/execute.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L70) + +Maximum time for task verification, executable grading, and receipt construction. + +*** + +### PrepareAgentCandidateExecutionOptions + +Defined in: [candidate-execution/prepare.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L88) + +#### Properties + +##### cleanupTimeoutMs? + +> `optional` **cleanupTimeoutMs?**: `number` + +Defined in: [candidate-execution/prepare.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L89) + +##### resultTimeoutMs? + +> `optional` **resultTimeoutMs?**: `number` + +Defined in: [candidate-execution/prepare.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L91) + +Maximum time for task verification, executable grading, and receipt construction. + +*** + +### RecoverExpiredAgentCandidateOptions + +Defined in: [candidate-execution/recover.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L23) + +#### Properties + +##### attempt + +> **attempt**: [`AgentCandidateExecutionAttemptRef`](#agentcandidateexecutionattemptref) + +Defined in: [candidate-execution/recover.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L24) + +##### claimStore + +> **claimStore**: [`AgentCandidateExecutionClaimStore`](#agentcandidateexecutionclaimstore) + +Defined in: [candidate-execution/recover.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L25) + +##### executor + +> **executor**: [`AgentCandidateExecutorPort`](#agentcandidateexecutorport) + +Defined in: [candidate-execution/recover.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L26) + +##### traceStore + +> **traceStore**: `TraceStore` + +Defined in: [candidate-execution/recover.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L27) + +##### ports + +> **ports**: `Pick`\<[`AgentCandidateExecutionPorts`](#agentcandidateexecutionports), `"models"` \| `"memory"`\> + +Defined in: [candidate-execution/recover.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L28) + +##### outputArtifacts + +> **outputArtifacts**: [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) + +Defined in: [candidate-execution/recover.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L29) + +##### cleanupTimeoutMs? + +> `optional` **cleanupTimeoutMs?**: `number` + +Defined in: [candidate-execution/recover.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L30) + +##### now? + +> `optional` **now?**: () => `number` + +Defined in: [candidate-execution/recover.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L32) + +Evaluator clock; must be the same clock used by the claim store. + +###### Returns + +`number` + +*** + +### AgentCandidateArtifactPort + +Defined in: [candidate-execution/types.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L33) + +Reads one content-addressed object from the closed S3/IPFS locator set. + +#### Extended by + +- [`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) + +#### Methods + +##### read() + +> **read**(`ref`): `Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> + +Defined in: [candidate-execution/types.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L34) + +###### Parameters + +###### ref + +`AgentCandidateArtifactRef` + +###### Returns + +`Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> + +*** + +### AgentCandidateOutputArtifactPort + +Defined in: [candidate-execution/types.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L52) + +Durable content-addressed evidence store controlled only by the evaluator. + +#### Extends + +- [`AgentCandidateArtifactPort`](#agentcandidateartifactport) + +#### Methods + +##### read() + +> **read**(`ref`): `Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> + +Defined in: [candidate-execution/types.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L34) + +###### Parameters + +###### ref + +`AgentCandidateArtifactRef` + +###### Returns + +`Promise`\<`Uint8Array`\<`ArrayBufferLike`\>\> + +###### Inherited from + +[`AgentCandidateArtifactPort`](#agentcandidateartifactport).[`read`](#read) + +##### put() + +> **put**(`input`): `Promise`\<`AgentCandidateArtifactRef`\> + +Defined in: [candidate-execution/types.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L54) + +Must be idempotent for identical bytes and return only a durable S3/IPFS locator. + +###### Parameters + +###### input + +###### executionId + +`string` + +###### purpose + +[`AgentCandidateOutputPurpose`](#agentcandidateoutputpurpose) + +###### bytes + +`Uint8Array` + +###### signal? + +`AbortSignal` + +Abort must prevent durable publication when it happens before resolution. + +###### Returns + +`Promise`\<`AgentCandidateArtifactRef`\> + +*** + +### AgentCandidateRepositoryPort + +Defined in: [candidate-execution/types.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L64) + +Resolves a declared GitHub repository to an already-present local Git object store. + +#### Methods + +##### resolve() + +> **resolve**(`repository`): `Promise`\<`string`\> + +Defined in: [candidate-execution/types.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L65) + +###### Parameters + +###### repository + +`AgentCandidateGitHubRepository` + +###### Returns + +`Promise`\<`string`\> + +*** + +### AgentCandidateVerificationPorts + +Defined in: [candidate-execution/types.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L68) + +#### Extended by + +- [`AgentCandidateExecutionPorts`](#agentcandidateexecutionports) + +#### Properties + +##### artifacts + +> **artifacts**: [`AgentCandidateArtifactPort`](#agentcandidateartifactport) + +Defined in: [candidate-execution/types.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L69) + +##### repositories + +> **repositories**: [`AgentCandidateRepositoryPort`](#agentcandidaterepositoryport) + +Defined in: [candidate-execution/types.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L70) + +*** + +### AgentCandidateWorkspacePort + +Defined in: [candidate-execution/types.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L80) + +Materializes an already-verified workspace archive. + +The runtime independently scans every resulting byte, mode, and path against +the signed manifest after this returns. Implementations may therefore unpack +any archive encoding, or no-op when the exact workspace is already present. + +#### Methods + +##### materialize() + +> **materialize**(`input`): `Promise`\<`void`\> + +Defined in: [candidate-execution/types.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L81) + +###### Parameters + +###### input + +###### role + +`"task"` \| `"memory"` \| `"candidate"` + +###### snapshot + +`AgentCandidateWorkspaceSnapshotEvidence` + +###### archive + +`Uint8Array` + +###### destination + +`string` + +###### Returns + +`Promise`\<`void`\> + +*** + +### ResolvedAgentCandidateContainer + +Defined in: [candidate-execution/types.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L89) + +#### Properties + +##### source + +> **source**: `"pinned-container"` \| `"evaluator-task-container"` + +Defined in: [candidate-execution/types.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L90) + +##### image + +> **image**: `string` + +Defined in: [candidate-execution/types.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L91) + +##### indexDigest + +> **indexDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L92) + +##### manifestDigest + +> **manifestDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L93) + +##### platform + +> **platform**: `AgentCandidateOciPlatform` + +Defined in: [candidate-execution/types.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L94) + +*** + +### AgentCandidateContainerPort + +Defined in: [candidate-execution/types.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L97) + +#### Methods + +##### resolve() + +> **resolve**(`input`): `Promise`\<[`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer)\> + +Defined in: [candidate-execution/types.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L98) + +###### Parameters + +###### input + +###### candidate + +`AgentCandidateContainer` \| `undefined` + +###### evaluatorTaskContainer + +[`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer) \| `undefined` + +###### Returns + +`Promise`\<[`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer)\> + +*** + +### AgentCandidateModelPort + +Defined in: [candidate-execution/types.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L104) + +#### Methods + +##### resolve() + +> **resolve**(`input`): `Promise`\<`AgentCandidateResolvedModel`\> + +Defined in: [candidate-execution/types.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L105) + +###### Parameters + +###### input + +###### requested + +`string` + +###### harness + +`HarnessType` + +###### reasoningEffort + +`ReasoningEffort` \| `undefined` + +###### Returns + +`Promise`\<`AgentCandidateResolvedModel`\> + +##### reserveGrant() + +> **reserveGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelReservation`](#agentcandidateprotectedmodelreservation)\> + +Defined in: [candidate-execution/types.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L115) + +Reserve a stable access identity without creating a live credential. +The reservation is scoped to `preparationId` and must automatically expire +at `expiresAtMs`, even if this call returns ambiguously to the runtime. + +###### Parameters + +###### input + +###### executionId + +`string` + +###### preparationId + +`string` + +###### expiresAtMs + +`number` + +###### attempt + +`AgentCandidateAttemptPolicy` + +###### bundleDigest + +`` `sha256:${string}` `` + +###### resolved + +`AgentCandidateResolvedModel` + +###### limits + +[`AgentCandidateModelLimits`](#agentcandidatemodellimits) + +###### Returns + +`Promise`\<[`AgentCandidateProtectedModelReservation`](#agentcandidateprotectedmodelreservation)\> + +##### activateGrant() + +> **activateGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelActivation`](#agentcandidateprotectedmodelactivation)\> + +Defined in: [candidate-execution/types.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L125) + +Create the live scoped credential only after the execution attempt is durably claimed. + +###### Parameters + +###### input + +###### executionId + +`string` + +###### preparationId + +`string` + +###### grantDigest + +`` `sha256:${string}` `` + +###### resolved + +`AgentCandidateResolvedModel` + +###### deadlineAtMs + +`number` + +###### Returns + +`Promise`\<[`AgentCandidateProtectedModelActivation`](#agentcandidateprotectedmodelactivation)\> + +##### settleGrant() + +> **settleGrant**(`input`): `Promise`\<[`AgentCandidateProtectedModelSettlement`](#agentcandidateprotectedmodelsettlement)\> + +Defined in: [candidate-execution/types.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L138) + +Atomically revoke the grant, drain in-flight calls, and return its immutable final ledger. +This operation must be idempotent for the exact preparation and must also +settle a reservation that was never activated. It must never affect a +different preparation, even when both reservations report the same digest. + +###### Parameters + +###### input + +###### executionId + +`string` + +###### preparationId + +`string` + +###### grantDigest + +`` `sha256:${string}` `` + +###### resolved + +`AgentCandidateResolvedModel` + +###### reason + +`"completed"` \| `"failed"` \| `"timeout"` \| `"replayed"` \| `"preparation-failed"` \| `"abandoned"` + +###### Returns + +`Promise`\<[`AgentCandidateProtectedModelSettlement`](#agentcandidateprotectedmodelsettlement)\> + +*** + +### AgentCandidateProtectedModelReservation + +Defined in: [candidate-execution/types.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L153) + +#### Properties + +##### preparationId + +> **preparationId**: `string` + +Defined in: [candidate-execution/types.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L154) + +##### digest + +> **digest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L155) + +##### expiresAtMs + +> **expiresAtMs**: `number` + +Defined in: [candidate-execution/types.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L157) + +Evaluator service must expire and revoke this reservation at this epoch millisecond. + +##### enforcedLimits + +> **enforcedLimits**: [`AgentCandidateModelLimits`](#agentcandidatemodellimits) + +Defined in: [candidate-execution/types.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L159) + +The gateway must stop calls before any one of these limits is exceeded. + +*** + +### AgentCandidateProtectedModelActivation + +Defined in: [candidate-execution/types.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L162) + +#### Properties + +##### env + +> **env**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [candidate-execution/types.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L164) + +Injected only into the trusted executor after all pre-launch checks pass. + +*** + +### AgentCandidateProtectedModelCall + +Defined in: [candidate-execution/types.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L168) + +One evaluator-gateway call in the final, revoked model-access ledger. + +#### Properties + +##### callId + +> **callId**: `string` + +Defined in: [candidate-execution/types.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L169) + +##### traceSpanId + +> **traceSpanId**: `string` + +Defined in: [candidate-execution/types.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L171) + +Exact protected agent-eval LLM span produced for this call. + +##### model + +> **model**: `string` + +Defined in: [candidate-execution/types.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L172) + +##### inputTokens + +> **inputTokens**: `number` + +Defined in: [candidate-execution/types.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L173) + +##### outputTokens + +> **outputTokens**: `number` + +Defined in: [candidate-execution/types.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L174) + +##### cachedInputTokens? + +> `optional` **cachedInputTokens?**: `number` + +Defined in: [candidate-execution/types.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L175) + +##### reasoningTokens? + +> `optional` **reasoningTokens?**: `number` + +Defined in: [candidate-execution/types.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L176) + +##### costUsdNanos + +> **costUsdNanos**: `number` + +Defined in: [candidate-execution/types.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L178) + +Integer billionths of one US dollar; avoids floating-point ledger drift. + +*** + +### AgentCandidateProtectedModelSettlement + +Defined in: [candidate-execution/types.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L181) + +#### Properties + +##### preparationId + +> **preparationId**: `string` + +Defined in: [candidate-execution/types.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L182) + +##### grantDigest + +> **grantDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L183) + +##### closed + +> **closed**: `true` + +Defined in: [candidate-execution/types.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L184) + +##### calls + +> **calls**: readonly [`AgentCandidateProtectedModelCall`](#agentcandidateprotectedmodelcall)[] + +Defined in: [candidate-execution/types.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L185) + +*** + +### AgentCandidateMemoryResetResult + +Defined in: [candidate-execution/types.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L188) + +#### Properties + +##### preparationId + +> **preparationId**: `string` + +Defined in: [candidate-execution/types.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L189) + +##### accessDigest + +> **accessDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L190) + +##### expiresAtMs + +> **expiresAtMs**: `number` + +Defined in: [candidate-execution/types.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L191) + +##### evidence + +> **evidence**: `AgentCandidateCapturedArtifact` + +Defined in: [candidate-execution/types.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L192) + +##### emptyStateDigest + +> **emptyStateDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L193) + +##### beforeState + +> **beforeState**: `AgentCandidateWorkspaceSnapshotEvidence` + +Defined in: [candidate-execution/types.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L194) + +*** + +### AgentCandidateMemoryPort + +Defined in: [candidate-execution/types.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L197) + +#### Methods + +##### reset() + +> **reset**(`input`): `Promise`\<[`AgentCandidateMemoryResetResult`](#agentcandidatememoryresetresult)\> + +Defined in: [candidate-execution/types.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L203) + +Reset and reserve exact task memory without returning live access. +The service must scope the reservation to `preparationId`, automatically +revoke it at `expiresAtMs`, and never reuse it for another preparation. + +###### Parameters + +###### input + +###### executionId + +`string` + +###### preparationId + +`string` + +###### expiresAtMs + +`number` + +###### effectiveNamespace + +`string` + +###### seed? + +`Uint8Array`\<`ArrayBufferLike`\> + +###### seedDigest? + +`` `sha256:${string}` `` + +###### Returns + +`Promise`\<[`AgentCandidateMemoryResetResult`](#agentcandidatememoryresetresult)\> + +##### activate() + +> **activate**(`input`): `Promise`\<\{ `env`: `Readonly`\<`Record`\<`string`, `string`\>\>; \}\> + +Defined in: [candidate-execution/types.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L215) + +Create live scoped access only after the execution attempt is durably claimed. +Activation must match the exact preparation/access pair and may not extend expiry. + +###### Parameters + +###### input + +###### executionId + +`string` + +###### preparationId + +`string` + +###### accessDigest + +`` `sha256:${string}` `` + +###### effectiveNamespace + +`string` + +###### deadlineAtMs + +`number` + +###### Returns + +`Promise`\<\{ `env`: `Readonly`\<`Record`\<`string`, `string`\>\>; \}\> + +##### close() + +> **close**(`input`): `Promise`\<\{ `closed`: `true`; \}\> + +Defined in: [candidate-execution/types.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L227) + +Revoke evaluator-owned access after process death or a failed preparation. +Must be idempotent and concurrency-safe for the exact preparation/access +pair and must never close a different preparation. + +###### Parameters + +###### input + +###### executionId + +`string` + +###### preparationId + +`string` + +###### accessDigest + +`` `sha256:${string}` `` + +###### effectiveNamespace + +`string` + +###### reason + +`"completed"` \| `"failed"` \| `"timeout"` \| `"replayed"` \| `"preparation-failed"` \| `"abandoned"` + +###### Returns + +`Promise`\<\{ `closed`: `true`; \}\> + +*** + +### AgentCandidateExecutionPorts + +Defined in: [candidate-execution/types.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L236) + +#### Extends + +- [`AgentCandidateVerificationPorts`](#agentcandidateverificationports) + +#### Properties + +##### artifacts + +> **artifacts**: [`AgentCandidateArtifactPort`](#agentcandidateartifactport) + +Defined in: [candidate-execution/types.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L69) + +###### Inherited from + +[`AgentCandidateVerificationPorts`](#agentcandidateverificationports).[`artifacts`](#artifacts) + +##### repositories + +> **repositories**: [`AgentCandidateRepositoryPort`](#agentcandidaterepositoryport) + +Defined in: [candidate-execution/types.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L70) + +###### Inherited from + +[`AgentCandidateVerificationPorts`](#agentcandidateverificationports).[`repositories`](#repositories) + +##### workspaces + +> **workspaces**: [`AgentCandidateWorkspacePort`](#agentcandidateworkspaceport) + +Defined in: [candidate-execution/types.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L237) + +##### containers + +> **containers**: [`AgentCandidateContainerPort`](#agentcandidatecontainerport) + +Defined in: [candidate-execution/types.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L238) + +##### models + +> **models**: [`AgentCandidateModelPort`](#agentcandidatemodelport) + +Defined in: [candidate-execution/types.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L239) + +##### memory + +> **memory**: [`AgentCandidateMemoryPort`](#agentcandidatememoryport) + +Defined in: [candidate-execution/types.ts:240](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L240) + +*** + +### AgentCandidateTaskExecution + +Defined in: [candidate-execution/types.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L243) + +#### Properties + +##### executionId + +> **executionId**: `string` + +Defined in: [candidate-execution/types.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L244) + +##### benchmark + +> **benchmark**: `string` + +Defined in: [candidate-execution/types.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L245) + +##### benchmarkVersion + +> **benchmarkVersion**: `string` + +Defined in: [candidate-execution/types.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L246) + +##### taskId + +> **taskId**: `string` + +Defined in: [candidate-execution/types.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L247) + +##### splitDigest + +> **splitDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L248) + +##### instruction + +> **instruction**: `string` + +Defined in: [candidate-execution/types.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L250) + +Exact agent-visible task instruction. The runtime rejects malformed Unicode. + +##### repository + +> **repository**: `object` + +Defined in: [candidate-execution/types.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L251) + +###### identity + +> **identity**: `string` + +###### rootIdentity + +> **rootIdentity**: `string` + +###### baseCommit + +> **baseCommit**: `string` + +###### baseTree + +> **baseTree**: `string` + +##### attempt + +> **attempt**: `AgentCandidateAttemptPolicy` + +Defined in: [candidate-execution/types.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L257) + +##### model + +> **model**: `object` + +Defined in: [candidate-execution/types.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L258) + +###### requested + +> **requested**: `string` + +###### reasoningEffort + +> **reasoningEffort**: `ReasoningEffort` + +##### executionRoots + +> **executionRoots**: `object` + +Defined in: [candidate-execution/types.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L263) + +Absolute paths inside the evaluator-owned execution environment. + +###### taskRoot + +> **taskRoot**: `string` + +###### candidateRoot? + +> `optional` **candidateRoot?**: `string` + +##### stagingRoots + +> **stagingRoots**: `object` + +Defined in: [candidate-execution/types.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L268) + +Host-side staging roots. These are verified but never signed as container paths. + +###### taskRoot + +> **taskRoot**: `string` + +###### candidateRoot? + +> `optional` **candidateRoot?**: `string` + +###### profileRoot + +> **profileRoot**: `string` + +##### workspace + +> **workspace**: `AgentCandidateWorkspaceSnapshotEvidence` + +Defined in: [candidate-execution/types.ts:273](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L273) + +##### evaluatorTaskContainer? + +> `optional` **evaluatorTaskContainer?**: [`ResolvedAgentCandidateContainer`](#resolvedagentcandidatecontainer) + +Defined in: [candidate-execution/types.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L274) + +##### limits + +> **limits**: `AgentCandidateExecutionLimits` + +Defined in: [candidate-execution/types.ts:275](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L275) + +*** + +### VerifiedAgentCandidate + +Defined in: [candidate-execution/types.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L278) + +#### Properties + +##### bundle + +> `readonly` **bundle**: `AgentCandidateBundleV1` + +Defined in: [candidate-execution/types.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L279) + +##### materializedTree? + +> `readonly` `optional` **materializedTree?**: `string` + +Defined in: [candidate-execution/types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L280) + +##### \[verifiedCandidateBrand\] + +> `readonly` **\[verifiedCandidateBrand\]**: `true` + +Defined in: [candidate-execution/types.ts:281](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L281) + +*** + +### CanonicalCandidateDocument + +Defined in: [candidate-execution/types.ts:284](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L284) + +#### Type Parameters + +##### T + +`T` + +#### Properties + +##### value + +> `readonly` **value**: `T` + +Defined in: [candidate-execution/types.ts:285](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L285) + +##### bytes + +> `readonly` **bytes**: `Uint8Array` + +Defined in: [candidate-execution/types.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L287) + +Canonical UTF-8 bytes of `value` with its top-level digest omitted. + +##### digest + +> `readonly` **digest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:288](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L288) + +*** + +### PreparedAgentCandidateLaunch + +Defined in: [candidate-execution/types.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L291) + +#### Properties + +##### executable + +> **executable**: `string` + +Defined in: [candidate-execution/types.ts:292](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L292) + +##### args + +> **args**: readonly `string`[] + +Defined in: [candidate-execution/types.ts:294](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L294) + +Complete fixed argv, including profile materializer flags but excluding task delivery. + +##### env + +> **env**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [candidate-execution/types.ts:295](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L295) + +##### flags + +> **flags**: readonly `string`[] + +Defined in: [candidate-execution/types.ts:297](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L297) + +Informational subset already present at the tail of `args`; executors must not append twice. + +##### cwd + +> **cwd**: `string` + +Defined in: [candidate-execution/types.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L298) + +*** + +### PreparedAgentCandidateInstruction + +Defined in: [candidate-execution/types.ts:301](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L301) + +#### Properties + +##### bytes + +> **bytes**: `Uint8Array` + +Defined in: [candidate-execution/types.ts:302](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L302) + +##### delivery + +> **delivery**: `AgentCandidateInstructionDelivery` + +Defined in: [candidate-execution/types.ts:303](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L303) + +*** + +### PreparedAgentCandidateTrace + +Defined in: [candidate-execution/types.ts:306](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L306) + +#### Properties + +##### runId + +> **runId**: `string` + +Defined in: [candidate-execution/types.ts:307](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L307) + +##### tags + +> **tags**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [candidate-execution/types.ts:308](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L308) + +##### env + +> **env**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [candidate-execution/types.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L309) + +*** + +### PreparedAgentCandidateExecution + +Defined in: [candidate-execution/types.ts:312](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L312) + +#### Properties + +##### bundle + +> `readonly` **bundle**: `AgentCandidateBundleV1` + +Defined in: [candidate-execution/types.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L313) + +##### executionId + +> `readonly` **executionId**: `string` + +Defined in: [candidate-execution/types.ts:314](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L314) + +##### roots + +> `readonly` **roots**: `object` + +Defined in: [candidate-execution/types.ts:315](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L315) + +###### execution + +> **execution**: `object` + +###### execution.taskRoot + +> **taskRoot**: `string` + +###### execution.candidateRoot? + +> `optional` **candidateRoot?**: `string` + +###### staging + +> **staging**: `object` + +###### staging.taskRoot + +> **taskRoot**: `string` + +###### staging.candidateRoot? + +> `optional` **candidateRoot?**: `string` + +###### staging.profileRoot + +> **profileRoot**: `string` + +##### profilePlan + +> `readonly` **profilePlan**: `object` + +Defined in: [candidate-execution/types.ts:326](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L326) + +###### value + +> **value**: `AgentCandidateProfilePlanEvidence` + +###### bytes + +> **bytes**: `Uint8Array` + +###### written + +> **written**: readonly `string`[] + +##### executionPlan + +> `readonly` **executionPlan**: `object` + +Defined in: [candidate-execution/types.ts:331](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L331) + +###### value + +> **value**: `AgentCandidateExecutionPlanEvidence` + +###### bytes + +> **bytes**: `Uint8Array` + +##### materializationReceipt + +> `readonly` **materializationReceipt**: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateMaterializationReceiptV1`\> + +Defined in: [candidate-execution/types.ts:335](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L335) + +##### launch + +> `readonly` **launch**: [`PreparedAgentCandidateLaunch`](#preparedagentcandidatelaunch) + +Defined in: [candidate-execution/types.ts:336](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L336) + +##### instruction + +> `readonly` **instruction**: [`PreparedAgentCandidateInstruction`](#preparedagentcandidateinstruction) + +Defined in: [candidate-execution/types.ts:337](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L337) + +##### resolvedModel + +> `readonly` **resolvedModel**: `AgentCandidateResolvedModel` + +Defined in: [candidate-execution/types.ts:338](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L338) + +##### knowledge? + +> `readonly` `optional` **knowledge?**: `object` + +Defined in: [candidate-execution/types.ts:339](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L339) + +###### snapshotId + +> **snapshotId**: `string` + +###### manifestDigest + +> **manifestDigest**: `` `sha256:${string}` `` + +###### manifest + +> **manifest**: `Uint8Array` + +##### trace + +> `readonly` **trace**: [`PreparedAgentCandidateTrace`](#preparedagentcandidatetrace) + +Defined in: [candidate-execution/types.ts:344](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L344) + +##### memory + +> `readonly` **memory**: `AgentCandidateEffectiveMemory` + +Defined in: [candidate-execution/types.ts:345](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L345) + +##### \[preparedCandidateBrand\] + +> `readonly` **\[preparedCandidateBrand\]**: `true` + +Defined in: [candidate-execution/types.ts:346](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L346) + +*** + +### AgentCandidateProtectedRunCapture + +Defined in: [candidate-execution/types.ts:349](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L349) + +#### Properties + +##### executionId + +> **executionId**: `string` + +Defined in: [candidate-execution/types.ts:350](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L350) + +##### termination + +> **termination**: `AgentCandidateTermination` + +Defined in: [candidate-execution/types.ts:351](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L351) + +*** + +### AgentCandidateExecutorTaskOutcomeCapture + +Defined in: [candidate-execution/types.ts:355](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L355) + +Raw evaluator capture made only after the candidate process is dead. + +#### Properties + +##### resultTree + +> **resultTree**: `string` + +Defined in: [candidate-execution/types.ts:357](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L357) + +Claimed final tree. The runtime recomputes it independently from `gitDiff`. + +##### afterState + +> **afterState**: `AgentCandidateWorkspaceManifestMaterialV1` + +Defined in: [candidate-execution/types.ts:359](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L359) + +Complete evaluator-captured workspace description after candidate execution. + +##### archive + +> **archive**: `Uint8Array` + +Defined in: [candidate-execution/types.ts:361](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L361) + +Reproducible workspace archive corresponding to `afterState`. + +##### gitDiff + +> **gitDiff**: `Uint8Array` + +Defined in: [candidate-execution/types.ts:363](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L363) + +Exact binary patch from the signed task base to `afterState`. + +*** + +### AgentCandidateExecutorMemoryCapture + +Defined in: [candidate-execution/types.ts:367](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L367) + +Raw isolated-memory capture made only after access has been revoked. + +#### Properties + +##### afterState + +> `readonly` **afterState**: `AgentCandidateWorkspaceManifestMaterialV1` + +Defined in: [candidate-execution/types.ts:368](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L368) + +##### archive + +> `readonly` **archive**: `Uint8Array` + +Defined in: [candidate-execution/types.ts:369](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L369) + +*** + +### AgentCandidateExecutorFinalCapture + +Defined in: [candidate-execution/types.ts:373](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L373) + +Idempotent executor result after process death and trace drain. + +#### Properties + +##### stopped + +> `readonly` **stopped**: `true` + +Defined in: [candidate-execution/types.ts:374](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L374) + +##### taskOutcome? + +> `readonly` `optional` **taskOutcome?**: [`AgentCandidateExecutorTaskOutcomeCapture`](#agentcandidateexecutortaskoutcomecapture) + +Defined in: [candidate-execution/types.ts:375](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L375) + +##### memoryAfter? + +> `readonly` `optional` **memoryAfter?**: [`AgentCandidateExecutorMemoryCapture`](#agentcandidateexecutormemorycapture) + +Defined in: [candidate-execution/types.ts:377](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L377) + +Required only when the prepared candidate uses isolated task memory. + +*** + +### VerifiedAgentCandidateTaskOutcome + +Defined in: [candidate-execution/types.ts:381](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L381) + +Branded task outcome that has survived independent patch and tree verification. + +#### Properties + +##### evidence + +> `readonly` **evidence**: `AgentCandidateTaskOutcomeEvidence` & `object` + +Defined in: [candidate-execution/types.ts:382](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L382) + +###### Type Declaration + +###### artifact + +> `readonly` **artifact**: `AgentCandidateArtifactRef` + +##### patch + +> `readonly` **patch**: `Uint8Array` + +Defined in: [candidate-execution/types.ts:385](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L385) + +##### \[verifiedTaskOutcomeBrand\] + +> `readonly` **\[verifiedTaskOutcomeBrand\]**: `true` + +Defined in: [candidate-execution/types.ts:386](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L386) + +*** + +### AgentCandidateBenchmarkGraderPort + +Defined in: [candidate-execution/types.ts:398](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L398) + +Evaluator-owned executable grader, pinned by immutable implementation bytes. + +`run` is an isolation boundary, not an arbitrary scoring callback. The +implementation admitted to that boundary is supplied by the runtime after +artifact verification. Implementations must derive every returned binding +digest from the bytes and task outcome they actually admitted, rather than +copying an expected digest from ambient configuration. + +#### Properties + +##### name + +> `readonly` **name**: `string` + +Defined in: [candidate-execution/types.ts:399](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L399) + +##### version + +> `readonly` **version**: `string` + +Defined in: [candidate-execution/types.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L400) + +##### artifact + +> `readonly` **artifact**: `AgentCandidateArtifactRef` + +Defined in: [candidate-execution/types.ts:401](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L401) + +#### Methods + +##### run() + +> **run**(`input`): `Promise`\<\{ `evaluation`: `BenchmarkEvaluation`; `evidence`: `Uint8Array`; `binding`: \{ `implementationDigest`: `` `sha256:${string}` ``; `taskOutcomeDigest`: `` `sha256:${string}` ``; `outputDigest`: `` `sha256:${string}` ``; \}; \}\> + +Defined in: [candidate-execution/types.ts:402](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L402) + +###### Parameters + +###### input + +###### executionId + +`string` + +###### termination + +`AgentCandidateTermination` + +###### outcome + +[`VerifiedAgentCandidateTaskOutcome`](#verifiedagentcandidatetaskoutcome) + +###### implementation + +\{ `byteLength`: `number`; `bytes`: `Uint8Array`; \} + +Exact verified artifact bytes. Each read returns a detached copy. + +###### implementation.byteLength + +`number` + +###### implementation.bytes + +`Uint8Array` + +###### signal + +`AbortSignal` + +Frozen result deadline; runners must stop work and side effects when aborted. + +###### Returns + +`Promise`\<\{ `evaluation`: `BenchmarkEvaluation`; `evidence`: `Uint8Array`; `binding`: \{ `implementationDigest`: `` `sha256:${string}` ``; `taskOutcomeDigest`: `` `sha256:${string}` ``; `outputDigest`: `` `sha256:${string}` ``; \}; \}\> + +*** + +### AgentCandidateExecutorRequest + +Defined in: [candidate-execution/types.ts:430](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L430) + +One detached request passed to the trusted environment-specific executor. + +#### Properties + +##### executionId + +> `readonly` **executionId**: `string` + +Defined in: [candidate-execution/types.ts:431](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L431) + +##### inputs + +> `readonly` **inputs**: `object` + +Defined in: [candidate-execution/types.ts:433](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L433) + +Immutable bytes from which the executor creates fresh isolated workspaces. + +###### task + +> `readonly` **task**: [`AgentCandidateExecutorWorkspaceInput`](#agentcandidateexecutorworkspaceinput) + +###### candidate? + +> `readonly` `optional` **candidate?**: [`AgentCandidateExecutorWorkspaceInput`](#agentcandidateexecutorworkspaceinput) + +###### profile + +> `readonly` **profile**: `object` + +###### profile.files + +> `readonly` **files**: readonly [`AgentCandidateExecutorProfileFile`](#agentcandidateexecutorprofilefile)[] + +##### roots + +> `readonly` **roots**: `object` + +Defined in: [candidate-execution/types.ts:440](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L440) + +###### taskRoot + +> **taskRoot**: `string` + +###### candidateRoot? + +> `optional` **candidateRoot?**: `string` + +##### profilePlan + +> `readonly` **profilePlan**: `object` + +Defined in: [candidate-execution/types.ts:441](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L441) + +###### value + +> **value**: `AgentCandidateProfilePlanEvidence` + +###### bytes + +> **bytes**: `Uint8Array` + +###### written + +> **written**: readonly `string`[] + +##### executionPlan + +> `readonly` **executionPlan**: `object` + +Defined in: [candidate-execution/types.ts:442](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L442) + +###### value + +> **value**: `AgentCandidateExecutionPlanEvidence` + +###### bytes + +> **bytes**: `Uint8Array` + +##### materializationReceipt + +> `readonly` **materializationReceipt**: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateMaterializationReceiptV1`\> + +Defined in: [candidate-execution/types.ts:443](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L443) + +##### launch + +> `readonly` **launch**: [`PreparedAgentCandidateLaunch`](#preparedagentcandidatelaunch) + +Defined in: [candidate-execution/types.ts:444](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L444) + +##### instruction + +> `readonly` **instruction**: [`PreparedAgentCandidateInstruction`](#preparedagentcandidateinstruction) + +Defined in: [candidate-execution/types.ts:445](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L445) + +##### resolvedModel + +> `readonly` **resolvedModel**: `AgentCandidateResolvedModel` + +Defined in: [candidate-execution/types.ts:446](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L446) + +##### hardLimits + +> `readonly` **hardLimits**: `Pick`\<`AgentCandidateExecutionLimits`, `"timeoutMs"`\> + +Defined in: [candidate-execution/types.ts:448](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L448) + +Mechanically enforced by the runtime plus executor process-death acknowledgement. + +##### observedLimits + +> `readonly` **observedLimits**: `Pick`\<`AgentCandidateExecutionLimits`, `"maxSteps"`\> + +Defined in: [candidate-execution/types.ts:450](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L450) + +Validity bound checked against protected traces; generic black-box executors cannot preempt it. + +##### knowledge? + +> `readonly` `optional` **knowledge?**: `object` + +Defined in: [candidate-execution/types.ts:451](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L451) + +###### snapshotId + +> **snapshotId**: `string` + +###### manifestDigest + +> **manifestDigest**: `` `sha256:${string}` `` + +###### manifest + +> **manifest**: `Uint8Array` + +##### trace + +> `readonly` **trace**: [`PreparedAgentCandidateTrace`](#preparedagentcandidatetrace) + +Defined in: [candidate-execution/types.ts:452](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L452) + +##### memory + +> `readonly` **memory**: `AgentCandidateEffectiveMemory` + +Defined in: [candidate-execution/types.ts:453](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L453) + +*** + +### AgentCandidateExecutorPort + +Defined in: [candidate-execution/types.ts:464](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L464) + +Executes one prepared request inside an evaluator-owned isolation boundary. + +`request.launch.env` is the complete allowlisted environment, including +protected model, memory, and trace bindings. Implementations must not merge +ambient host variables into it. The returned capture deliberately contains +no candidate-authored usage or score fields. + +#### Methods + +##### execute() + +> **execute**(`request`, `context`): `Promise`\<[`AgentCandidateProtectedRunCapture`](#agentcandidateprotectedruncapture)\> + +Defined in: [candidate-execution/types.ts:465](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L465) + +###### Parameters + +###### request + +[`AgentCandidateExecutorRequest`](#agentcandidateexecutorrequest) + +###### context + +###### traceStore + +`TraceStore` + +###### signal + +`AbortSignal` + +Aborted by the runtime at the exact frozen wall-time deadline. + +###### deadlineAtMs + +`number` + +Absolute epoch-millisecond deadline owned by the runtime. + +###### Returns + +`Promise`\<[`AgentCandidateProtectedRunCapture`](#agentcandidateprotectedruncapture)\> + +##### stopAndCapture() + +> **stopAndCapture**(`request`, `context`): `Promise`\<[`AgentCandidateExecutorFinalCapture`](#agentcandidateexecutorfinalcapture)\> + +Defined in: [candidate-execution/types.ts:482](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L482) + +Kill any process/container still associated with the request, drain trace +writes, and capture the final task workspace before teardown. +The runtime calls this on success, failure, and timeout before model settlement. +Implementations must be idempotent and concurrency-safe for this exact +execution/plan pair because a fresh worker may repeat crash recovery. + +###### Parameters + +###### request + +[`AgentCandidateExecutorStopRequest`](#agentcandidateexecutorstoprequest) + +###### context + +###### traceStore + +`TraceStore` + +###### reason + +`"completed"` \| `"failed"` \| `"timeout"` + +###### signal + +`AbortSignal` + +Aborted at the frozen execution deadline or evaluator cleanup deadline. + +###### deadlineAtMs + +`number` + +Absolute execution deadline; a later stop acknowledgement cannot produce success. + +###### Returns + +`Promise`\<[`AgentCandidateExecutorFinalCapture`](#agentcandidateexecutorfinalcapture)\> + +*** + +### AgentCandidateExecutorStopRequest + +Defined in: [candidate-execution/types.ts:496](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L496) + +Opaque process identity used for termination without re-exposing launch credentials. + +#### Properties + +##### executionId + +> `readonly` **executionId**: `string` + +Defined in: [candidate-execution/types.ts:497](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L497) + +##### executionPlanDigest + +> `readonly` **executionPlanDigest**: `` `sha256:${string}` `` + +Defined in: [candidate-execution/types.ts:498](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L498) + +*** + +### AgentCandidateExecutorWorkspaceInput + +Defined in: [candidate-execution/types.ts:501](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L501) + +#### Properties + +##### snapshot -> **put**(`session`): `void` +> `readonly` **snapshot**: `AgentCandidateWorkspaceSnapshotEvidence` -Defined in: [sessions.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/sessions.ts#L49) +Defined in: [candidate-execution/types.ts:502](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L502) -###### Parameters +##### files -###### session +> `readonly` **files**: readonly [`AgentCandidateExecutorWorkspaceFile`](#agentcandidateexecutorworkspacefile)[] -`RuntimeSession` +Defined in: [candidate-execution/types.ts:503](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L503) -###### Returns +*** -`void` +### AgentCandidateExecutorWorkspaceFile -###### Implementation of +Defined in: [candidate-execution/types.ts:506](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L506) -[`RuntimeSessionStore`](#runtimesessionstore).[`put`](#put-1) +#### Properties -##### appendEvent() +##### path -> **appendEvent**(`sessionId`, `event`): `void` +> `readonly` **path**: `string` -Defined in: [sessions.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/sessions.ts#L53) +Defined in: [candidate-execution/types.ts:507](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L507) -###### Parameters +##### mode -###### sessionId +> `readonly` **mode**: `420` \| `493` -`string` +Defined in: [candidate-execution/types.ts:508](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L508) -###### event +##### bytes -[`RuntimeStreamEvent`](#runtimestreamevent) +> `readonly` **bytes**: `Uint8Array` -###### Returns +Defined in: [candidate-execution/types.ts:509](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L509) -`void` +*** -###### Implementation of +### AgentCandidateExecutorProfileFile -[`RuntimeSessionStore`](#runtimesessionstore).[`appendEvent`](#appendevent-1) +Defined in: [candidate-execution/types.ts:512](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L512) -##### listEvents() +#### Properties -> **listEvents**(`sessionId`): [`RuntimeStreamEvent`](#runtimestreamevent)[] +##### path -Defined in: [sessions.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/sessions.ts#L59) +> `readonly` **path**: `string` -###### Parameters +Defined in: [candidate-execution/types.ts:513](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L513) -###### sessionId +##### mode -`string` +> `readonly` **mode**: `420` \| `493` -###### Returns +Defined in: [candidate-execution/types.ts:514](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L514) -[`RuntimeStreamEvent`](#runtimestreamevent)[] +##### bytes -###### Implementation of +> `readonly` **bytes**: `Uint8Array` -[`RuntimeSessionStore`](#runtimesessionstore).[`listEvents`](#listevents-1) +Defined in: [candidate-execution/types.ts:515](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L515) -## Interfaces +*** ### CircuitBreakerConfig @@ -6775,165 +9587,417 @@ Defined in: [types.ts:511](https://github.com/tangle-network/agent-runtime/blob/ ###### Parameters -###### input +###### input + +`TInput` + +###### context + +`Omit`\<[`AgentBackendContext`](#agentbackendcontext), `"session"`\> & `object` + +###### Returns + +`RuntimeSession` \| `Promise`\<`RuntimeSession`\> + +##### resume()? + +> `optional` **resume**(`session`, `input`, `context`): `RuntimeSession` \| `Promise`\<`RuntimeSession`\> + +Defined in: [types.ts:515](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L515) + +###### Parameters + +###### session + +`RuntimeSession` + +###### input + +`TInput` + +###### context + +`Omit`\<[`AgentBackendContext`](#agentbackendcontext), `"session"`\> + +###### Returns + +`RuntimeSession` \| `Promise`\<`RuntimeSession`\> + +##### stream() + +> **stream**(`input`, `context`): `AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> + +Defined in: [types.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L520) + +###### Parameters + +###### input + +`TInput` + +###### context + +[`AgentBackendContext`](#agentbackendcontext) + +###### Returns + +`AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> + +##### stop()? + +> `optional` **stop**(`session`, `reason`): `void` \| `Promise`\<`void`\> + +Defined in: [types.ts:521](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L521) + +###### Parameters + +###### session + +`RuntimeSession` + +###### reason + +`string` + +###### Returns + +`void` \| `Promise`\<`void`\> + +*** + +### AgentTaskRunResult + +Defined in: [types.ts:557](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L557) + +#### Stable + +#### Type Parameters + +##### TState + +`TState` + +##### TAction + +`TAction` + +##### TActionResult + +`TActionResult` + +##### TEval + +`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` + +#### Properties + +##### task + +> **task**: [`AgentTaskSpec`](#agenttaskspec) + +Defined in: [types.ts:563](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L563) + +##### status + +> **status**: [`AgentTaskStatus`](#agenttaskstatus) + +Defined in: [types.ts:564](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L564) + +##### knowledge + +> **knowledge**: `KnowledgeReadinessReport` + +Defined in: [types.ts:565](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L565) + +##### questions + +> **questions**: `UserQuestion`[] + +Defined in: [types.ts:566](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L566) + +##### acquisitionPlans + +> **acquisitionPlans**: `DataAcquisitionPlan`[] + +Defined in: [types.ts:567](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L567) + +##### userAnswers + +> **userAnswers**: `Record`\<`string`, `string`\> + +Defined in: [types.ts:568](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L568) + +##### acquiredEvidenceIds + +> **acquiredEvidenceIds**: `string`[] + +Defined in: [types.ts:569](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L569) + +##### control + +> **control**: `ControlRunResult`\<`TState`, `TAction`, `TActionResult`, `TEval`\> + +Defined in: [types.ts:570](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L570) + +##### runRecords + +> **runRecords**: `RunRecord`[] + +Defined in: [types.ts:571](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L571) + +## Type Aliases + +### AgentCandidateExecutionFailureClass + +> **AgentCandidateExecutionFailureClass** = `"pre-model-infrastructure"` \| `"execution"` \| `"post-model-infrastructure"` \| `"unknown"` + +Defined in: [candidate-execution/claim.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L78) + +Only the first class is retryable, and only when the closed model ledger has zero calls. + +*** + +### AgentCandidateExecutionTerminalResult + +> **AgentCandidateExecutionTerminalResult** = \{ `schemaVersion`: `1`; `status`: `"succeeded"`; `usage`: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage); `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \} \| \{ `schemaVersion`: `1`; `status`: `"failed"`; `failureClass`: [`AgentCandidateExecutionFailureClass`](#agentcandidateexecutionfailureclass); `usage`: [`AgentCandidateExecutionUsage`](#agentcandidateexecutionusage); `modelSettlement`: `AgentCandidateArtifactRef`; `failureEvidence?`: `AgentCandidateArtifactRef`; \} + +Defined in: [candidate-execution/claim.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L95) + +Evaluator-owned terminal facts staged durably before the terminal CAS. + +*** + +### AgentCandidateExecutionTerminalRecord + +> **AgentCandidateExecutionTerminalRecord** = [`AgentCandidateExecutionTerminalResult`](#agentcandidateexecutionterminalresult) & `object` + +Defined in: [candidate-execution/claim.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L115) + +Durable terminal record for one acquired execution attempt. + +#### Type Declaration + +##### executionId + +> `readonly` **executionId**: `string` + +##### attempt + +> `readonly` **attempt**: `number` + +##### bundleDigest + +> `readonly` **bundleDigest**: `Sha256Digest` + +##### executionPlanDigest + +> `readonly` **executionPlanDigest**: `Sha256Digest` + +##### terminalDigest + +> `readonly` **terminalDigest**: `Sha256Digest` + +RFC 8785 SHA-256 of this record with `terminalDigest` omitted. + +*** + +### AgentCandidateExecutionPhase + +> **AgentCandidateExecutionPhase** = `"claimed"` \| `"candidate-may-run"` + +Defined in: [candidate-execution/claim.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L125) + +Monotonic durable phase: the second value means candidate code could have started. + +*** + +### AgentCandidateExecutionClaimResult + +> **AgentCandidateExecutionClaimResult** = \{ `acquired`: `true`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `lease`: [`AgentCandidateExecutionLease`](#agentcandidateexecutionlease); \} \| \{ `acquired`: `false`; `reason`: `"already-claimed"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `exactReplay`: `boolean`; \} \| \{ `acquired`: `false`; `reason`: `"retry-not-eligible"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `detail`: [`AgentCandidateRetryRejection`](#agentcandidateretryrejection); \} + +Defined in: [candidate-execution/claim.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L165) + +Result of atomically claiming one execution attempt. + +#### Union Members + +##### Type Literal + +\{ `acquired`: `true`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `lease`: [`AgentCandidateExecutionLease`](#agentcandidateexecutionlease); \} + +*** + +##### Type Literal + +\{ `acquired`: `false`; `reason`: `"already-claimed"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `exactReplay`: `boolean`; \} + +###### acquired + +> `readonly` **acquired**: `false` + +###### reason + +> `readonly` **reason**: `"already-claimed"` -`TInput` +###### claim -###### context +> `readonly` **claim**: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) -`Omit`\<[`AgentBackendContext`](#agentbackendcontext), `"session"`\> & `object` +The durable winner already occupying this execution-attempt slot. -###### Returns +###### exactReplay -`RuntimeSession` \| `Promise`\<`RuntimeSession`\> +> `readonly` **exactReplay**: `boolean` -##### resume()? +True only when every signed claim field matches the durable winner. -> `optional` **resume**(`session`, `input`, `context`): `RuntimeSession` \| `Promise`\<`RuntimeSession`\> +*** -Defined in: [types.ts:515](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L515) +##### Type Literal -###### Parameters +\{ `acquired`: `false`; `reason`: `"retry-not-eligible"`; `claim`: [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim); `detail`: [`AgentCandidateRetryRejection`](#agentcandidateretryrejection); \} -###### session +*** -`RuntimeSession` +### AgentCandidateExecutionFinishResult -###### input +> **AgentCandidateExecutionFinishResult** = \{ `finished`: `true`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); \} \| \{ `finished`: `false`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); `exactReplay`: `boolean`; \} -`TInput` +Defined in: [candidate-execution/claim.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L187) -###### context +Result of atomically recording an attempt's terminal facts. -`Omit`\<[`AgentBackendContext`](#agentbackendcontext), `"session"`\> +#### Union Members -###### Returns +##### Type Literal -`RuntimeSession` \| `Promise`\<`RuntimeSession`\> +\{ `finished`: `true`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); \} -##### stream() +*** -> **stream**(`input`, `context`): `AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> +##### Type Literal -Defined in: [types.ts:520](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L520) +\{ `finished`: `false`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); `exactReplay`: `boolean`; \} -###### Parameters +###### finished -###### input +> `readonly` **finished**: `false` -`TInput` +###### terminal -###### context +> `readonly` **terminal**: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord) -[`AgentBackendContext`](#agentbackendcontext) +###### exactReplay -###### Returns +> `readonly` **exactReplay**: `boolean` -`AsyncIterable`\<[`RuntimeStreamEvent`](#runtimestreamevent)\> +True when a repeated finish supplied the same terminal digest. -##### stop()? +*** -> `optional` **stop**(`session`, `reason`): `void` \| `Promise`\<`void`\> +### AgentCandidateExecutionStageResult -Defined in: [types.ts:521](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L521) +> **AgentCandidateExecutionStageResult** = \{ `staged`: `true`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); \} \| \{ `staged`: `false`; `terminal`: [`AgentCandidateExecutionTerminalRecord`](#agentcandidateexecutionterminalrecord); `exactReplay`: `boolean`; \} -###### Parameters +Defined in: [candidate-execution/claim.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L200) -###### session +Result of durably staging the one immutable terminal outbox entry. -`RuntimeSession` +*** -###### reason +### AgentCandidateExecutionPhaseResult -`string` +> **AgentCandidateExecutionPhaseResult** = \{ `marked`: `true`; `phase`: `"candidate-may-run"`; \} \| \{ `marked`: `false`; `phase`: `"candidate-may-run"`; \} -###### Returns +Defined in: [candidate-execution/claim.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L212) -`void` \| `Promise`\<`void`\> +Result of crossing the irreversible candidate-may-run boundary. *** -### AgentTaskRunResult +### AgentCandidateRetryRejection -Defined in: [types.ts:557](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L557) +> **AgentCandidateRetryRejection** = `"prior-attempt-missing"` \| `"prior-attempt-running"` \| `"prior-attempt-succeeded"` \| `"prior-attempt-spent-model-calls"` \| `"prior-attempt-not-pre-model-infrastructure"` \| `"retry-lineage-mismatch"` -#### Stable +Defined in: [candidate-execution/claim.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim.ts#L216) -#### Type Parameters +*** -##### TState +### AgentCandidateOutputPurpose -`TState` +> **AgentCandidateOutputPurpose** = `"task-manifest"` \| `"task-archive"` \| `"task-patch"` \| `"task-outcome"` \| `"memory-after-manifest"` \| `"memory-after-archive"` \| `"grader-evidence"` \| `"benchmark-result"` \| `"model-settlement"` \| `"trace"` \| `"run-receipt"` \| `"failure-evidence"` -##### TAction +Defined in: [candidate-execution/types.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L37) -`TAction` +*** -##### TActionResult +### AgentCandidateModelLimits -`TActionResult` +> **AgentCandidateModelLimits** = `Pick`\<`AgentCandidateExecutionLimits`, `"maxModelCalls"` \| `"maxInputTokens"` \| `"maxOutputTokens"` \| `"maxCostUsd"`\> -##### TEval +Defined in: [candidate-execution/types.ts:148](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L148) -`TEval` *extends* `ControlEvalResult` = `ControlEvalResult` +Limits mechanically enforced by the evaluator-owned model gateway. -#### Properties +*** -##### task +### AgentCandidateRunFinalization -> **task**: [`AgentTaskSpec`](#agenttaskspec) +> **AgentCandidateRunFinalization** = \{ `succeeded`: `true`; `receipt`: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateRunReceiptV2`\>; `artifacts`: \{ `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \}; \} \| \{ `succeeded`: `false`; `reason`: `string`; `partial`: \{ `executionId`: `string`; `bundleDigest`: `Sha256Digest`; `executionPlanDigest`: `Sha256Digest`; `materializationReceiptDigest`: `Sha256Digest`; `termination?`: `AgentCandidateTermination`; \}; `usage`: `AgentCandidateSpend` \| `null`; \} -Defined in: [types.ts:563](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L563) +Defined in: [candidate-execution/types.ts:518](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L518) -##### status +#### Union Members -> **status**: [`AgentTaskStatus`](#agenttaskstatus) +##### Type Literal -Defined in: [types.ts:564](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L564) +\{ `succeeded`: `true`; `receipt`: [`CanonicalCandidateDocument`](#canonicalcandidatedocument)\<`AgentCandidateRunReceiptV2`\>; `artifacts`: \{ `modelSettlement`: `AgentCandidateArtifactRef`; `taskOutcome`: `AgentCandidateArtifactRef`; `benchmarkResult`: `AgentCandidateArtifactRef`; `runReceipt`: `AgentCandidateArtifactRef`; \}; \} -##### knowledge +*** -> **knowledge**: `KnowledgeReadinessReport` +##### Type Literal -Defined in: [types.ts:565](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L565) +\{ `succeeded`: `false`; `reason`: `string`; `partial`: \{ `executionId`: `string`; `bundleDigest`: `Sha256Digest`; `executionPlanDigest`: `Sha256Digest`; `materializationReceiptDigest`: `Sha256Digest`; `termination?`: `AgentCandidateTermination`; \}; `usage`: `AgentCandidateSpend` \| `null`; \} -##### questions +###### succeeded -> **questions**: `UserQuestion`[] +> **succeeded**: `false` -Defined in: [types.ts:566](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L566) +###### reason -##### acquisitionPlans +> **reason**: `string` -> **acquisitionPlans**: `DataAcquisitionPlan`[] +###### partial -Defined in: [types.ts:567](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L567) +> **partial**: `object` -##### userAnswers +###### partial.executionId -> **userAnswers**: `Record`\<`string`, `string`\> +> **executionId**: `string` -Defined in: [types.ts:568](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L568) +###### partial.bundleDigest -##### acquiredEvidenceIds +> **bundleDigest**: `Sha256Digest` -> **acquiredEvidenceIds**: `string`[] +###### partial.executionPlanDigest -Defined in: [types.ts:569](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L569) +> **executionPlanDigest**: `Sha256Digest` -##### control +###### partial.materializationReceiptDigest -> **control**: `ControlRunResult`\<`TState`, `TAction`, `TActionResult`, `TEval`\> +> **materializationReceiptDigest**: `Sha256Digest` -Defined in: [types.ts:570](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L570) +###### partial.termination? -##### runRecords +> `optional` **termination?**: `AgentCandidateTermination` -> **runRecords**: `RunRecord`[] +###### usage -Defined in: [types.ts:571](https://github.com/tangle-network/agent-runtime/blob/main/src/types.ts#L571) +> **usage**: `AgentCandidateSpend` \| `null` -## Type Aliases +Independent evaluator-gateway usage, even when execution or trace capture failed. + +*** ### RetryableErrorPredicate @@ -7651,6 +10715,66 @@ MUST map this to `RunRecord.error` rather than recording silent ## Variables +### CANDIDATE\_TRACE\_TAGS + +> `const` **CANDIDATE\_TRACE\_TAGS**: `object` + +Defined in: [candidate-execution/types.ts:544](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L544) + +Protected trace tags that bind a run to one prepared candidate execution. + +#### Type Declaration + +##### executionId + +> `readonly` **executionId**: `"tangle.candidate.execution_id"` = `'tangle.candidate.execution_id'` + +##### bundleDigest + +> `readonly` **bundleDigest**: `"tangle.candidate.bundle_digest"` = `'tangle.candidate.bundle_digest'` + +##### executionPlanDigest + +> `readonly` **executionPlanDigest**: `"tangle.candidate.execution_plan_digest"` = `'tangle.candidate.execution_plan_digest'` + +##### materializationReceiptDigest + +> `readonly` **materializationReceiptDigest**: `"tangle.candidate.materialization_receipt_digest"` = `'tangle.candidate.materialization_receipt_digest'` + +*** + +### CANDIDATE\_TRACE\_ENV + +> `const` **CANDIDATE\_TRACE\_ENV**: `object` + +Defined in: [candidate-execution/types.ts:552](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/types.ts#L552) + +Environment keys used to propagate immutable candidate trace identity. + +#### Type Declaration + +##### executionId + +> `readonly` **executionId**: `"TANGLE_CANDIDATE_EXECUTION_ID"` = `'TANGLE_CANDIDATE_EXECUTION_ID'` + +##### bundleDigest + +> `readonly` **bundleDigest**: `"TANGLE_CANDIDATE_BUNDLE_DIGEST"` = `'TANGLE_CANDIDATE_BUNDLE_DIGEST'` + +##### executionPlanDigest + +> `readonly` **executionPlanDigest**: `"TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST"` = `'TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST'` + +##### materializationReceiptDigest + +> `readonly` **materializationReceiptDigest**: `"TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST"` = `'TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST'` + +##### traceRunId + +> `readonly` **traceRunId**: `"TANGLE_TRACE_RUN_ID"` = `'TANGLE_TRACE_RUN_ID'` + +*** + ### defaultIsRetryable > `const` **defaultIsRetryable**: [`RetryableErrorPredicate`](#retryableerrorpredicate) @@ -8043,6 +11167,188 @@ Maximum completion tokens, sent as OpenAI-compatible `max_tokens`. Omit for prov *** +### candidateExecutionClaim() + +> **candidateExecutionClaim**(`prepared`): [`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) + +Defined in: [candidate-execution/claim-plan.ts:10](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/claim-plan.ts#L10) + +Extract the complete durable claim from a prepared execution. + +#### Parameters + +##### prepared + +[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution) + +#### Returns + +[`AgentCandidateExecutionClaim`](#agentcandidateexecutionclaim) + +*** + +### disposePreparedAgentCandidateExecution() + +> **disposePreparedAgentCandidateExecution**(`prepared`, `options?`): `Promise`\<\{ `disposed`: `true`; \}\> + +Defined in: [candidate-execution/dispose.ts:15](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/dispose.ts#L15) + +Revoke reservations held by a prepared candidate that will not be executed. + +#### Parameters + +##### prepared + +[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution) + +##### options? + +[`DisposePreparedAgentCandidateOptions`](#disposepreparedagentcandidateoptions) = `{}` + +#### Returns + +`Promise`\<\{ `disposed`: `true`; \}\> + +*** + +### executePreparedAgentCandidate() + +> **executePreparedAgentCandidate**(`prepared`, `options`): `Promise`\<[`AgentCandidateRunFinalization`](#agentcandidaterunfinalization)\> + +Defined in: [candidate-execution/execute.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/execute.ts#L74) + +Executes and finalizes one durably claimed candidate without exposing an unproven result. + +#### Parameters + +##### prepared + +[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution) + +##### options + +[`ExecutePreparedAgentCandidateOptions`](#executepreparedagentcandidateoptions) + +#### Returns + +`Promise`\<[`AgentCandidateRunFinalization`](#agentcandidaterunfinalization)\> + +*** + +### persistCandidateOutputArtifact() + +> **persistCandidateOutputArtifact**(`port`, `input`): `Promise`\<`AgentCandidateArtifactRef`\> + +Defined in: [candidate-execution/output-artifacts.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/output-artifacts.ts#L11) + +Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. + +#### Parameters + +##### port + +[`AgentCandidateOutputArtifactPort`](#agentcandidateoutputartifactport) + +##### input + +###### executionId + +`string` + +###### purpose + +[`AgentCandidateOutputPurpose`](#agentcandidateoutputpurpose) + +###### bytes + +`Uint8Array` + +###### signal? + +`AbortSignal` + +#### Returns + +`Promise`\<`AgentCandidateArtifactRef`\> + +*** + +### prepareAgentCandidateExecution() + +> **prepareAgentCandidateExecution**(`candidate`, `task`, `ports`, `options?`): `Promise`\<[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution)\> + +Defined in: [candidate-execution/prepare.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/prepare.ts#L95) + +Materializes a verified candidate into one immutable evaluator-owned execution plan. + +#### Parameters + +##### candidate + +[`VerifiedAgentCandidate`](#verifiedagentcandidate) + +##### task + +[`AgentCandidateTaskExecution`](#agentcandidatetaskexecution) + +##### ports + +[`AgentCandidateExecutionPorts`](#agentcandidateexecutionports) + +##### options? + +[`PrepareAgentCandidateExecutionOptions`](#prepareagentcandidateexecutionoptions) = `{}` + +#### Returns + +`Promise`\<[`PreparedAgentCandidateExecution`](#preparedagentcandidateexecution)\> + +*** + +### recoverExpiredAgentCandidateExecution() + +> **recoverExpiredAgentCandidateExecution**(`options`): `Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +Defined in: [candidate-execution/recover.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/recover.ts#L36) + +Close an expired crashed attempt from persisted non-secret handles, then record failure. + +#### Parameters + +##### options + +[`RecoverExpiredAgentCandidateOptions`](#recoverexpiredagentcandidateoptions) + +#### Returns + +`Promise`\<[`AgentCandidateExecutionFinishResult`](#agentcandidateexecutionfinishresult)\> + +*** + +### verifyAgentCandidateBundle() + +> **verifyAgentCandidateBundle**(`input`, `ports`): `Promise`\<[`VerifiedAgentCandidate`](#verifiedagentcandidate)\> + +Defined in: [candidate-execution/verify.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/candidate-execution/verify.ts#L37) + +Verifies every digest, resource, workspace, and Git object in a candidate bundle. + +#### Parameters + +##### input + +`unknown` + +##### ports + +[`AgentCandidateVerificationPorts`](#agentcandidateverificationports) + +#### Returns + +`Promise`\<[`VerifiedAgentCandidate`](#verifiedagentcandidate)\> + +*** + ### makePerAttemptSignal() > **makePerAttemptSignal**(`parentSignal`, `deadlineMs`): `object` diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index f0a48d11..525f268f 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 240 exports. +Import from `@tangle-network/agent-runtime` — 310 exports. | Symbol | Kind | Summary | |---|---|---| @@ -26,6 +26,7 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `buildForwardHeaders` | function | Build the headers to emit on an outbound participant call, given the | | `buildLoopOtelSpans` | function | Build a nested, real-duration OTLP span tree for ONE loop run from its full | | `buildLoopSpanNodes` | function | Sink-neutral core behind {@link buildLoopOtelSpans}: reconstruct the | +| `candidateExecutionClaim` | function | Extract the complete durable claim from a prepared execution. | | `cleanModelId` | function | Trim a candidate model id; `undefined` for non-strings and blanks. | | `commandVerifier` | function | A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other | | `composeRuntimeHooks` | function | Merge several {@link RuntimeHooks} into one. Falsy entries are dropped (so you can | @@ -44,7 +45,9 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `defineConversation` | function | Declarative constructor for a multi-agent `Conversation`. Validates inputs | | `defineRuntimeHooks` | function | Identity helper that types a {@link RuntimeHooks} literal so the fields are inferred. | | `deriveExecutionId` | function | Derive a stable executionId from the run identity. The same | +| `disposePreparedAgentCandidateExecution` | function | Revoke reservations held by a prepared candidate that will not be executed. | | `enumerateNeighborPolicies` | function | All bounded single-dial neighbors of `policy`, in a fixed priority order: k | +| `executePreparedAgentCandidate` | function | Executes and finalizes one durably claimed candidate without exposing an unproven result. | | `exportEvalRuns` | function | Ship self-improvement eval-run events to Tangle Intelligence. Unlike the | | `formatSupervisedKnowledgeTask` | function | Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. | | `getModels` | function | Fetch the model catalog from the router's `/v1/models`. Throws on a non-2xx | @@ -65,9 +68,12 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `notifyRuntimeHookEvent` | function | Fire `hooks.onEvent`, swallowing sync throws and surfacing async failures to `onError`. | | `parseLoopRunnerArgv` | function | Parse `--mode X --config Y` from an argv tail (`process.argv.slice(2)`). | | `parseRolloutPolicy` | function | Parse a serialized policy surface. Defensive by design — the proposer reads | +| `persistCandidateOutputArtifact` | function | Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. | +| `prepareAgentCandidateExecution` | function | Materializes a verified candidate into one immutable evaluator-owned execution plan. | | `rawTraceDistiller` | function | Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE | | `readDepth` | function | Read the depth counter off an inbound request. Missing → 0 (caller is the | | `readinessServerSentEvent` | function | Serialize a `KnowledgeReadinessReport` as a Server-Sent Event string. | +| `recoverExpiredAgentCandidateExecution` | function | Close an expired crashed attempt from persisted non-secret handles, then record failure. | | `reflectiveGenerator` | function | Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edits via the improvement adapter and apply them as one coherent candidate. | | `researchLoopRunner` | function | `research` mode — research-in-a-loop with valid-only KB growth. | | `resolveAgentBackend` | function | Resolve the `AgentExecutionBackend` for the chosen `kind`. Reuse this instead | @@ -99,7 +105,10 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `toolBuildPrompt` | function | Build the starting instruction for a coder agent tasked with implementing a new tool. | | `turnId` | function | Deterministic turn identifier. Stable across retries of the same logical | | `validateChatModelId` | function | Validate a caller-supplied chat-model id. Rejects non-strings, malformed | +| `verifyAgentCandidateBundle` | function | Verifies every digest, resource, workspace, and Git object in a candidate bundle. | | `worktreeLoopRunner` | function | `code` mode on the GENERIC recursive path: author one `AgentProfile` per harness, run them as a | +| `CANDIDATE_TRACE_ENV` | const | Environment keys used to propagate immutable candidate trace identity. | +| `CANDIDATE_TRACE_TAGS` | const | Protected trace tags that bind a run to one prepared candidate execution. | | `DEFAULT_MAX_DEPTH` | const | Hard cap on chained gateway hops; refused beyond this. Default keeps recursion bounded. | | `DEFAULT_ROUTER_BASE_URL` | const | Default Tangle Router base URL used when no env override is set. | | `defaultIsRetryable` | const | Default retryable classification — network/timeout class errors. Errors | @@ -115,7 +124,9 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `CircuitOpenError` | class | Thrown when the circuit breaker is open for a participant and no retry is allowed yet. | | `ConfigError` | class | Configuration missing or malformed (`HOME` unset, required image not supplied, env var absent). | | `DeadlineExceededError` | class | Thrown when a backend call exceeds its per-attempt deadline. | +| `FileAgentCandidateExecutionClaimStore` | class | Cross-process lifecycle implemented as fsynced, create-if-absent records. | | `FileConversationJournal` | class | JSONL on disk. One line per record; first line is the `begin`, subsequent | +| `InMemoryAgentCandidateExecutionClaimStore` | class | Single-process lifecycle implementation. | | `InMemoryConversationJournal` | class | In-memory `ConversationJournal` — suitable for testing and single-process runs. | | `InMemoryRuntimeSessionStore` | class | In-memory `RuntimeSessionStore` for single-process use and tests. | | `JudgeError` | class | A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. | @@ -124,6 +135,25 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `RuntimeRunStateError` | class | A runtime-run lifecycle method was called in an order the state machine does | | `SqlConversationJournal` | class | SQL-backed ConversationJournal. Two tables — runs (one row per runId, holds | | `ValidationError` | class | Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). | +| `AgentCandidateArtifactPort` | interface | Reads one content-addressed object from the closed S3/IPFS locator set. | +| `AgentCandidateBenchmarkGraderPort` | interface | Evaluator-owned executable grader, pinned by immutable implementation bytes. | +| `AgentCandidateExecutionAttemptRecord` | interface | Persisted state available to a fresh trusted recovery worker after a crash. | +| `AgentCandidateExecutionClaim` | interface | Immutable signed identity stored for one execution attempt. | +| `AgentCandidateExecutionClaimStore` | interface | Atomic one-shot store for candidate execution attempts. | +| `AgentCandidateExecutionCleanupHandles` | interface | Non-secret identities a trusted recovery worker needs to close an abandoned attempt. | +| `AgentCandidateExecutionLease` | interface | Secret capability required to finish the acquired attempt. | +| `AgentCandidateExecutionRecoveryEvidence` | interface | Trusted, independently observed closure facts for one expired winning lease. | +| `AgentCandidateExecutionUsage` | interface | Exact fixed-point usage proven by the closed evaluator model ledger. | +| `AgentCandidateExecutorFinalCapture` | interface | Idempotent executor result after process death and trace drain. | +| `AgentCandidateExecutorMemoryCapture` | interface | Raw isolated-memory capture made only after access has been revoked. | +| `AgentCandidateExecutorPort` | interface | Executes one prepared request inside an evaluator-owned isolation boundary. | +| `AgentCandidateExecutorRequest` | interface | One detached request passed to the trusted environment-specific executor. | +| `AgentCandidateExecutorStopRequest` | interface | Opaque process identity used for termination without re-exposing launch credentials. | +| `AgentCandidateExecutorTaskOutcomeCapture` | interface | Raw evaluator capture made only after the candidate process is dead. | +| `AgentCandidateOutputArtifactPort` | interface | Durable content-addressed evidence store controlled only by the evaluator. | +| `AgentCandidateProtectedModelCall` | interface | One evaluator-gateway call in the final, revoked model-access ledger. | +| `AgentCandidateRepositoryPort` | interface | Resolves a declared GitHub repository to an already-present local Git object store. | +| `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | | `BackendErrorDetail` | interface | Typed transport / backend failure detail. Carried on `backend_error` and | | `CandidateGenerator` | interface | The byte-producing seam — the ONE thing that differs between the cheap | | `ChatStreamEvent` | interface | The NDJSON line protocol every product chat client already speaks. | @@ -145,8 +175,18 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `SqlAdapter` | interface | Minimal SQL driver shape. Implementations forward to whichever client the | | `ToolLoopAssistantToolCall` | interface | One OpenAI-shaped tool-call entry carried on an assistant message. | | `ToolLoopCall` | interface | Bounded turn-level tool-dispatch loop. | +| `VerifiedAgentCandidateTaskOutcome` | interface | Branded task outcome that has survived independent patch and tree verification. | | `VerifyResult` | interface | Outcome of verifying a candidate worktree. `feedback` (compiler errors, | | `AgentBackendKind` | type | The transport a chat backend runs on. | +| `AgentCandidateExecutionClaimResult` | type | Result of atomically claiming one execution attempt. | +| `AgentCandidateExecutionFailureClass` | type | Only the first class is retryable, and only when the closed model ledger has zero calls. | +| `AgentCandidateExecutionFinishResult` | type | Result of atomically recording an attempt's terminal facts. | +| `AgentCandidateExecutionPhase` | type | Monotonic durable phase: the second value means candidate code could have started. | +| `AgentCandidateExecutionPhaseResult` | type | Result of crossing the irreversible candidate-may-run boundary. | +| `AgentCandidateExecutionStageResult` | type | Result of durably staging the one immutable terminal outbox entry. | +| `AgentCandidateExecutionTerminalRecord` | type | Durable terminal record for one acquired execution attempt. | +| `AgentCandidateExecutionTerminalResult` | type | Evaluator-owned terminal facts staged durably before the terminal CAS. | +| `AgentCandidateModelLimits` | type | Limits mechanically enforced by the evaluator-owned model gateway. | | `AgentEvalErrorCode` | type | Error taxonomy for `@tangle-network/agent-eval`. | | `ImproveSurface` | type | The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law | | `OpenAIChatResponseFormat` | type | `response_format` parameter for OpenAI-compatible chat endpoints. Use | @@ -161,7 +201,7 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `ToolLoopStopReason` | type | Why the loop stopped. `completed` = model finished naturally; `stuck-loop` = | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `HaltContext`, `HaltSignal`, `ImprovementDriverOptions`, `ImproveOptions`, `ImproveResult`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorProfileFile`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateTaskExecution`, `AgentCandidateVerificationPorts`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImprovementDriverOptions`, `ImproveOptions`, `ImproveResult`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. ### Vertical agent — manifest + improvement adapter diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 1bf5d9ff..34eadb26 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -2667,7 +2667,7 @@ The worker's trace — any event array (sandbox events, tool-call records). ##### outcome? -> `optional` **outcome?**: `"failed"` \| `"passed"` \| `"unknown"` +> `optional` **outcome?**: `"failed"` \| `"unknown"` \| `"passed"` Defined in: [runtime/observe.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/observe.ts#L32) @@ -12977,7 +12977,7 @@ Why this candidate was (or was not) selected, when the selector states it. ##### selector -> **selector**: `"driver"` \| `"caller"` \| `"default"` +> **selector**: `"default"` \| `"driver"` \| `"caller"` Defined in: [runtime/types.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/types.ts#L172) @@ -17408,7 +17408,7 @@ One OpenAI-compatible chat completion through the Tangle router, returning text ###### reasoningEffort? -`"none"` \| `"high"` \| `"medium"` \| `"low"` +`"none"` \| `"low"` \| `"medium"` \| `"high"` Reasoning control for thinking models, forwarded as `reasoning_effort`. 'none' is the load-bearing value: binary/single-token decisions (routing, diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 478655f8..afb5a79e 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -2,7 +2,7 @@ -> **Version 0.90.1.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.101.0 <1.0.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.14.0 <1.0.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. +> **Version 0.90.1.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.101.0 <1.0.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.22.0 <1.0.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. > > **`./loops` is the runtime barrel** — `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > diff --git a/package.json b/package.json index 2a03120d..0d52b565 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,7 @@ "devDependencies": { "@biomejs/biome": "^2.4.15", "@tangle-network/agent-eval": "^0.108.1", - "@tangle-network/agent-interface": "^0.19.0", + "@tangle-network/agent-interface": "^0.22.0", "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.9.3", "playwright": "^1.61.0", @@ -116,6 +116,7 @@ "minimumReleaseAgeExclude": [ "@tangle-network/agent-eval", "@tangle-network/agent-interface", + "@tangle-network/agent-profile-materialize", "@tangle-network/sandbox" ], "onlyBuiltDependencies": [ @@ -129,14 +130,11 @@ "packageManager": "pnpm@10.28.0", "peerDependencies": { "@tangle-network/agent-eval": ">=0.101.0 <1.0.0", - "@tangle-network/agent-interface": ">=0.14.0 <1.0.0", + "@tangle-network/agent-interface": ">=0.22.0 <1.0.0", "@tangle-network/sandbox": ">=0.8.0 <1.0.0", "playwright": "^1.40.0" }, "peerDependenciesMeta": { - "@tangle-network/agent-interface": { - "optional": true - }, "@tangle-network/sandbox": { "optional": true }, @@ -145,6 +143,7 @@ } }, "dependencies": { - "@tangle-network/agent-knowledge": "^1.11.1" + "@tangle-network/agent-knowledge": "^1.11.1", + "@tangle-network/agent-profile-materialize": "0.3.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf4d276a..ba8accf6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@tangle-network/agent-knowledge': specifier: ^1.11.1 version: 1.11.1(typescript@5.9.3) + '@tangle-network/agent-profile-materialize': + specifier: 0.3.0 + version: 0.3.0 devDependencies: '@biomejs/biome': specifier: ^2.4.15 @@ -19,8 +22,8 @@ importers: specifier: ^0.108.1 version: 0.108.1(typescript@5.9.3) '@tangle-network/agent-interface': - specifier: ^0.19.0 - version: 0.19.0 + specifier: ^0.22.0 + version: 0.22.0 '@tangle-network/sandbox': specifier: ^0.9.7 version: 0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3)) @@ -654,14 +657,20 @@ packages: '@tangle-network/agent-interface@0.17.1': resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} - '@tangle-network/agent-interface@0.19.0': - resolution: {integrity: sha512-WCN1j9dVmPlCDAU9I1R6OvGdZ9uvKrxz+3gsh61DbKoLvnxxRQEUWuZ/JByzSZGq4J84zdzryr6aET7KTZ6z+w==} + '@tangle-network/agent-interface@0.21.0': + resolution: {integrity: sha512-jDxhVJgxymrvU1RLWxWKueuaWQIpBAfrW8BuVTB5m2Y4eFMLo1SawDBEMDLdZN4/Gf34xrFrsRk+PAj9brGKMQ==} + + '@tangle-network/agent-interface@0.22.0': + resolution: {integrity: sha512-7fsJhNdvTmOB1X9E2owl06jzyrqaN+jMkOPVKbK7dvNqQvf627PowCtL/edhUqEEv+K0FWtkwG3R3xqjX7VNZg==} '@tangle-network/agent-knowledge@1.11.1': resolution: {integrity: sha512-2vvNHHsb4TQFnY+SrL7rYaMhwTxTK8R/lD9IAteonmgwcE5amhpPQZqftw+UDMl1d6OBe4QX+Qv1pNmWQDXzqQ==} engines: {node: '>=20'} hasBin: true + '@tangle-network/agent-profile-materialize@0.3.0': + resolution: {integrity: sha512-zEVNjhfDFGPUjlSqwFitlzEjczerkw60ucxUGsjmz7LhUw7kI86SxCCpfK0rBmeaHjhDYNQfVE7VTwEJxAfyfg==} + '@tangle-network/sandbox@0.9.7': resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} peerDependencies: @@ -1646,7 +1655,11 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.19.0': + '@tangle-network/agent-interface@0.21.0': + dependencies: + zod: 4.4.3 + + '@tangle-network/agent-interface@0.22.0': dependencies: zod: 4.4.3 @@ -1663,6 +1676,10 @@ snapshots: - typescript - utf-8-validate + '@tangle-network/agent-profile-materialize@0.3.0': + dependencies: + '@tangle-network/agent-interface': 0.21.0 + '@tangle-network/sandbox@0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3))': dependencies: '@tangle-network/agent-core': 0.3.8 diff --git a/src/candidate-execution/artifacts.ts b/src/candidate-execution/artifacts.ts new file mode 100644 index 00000000..f8f3b678 --- /dev/null +++ b/src/candidate-execution/artifacts.ts @@ -0,0 +1,201 @@ +import { constants as fsConstants } from 'node:fs' +import { lstat, open, readdir, realpath } from 'node:fs/promises' +import { relative, resolve, sep } from 'node:path' + +import type { + AgentCandidateArtifactRef, + AgentCandidateCapturedArtifact, + AgentCandidateProfilePlanMaterialV1, + AgentCandidateWorkspaceManifestMaterialV1, + AgentCandidateWorkspaceSnapshotEvidence, +} from '@tangle-network/agent-interface' + +import { canonicalCandidateBytes, sha256Bytes } from './digest' +import type { AgentCandidateArtifactPort } from './types' + +export function artifactCacheKey(artifact: AgentCandidateCapturedArtifact): string { + return `${artifact.sha256}:${artifact.byteLength}` +} + +export async function readVerifiedArtifact( + artifact: AgentCandidateCapturedArtifact, + port: AgentCandidateArtifactPort, +): Promise { + const bytes = + 'content' in artifact + ? Buffer.from(artifact.content, 'base64') + : await port.read(artifact as AgentCandidateArtifactRef) + verifyBytes(bytes, artifact.sha256, artifact.byteLength, 'candidate artifact') + return Uint8Array.from(bytes) +} + +export function verifyBytes( + bytes: Uint8Array, + digest: string, + byteLength: number, + label: string, +): void { + if (bytes.byteLength !== byteLength) { + throw new Error(`${label} byte length ${bytes.byteLength} does not match ${byteLength}`) + } + const actual = sha256Bytes(bytes) + if (actual !== digest) { + throw new Error(`${label} digest ${actual} does not match ${digest}`) + } +} + +export async function verifyWorkspaceSnapshotArtifacts( + snapshot: AgentCandidateWorkspaceSnapshotEvidence, + port: AgentCandidateArtifactPort, +): Promise<{ manifest: Uint8Array; archive: Uint8Array }> { + const [manifest, archive] = await Promise.all([ + readVerifiedArtifact(snapshot.manifest, port), + readVerifiedArtifact(snapshot.archive, port), + ]) + const canonicalManifest = canonicalCandidateBytes(snapshot.material) + if (!Buffer.from(manifest).equals(Buffer.from(canonicalManifest))) { + throw new Error('workspace manifest artifact is not the exact canonical manifest material') + } + if (sha256Bytes(canonicalManifest) !== snapshot.digest) { + throw new Error('workspace snapshot digest does not match its canonical manifest material') + } + return { manifest, archive } +} + +export async function verifyMaterializedWorkspace( + root: string, + expected: AgentCandidateWorkspaceManifestMaterialV1, + options: { ignoredProtectedRootEntries?: readonly ('.git' | '.sidecar')[] } = {}, +): Promise { + const observed = await scanWorkspace(root, new Set(options.ignoredProtectedRootEntries ?? [])) + assertWorkspaceManifest(observed.manifest, expected) +} + +/** Capture exact verified regular-file bytes for fresh isolated materialization. */ +export async function readMaterializedWorkspaceFiles( + root: string, + expected: AgentCandidateWorkspaceManifestMaterialV1, + options: { ignoredProtectedRootEntries?: readonly ('.git' | '.sidecar')[] } = {}, +): Promise> { + const observed = await scanWorkspace(root, new Set(options.ignoredProtectedRootEntries ?? [])) + assertWorkspaceManifest(observed.manifest, expected) + return observed.files.map((file) => + Object.freeze({ path: file.path, mode: file.mode, bytes: Uint8Array.from(file.bytes) }), + ) +} + +function assertWorkspaceManifest( + observed: AgentCandidateWorkspaceManifestMaterialV1, + expected: AgentCandidateWorkspaceManifestMaterialV1, +): void { + if (!Buffer.from(canonicalCandidateBytes(observed)).equals(canonicalCandidateBytes(expected))) { + throw new Error( + 'materialized workspace files, modes, or bytes do not match the signed manifest', + ) + } +} + +export async function verifyMaterializedProfileWorkspace( + root: string, + expected: AgentCandidateProfilePlanMaterialV1, +): Promise { + const observed = await scanWorkspace(root, new Set()) + const observedProfile = observed.manifest.files.map(({ path, mode, sha256 }) => ({ + relPath: path, + mode, + contentSha256: sha256, + })) + if ( + !Buffer.from(canonicalCandidateBytes(observedProfile)).equals( + canonicalCandidateBytes(expected.files), + ) + ) { + throw new Error('profile staging files, modes, or bytes do not match the signed profile plan') + } +} + +async function scanWorkspace( + root: string, + ignoredProtectedRootEntries: ReadonlySet, +): Promise<{ + manifest: AgentCandidateWorkspaceManifestMaterialV1 + files: Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> +}> { + const absoluteRoot = resolve(root) + const rootStats = await lstat(absoluteRoot) + if (!rootStats.isDirectory() || rootStats.isSymbolicLink()) { + throw new Error('workspace root must be a real directory') + } + if ((await realpath(absoluteRoot)) !== absoluteRoot) { + throw new Error('workspace root has a symlinked path component') + } + const files: AgentCandidateWorkspaceManifestMaterialV1['files'] = [] + const capturedFiles: Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> = [] + + async function visit(directory: string): Promise { + const entries = await readdir(directory, { withFileTypes: true }) + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + for (const entry of entries) { + if (directory === absoluteRoot && ignoredProtectedRootEntries.has(entry.name)) { + continue + } + const absolute = resolve(directory, entry.name) + const relPath = relative(absoluteRoot, absolute).split(sep).join('/') + if (!relPath || relPath.startsWith('../') || relPath.includes('/../')) { + throw new Error(`workspace entry escapes root: ${relPath}`) + } + const stats = await lstat(absolute) + if (stats.isSymbolicLink()) { + throw new Error(`workspace contains a symlink: ${relPath}`) + } + if (stats.isDirectory()) { + await visit(absolute) + continue + } + if (!stats.isFile()) { + throw new Error(`workspace contains a non-regular entry: ${relPath}`) + } + const descriptor = await open( + absolute, + fsConstants.O_RDONLY | + (typeof fsConstants.O_NOFOLLOW === 'number' ? fsConstants.O_NOFOLLOW : 0), + ) + try { + const openedStats = await descriptor.stat() + if (!openedStats.isFile()) { + throw new Error(`workspace contains a non-regular entry: ${relPath}`) + } + if (openedStats.nlink !== 1) { + throw new Error(`workspace contains a hard-linked file: ${relPath}`) + } + const mode = openedStats.mode & 0o777 + if (mode !== 0o644 && mode !== 0o755) { + throw new Error(`workspace file has unsupported mode ${mode.toString(8)}: ${relPath}`) + } + const bytes = await descriptor.readFile() + const supportedMode = mode as 0o644 | 0o755 + files.push({ + path: relPath, + mode: supportedMode, + sha256: sha256Bytes(bytes), + byteLength: bytes.byteLength, + }) + capturedFiles.push({ path: relPath, mode: supportedMode, bytes: Uint8Array.from(bytes) }) + } finally { + await descriptor.close() + } + } + } + + await visit(absoluteRoot) + files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + capturedFiles.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + return { + manifest: { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files, + }, + files: capturedFiles, + } +} diff --git a/src/candidate-execution/benchmark-grader.ts b/src/candidate-execution/benchmark-grader.ts new file mode 100644 index 00000000..26981081 --- /dev/null +++ b/src/candidate-execution/benchmark-grader.ts @@ -0,0 +1,142 @@ +import type { BenchmarkEvaluation } from '@tangle-network/agent-eval' +import type { + AgentCandidateArtifactRef, + AgentCandidateTermination, +} from '@tangle-network/agent-interface' + +import { readVerifiedArtifact } from './artifacts' +import { immutableCandidateValue, sha256Bytes } from './digest' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateOutputArtifactPort, + VerifiedAgentCandidateTaskOutcome, +} from './types' + +export interface BoundAgentCandidateBenchmarkRun { + readonly grader: { + readonly name: string + readonly version: string + readonly artifact: AgentCandidateArtifactRef + } + readonly evaluation: BenchmarkEvaluation + readonly evidence: Uint8Array +} + +/** + * Admit verified grader bytes to the evaluator runner and reject any result + * that is not bound to those bytes, the exact task outcome, and its raw output. + */ +export async function runBoundCandidateBenchmarkGrader(input: { + executionId: string + termination: AgentCandidateTermination + outcome: VerifiedAgentCandidateTaskOutcome + grader: AgentCandidateBenchmarkGraderPort + artifacts: AgentCandidateOutputArtifactPort + signal?: AbortSignal +}): Promise { + input.signal?.throwIfAborted() + const descriptor = snapshotGraderDescriptor(input.grader) + const implementationBytes = await readVerifiedArtifact(descriptor.artifact, input.artifacts) + if (implementationBytes.byteLength === 0) { + throw new Error('candidate benchmark grader implementation cannot be empty') + } + const expectedImplementationDigest = sha256Bytes(implementationBytes) + const termination = immutableCandidateValue(input.termination) + const run = input.grader.run + const result = await run( + Object.freeze({ + executionId: input.executionId, + termination, + outcome: input.outcome, + implementation: detachedImplementation(implementationBytes), + signal: input.signal ?? new AbortController().signal, + }), + ) + input.signal?.throwIfAborted() + assertExactRunnerResult(result) + + const evidence = Uint8Array.from(result.evidence) + const outputDigest = sha256Bytes(evidence) + if (result.binding.implementationDigest !== expectedImplementationDigest) { + throw new Error( + 'candidate benchmark grader executed implementation digest does not match its verified artifact', + ) + } + if (result.binding.taskOutcomeDigest !== input.outcome.evidence.digest) { + throw new Error( + 'candidate benchmark grader task outcome digest does not match verified outcome', + ) + } + if (result.binding.outputDigest !== outputDigest) { + throw new Error('candidate benchmark grader raw output digest does not match returned evidence') + } + + return Object.freeze({ + grader: descriptor, + evaluation: result.evaluation, + evidence, + }) +} + +function snapshotGraderDescriptor( + grader: AgentCandidateBenchmarkGraderPort, +): BoundAgentCandidateBenchmarkRun['grader'] { + if (!grader.name || !grader.version) { + throw new Error('candidate benchmark grader name and version must be non-empty') + } + return immutableCandidateValue({ + name: grader.name, + version: grader.version, + artifact: grader.artifact, + }) +} + +function detachedImplementation(bytes: Uint8Array): { + readonly byteLength: number + readonly bytes: Uint8Array +} { + const stored = Uint8Array.from(bytes) + return Object.freeze({ + byteLength: stored.byteLength, + get bytes(): Uint8Array { + return Uint8Array.from(stored) + }, + }) +} + +function assertExactRunnerResult( + value: unknown, +): asserts value is Awaited> { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('candidate benchmark grader result must be an object') + } + const result = value as Record + const keys = Object.keys(result).sort() + if ( + keys.length !== 3 || + keys[0] !== 'binding' || + keys[1] !== 'evaluation' || + keys[2] !== 'evidence' + ) { + throw new Error('candidate benchmark grader returned unknown or missing fields') + } + if (!(result.evidence instanceof Uint8Array)) { + throw new Error('candidate benchmark grader evidence must be bytes') + } + if ( + result.binding === null || + typeof result.binding !== 'object' || + Array.isArray(result.binding) + ) { + throw new Error('candidate benchmark grader binding must be an object') + } + const bindingKeys = Object.keys(result.binding).sort() + if ( + bindingKeys.length !== 3 || + bindingKeys[0] !== 'implementationDigest' || + bindingKeys[1] !== 'outputDigest' || + bindingKeys[2] !== 'taskOutcomeDigest' + ) { + throw new Error('candidate benchmark grader binding returned unknown or missing fields') + } +} diff --git a/src/candidate-execution/claim-file-formats.ts b/src/candidate-execution/claim-file-formats.ts new file mode 100644 index 00000000..54b99463 --- /dev/null +++ b/src/candidate-execution/claim-file-formats.ts @@ -0,0 +1,34 @@ +import type { Sha256Digest } from '@tangle-network/agent-interface' + +import type { AgentCandidateExecutionClaim, AgentCandidateExecutionTerminalRecord } from './claim' + +export const CLAIM_FORMAT_VERSION = 7 +export const PENDING_FORMAT_VERSION = 1 +export const TERMINAL_FORMAT_VERSION = 3 +export const PHASE_FORMAT_VERSION = 1 + +export interface PersistedAgentCandidateExecutionClaim extends AgentCandidateExecutionClaim { + version: typeof CLAIM_FORMAT_VERSION + phase: 'claimed' + leaseDigest: Sha256Digest +} + +export interface PersistedAgentCandidateExecutionPending { + version: typeof PENDING_FORMAT_VERSION + kind: 'candidate-execution-pending-terminal' + terminal: AgentCandidateExecutionTerminalRecord +} + +export interface PersistedAgentCandidateExecutionPhase { + version: typeof PHASE_FORMAT_VERSION + kind: 'candidate-execution-phase' + executionId: string + attempt: number + executionPlanDigest: Sha256Digest + phase: 'candidate-may-run' +} + +export interface PersistedAgentCandidateExecutionTerminal { + version: typeof TERMINAL_FORMAT_VERSION + terminal: AgentCandidateExecutionTerminalRecord +} diff --git a/src/candidate-execution/claim-file-store.ts b/src/candidate-execution/claim-file-store.ts new file mode 100644 index 00000000..d4d9aeba --- /dev/null +++ b/src/candidate-execution/claim-file-store.ts @@ -0,0 +1,417 @@ +import { randomUUID } from 'node:crypto' +import { linkSync } from 'node:fs' +import { mkdir, open, unlink } from 'node:fs/promises' +import { join } from 'node:path' +import type { Sha256Digest } from '@tangle-network/agent-interface' + +import { + type AgentCandidateExecutionAttemptRecord, + type AgentCandidateExecutionAttemptRef, + type AgentCandidateExecutionClaim, + type AgentCandidateExecutionClaimResult, + type AgentCandidateExecutionClaimStore, + type AgentCandidateExecutionFinishResult, + type AgentCandidateExecutionLease, + type AgentCandidateExecutionPhase, + type AgentCandidateExecutionPhaseResult, + type AgentCandidateExecutionRecoveryEvidence, + type AgentCandidateExecutionStageResult, + type AgentCandidateExecutionTerminalRecord, + type AgentCandidateExecutionTerminalResult, + type AgentCandidateRetryRejection, + candidateClaimFileInternals, +} from './claim' +import { + CLAIM_FORMAT_VERSION, + PENDING_FORMAT_VERSION, + PHASE_FORMAT_VERSION, + TERMINAL_FORMAT_VERSION, +} from './claim-file-formats' +import { + assertRecoveryMatchesStaged, + assertTerminalAllowedInPhase, + assertTerminalMatchesClaim, + assertTerminalMatchesStaged, + recoveredTerminalRecord, + rejectedFinish, + rejectedStage, + requireStagedTerminal, + sealTerminalDigest, + terminalRecord, +} from './claim-terminal' +import { canonicalCandidateBytes } from './digest' + +const { + assertExpiredLease, + assertLease, + assertSameSlot, + assertUnexpiredLease, + attemptRecord, + claimSlot, + leaseDigest, + newLease, + readClaim, + readClaimIfPresent, + readTerminal, + readTerminalIfPresent, + readTransitionIfPresent, + rejectedExistingClaim, + rejectedRetry, + retryRejection, + sealAttemptRef, + sealClaim, + sealLease, +} = candidateClaimFileInternals + +export interface FileAgentCandidateExecutionClaimStoreOptions { + /** Evaluator-owned directory shared by every process allowed to execute candidates. */ + directory: string + /** Testable evaluator clock; defaults to `Date.now`. */ + now?: () => number +} + +/** Cross-process lifecycle implemented as fsynced, create-if-absent records. */ +export class FileAgentCandidateExecutionClaimStore implements AgentCandidateExecutionClaimStore { + private readonly directory: string + private readonly now: () => number + + constructor(options: FileAgentCandidateExecutionClaimStoreOptions) { + if (options.directory.length === 0) { + throw new Error('candidate execution claim directory must not be empty') + } + this.directory = options.directory + this.now = options.now ?? Date.now + } + + async tryClaim( + requested: AgentCandidateExecutionClaim, + ): Promise { + const claim = sealClaim(requested) + await mkdir(this.directory, { recursive: true }) + const claimPath = this.claimPath(claim) + const existing = await readClaimIfPresent(claimPath) + if (existing) { + assertSameSlot(existing.claim, claim, claimPath) + return rejectedExistingClaim(existing.claim, claim) + } + assertUnexpiredLease(claim.leaseExpiresAtMs, this.now()) + const retryFailure = await this.retryFailure(claim) + if (retryFailure) return rejectedRetry(claim, retryFailure) + + const lease = newLease(claim) + const acquired = await writeRecordIfAbsent(this.directory, claimPath, { + version: CLAIM_FORMAT_VERSION, + ...claim, + phase: 'claimed', + leaseDigest: leaseDigest(lease), + }) + if (acquired) return Object.freeze({ acquired: true, claim, lease }) + + const winner = await readClaim(claimPath) + assertSameSlot(winner.claim, claim, claimPath) + return rejectedExistingClaim(winner.claim, claim) + } + + async getAttempt( + requestedAttempt: AgentCandidateExecutionAttemptRef, + ): Promise { + return await this.storedAttempt(sealAttemptRef(requestedAttempt)) + } + + async markCandidateMayRun( + requestedLease: AgentCandidateExecutionLease, + ): Promise { + const lease = sealLease(requestedLease) + const claimPath = this.claimPath(lease) + const stored = await readClaim(claimPath) + assertSameSlot(stored.claim, lease, claimPath) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const state = await this.transitionState(stored.claim) + if (state.phase === 'candidate-may-run') { + return Object.freeze({ marked: false, phase: 'candidate-may-run' }) + } + if (state.staged) { + throw new Error('candidate execution terminal was staged before candidate-may-run phase') + } + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + const marked = await writeRecordIfAbsent( + this.directory, + this.transitionPath(stored.claim, 1), + { + version: PHASE_FORMAT_VERSION, + kind: 'candidate-execution-phase', + executionId: stored.claim.executionId, + attempt: stored.claim.attempt, + executionPlanDigest: stored.claim.executionPlanDigest, + phase: 'candidate-may-run', + }, + this.ownerPublication(stored.claim), + ) + if (marked) return Object.freeze({ marked: true, phase: 'candidate-may-run' }) + const winner = await this.transitionState(stored.claim) + if (winner.phase === 'candidate-may-run') { + return Object.freeze({ marked: false, phase: 'candidate-may-run' }) + } + throw new Error('candidate execution terminal was staged before candidate-may-run phase') + } + + async stageTerminal( + requestedLease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, + ): Promise { + const lease = sealLease(requestedLease) + const claimPath = this.claimPath(lease) + const stored = await readClaim(claimPath) + assertSameSlot(stored.claim, lease, claimPath) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const terminal = terminalRecord(stored.claim, result) + const state = await this.transitionState(stored.claim) + if (state.staged) return rejectedStage(state.staged, terminal) + assertTerminalAllowedInPhase(state.phase, terminal) + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + const transition = state.phase === 'claimed' ? 1 : 2 + const staged = await writeRecordIfAbsent( + this.directory, + this.transitionPath(stored.claim, transition), + { + version: PENDING_FORMAT_VERSION, + kind: 'candidate-execution-pending-terminal', + terminal, + }, + this.ownerPublication(stored.claim), + ) + if (staged) return Object.freeze({ staged: true, terminal }) + const winner = await this.transitionState(stored.claim) + if (!winner.staged) { + throw new Error('candidate execution phase transition won while staging terminal') + } + return rejectedStage(winner.staged, terminal) + } + + async finish( + requestedLease: AgentCandidateExecutionLease, + requestedTerminalDigest: Sha256Digest, + ): Promise { + const lease = sealLease(requestedLease) + const terminalDigest = sealTerminalDigest(requestedTerminalDigest) + const claimPath = this.claimPath(lease) + const stored = await readClaim(claimPath) + assertSameSlot(stored.claim, lease, claimPath) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const state = await this.transitionState(stored.claim) + const staged = requireStagedTerminal(state.staged, terminalDigest) + const terminalPath = this.terminalPath(lease) + const existing = await readTerminalIfPresent(terminalPath) + if (existing) { + assertTerminalMatchesClaim(existing, stored.claim, terminalPath) + assertTerminalMatchesStaged(existing, staged, terminalPath) + return rejectedFinish(existing, terminalDigest) + } + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + const finished = await writeRecordIfAbsent( + this.directory, + terminalPath, + { + version: TERMINAL_FORMAT_VERSION, + terminal: staged, + }, + this.ownerPublication(stored.claim), + ) + if (finished) return Object.freeze({ finished: true, terminal: staged }) + const winner = await readTerminal(terminalPath) + assertSameSlot(winner, lease, terminalPath) + assertTerminalMatchesClaim(winner, stored.claim, terminalPath) + assertTerminalMatchesStaged(winner, staged, terminalPath) + return rejectedFinish(winner, terminalDigest) + } + + async recoverExpired( + requestedAttempt: AgentCandidateExecutionAttemptRef, + evidence: AgentCandidateExecutionRecoveryEvidence, + ): Promise { + const attempt = sealAttemptRef(requestedAttempt) + const record = await this.storedAttempt(attempt) + if (!record) throw new Error('candidate execution recovery does not name an acquired attempt') + const recovered = recoveredTerminalRecord(record.claim, record.phase, evidence) + if (record.staged) assertRecoveryMatchesStaged(record.staged, recovered) + const requestedDigest = record.staged?.terminalDigest ?? recovered.terminalDigest + if (record.terminal) return rejectedFinish(record.terminal, requestedDigest) + assertExpiredLease(record.claim.leaseExpiresAtMs, this.now()) + + let staged = record.staged + if (!staged) { + const transition = record.phase === 'claimed' ? 1 : 2 + const didStage = await writeRecordIfAbsent( + this.directory, + this.transitionPath(record.claim, transition), + { + version: PENDING_FORMAT_VERSION, + kind: 'candidate-execution-pending-terminal', + terminal: recovered, + }, + ) + if (didStage) { + staged = recovered + } else { + const winner = await this.transitionState(record.claim) + if (!winner.staged) { + throw new Error( + 'candidate execution recovery lost terminal staging to a phase transition', + ) + } + assertRecoveryMatchesStaged(winner.staged, recovered) + staged = winner.staged + } + } + + const terminalPath = this.terminalPath(attempt) + const didFinish = await writeRecordIfAbsent(this.directory, terminalPath, { + version: TERMINAL_FORMAT_VERSION, + terminal: staged, + }) + if (didFinish) return Object.freeze({ finished: true, terminal: staged }) + const winner = await readTerminal(terminalPath) + assertSameSlot(winner, attempt, terminalPath) + assertTerminalMatchesClaim(winner, record.claim, terminalPath) + assertTerminalMatchesStaged(winner, staged, terminalPath) + return rejectedFinish(winner, staged.terminalDigest) + } + + private async storedAttempt( + claim: Pick, + ): Promise { + const claimPath = this.claimPath(claim) + const stored = await readClaimIfPresent(claimPath) + if (!stored) return undefined + assertSameSlot(stored.claim, claim, claimPath) + const state = await this.transitionState(stored.claim) + const terminalPath = this.terminalPath(claim) + const terminal = await readTerminalIfPresent(terminalPath) + if (terminal) { + assertTerminalMatchesClaim(terminal, stored.claim, terminalPath) + if (!state.staged) { + throw new Error( + `candidate execution terminal record at ${terminalPath} has no staged outbox`, + ) + } + assertTerminalMatchesStaged(terminal, state.staged, terminalPath) + } + return attemptRecord(stored.claim, state.phase, state.staged, terminal) + } + + private async transitionState(claim: AgentCandidateExecutionClaim): Promise<{ + phase: AgentCandidateExecutionPhase + staged?: AgentCandidateExecutionTerminalRecord + }> { + const firstPath = this.transitionPath(claim, 1) + const first = await readTransitionIfPresent(firstPath, claim) + if (!first) return { phase: 'claimed' } + if (first.kind === 'pending') return { phase: 'claimed', staged: first.terminal } + const secondPath = this.transitionPath(claim, 2) + const second = await readTransitionIfPresent(secondPath, claim) + if (!second) return { phase: 'candidate-may-run' } + if (second.kind !== 'pending') { + throw new Error(`candidate execution transition at ${secondPath} repeats the phase marker`) + } + return { phase: 'candidate-may-run', staged: second.terminal } + } + + private async retryFailure( + claim: AgentCandidateExecutionClaim, + ): Promise { + if (claim.attempt === 1) return undefined + return retryRejection( + claim, + await this.storedAttempt({ + executionId: claim.executionId, + attempt: claim.attempt - 1, + }), + ) + } + + private ownerPublication(claim: AgentCandidateExecutionClaim): { + authorizePublish: () => true + } { + return { + authorizePublish: () => { + assertUnexpiredLease(claim.leaseExpiresAtMs, this.now()) + return true + }, + } + } + + private claimPath(claim: Pick): string { + return join(this.directory, `${claimSlot(claim)}.claim.json`) + } + + private terminalPath( + claim: Pick, + ): string { + return join(this.directory, `${claimSlot(claim)}.terminal.json`) + } + + private transitionPath( + claim: Pick, + ordinal: 1 | 2, + ): string { + return join(this.directory, `${claimSlot(claim)}.transition-${ordinal}.json`) + } +} + +async function writeRecordIfAbsent( + directory: string, + destination: string, + record: object, + options: { + /** Synchronous authorization checked after temp fsync, immediately before atomic publication. */ + authorizePublish?: () => true + } = {}, +): Promise { + const temporaryPath = join(directory, `.candidate-execution-${process.pid}-${randomUUID()}.tmp`) + const handle = await open(temporaryPath, 'wx', 0o600) + try { + await handle.writeFile( + Buffer.concat([Buffer.from(canonicalCandidateBytes(record)), Buffer.from('\n')]), + ) + await handle.sync() + } finally { + await handle.close() + } + + let written = false + try { + if (options.authorizePublish && options.authorizePublish() !== true) { + throw new Error('candidate execution record publication was not authorized') + } + // Authorization and the atomic filesystem call are one synchronous critical + // section so the event loop cannot advance an injected lease clock between them. + linkSync(temporaryPath, destination) + written = true + await syncDirectory(directory) + } catch (error) { + if (!isNodeError(error, 'EEXIST')) throw error + } finally { + await unlink(temporaryPath).catch((error: unknown) => { + if (!isNodeError(error, 'ENOENT')) throw error + }) + } + return written +} + +async function syncDirectory(directory: string): Promise { + const handle = await open(directory, 'r') + try { + await handle.sync() + } finally { + await handle.close() + } +} + +function isNodeError(error: unknown, code: string): boolean { + return ( + error !== null && + typeof error === 'object' && + 'code' in error && + (error as { code?: unknown }).code === code + ) +} diff --git a/src/candidate-execution/claim-plan.ts b/src/candidate-execution/claim-plan.ts new file mode 100644 index 00000000..8d8333a3 --- /dev/null +++ b/src/candidate-execution/claim-plan.ts @@ -0,0 +1,107 @@ +import type { Sha256Digest } from '@tangle-network/agent-interface' + +import { type AgentCandidateExecutionClaim, candidateClaimFileInternals } from './claim' +import { canonicalCandidateDigest } from './digest' +import { candidateExecutionOwnerWindowMs } from './execution-window' +import { assertPreparedCandidateIntegrity } from './prepared-state' +import type { PreparedAgentCandidateExecution } from './types' + +/** Extract the complete durable claim from a prepared execution. */ +export function candidateExecutionClaim( + prepared: PreparedAgentCandidateExecution, +): AgentCandidateExecutionClaim { + const state = assertPreparedCandidateIntegrity(prepared) + const material = prepared.executionPlan.value.material + const attempt = material.attempt + const nowMs = Date.now() + if (!Number.isSafeInteger(nowMs) || nowMs < 0) { + throw new Error('candidate execution claim-store clock returned an invalid timestamp') + } + const leaseExpiresAtMs = + nowMs + + candidateExecutionOwnerWindowMs( + material.limits.timeoutMs, + state.cleanupTimeoutMs, + state.resultTimeoutMs, + ) + if (!Number.isSafeInteger(leaseExpiresAtMs) || leaseExpiresAtMs <= 0) { + throw new Error('candidate execution leaseExpiresAtMs must be a positive safe timestamp') + } + if (leaseExpiresAtMs > state.reservationExpiresAtMs) { + throw new Error( + 'candidate preparation expires before its full execution and cleanup owner window', + ) + } + return candidateClaimFileInternals.sealClaim({ + executionId: prepared.executionId, + attempt: attempt.number, + maxAttempts: attempt.maxAttempts, + retryPolicy: attempt.retryPolicy, + bundleDigest: prepared.bundle.digest, + executionPlanDigest: prepared.executionPlan.value.digest, + retryLineageDigest: retryLineageDigest(prepared, state.resultTimeoutMs), + leaseExpiresAtMs, + resultTimeoutMs: state.resultTimeoutMs, + cleanup: { + preparationId: state.preparationId, + modelGrantDigest: state.modelReservation.digest, + resolvedModel: state.resolvedModel, + traceRunId: state.trace.runId, + cleanupTimeoutMs: state.cleanupTimeoutMs, + ...(state.memoryReservation + ? { + memory: { + accessDigest: state.memoryReservation.accessDigest, + effectiveNamespace: state.memoryReservation.effectiveNamespace, + }, + } + : {}), + }, + }) +} + +function retryLineageDigest( + prepared: PreparedAgentCandidateExecution, + resultTimeoutMs: number, +): Sha256Digest { + const material = prepared.executionPlan.value.material + return canonicalCandidateDigest({ + resultTimeoutMs, + executionPlan: { + ...material, + attempt: { ...material.attempt, number: 0 }, + model: { + ...material.model, + access: { + ...material.model.access, + grantDigest: `sha256:${'0'.repeat(64)}`, + }, + }, + memory: + material.memory.mode === 'disabled' + ? material.memory + : { + mode: 'isolated', + scope: 'task', + effectiveNamespace: 'candidate/retry-lineage-normalized', + reset: { + kind: 'fresh', + emptyStateDigest: material.memory.reset.emptyStateDigest, + }, + beforeState: { + digest: material.memory.beforeState.digest, + material: material.memory.beforeState.material, + manifest: { + sha256: material.memory.beforeState.manifest.sha256, + byteLength: material.memory.beforeState.manifest.byteLength, + }, + archive: { + sha256: material.memory.beforeState.archive.sha256, + byteLength: material.memory.beforeState.archive.byteLength, + }, + }, + ...(material.memory.seedDigest ? { seedDigest: material.memory.seedDigest } : {}), + }, + }, + }) +} diff --git a/src/candidate-execution/claim-terminal.ts b/src/candidate-execution/claim-terminal.ts new file mode 100644 index 00000000..d71b1a9c --- /dev/null +++ b/src/candidate-execution/claim-terminal.ts @@ -0,0 +1,516 @@ +import type { AgentCandidateArtifactRef, Sha256Digest } from '@tangle-network/agent-interface' +import { agentCandidateArtifactRefSchema } from '@tangle-network/agent-interface' + +import type { + AgentCandidateExecutionClaim, + AgentCandidateExecutionFailureClass, + AgentCandidateExecutionFinishResult, + AgentCandidateExecutionPhase, + AgentCandidateExecutionRecoveryEvidence, + AgentCandidateExecutionStageResult, + AgentCandidateExecutionTerminalRecord, + AgentCandidateExecutionTerminalResult, + AgentCandidateExecutionUsage, +} from './claim' +import { canonicalCandidateDigest, immutableCandidateValue } from './digest' +import { assertExactObjectKeys as assertExactKeys } from './exact-object' + +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/ + +export function terminalRecord( + claim: AgentCandidateExecutionClaim, + result: AgentCandidateExecutionTerminalResult, +): AgentCandidateExecutionTerminalRecord { + const terminal = sealTerminalResult(result) + const value = { + executionId: claim.executionId, + attempt: claim.attempt, + bundleDigest: claim.bundleDigest, + executionPlanDigest: claim.executionPlanDigest, + ...terminal, + } + return immutableCandidateValue({ + ...value, + terminalDigest: canonicalCandidateDigest(value), + }) as AgentCandidateExecutionTerminalRecord +} + +export function recoveredTerminalRecord( + claim: AgentCandidateExecutionClaim, + phase: AgentCandidateExecutionPhase, + evidence: AgentCandidateExecutionRecoveryEvidence, +): AgentCandidateExecutionTerminalRecord { + const recovered = sealRecoveryEvidence(evidence, claim) + return terminalRecord(claim, { + schemaVersion: 1, + status: 'failed', + failureClass: + recovered.failureClass === 'pre-model-infrastructure' && phase !== 'claimed' + ? 'unknown' + : recovered.failureClass, + usage: recovered.usage, + modelSettlement: recovered.modelSettlement, + ...(recovered.failureEvidence ? { failureEvidence: recovered.failureEvidence } : {}), + }) +} + +export function assertTerminalAllowedInPhase( + phase: AgentCandidateExecutionPhase, + terminal: AgentCandidateExecutionTerminalRecord, +): void { + if ( + phase === 'candidate-may-run' && + terminal.status === 'failed' && + terminal.failureClass === 'pre-model-infrastructure' + ) { + throw new Error('candidate execution crossed candidate-may-run before pre-model failure') + } + if ( + phase === 'claimed' && + (terminal.status === 'succeeded' || + (terminal.status === 'failed' && + (terminal.failureClass === 'execution' || + terminal.failureClass === 'post-model-infrastructure'))) + ) { + throw new Error('candidate execution terminal requires candidate-may-run phase') + } +} + +export function rejectedFinish( + existing: AgentCandidateExecutionTerminalRecord, + requestedTerminalDigest: Sha256Digest, +): AgentCandidateExecutionFinishResult { + return Object.freeze({ + finished: false, + terminal: existing, + exactReplay: existing.terminalDigest === requestedTerminalDigest, + }) +} + +export function rejectedStage( + existing: AgentCandidateExecutionTerminalRecord, + requested: AgentCandidateExecutionTerminalRecord, +): AgentCandidateExecutionStageResult { + return Object.freeze({ + staged: false, + terminal: existing, + exactReplay: existing.terminalDigest === requested.terminalDigest, + }) +} + +export function requireStagedTerminal( + staged: AgentCandidateExecutionTerminalRecord | undefined, + terminalDigest: Sha256Digest, +): AgentCandidateExecutionTerminalRecord { + if (!staged) throw new Error('candidate execution terminal has not been staged') + if (staged.terminalDigest !== terminalDigest) { + throw new Error('candidate execution terminal digest does not match staged outbox') + } + return staged +} + +export function sealTerminalDigest(value: Sha256Digest): Sha256Digest { + assertSha256Digest(value, 'terminalDigest') + return value +} + +export function assertRecoveryMatchesStaged( + staged: AgentCandidateExecutionTerminalRecord, + recovered: AgentCandidateExecutionTerminalRecord, +): void { + if ( + canonicalCandidateDigest(staged.usage) !== canonicalCandidateDigest(recovered.usage) || + canonicalCandidateDigest(staged.modelSettlement) !== + canonicalCandidateDigest(recovered.modelSettlement) + ) { + throw new Error('candidate execution recovery evidence does not match staged model evidence') + } +} + +export function sealTerminalRecordValue( + value: Record, + label: string, +): AgentCandidateExecutionTerminalRecord { + const status = requireTerminalStatus(value.status, label) + assertExactKeys( + value, + status === 'succeeded' + ? [ + 'executionId', + 'attempt', + 'bundleDigest', + 'executionPlanDigest', + 'terminalDigest', + 'schemaVersion', + 'status', + 'usage', + 'modelSettlement', + 'taskOutcome', + 'benchmarkResult', + 'runReceipt', + ] + : [ + 'executionId', + 'attempt', + 'bundleDigest', + 'executionPlanDigest', + 'terminalDigest', + 'schemaVersion', + 'status', + 'failureClass', + 'usage', + 'modelSettlement', + ...(value.failureEvidence ? ['failureEvidence'] : []), + ], + label, + ) + const identity = { + executionId: requireString(value.executionId, label, 'executionId'), + attempt: requireNumber(value.attempt, label, 'attempt'), + bundleDigest: requireString(value.bundleDigest, label, 'bundleDigest') as Sha256Digest, + executionPlanDigest: requireString( + value.executionPlanDigest, + label, + 'executionPlanDigest', + ) as Sha256Digest, + } + assertExecutionId(identity.executionId) + if (!Number.isSafeInteger(identity.attempt) || identity.attempt < 1) { + throw new Error(`${label} has invalid attempt`) + } + assertSha256Digest(identity.bundleDigest, 'bundleDigest') + assertSha256Digest(identity.executionPlanDigest, 'executionPlanDigest') + const result = sealTerminalResult( + status === 'succeeded' + ? { + schemaVersion: requireNumber(value.schemaVersion, label, 'schemaVersion') as 1, + status, + usage: requireObject( + value.usage, + label, + 'usage', + ) as unknown as AgentCandidateExecutionUsage, + modelSettlement: requireArtifactRef(value.modelSettlement, label, 'modelSettlement'), + taskOutcome: requireArtifactRef(value.taskOutcome, label, 'taskOutcome'), + benchmarkResult: requireArtifactRef(value.benchmarkResult, label, 'benchmarkResult'), + runReceipt: requireArtifactRef(value.runReceipt, label, 'runReceipt'), + } + : { + schemaVersion: requireNumber(value.schemaVersion, label, 'schemaVersion') as 1, + status, + failureClass: requireFailureClass(value.failureClass, label), + usage: requireObject( + value.usage, + label, + 'usage', + ) as unknown as AgentCandidateExecutionUsage, + modelSettlement: requireArtifactRef(value.modelSettlement, label, 'modelSettlement'), + ...(value.failureEvidence + ? { + failureEvidence: requireArtifactRef( + value.failureEvidence, + label, + 'failureEvidence', + ), + } + : {}), + }, + ) + const material = { ...identity, ...result } + const terminalDigest = requireString( + value.terminalDigest, + label, + 'terminalDigest', + ) as Sha256Digest + assertSha256Digest(terminalDigest, 'terminalDigest') + if (terminalDigest !== canonicalCandidateDigest(material)) { + throw new Error(`${label} has invalid terminalDigest`) + } + return immutableCandidateValue({ + ...material, + terminalDigest, + }) as AgentCandidateExecutionTerminalRecord +} + +export function assertTerminalMatchesClaim( + terminal: AgentCandidateExecutionTerminalRecord, + claim: AgentCandidateExecutionClaim, + path: string, +): void { + if ( + terminal.executionId !== claim.executionId || + terminal.attempt !== claim.attempt || + terminal.bundleDigest !== claim.bundleDigest || + terminal.executionPlanDigest !== claim.executionPlanDigest + ) { + throw new Error(`candidate execution terminal record at ${path} does not match its claim`) + } +} + +export function assertTerminalMatchesStaged( + terminal: AgentCandidateExecutionTerminalRecord, + staged: AgentCandidateExecutionTerminalRecord, + path: string, +): void { + if ( + terminal.terminalDigest !== staged.terminalDigest || + canonicalCandidateDigest(terminal) !== canonicalCandidateDigest(staged) + ) { + throw new Error(`candidate execution terminal record at ${path} differs from staged outbox`) + } +} + +function sealRecoveryEvidence( + evidence: AgentCandidateExecutionRecoveryEvidence, + claim: AgentCandidateExecutionClaim, +): AgentCandidateExecutionRecoveryEvidence { + assertExactKeys( + evidence, + [ + 'failureClass', + 'usage', + 'modelSettlement', + 'process', + 'model', + ...(evidence.failureEvidence ? ['failureEvidence'] : []), + ...(evidence.memory ? ['memory'] : []), + ], + 'candidate execution recovery evidence', + ) + assertFailureClass(evidence.failureClass) + const usage = sealUsage(evidence.usage) + if (evidence.failureClass === 'pre-model-infrastructure' && usage.modelCalls !== 0) { + throw new Error('pre-model infrastructure failure cannot contain model calls') + } + const modelSettlement = sealArtifactRef(evidence.modelSettlement, 'modelSettlement') + const failureEvidence = evidence.failureEvidence + ? sealArtifactRef(evidence.failureEvidence, 'failureEvidence') + : undefined + assertExactKeys( + evidence.process, + ['stopped', 'executionPlanDigest'], + 'candidate execution process closure evidence', + ) + if ( + evidence.process.stopped !== true || + evidence.process.executionPlanDigest !== claim.executionPlanDigest + ) { + throw new Error('candidate execution recovery does not prove the claimed process stopped') + } + assertExactKeys( + evidence.model, + ['closed', 'preparationId', 'grantDigest'], + 'candidate execution model closure evidence', + ) + if ( + evidence.model.closed !== true || + evidence.model.preparationId !== claim.cleanup.preparationId || + evidence.model.grantDigest !== claim.cleanup.modelGrantDigest + ) { + throw new Error('candidate execution recovery does not prove the claimed model grant closed') + } + if (claim.cleanup.memory) { + if (!evidence.memory) { + throw new Error('candidate execution recovery is missing memory closure evidence') + } + assertExactKeys( + evidence.memory, + ['closed', 'preparationId', 'accessDigest', 'effectiveNamespace'], + 'candidate execution memory closure evidence', + ) + if ( + evidence.memory.closed !== true || + evidence.memory.preparationId !== claim.cleanup.preparationId || + evidence.memory.accessDigest !== claim.cleanup.memory.accessDigest || + evidence.memory.effectiveNamespace !== claim.cleanup.memory.effectiveNamespace + ) { + throw new Error( + 'candidate execution recovery does not prove the claimed memory access closed', + ) + } + } else if (evidence.memory !== undefined) { + throw new Error('candidate execution recovery has unexpected memory closure evidence') + } + return Object.freeze({ + failureClass: evidence.failureClass, + usage, + modelSettlement, + ...(failureEvidence ? { failureEvidence } : {}), + process: Object.freeze({ ...evidence.process }), + model: Object.freeze({ ...evidence.model }), + ...(evidence.memory ? { memory: Object.freeze({ ...evidence.memory }) } : {}), + }) +} + +function sealTerminalResult( + result: AgentCandidateExecutionTerminalResult, +): AgentCandidateExecutionTerminalResult { + if (result.status !== 'succeeded' && result.status !== 'failed') { + throw new Error('candidate execution terminal status is invalid') + } + assertExactKeys( + result, + result.status === 'succeeded' + ? [ + 'schemaVersion', + 'status', + 'usage', + 'modelSettlement', + 'taskOutcome', + 'benchmarkResult', + 'runReceipt', + ] + : [ + 'schemaVersion', + 'status', + 'failureClass', + 'usage', + 'modelSettlement', + ...(result.failureEvidence ? ['failureEvidence'] : []), + ], + 'candidate execution terminal result', + ) + if (result.schemaVersion !== 1) { + throw new Error('candidate execution terminal schemaVersion must be 1') + } + const usage = sealUsage(result.usage) + const modelSettlement = sealArtifactRef(result.modelSettlement, 'modelSettlement') + if (result.status === 'succeeded') { + return Object.freeze({ + schemaVersion: 1, + status: 'succeeded', + usage, + modelSettlement, + taskOutcome: sealArtifactRef(result.taskOutcome, 'taskOutcome'), + benchmarkResult: sealArtifactRef(result.benchmarkResult, 'benchmarkResult'), + runReceipt: sealArtifactRef(result.runReceipt, 'runReceipt'), + }) + } + assertFailureClass(result.failureClass) + if (result.failureClass === 'pre-model-infrastructure' && usage.modelCalls !== 0) { + throw new Error('pre-model infrastructure failure cannot contain model calls') + } + return Object.freeze({ + schemaVersion: 1, + status: 'failed', + failureClass: result.failureClass, + usage, + modelSettlement, + ...(result.failureEvidence + ? { failureEvidence: sealArtifactRef(result.failureEvidence, 'failureEvidence') } + : {}), + }) +} + +function sealUsage(usage: AgentCandidateExecutionUsage): AgentCandidateExecutionUsage { + assertExactKeys( + usage, + [ + 'costUsdNanos', + 'inputTokens', + 'outputTokens', + 'cachedInputTokens', + 'reasoningTokens', + 'modelCalls', + ], + 'candidate execution terminal usage', + ) + for (const [field, value] of Object.entries(usage)) { + assertCount(value, `terminal usage ${field}`) + } + return Object.freeze({ + costUsdNanos: usage.costUsdNanos, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + cachedInputTokens: usage.cachedInputTokens, + reasoningTokens: usage.reasoningTokens, + modelCalls: usage.modelCalls, + }) +} + +function sealArtifactRef(ref: AgentCandidateArtifactRef, label: string): AgentCandidateArtifactRef { + const parsed = agentCandidateArtifactRefSchema.parse(ref) + if (!Number.isSafeInteger(parsed.byteLength)) { + throw new Error(`candidate execution terminal ${label} byteLength exceeds safe integer range`) + } + return immutableCandidateValue(parsed) +} + +function requireArtifactRef( + value: unknown, + path: string, + field: string, +): AgentCandidateArtifactRef { + return requireObject(value, path, field) as unknown as AgentCandidateArtifactRef +} + +function requireString(value: unknown, path: string, field: string): string { + if (typeof value !== 'string') { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value +} + +function requireNumber(value: unknown, path: string, field: string): number { + if (typeof value !== 'number') { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value +} + +function requireObject(value: unknown, path: string, field: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value as Record +} + +function requireTerminalStatus( + value: unknown, + path: string, +): AgentCandidateExecutionTerminalResult['status'] { + if (value !== 'succeeded' && value !== 'failed') { + throw new Error(`candidate execution terminal record at ${path} has invalid status`) + } + return value +} + +function requireFailureClass(value: unknown, path: string): AgentCandidateExecutionFailureClass { + try { + assertFailureClass(value) + return value + } catch (error) { + throw new Error(`candidate execution terminal record at ${path} has invalid failureClass`, { + cause: error, + }) + } +} + +function assertFailureClass(value: unknown): asserts value is AgentCandidateExecutionFailureClass { + if ( + value !== 'pre-model-infrastructure' && + value !== 'execution' && + value !== 'post-model-infrastructure' && + value !== 'unknown' + ) { + throw new Error('candidate execution failureClass is invalid') + } +} + +function assertExecutionId(value: unknown): asserts value is string { + if (typeof value !== 'string' || !/^[A-Za-z0-9._:-]{1,200}$/.test(value)) { + throw new Error('candidate execution claim executionId is invalid') + } +} + +function assertSha256Digest(value: string, field: string): void { + if (!SHA256_PATTERN.test(value)) { + throw new Error(`candidate execution claim ${field} must be a lowercase sha256 digest`) + } +} + +function assertCount(value: unknown, label: string): asserts value is number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error(`candidate execution ${label} must be a non-negative safe integer`) + } +} diff --git a/src/candidate-execution/claim.ts b/src/candidate-execution/claim.ts new file mode 100644 index 00000000..e31ac79d --- /dev/null +++ b/src/candidate-execution/claim.ts @@ -0,0 +1,946 @@ +/** Durable one-shot lifecycle for candidate execution attempts. */ + +import { createHash, randomBytes, timingSafeEqual } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { + type AgentCandidateArtifactRef, + type AgentCandidateAttemptPolicy, + type AgentCandidateResolvedModel, + agentCandidateResolvedModelSchema, + type Sha256Digest, +} from '@tangle-network/agent-interface' +import { + CLAIM_FORMAT_VERSION, + PENDING_FORMAT_VERSION, + type PersistedAgentCandidateExecutionClaim, + type PersistedAgentCandidateExecutionPending, + type PersistedAgentCandidateExecutionPhase, + type PersistedAgentCandidateExecutionTerminal, + PHASE_FORMAT_VERSION, + TERMINAL_FORMAT_VERSION, +} from './claim-file-formats' +import { + assertRecoveryMatchesStaged, + assertTerminalAllowedInPhase, + assertTerminalMatchesClaim, + recoveredTerminalRecord, + rejectedFinish, + rejectedStage, + requireStagedTerminal, + sealTerminalDigest, + sealTerminalRecordValue, + terminalRecord, +} from './claim-terminal' +import { candidateCleanupTimeout, candidateResultTimeout } from './cleanup' +import { canonicalCandidateDigest, immutableCandidateValue } from './digest' +import { assertExactObjectKeys as assertExactKeys } from './exact-object' + +/** Non-secret identities a trusted recovery worker needs to close an abandoned attempt. */ +export interface AgentCandidateExecutionCleanupHandles { + readonly preparationId: string + readonly modelGrantDigest: Sha256Digest + readonly resolvedModel: AgentCandidateResolvedModel + readonly traceRunId: string + readonly cleanupTimeoutMs: number + readonly memory?: { + readonly accessDigest: Sha256Digest + readonly effectiveNamespace: string + } +} + +/** Immutable signed identity stored for one execution attempt. */ +export interface AgentCandidateExecutionClaim { + readonly executionId: string + readonly attempt: number + readonly maxAttempts: number + readonly retryPolicy: AgentCandidateAttemptPolicy['retryPolicy'] + readonly bundleDigest: Sha256Digest + readonly executionPlanDigest: Sha256Digest + /** Frozen plan identity with only attempt number and per-attempt grant identity normalized. */ + readonly retryLineageDigest: Sha256Digest + /** The winning lease stops authorizing a new terminal write at this instant. */ + readonly leaseExpiresAtMs: number + /** Frozen budget for task verification, executable grading, and receipt construction. */ + readonly resultTimeoutMs: number + /** Non-secret handles retained so an expired attempt can be closed and reconciled. */ + readonly cleanup: AgentCandidateExecutionCleanupHandles +} + +/** Secret capability required to finish the acquired attempt. */ +export interface AgentCandidateExecutionLease { + readonly executionId: string + readonly attempt: number + readonly token: string + readonly expiresAtMs: number +} + +/** Only the first class is retryable, and only when the closed model ledger has zero calls. */ +export type AgentCandidateExecutionFailureClass = + | 'pre-model-infrastructure' + | 'execution' + | 'post-model-infrastructure' + | 'unknown' + +/** Exact fixed-point usage proven by the closed evaluator model ledger. */ +export interface AgentCandidateExecutionUsage { + readonly costUsdNanos: number + readonly inputTokens: number + readonly outputTokens: number + readonly cachedInputTokens: number + readonly reasoningTokens: number + readonly modelCalls: number +} + +/** Evaluator-owned terminal facts staged durably before the terminal CAS. */ +export type AgentCandidateExecutionTerminalResult = + | { + readonly schemaVersion: 1 + readonly status: 'succeeded' + readonly usage: AgentCandidateExecutionUsage + readonly modelSettlement: AgentCandidateArtifactRef + readonly taskOutcome: AgentCandidateArtifactRef + readonly benchmarkResult: AgentCandidateArtifactRef + readonly runReceipt: AgentCandidateArtifactRef + } + | { + readonly schemaVersion: 1 + readonly status: 'failed' + readonly failureClass: AgentCandidateExecutionFailureClass + readonly usage: AgentCandidateExecutionUsage + readonly modelSettlement: AgentCandidateArtifactRef + readonly failureEvidence?: AgentCandidateArtifactRef + } + +/** Durable terminal record for one acquired execution attempt. */ +export type AgentCandidateExecutionTerminalRecord = AgentCandidateExecutionTerminalResult & { + readonly executionId: string + readonly attempt: number + readonly bundleDigest: Sha256Digest + readonly executionPlanDigest: Sha256Digest + /** RFC 8785 SHA-256 of this record with `terminalDigest` omitted. */ + readonly terminalDigest: Sha256Digest +} + +/** Monotonic durable phase: the second value means candidate code could have started. */ +export type AgentCandidateExecutionPhase = 'claimed' | 'candidate-may-run' + +/** Trusted, independently observed closure facts for one expired winning lease. */ +export interface AgentCandidateExecutionRecoveryEvidence { + readonly failureClass: AgentCandidateExecutionFailureClass + readonly usage: AgentCandidateExecutionUsage + readonly modelSettlement: AgentCandidateArtifactRef + readonly failureEvidence?: AgentCandidateArtifactRef + readonly process: { + readonly stopped: true + readonly executionPlanDigest: Sha256Digest + } + readonly model: { + readonly closed: true + readonly preparationId: string + readonly grantDigest: Sha256Digest + } + readonly memory?: { + readonly closed: true + readonly preparationId: string + readonly accessDigest: Sha256Digest + readonly effectiveNamespace: string + } +} + +export interface AgentCandidateExecutionAttemptRef { + readonly executionId: string + readonly attempt: number +} + +/** Persisted state available to a fresh trusted recovery worker after a crash. */ +export interface AgentCandidateExecutionAttemptRecord { + readonly claim: AgentCandidateExecutionClaim + readonly phase: AgentCandidateExecutionPhase + /** Durable outbox content written before the terminal compare-and-set. */ + readonly staged?: AgentCandidateExecutionTerminalRecord + readonly terminal?: AgentCandidateExecutionTerminalRecord +} + +/** Result of atomically claiming one execution attempt. */ +export type AgentCandidateExecutionClaimResult = + | { + readonly acquired: true + readonly claim: AgentCandidateExecutionClaim + readonly lease: AgentCandidateExecutionLease + } + | { + readonly acquired: false + readonly reason: 'already-claimed' + /** The durable winner already occupying this execution-attempt slot. */ + readonly claim: AgentCandidateExecutionClaim + /** True only when every signed claim field matches the durable winner. */ + readonly exactReplay: boolean + } + | { + readonly acquired: false + readonly reason: 'retry-not-eligible' + readonly claim: AgentCandidateExecutionClaim + readonly detail: AgentCandidateRetryRejection + } + +/** Result of atomically recording an attempt's terminal facts. */ +export type AgentCandidateExecutionFinishResult = + | { + readonly finished: true + readonly terminal: AgentCandidateExecutionTerminalRecord + } + | { + readonly finished: false + readonly terminal: AgentCandidateExecutionTerminalRecord + /** True when a repeated finish supplied the same terminal digest. */ + readonly exactReplay: boolean + } + +/** Result of durably staging the one immutable terminal outbox entry. */ +export type AgentCandidateExecutionStageResult = + | { + readonly staged: true + readonly terminal: AgentCandidateExecutionTerminalRecord + } + | { + readonly staged: false + readonly terminal: AgentCandidateExecutionTerminalRecord + readonly exactReplay: boolean + } + +/** Result of crossing the irreversible candidate-may-run boundary. */ +export type AgentCandidateExecutionPhaseResult = + | { readonly marked: true; readonly phase: 'candidate-may-run' } + | { readonly marked: false; readonly phase: 'candidate-may-run' } + +export type AgentCandidateRetryRejection = + | 'prior-attempt-missing' + | 'prior-attempt-running' + | 'prior-attempt-succeeded' + | 'prior-attempt-spent-model-calls' + | 'prior-attempt-not-pre-model-infrastructure' + | 'retry-lineage-mismatch' + +/** + * Atomic one-shot store for candidate execution attempts. + * + * Implementations must linearize both methods across every process sharing the + * store. Terminal publication is deliberately two-step: `stageTerminal` + * fsyncs the complete immutable outbox record, then `finish` publishes exactly + * those staged bytes by digest. A crash between the two leaves recoverable + * evidence rather than an ambiguous completed run. + */ +export interface AgentCandidateExecutionClaimStore { + tryClaim(claim: AgentCandidateExecutionClaim): Promise + getAttempt( + attempt: AgentCandidateExecutionAttemptRef, + ): Promise + /** Persist the point after which candidate code may have run. */ + markCandidateMayRun( + lease: AgentCandidateExecutionLease, + ): Promise + /** Fsync the complete terminal record into the durable outbox. */ + stageTerminal( + lease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, + ): Promise + /** Publish exactly the staged terminal identified by `terminalDigest`. */ + finish( + lease: AgentCandidateExecutionLease, + terminalDigest: Sha256Digest, + ): Promise + /** + * Write a failed terminal only after the lease expired and a trusted worker + * independently proved process death plus model and memory closure. + */ + recoverExpired( + attempt: AgentCandidateExecutionAttemptRef, + evidence: AgentCandidateExecutionRecoveryEvidence, + ): Promise +} + +interface StoredClaim { + claim: AgentCandidateExecutionClaim + leaseDigest: Sha256Digest + phase: AgentCandidateExecutionPhase + staged?: AgentCandidateExecutionTerminalRecord + terminal?: AgentCandidateExecutionTerminalRecord +} + +function attemptRecord( + claim: AgentCandidateExecutionClaim, + phase: AgentCandidateExecutionPhase, + staged?: AgentCandidateExecutionTerminalRecord, + terminal?: AgentCandidateExecutionTerminalRecord, +): AgentCandidateExecutionAttemptRecord { + return Object.freeze({ + claim, + phase, + ...(staged ? { staged } : {}), + ...(terminal ? { terminal } : {}), + }) +} + +export interface InMemoryAgentCandidateExecutionClaimStoreOptions { + /** Testable evaluator clock; defaults to `Date.now`. */ + now?: () => number +} + +/** Single-process lifecycle implementation. */ +export class InMemoryAgentCandidateExecutionClaimStore + implements AgentCandidateExecutionClaimStore +{ + private readonly claims = new Map() + private readonly now: () => number + + constructor(options: InMemoryAgentCandidateExecutionClaimStoreOptions = {}) { + this.now = options.now ?? Date.now + } + + async tryClaim( + requested: AgentCandidateExecutionClaim, + ): Promise { + const claim = sealClaim(requested) + const slot = claimSlot(claim) + const existing = this.claims.get(slot) + if (existing) return rejectedExistingClaim(existing.claim, claim) + assertUnexpiredLease(claim.leaseExpiresAtMs, this.now()) + + const retryRejection = retryRejectionFromMemory(this.claims, claim) + if (retryRejection) return rejectedRetry(claim, retryRejection) + + const lease = newLease(claim) + // No await may occur between the read and write: this is the linearization + // point for every caller sharing this store instance. + this.claims.set(slot, { + claim, + leaseDigest: leaseDigest(lease), + phase: 'claimed', + }) + return Object.freeze({ acquired: true, claim, lease }) + } + + async getAttempt( + requestedAttempt: AgentCandidateExecutionAttemptRef, + ): Promise { + const attempt = sealAttemptRef(requestedAttempt) + const stored = this.claims.get(claimSlot(attempt)) + return stored + ? attemptRecord(stored.claim, stored.phase, stored.staged, stored.terminal) + : undefined + } + + async markCandidateMayRun( + requestedLease: AgentCandidateExecutionLease, + ): Promise { + const lease = sealLease(requestedLease) + const stored = this.requireClaim(lease) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + if (stored.phase === 'candidate-may-run') { + return Object.freeze({ marked: false, phase: 'candidate-may-run' }) + } + if (stored.staged || stored.terminal) { + throw new Error('candidate execution terminal was staged before candidate-may-run phase') + } + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + stored.phase = 'candidate-may-run' + return Object.freeze({ marked: true, phase: 'candidate-may-run' }) + } + + async stageTerminal( + requestedLease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, + ): Promise { + const lease = sealLease(requestedLease) + const stored = this.requireClaim(lease) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const terminal = terminalRecord(stored.claim, result) + if (stored.staged) return rejectedStage(stored.staged, terminal) + assertTerminalAllowedInPhase(stored.phase, terminal) + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + stored.staged = terminal + return Object.freeze({ staged: true, terminal }) + } + + async finish( + requestedLease: AgentCandidateExecutionLease, + requestedTerminalDigest: Sha256Digest, + ): Promise { + const lease = sealLease(requestedLease) + const terminalDigest = sealTerminalDigest(requestedTerminalDigest) + const stored = this.requireClaim(lease) + assertLease(stored.leaseDigest, stored.claim.leaseExpiresAtMs, lease) + const staged = requireStagedTerminal(stored.staged, terminalDigest) + if (stored.terminal) return rejectedFinish(stored.terminal, terminalDigest) + assertUnexpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + + // The terminal assignment is the in-memory finish linearization point and + // publishes the exact immutable object already present in the outbox. + stored.terminal = staged + return Object.freeze({ finished: true, terminal: staged }) + } + + async recoverExpired( + requestedAttempt: AgentCandidateExecutionAttemptRef, + evidence: AgentCandidateExecutionRecoveryEvidence, + ): Promise { + const attempt = sealAttemptRef(requestedAttempt) + const stored = this.requireClaim(attempt, 'candidate execution recovery') + const recovered = recoveredTerminalRecord(stored.claim, stored.phase, evidence) + if (stored.staged) assertRecoveryMatchesStaged(stored.staged, recovered) + const requestedDigest = stored.staged?.terminalDigest ?? recovered.terminalDigest + if (stored.terminal) return rejectedFinish(stored.terminal, requestedDigest) + assertExpiredLease(stored.claim.leaseExpiresAtMs, this.now()) + const terminal = stored.staged ?? recovered + assertRecoveryMatchesStaged(terminal, recovered) + stored.staged ??= terminal + stored.terminal = terminal + return Object.freeze({ finished: true, terminal }) + } + + private requireClaim( + attempt: Pick, + operation = 'candidate execution lease', + ): StoredClaim { + const stored = this.claims.get(claimSlot(attempt)) + if (!stored) throw new Error(`${operation} does not name an acquired attempt`) + return stored + } +} + +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/ +const LEASE_TOKEN_PATTERN = /^candidate-execution-lease-v1\.[A-Za-z0-9_-]{43}$/ +const PREPARATION_ID_PATTERN = /^candidate-preparation-v1\.[A-Za-z0-9_-]{43}$/ + +function sealClaim(claim: AgentCandidateExecutionClaim): AgentCandidateExecutionClaim { + assertExactKeys( + claim, + [ + 'executionId', + 'attempt', + 'maxAttempts', + 'retryPolicy', + 'bundleDigest', + 'executionPlanDigest', + 'retryLineageDigest', + 'leaseExpiresAtMs', + 'resultTimeoutMs', + 'cleanup', + ], + 'candidate execution claim', + ) + assertExecutionId(claim.executionId) + if (!Number.isSafeInteger(claim.attempt) || claim.attempt < 1) { + throw new Error('candidate execution claim attempt must be a positive safe integer') + } + if (!Number.isSafeInteger(claim.maxAttempts) || claim.maxAttempts < 1) { + throw new Error('candidate execution claim maxAttempts must be a positive safe integer') + } + if (claim.attempt > claim.maxAttempts) { + throw new Error('candidate execution claim attempt exceeds maxAttempts') + } + if (!['none', 'pre-model-infrastructure-only'].includes(claim.retryPolicy)) { + throw new Error('candidate execution claim retryPolicy is invalid') + } + if (claim.retryPolicy === 'none' && claim.maxAttempts !== 1) { + throw new Error('candidate execution claim retryPolicy none requires maxAttempts 1') + } + assertSha256Digest(claim.bundleDigest, 'bundleDigest') + assertSha256Digest(claim.executionPlanDigest, 'executionPlanDigest') + assertSha256Digest(claim.retryLineageDigest, 'retryLineageDigest') + assertPositiveTimestamp(claim.leaseExpiresAtMs, 'leaseExpiresAtMs') + candidateResultTimeout(claim.resultTimeoutMs, claim.resultTimeoutMs) + const cleanup = sealCleanupHandles(claim.cleanup) + return Object.freeze({ + executionId: claim.executionId, + attempt: claim.attempt, + maxAttempts: claim.maxAttempts, + retryPolicy: claim.retryPolicy, + bundleDigest: claim.bundleDigest, + executionPlanDigest: claim.executionPlanDigest, + retryLineageDigest: claim.retryLineageDigest, + leaseExpiresAtMs: claim.leaseExpiresAtMs, + resultTimeoutMs: claim.resultTimeoutMs, + cleanup, + }) +} + +function sealCleanupHandles( + cleanup: AgentCandidateExecutionCleanupHandles, +): AgentCandidateExecutionCleanupHandles { + assertExactKeys( + cleanup, + cleanup.memory + ? [ + 'preparationId', + 'modelGrantDigest', + 'resolvedModel', + 'traceRunId', + 'cleanupTimeoutMs', + 'memory', + ] + : ['preparationId', 'modelGrantDigest', 'resolvedModel', 'traceRunId', 'cleanupTimeoutMs'], + 'candidate execution cleanup handles', + ) + if (!PREPARATION_ID_PATTERN.test(cleanup.preparationId)) { + throw new Error('candidate execution cleanup preparationId is invalid') + } + assertSha256Digest(cleanup.modelGrantDigest, 'cleanup modelGrantDigest') + const resolvedModel = immutableCandidateValue( + agentCandidateResolvedModelSchema.parse(cleanup.resolvedModel), + ) + assertBoundedIdentifier(cleanup.traceRunId, 'cleanup traceRunId', 512) + const cleanupTimeoutMs = candidateCleanupTimeout(cleanup.cleanupTimeoutMs) + const memory = cleanup.memory ? sealMemoryCleanupHandle(cleanup.memory) : undefined + return Object.freeze({ + preparationId: cleanup.preparationId, + modelGrantDigest: cleanup.modelGrantDigest, + resolvedModel, + traceRunId: cleanup.traceRunId, + cleanupTimeoutMs, + ...(memory ? { memory } : {}), + }) +} + +function sealMemoryCleanupHandle( + memory: NonNullable, +): NonNullable { + assertExactKeys( + memory, + ['accessDigest', 'effectiveNamespace'], + 'candidate execution memory cleanup handle', + ) + assertSha256Digest(memory.accessDigest, 'memory accessDigest') + assertBoundedIdentifier(memory.effectiveNamespace, 'memory effectiveNamespace', 1_024) + return Object.freeze({ + accessDigest: memory.accessDigest, + effectiveNamespace: memory.effectiveNamespace, + }) +} + +function sealAttemptRef( + attempt: AgentCandidateExecutionAttemptRef, +): AgentCandidateExecutionAttemptRef { + assertExactKeys(attempt, ['executionId', 'attempt'], 'candidate execution attempt reference') + assertExecutionId(attempt.executionId) + if (!Number.isSafeInteger(attempt.attempt) || attempt.attempt < 1) { + throw new Error('candidate execution attempt reference must have a positive safe attempt') + } + return Object.freeze({ executionId: attempt.executionId, attempt: attempt.attempt }) +} + +function sealLease(lease: AgentCandidateExecutionLease): AgentCandidateExecutionLease { + assertExactKeys( + lease, + ['executionId', 'attempt', 'token', 'expiresAtMs'], + 'candidate execution lease', + ) + if (lease.executionId.length === 0 || !Number.isSafeInteger(lease.attempt) || lease.attempt < 1) { + throw new Error('candidate execution lease identity is invalid') + } + if (!LEASE_TOKEN_PATTERN.test(lease.token)) { + throw new Error('candidate execution lease token is invalid') + } + assertPositiveTimestamp(lease.expiresAtMs, 'lease expiresAtMs') + return Object.freeze({ + executionId: lease.executionId, + attempt: lease.attempt, + token: lease.token, + expiresAtMs: lease.expiresAtMs, + }) +} + +function newLease(claim: AgentCandidateExecutionClaim): AgentCandidateExecutionLease { + return Object.freeze({ + executionId: claim.executionId, + attempt: claim.attempt, + token: `candidate-execution-lease-v1.${randomBytes(32).toString('base64url')}`, + expiresAtMs: claim.leaseExpiresAtMs, + }) +} + +function leaseDigest(lease: AgentCandidateExecutionLease): Sha256Digest { + return sha256(lease.token) +} + +function assertLease( + expectedDigest: Sha256Digest, + expectedExpiresAtMs: number, + lease: AgentCandidateExecutionLease, +): void { + const expected = Buffer.from(expectedDigest) + const actual = Buffer.from(leaseDigest(lease)) + if ( + expected.length !== actual.length || + !timingSafeEqual(expected, actual) || + lease.expiresAtMs !== expectedExpiresAtMs + ) { + throw new Error('candidate execution lease is invalid') + } +} + +function claimSlot(claim: Pick): string { + return createHash('sha256') + .update(JSON.stringify([claim.executionId, claim.attempt]), 'utf8') + .digest('hex') +} + +function retryRejectionFromMemory( + claims: ReadonlyMap, + claim: AgentCandidateExecutionClaim, +): AgentCandidateRetryRejection | undefined { + if (claim.attempt === 1) return undefined + const prior = claims.get( + claimSlot({ executionId: claim.executionId, attempt: claim.attempt - 1 }), + ) + return retryRejection(claim, prior) +} + +function retryRejection( + claim: AgentCandidateExecutionClaim, + prior: + | { + claim: AgentCandidateExecutionClaim + terminal?: AgentCandidateExecutionTerminalRecord + } + | undefined, +): AgentCandidateRetryRejection | undefined { + if (!prior) return 'prior-attempt-missing' + if ( + claim.retryPolicy !== 'pre-model-infrastructure-only' || + prior.claim.retryPolicy !== claim.retryPolicy || + prior.claim.maxAttempts !== claim.maxAttempts || + prior.claim.bundleDigest !== claim.bundleDigest || + prior.claim.retryLineageDigest !== claim.retryLineageDigest + ) { + return 'retry-lineage-mismatch' + } + if (!prior.terminal) return 'prior-attempt-running' + if (prior.terminal.status === 'succeeded') return 'prior-attempt-succeeded' + if (prior.terminal.usage.modelCalls !== 0) return 'prior-attempt-spent-model-calls' + if (prior.terminal.failureClass !== 'pre-model-infrastructure') { + return 'prior-attempt-not-pre-model-infrastructure' + } + return undefined +} + +function rejectedExistingClaim( + existing: AgentCandidateExecutionClaim, + requested: AgentCandidateExecutionClaim, +): AgentCandidateExecutionClaimResult { + return Object.freeze({ + acquired: false, + reason: 'already-claimed', + claim: existing, + exactReplay: canonicalCandidateDigest(existing) === canonicalCandidateDigest(requested), + }) +} + +function rejectedRetry( + claim: AgentCandidateExecutionClaim, + detail: AgentCandidateRetryRejection, +): AgentCandidateExecutionClaimResult { + return Object.freeze({ acquired: false, reason: 'retry-not-eligible', claim, detail }) +} + +async function readClaim(path: string): Promise { + const parsed = await readJsonObject(path, 'claim') + const record = parsed as Partial + if (record.version !== CLAIM_FORMAT_VERSION) { + throw new Error(`candidate execution claim at ${path} has unsupported version`) + } + assertExactKeys( + parsed, + [ + 'version', + 'executionId', + 'attempt', + 'maxAttempts', + 'retryPolicy', + 'bundleDigest', + 'executionPlanDigest', + 'retryLineageDigest', + 'leaseExpiresAtMs', + 'resultTimeoutMs', + 'cleanup', + 'phase', + 'leaseDigest', + ], + `candidate execution claim at ${path}`, + ) + const claim = sealClaim({ + executionId: requireString(record.executionId, path, 'executionId'), + attempt: requireNumber(record.attempt, path, 'attempt'), + maxAttempts: requireNumber(record.maxAttempts, path, 'maxAttempts'), + retryPolicy: requireRetryPolicy(record.retryPolicy, path), + bundleDigest: requireString(record.bundleDigest, path, 'bundleDigest') as Sha256Digest, + executionPlanDigest: requireString( + record.executionPlanDigest, + path, + 'executionPlanDigest', + ) as Sha256Digest, + retryLineageDigest: requireString( + record.retryLineageDigest, + path, + 'retryLineageDigest', + ) as Sha256Digest, + leaseExpiresAtMs: requireNumber(record.leaseExpiresAtMs, path, 'leaseExpiresAtMs'), + resultTimeoutMs: requireNumber(record.resultTimeoutMs, path, 'resultTimeoutMs'), + cleanup: requireObject( + record.cleanup, + path, + 'cleanup', + ) as unknown as AgentCandidateExecutionCleanupHandles, + }) + const persistedLeaseDigest = requireString( + record.leaseDigest, + path, + 'leaseDigest', + ) as Sha256Digest + assertSha256Digest(persistedLeaseDigest, 'leaseDigest') + if (record.phase !== 'claimed') { + throw new Error(`candidate execution claim at ${path} has invalid initial phase`) + } + return { claim, leaseDigest: persistedLeaseDigest, phase: 'claimed' } +} + +async function readClaimIfPresent(path: string): Promise { + try { + return await readClaim(path) + } catch (error) { + if (isMissingError(error)) return undefined + throw error + } +} + +async function readTerminal(path: string): Promise { + const parsed = await readJsonObject(path, 'terminal record') + const record = parsed as Partial + if (record.version !== TERMINAL_FORMAT_VERSION) { + throw new Error(`candidate execution terminal record at ${path} has unsupported version`) + } + assertExactKeys(parsed, ['version', 'terminal'], `candidate execution terminal record at ${path}`) + return sealTerminalRecordValue( + requireObject(record.terminal, path, 'terminal'), + `candidate execution terminal record at ${path}`, + ) +} + +async function readTerminalIfPresent( + path: string, +): Promise { + try { + return await readTerminal(path) + } catch (error) { + if (isMissingError(error)) return undefined + throw error + } +} + +type ReadCandidateExecutionTransition = + | { kind: 'phase' } + | { kind: 'pending'; terminal: AgentCandidateExecutionTerminalRecord } + +async function readTransitionIfPresent( + path: string, + claim: AgentCandidateExecutionClaim, +): Promise { + let parsed: Record + try { + parsed = await readJsonObject(path, 'transition record') + } catch (error) { + if (isMissingError(error)) return undefined + throw error + } + if (parsed.kind === 'candidate-execution-phase') { + const record = parsed as unknown as PersistedAgentCandidateExecutionPhase + if (record.version !== PHASE_FORMAT_VERSION) { + throw new Error(`candidate execution phase record at ${path} has unsupported version`) + } + assertExactKeys( + parsed, + ['version', 'kind', 'executionId', 'attempt', 'executionPlanDigest', 'phase'], + `candidate execution phase record at ${path}`, + ) + if ( + record.executionId !== claim.executionId || + record.attempt !== claim.attempt || + record.executionPlanDigest !== claim.executionPlanDigest || + record.phase !== 'candidate-may-run' + ) { + throw new Error(`candidate execution phase record at ${path} does not match its claim`) + } + return { kind: 'phase' } + } + if (parsed.kind === 'candidate-execution-pending-terminal') { + const record = parsed as unknown as PersistedAgentCandidateExecutionPending + if (record.version !== PENDING_FORMAT_VERSION) { + throw new Error(`candidate execution pending record at ${path} has unsupported version`) + } + assertExactKeys( + parsed, + ['version', 'kind', 'terminal'], + `candidate execution pending record at ${path}`, + ) + const terminal = sealTerminalRecordValue( + requireObject(record.terminal, path, 'terminal'), + `candidate execution pending record at ${path}`, + ) + assertTerminalMatchesClaim(terminal, claim, path) + return { kind: 'pending', terminal } + } + throw new Error(`candidate execution transition record at ${path} has invalid kind`) +} + +async function readJsonObject(path: string, kind: string): Promise> { + let parsed: unknown + try { + parsed = JSON.parse(await readFile(path, 'utf8')) + } catch (error) { + if (isMissingError(error)) throw error + throw new Error(`candidate execution ${kind} at ${path} is unreadable`, { cause: error }) + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`candidate execution ${kind} at ${path} is not an object`) + } + return parsed as Record +} + +function assertSameSlot( + existing: Pick, + requested: Pick, + path: string, +): void { + if (existing.executionId !== requested.executionId || existing.attempt !== requested.attempt) { + throw new Error(`candidate execution record at ${path} does not match its claim slot`) + } +} + +function assertSha256Digest(value: string, field: string): void { + if (!SHA256_PATTERN.test(value)) { + throw new Error(`candidate execution claim ${field} must be a lowercase sha256 digest`) + } +} + +function requireString(value: unknown, path: string, field: string): string { + if (typeof value !== 'string') { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value +} + +function requireNumber(value: unknown, path: string, field: string): number { + if (typeof value !== 'number') { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value +} + +function requireObject(value: unknown, path: string, field: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`candidate execution record at ${path} has invalid ${field}`) + } + return value as Record +} + +function requireRetryPolicy( + value: unknown, + path: string, +): AgentCandidateAttemptPolicy['retryPolicy'] { + if (value !== 'none' && value !== 'pre-model-infrastructure-only') { + throw new Error(`candidate execution record at ${path} has invalid retryPolicy`) + } + return value +} + +function assertExecutionId(value: unknown): asserts value is string { + if (typeof value !== 'string' || !/^[A-Za-z0-9._:-]{1,200}$/.test(value)) { + throw new Error('candidate execution claim executionId is invalid') + } +} + +function assertBoundedIdentifier( + value: unknown, + label: string, + maxLength: number, +): asserts value is string { + if ( + typeof value !== 'string' || + value.length === 0 || + value.length > maxLength || + hasControlCharacter(value) + ) { + throw new Error(`candidate execution ${label} is invalid`) + } +} + +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code < 0x20 || code === 0x7f) return true + } + return false +} + +function assertPositiveTimestamp(value: unknown, label: string): asserts value is number { + if (!Number.isSafeInteger(value) || (value as number) <= 0) { + throw new Error(`candidate execution ${label} must be a positive safe timestamp`) + } +} + +function assertClock(value: number): void { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error('candidate execution claim-store clock returned an invalid timestamp') + } +} + +function assertUnexpiredLease(expiresAtMs: number, nowMs: number): void { + assertClock(nowMs) + if (nowMs >= expiresAtMs) throw new Error('candidate execution lease has expired') +} + +function assertExpiredLease(expiresAtMs: number, nowMs: number): void { + assertClock(nowMs) + if (nowMs < expiresAtMs) throw new Error('candidate execution lease has not expired') +} + +function sha256(value: string): Sha256Digest { + return `sha256:${createHash('sha256').update(value, 'utf8').digest('hex')}` +} + +function isMissingError(error: unknown): boolean { + return isNodeError(error, 'ENOENT') +} + +function isNodeError(error: unknown, code: string): boolean { + return ( + error !== null && + typeof error === 'object' && + 'code' in error && + (error as { code?: unknown }).code === code + ) +} + +export const candidateClaimFileInternals = Object.freeze({ + assertExpiredLease, + assertLease, + assertSameSlot, + assertUnexpiredLease, + attemptRecord, + claimSlot, + leaseDigest, + newLease, + readClaim, + readClaimIfPresent, + readTerminal, + readTerminalIfPresent, + readTransitionIfPresent, + rejectedExistingClaim, + rejectedRetry, + retryRejection, + sealAttemptRef, + sealClaim, + sealLease, +}) + +export { candidateExecutionClaim } from './claim-plan' diff --git a/src/candidate-execution/cleanup.ts b/src/candidate-execution/cleanup.ts new file mode 100644 index 00000000..daea4b9e --- /dev/null +++ b/src/candidate-execution/cleanup.ts @@ -0,0 +1,114 @@ +export const DEFAULT_CANDIDATE_CLEANUP_TIMEOUT_MS = 30_000 +/** Largest delay Node schedules without clamping to one millisecond. */ +export const MAX_CANDIDATE_TIMER_INTERVAL_MS = 2_147_483_647 + +export function candidateCleanupTimeout(timeoutMs: number | undefined): number { + const effective = timeoutMs ?? DEFAULT_CANDIDATE_CLEANUP_TIMEOUT_MS + if ( + !Number.isSafeInteger(effective) || + effective <= 0 || + effective > MAX_CANDIDATE_TIMER_INTERVAL_MS + ) { + throw new Error('candidate cleanup timeout is outside the supported timer range') + } + return effective +} + +export function candidateCleanupDeadline(timeoutMs: number | undefined): number { + return Date.now() + candidateCleanupTimeout(timeoutMs) +} + +/** Freeze a separate result-construction budget; defaults to the task wall limit. */ +export function candidateResultTimeout( + timeoutMs: number | undefined, + taskTimeoutMs: number, +): number { + const effective = timeoutMs ?? taskTimeoutMs + if ( + !Number.isSafeInteger(effective) || + effective <= 0 || + effective > MAX_CANDIDATE_TIMER_INTERVAL_MS + ) { + throw new Error('candidate result timeout is outside the supported timer range') + } + return effective +} + +/** Bound an evaluator cleanup call while keeping late rejection observed. */ +export async function withinCandidateCleanupDeadline( + operation: () => Promise, + deadlineAtMs: number, + label: string, +): Promise { + const remainingMs = deadlineAtMs - Date.now() + if (remainingMs <= 0) throw new CandidateCleanupTimeoutError(label) + + const pending = Promise.resolve().then(operation) + void pending.catch(() => undefined) + let timer: ReturnType | undefined + try { + const result = await Promise.race([ + pending, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new CandidateCleanupTimeoutError(label)), remainingMs) + }), + ]) + // Exact-boundary completion is ambiguous under event-loop delay. + if (Date.now() >= deadlineAtMs) throw new CandidateCleanupTimeoutError(label) + return result + } finally { + if (timer) clearTimeout(timer) + } +} + +/** + * Bound cancellable scoring/result work. Every side-effecting port called by + * `operation` must honor the supplied signal before durable publication. + */ +export async function withinCandidateResultDeadline( + operation: (signal: AbortSignal) => Promise, + deadlineAtMs: number, + label: string, +): Promise { + const remainingMs = deadlineAtMs - Date.now() + if (remainingMs <= 0) throw new CandidateResultTimeoutError(label) + + const controller = new AbortController() + const timeoutError = new CandidateResultTimeoutError(label) + const pending = Promise.resolve().then(() => operation(controller.signal)) + void pending.catch(() => undefined) + let timer: ReturnType | undefined + try { + const result = await Promise.race([ + pending, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + controller.abort(timeoutError) + reject(timeoutError) + }, remainingMs) + }), + ]) + // Exact-boundary completion is ambiguous under event-loop delay. + if (Date.now() >= deadlineAtMs) { + controller.abort(timeoutError) + throw timeoutError + } + return result + } finally { + if (timer) clearTimeout(timer) + } +} + +export class CandidateCleanupTimeoutError extends Error { + constructor(label: string) { + super(`${label} did not complete before the evaluator cleanup deadline`) + this.name = 'CandidateCleanupTimeoutError' + } +} + +export class CandidateResultTimeoutError extends Error { + constructor(label: string) { + super(`${label} did not complete before the evaluator result deadline`) + this.name = 'CandidateResultTimeoutError' + } +} diff --git a/src/candidate-execution/digest.ts b/src/candidate-execution/digest.ts new file mode 100644 index 00000000..47fe0c7b --- /dev/null +++ b/src/candidate-execution/digest.ts @@ -0,0 +1,76 @@ +import { createHash } from 'node:crypto' +import { canonicalJson } from '@tangle-network/agent-eval' +import type { AgentCandidateEmbeddedArtifact, Sha256Digest } from '@tangle-network/agent-interface' + +import { contentAddress } from '../durable/spawn-journal' +import type { CanonicalCandidateDocument } from './types' + +export function sha256Bytes(bytes: Uint8Array): Sha256Digest { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +export function canonicalCandidateBytes(value: unknown): Uint8Array { + return Buffer.from(canonicalJson(value), 'utf8') +} + +export function canonicalCandidateDigest(value: unknown): Sha256Digest { + return contentAddress(value) as Sha256Digest +} + +/** Returns a detached, deeply frozen JSON value with canonical number normalization. */ +export function immutableCandidateValue(value: T): T { + return deepFreezeCandidate( + JSON.parse(Buffer.from(canonicalCandidateBytes(value)).toString('utf8')) as T, + ) +} + +export function canonicalCandidateDocument( + valueWithoutDigest: Omit, +): CanonicalCandidateDocument { + const bytes = canonicalCandidateBytes(valueWithoutDigest) + const digest = canonicalCandidateDigest(valueWithoutDigest) + if (sha256Bytes(bytes) !== digest) { + throw new Error('canonical candidate serializers disagree on document digest') + } + const storedBytes = Uint8Array.from(bytes) + const value = immutableCandidateValue({ ...valueWithoutDigest, digest }) as T + return Object.freeze({ + value, + get bytes(): Uint8Array { + return Uint8Array.from(storedBytes) + }, + digest, + }) +} + +export function embeddedCandidateArtifact(bytes: Uint8Array): AgentCandidateEmbeddedArtifact { + return { + encoding: 'base64', + content: Buffer.from(bytes).toString('base64'), + sha256: sha256Bytes(bytes), + byteLength: bytes.byteLength, + } +} + +export function omitTopLevelDigest( + value: T, +): Omit { + const { digest: _digest, ...rest } = value + return rest +} + +export function deepFreezeCandidate(value: T, seen = new Set()): T { + if ( + value === null || + typeof value !== 'object' || + ArrayBuffer.isView(value) || + seen.has(value as object) + ) { + return value + } + seen.add(value as object) + for (const child of Object.values(value as Record)) { + deepFreezeCandidate(child, seen) + } + return Object.freeze(value) +} diff --git a/src/candidate-execution/dispose.ts b/src/candidate-execution/dispose.ts new file mode 100644 index 00000000..c1899013 --- /dev/null +++ b/src/candidate-execution/dispose.ts @@ -0,0 +1,88 @@ +import { candidateCleanupDeadline, withinCandidateCleanupDeadline } from './cleanup' +import { sealAgentCandidateModelSettlement } from './model-settlement' +import { + assertPreparedCandidateIntegrity, + beginPreparedCandidateDisposal, + consumePreparedCandidateExecution, +} from './prepared-state' +import type { PreparedAgentCandidateExecution } from './types' + +export interface DisposePreparedAgentCandidateOptions { + cleanupTimeoutMs?: number +} + +/** Revoke reservations held by a prepared candidate that will not be executed. */ +export async function disposePreparedAgentCandidateExecution( + prepared: PreparedAgentCandidateExecution, + options: DisposePreparedAgentCandidateOptions = {}, +): Promise<{ disposed: true }> { + const initialState = assertPreparedCandidateIntegrity(prepared) + const cleanupTimeoutMs = options.cleanupTimeoutMs ?? initialState.cleanupTimeoutMs + if (cleanupTimeoutMs > initialState.cleanupTimeoutMs) { + throw new Error('disposal cleanup timeout exceeds the frozen preparation bound') + } + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const state = beginPreparedCandidateDisposal(prepared) + const cleanup: Array> = [] + + if (state.memory.mode === 'isolated') { + const reservation = state.memoryReservation + if (!reservation) throw new Error('isolated memory reservation is missing') + cleanup.push( + withinCandidateCleanupDeadline( + async () => { + const closed = await state.ports.memory.close({ + executionId: state.executionId, + preparationId: reservation.preparationId, + accessDigest: reservation.accessDigest, + effectiveNamespace: reservation.effectiveNamespace, + reason: 'abandoned', + }) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('abandoned isolated memory access did not acknowledge closure') + } + }, + cleanupDeadlineAtMs, + 'isolated memory disposal', + ), + ) + } + + cleanup.push( + withinCandidateCleanupDeadline( + async () => { + const settlement = sealAgentCandidateModelSettlement( + await state.ports.models.settleGrant({ + executionId: state.executionId, + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + resolved: state.resolvedModel, + reason: 'abandoned', + }), + { + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + model: state.resolvedModel.model, + }, + ) + if (settlement.usage.modelCalls !== 0) { + throw new Error('unexecuted model reservation unexpectedly contains calls') + } + }, + cleanupDeadlineAtMs, + 'model reservation disposal', + ), + ) + + const results = await Promise.allSettled(cleanup) + const failures = results + .filter((result): result is PromiseRejectedResult => result.status === 'rejected') + .map((result) => result.reason) + if (failures.length > 0) { + consumePreparedCandidateExecution(prepared, 'disposal-failed') + throw new AggregateError(failures, 'candidate preparation disposal failed') + } + + consumePreparedCandidateExecution(prepared, 'disposed') + return Object.freeze({ disposed: true as const }) +} diff --git a/src/candidate-execution/exact-object.ts b/src/candidate-execution/exact-object.ts new file mode 100644 index 00000000..2eecc367 --- /dev/null +++ b/src/candidate-execution/exact-object.ts @@ -0,0 +1,21 @@ +/** Reject unknown fields while requiring every declared non-optional field. */ +export function assertExactObjectKeys( + value: unknown, + required: readonly string[], + label: string, + optional: readonly string[] = [], +): void { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + const allowed = new Set([...required, ...optional]) + if (allowed.size !== required.length + optional.length) { + throw new Error(`${label} exact-key contract contains duplicate fields`) + } + for (const key of Object.keys(value)) { + if (!allowed.has(key)) throw new Error(`${label} contains unknown field ${key}`) + } + for (const key of required) { + if (!(key in value)) throw new Error(`${label} is missing field ${key}`) + } +} diff --git a/src/candidate-execution/execute.ts b/src/candidate-execution/execute.ts new file mode 100644 index 00000000..524bc5d7 --- /dev/null +++ b/src/candidate-execution/execute.ts @@ -0,0 +1,912 @@ +import type { TraceStore } from '@tangle-network/agent-eval' +import type { AgentCandidateTermination } from '@tangle-network/agent-interface' + +import type { + AgentCandidateExecutionClaimStore, + AgentCandidateExecutionFailureClass, + AgentCandidateExecutionLease, + AgentCandidateExecutionTerminalResult, +} from './claim' +import { candidateExecutionClaim } from './claim-plan' +import { + candidateCleanupDeadline, + candidateCleanupTimeout, + candidateResultTimeout, + withinCandidateCleanupDeadline, + withinCandidateResultDeadline, +} from './cleanup' +import { canonicalCandidateBytes } from './digest' +import { candidatePostRunWindowMs, candidateTerminalWindowMs } from './execution-window' +import { + sealAgentCandidateExecutorFinalCapture, + sealAgentCandidateProtectedRunCapture, +} from './executor-capture' +import { failedAgentCandidateRun, finalizeAgentCandidateRun } from './finalize' +import { + type SealedAgentCandidateModelSettlement, + sealAgentCandidateModelSettlement, +} from './model-settlement' +import { + type PersistedAgentCandidateModelSettlement, + persistCandidateBenchmarkResult, + persistCandidateModelSettlement, + persistVerifiedCandidateTaskOutcome, +} from './outcome-evidence' +import { persistCandidateOutputArtifact } from './output-artifacts' +import { + assertPreparedCandidateIntegrity, + assertPreparedCandidateWorkspaces, + beginPreparedCandidateClaim, + beginPreparedCandidateRun, + beginPreparedCandidateSettlement, + consumePreparedCandidateExecution, + markPreparedCandidateClaimed, + type PreparedCandidateState, +} from './prepared-state' +import { redactProtectedReason } from './protected-redaction' +import { ProtectedAgentCandidateTraceStore } from './protected-trace-store' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutorFinalCapture, + AgentCandidateExecutorPort, + AgentCandidateExecutorRequest, + AgentCandidateOutputArtifactPort, + AgentCandidateProtectedModelActivation, + AgentCandidateProtectedRunCapture, + AgentCandidateRunFinalization, + PreparedAgentCandidateExecution, +} from './types' + +export interface ExecutePreparedAgentCandidateOptions { + executor: AgentCandidateExecutorPort + grader: AgentCandidateBenchmarkGraderPort + outputArtifacts: AgentCandidateOutputArtifactPort + traceStore: TraceStore + /** Long-lived evaluator-owned store shared by every process that can run this benchmark. */ + claimStore: AgentCandidateExecutionClaimStore + /** Maximum time to prove process death and revoke protected access after a run ends. */ + cleanupTimeoutMs?: number + /** Maximum time for task verification, executable grading, and receipt construction. */ + resultTimeoutMs?: number +} + +/** Executes and finalizes one durably claimed candidate without exposing an unproven result. */ +export async function executePreparedAgentCandidate( + prepared: PreparedAgentCandidateExecution, + options: ExecutePreparedAgentCandidateOptions, +): Promise { + const initialState = assertPreparedCandidateIntegrity(prepared) + const cleanupTimeoutMs = candidateCleanupTimeout( + options.cleanupTimeoutMs ?? initialState.cleanupTimeoutMs, + ) + if (cleanupTimeoutMs > initialState.cleanupTimeoutMs) { + throw new Error('execution cleanup timeout exceeds the frozen preparation bound') + } + const resultTimeoutMs = candidateResultTimeout( + options.resultTimeoutMs ?? initialState.resultTimeoutMs, + initialState.resultTimeoutMs, + ) + if (resultTimeoutMs > initialState.resultTimeoutMs) { + throw new Error('execution result timeout exceeds the frozen preparation bound') + } + let state: PreparedCandidateState + try { + state = beginPreparedCandidateClaim(prepared) + } catch (error) { + return failedAgentCandidateRun(initialState, errorMessage(error)) + } + + try { + // Mutable staging is only a preparation check. The executor receives the + // detached file bytes sealed in private state, never these host paths. + await assertPreparedCandidateWorkspaces(state) + } catch (error) { + return await failBeforeActivation(prepared, state, error, 'failed', cleanupTimeoutMs) + } + + let acquired: Awaited> + try { + acquired = await options.claimStore.tryClaim(candidateExecutionClaim(prepared)) + } catch (error) { + return await failBeforeActivation(prepared, state, error, 'failed', cleanupTimeoutMs) + } + if (!acquired.acquired) { + return await failBeforeActivation( + prepared, + state, + new Error( + acquired.reason === 'retry-not-eligible' + ? `candidate execution retry is not eligible: ${acquired.detail}` + : 'candidate execution attempt is already claimed', + ), + 'replayed', + cleanupTimeoutMs, + ) + } + + markPreparedCandidateClaimed(prepared) + const postRunWindowMs = candidatePostRunWindowMs(cleanupTimeoutMs, resultTimeoutMs) + // A caller may shorten cleanup for this invocation, but that must never buy + // the candidate more execution time. The claim was frozen with the prepared + // cleanup window, so recover the original task deadline from that value. + const deadlineAtMs = + acquired.claim.leaseExpiresAtMs - + candidatePostRunWindowMs(state.cleanupTimeoutMs, state.resultTimeoutMs) + const requiredLeaseExpiry = deadlineAtMs + postRunWindowMs + if ( + Date.now() >= deadlineAtMs || + deadlineAtMs > state.reservationExpiresAtMs || + requiredLeaseExpiry > acquired.lease.expiresAtMs + ) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('candidate claim no longer covers its full execution and cleanup window'), + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + ) + } + let activation: AgentCandidateProtectedModelActivation + try { + const activated = await withinCandidateCleanupDeadline( + () => + state.ports.models.activateGrant({ + executionId: state.executionId, + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + resolved: state.resolvedModel, + deadlineAtMs, + }), + Math.min(deadlineAtMs, candidateCleanupDeadline(cleanupTimeoutMs)), + 'protected model activation', + ) + activation = Object.freeze({ env: Object.freeze({ ...activated.env }) }) + } catch (error) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('protected model activation failed', { cause: error }), + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + ) + } + + let memoryActivation: { env: Readonly> } | undefined + if (state.memory.mode === 'isolated') { + try { + const reservation = state.memoryReservation + if (!reservation) throw new Error('isolated memory reservation is missing') + const activated = await withinCandidateCleanupDeadline( + () => + state.ports.memory.activate({ + executionId: state.executionId, + preparationId: reservation.preparationId, + accessDigest: reservation.accessDigest, + effectiveNamespace: reservation.effectiveNamespace, + deadlineAtMs, + }), + Math.min(deadlineAtMs, candidateCleanupDeadline(cleanupTimeoutMs)), + 'isolated memory activation', + ) + memoryActivation = Object.freeze({ env: Object.freeze({ ...activated.env }) }) + } catch (error) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('isolated memory activation failed', { cause: error }), + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + activation, + ) + } + } + + let request: AgentCandidateExecutorRequest + try { + request = beginPreparedCandidateRun(prepared, activation, memoryActivation).request + } catch (error) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + error, + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + activation, + memoryActivation, + ) + } + + try { + const phase = await options.claimStore.markCandidateMayRun(acquired.lease) + if (phase.phase !== 'candidate-may-run') { + throw new Error('candidate claim did not persist the candidate-may-run phase') + } + } catch (error) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('candidate execution phase persistence failed', { cause: error }), + 'failed', + cleanupTimeoutMs, + 'pre-model-infrastructure', + activation, + memoryActivation, + ) + } + if (Date.now() >= deadlineAtMs) { + return await failClaimedExecution( + prepared, + state, + acquired.lease, + options.claimStore, + options.outputArtifacts, + new Error('candidate execution deadline elapsed while persisting its launch phase'), + 'failed', + cleanupTimeoutMs, + 'unknown', + activation, + memoryActivation, + ) + } + + const protectedValues = protectedEnvironmentValues(activation, memoryActivation) + const protectedTraceStore = new ProtectedAgentCandidateTraceStore( + options.traceStore, + protectedValues, + ) + const execution = await runAndStopExecutor( + options.executor, + request, + protectedTraceStore, + deadlineAtMs, + cleanupTimeoutMs, + ) + beginPreparedCandidateSettlement(prepared) + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + + const accessReason = + execution.kind === 'timeout' || execution.termination?.kind === 'timeout' + ? 'timeout' + : execution.kind === 'capture' + ? 'completed' + : 'failed' + const [memoryClose, settlementResult] = await Promise.all([ + closeMemoryAccess(state, accessReason, cleanupDeadlineAtMs), + settleModelGrant(state, accessReason, cleanupDeadlineAtMs), + ]) + if (!execution.processStopped || !settlementResult.settlement || !memoryClose.closed) { + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + execution.error, + !execution.processStopped + ? new Error('candidate process termination is not proven') + : undefined, + memoryClose.error, + settlementResult.error ?? + (!settlementResult.settlement ? new Error('model settlement failed') : undefined), + new Error('candidate claim remains recoverable until protected cleanup is proven'), + ), + protectedValues, + ), + execution.termination, + settlementResult.settlement?.usage ?? null, + ) + } + + let modelSettlement: PersistedAgentCandidateModelSettlement + try { + modelSettlement = await withinCandidateCleanupDeadline( + () => + persistCandidateModelSettlement( + state, + settlementResult.settlement as SealedAgentCandidateModelSettlement, + options.outputArtifacts, + ), + candidateCleanupDeadline(cleanupTimeoutMs), + 'candidate model settlement persistence', + ) + } catch (error) { + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + error, + new Error('candidate claim remains recoverable until settlement evidence is durable'), + ), + protectedValues, + ), + execution.termination, + settlementResult.settlement.usage, + ) + } + + let result: AgentCandidateRunFinalization + const failureClass: AgentCandidateExecutionFailureClass = + execution.kind === 'error' ? 'execution' : 'post-model-infrastructure' + if (execution.kind === 'error') { + result = failedAgentCandidateRun( + state, + redactProtectedReason(errorMessage(execution.error), protectedValues), + execution.termination, + settlementResult.settlement.usage, + ) + } else if (!execution.finalCapture.taskOutcome) { + result = failedAgentCandidateRun( + state, + 'candidate executor stopped without a captured task outcome', + execution.termination, + settlementResult.settlement.usage, + ) + } else { + const capture = + execution.kind === 'capture' + ? execution.capture + : { + executionId: state.executionId, + termination: execution.termination, + } + try { + const resultDeadlineAtMs = Math.min( + Date.now() + resultTimeoutMs, + acquired.lease.expiresAtMs - candidateTerminalWindowMs(cleanupTimeoutMs), + ) + result = await withinCandidateResultDeadline( + async (signal) => { + const taskOutcome = await persistVerifiedCandidateTaskOutcome( + state, + execution.finalCapture.taskOutcome!, + options.outputArtifacts, + protectedValues, + signal, + ) + const benchmarkResult = await persistCandidateBenchmarkResult( + state, + capture.termination, + taskOutcome, + options.grader, + options.outputArtifacts, + protectedValues, + signal, + ) + return await finalizeAgentCandidateRun( + state, + capture, + options.traceStore, + settlementResult.settlement as SealedAgentCandidateModelSettlement, + { + finalCapture: execution.finalCapture, + modelSettlement, + taskOutcome, + benchmarkResult, + outputArtifacts: options.outputArtifacts, + }, + protectedValues, + protectedTraceStore.report(), + signal, + ) + }, + resultDeadlineAtMs, + 'candidate evidence finalization', + ) + } catch (error) { + result = failedAgentCandidateRun( + state, + redactProtectedReason(errorMessage(error), protectedValues), + execution.termination, + settlementResult.settlement.usage, + ) + } + } + + let terminal: AgentCandidateExecutionTerminalResult + try { + terminal = result.succeeded + ? { + schemaVersion: 1, + status: 'succeeded', + usage: settlementResult.settlement.fixedUsage, + modelSettlement: result.artifacts.modelSettlement, + taskOutcome: result.artifacts.taskOutcome, + benchmarkResult: result.artifacts.benchmarkResult, + runReceipt: result.artifacts.runReceipt, + } + : { + schemaVersion: 1, + status: 'failed', + failureClass, + usage: settlementResult.settlement.fixedUsage, + modelSettlement: modelSettlement.artifact, + failureEvidence: await withinCandidateCleanupDeadline( + () => + persistFailureEvidence( + state, + result.reason, + failureClass, + execution.termination, + options.outputArtifacts, + ), + acquired.lease.expiresAtMs, + 'candidate failure-evidence persistence', + ), + } + } catch (error) { + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + error, + new Error('candidate claim remains recoverable until terminal evidence is durable'), + ), + protectedValues, + ), + execution.termination, + settlementResult.settlement.usage, + ) + } + + const finished = await finishClaim( + options.claimStore, + acquired.lease, + terminal, + acquired.lease.expiresAtMs, + ) + if (finished) { + consumePreparedCandidateExecution(prepared, result.succeeded ? 'succeeded' : 'failed') + return result + } + + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + 'candidate execution terminal record could not be persisted', + execution.termination, + settlementResult.settlement.usage, + ) +} + +type ExecutorOutcome = + | { + kind: 'capture' + capture: AgentCandidateProtectedRunCapture + termination: AgentCandidateTermination + finalCapture: AgentCandidateExecutorFinalCapture + processStopped: true + error?: undefined + } + | { + kind: 'timeout' + termination: AgentCandidateTermination & { kind: 'timeout' } + finalCapture: AgentCandidateExecutorFinalCapture + processStopped: true + error?: undefined + } + | { + kind: 'error' + error: unknown + termination?: AgentCandidateTermination + finalCapture?: AgentCandidateExecutorFinalCapture + processStopped: boolean + } + +async function runAndStopExecutor( + executor: AgentCandidateExecutorPort, + request: AgentCandidateExecutorRequest, + traceStore: TraceStore, + deadlineAtMs: number, + cleanupTimeoutMs: number, +): Promise { + const timeoutMs = request.hardLimits.timeoutMs + const timeoutError = new CandidateExecutionDeadlineError(timeoutMs) + if (Date.now() >= deadlineAtMs) { + return { + kind: 'error', + error: timeoutError, + termination: { kind: 'timeout', timeoutMs }, + processStopped: true, + } + } + const controller = new AbortController() + let timer: ReturnType | undefined + let timedOut = false + let capture: AgentCandidateProtectedRunCapture | undefined + let executionError: unknown + const executionPromise = Promise.resolve().then(() => + executor.execute(request, { + traceStore, + signal: controller.signal, + deadlineAtMs, + }), + ) + // A stopped process may still leave a buggy adapter promise pending. Attach a + // terminal handler now so the deadline path cannot create an unhandled rejection. + void executionPromise.catch(() => undefined) + const deadlinePromise = new Promise((_resolve, reject) => { + timer = setTimeout( + () => { + timedOut = true + controller.abort(timeoutError) + reject(timeoutError) + }, + Math.max(0, deadlineAtMs - Date.now()), + ) + }) + void deadlinePromise.catch(() => undefined) + try { + capture = sealAgentCandidateProtectedRunCapture( + await Promise.race([executionPromise, deadlinePromise]), + ) + } catch (error) { + executionError = error + } + + if (!timedOut && capture && capture.executionId !== request.executionId) { + executionError = new Error('candidate execution capture id does not match the request') + capture = undefined + } else if (!timedOut && capture && Date.now() >= deadlineAtMs) { + // Promise resolution at the boundary is ambiguous under event-loop delay. + // Fail closed unless completion is observed strictly before the deadline. + timedOut = true + executionError = timeoutError + capture = undefined + } else if (!timedOut && capture?.termination.kind === 'timeout') { + executionError = new Error('candidate executor cannot declare the runtime-owned timeout') + capture = undefined + } + + if (!capture || timedOut) controller.abort(executionError) + const stopReason = timedOut ? 'timeout' : capture ? 'completed' : 'failed' + let stopped: unknown + try { + stopped = await withinCandidateCleanupDeadline( + () => + executor.stopAndCapture( + { + executionId: request.executionId, + executionPlanDigest: request.executionPlan.value.digest, + }, + { + traceStore, + reason: stopReason, + signal: controller.signal, + deadlineAtMs, + }, + ), + Date.now() + cleanupTimeoutMs, + 'candidate process termination', + ) + if ( + !stopped || + typeof stopped !== 'object' || + (stopped as { stopped?: unknown }).stopped !== true + ) { + throw new Error('candidate executor did not acknowledge exact process termination') + } + } catch (stopError) { + if (timer) clearTimeout(timer) + controller.abort(stopError) + return { + kind: 'error', + error: new Error(joinErrors(executionError, stopError)), + ...(timedOut ? { termination: { kind: 'timeout', timeoutMs } } : {}), + processStopped: false, + } + } + + let finalCapture: AgentCandidateExecutorFinalCapture + try { + finalCapture = sealAgentCandidateExecutorFinalCapture(stopped) + } catch (captureError) { + if (timer) clearTimeout(timer) + controller.abort(captureError) + return { + kind: 'error', + error: new Error(joinErrors(executionError, captureError)), + ...(timedOut ? { termination: { kind: 'timeout', timeoutMs } } : {}), + processStopped: true, + } + } + + if (Date.now() >= deadlineAtMs) { + timedOut = true + executionError = timeoutError + capture = undefined + controller.abort(timeoutError) + } + if (timer) clearTimeout(timer) + + if (timedOut) { + return { + kind: 'timeout', + termination: { kind: 'timeout', timeoutMs }, + finalCapture, + processStopped: true, + } + } + if (executionError || !capture) { + return { + kind: 'error', + error: executionError ?? new Error('candidate executor returned no capture'), + finalCapture, + processStopped: true, + } + } + return { + kind: 'capture', + capture, + termination: capture.termination, + finalCapture, + processStopped: true, + } +} + +async function failBeforeActivation( + prepared: PreparedAgentCandidateExecution, + state: PreparedCandidateState, + error: unknown, + reason: 'failed' | 'replayed', + cleanupTimeoutMs: number, +): Promise { + beginPreparedCandidateSettlement(prepared) + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const [memoryClose, settled] = await Promise.all([ + closeMemoryAccess(state, reason, cleanupDeadlineAtMs), + settleModelGrant(state, reason, cleanupDeadlineAtMs), + ]) + const cleanupProven = memoryClose.closed && settled.settlement !== undefined + consumePreparedCandidateExecution(prepared, cleanupProven ? 'failed' : 'cleanup-failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + error, + memoryClose.error, + settled.error, + !cleanupProven + ? new Error('prepared access cleanup remains incomplete and may be retried by disposal') + : undefined, + ), + [], + ), + undefined, + settled.settlement?.usage ?? null, + ) +} + +async function failClaimedExecution( + prepared: PreparedAgentCandidateExecution, + state: PreparedCandidateState, + lease: AgentCandidateExecutionLease, + claimStore: AgentCandidateExecutionClaimStore, + outputArtifacts: AgentCandidateOutputArtifactPort, + error: unknown, + reason: 'failed', + cleanupTimeoutMs: number, + failureClass: AgentCandidateExecutionFailureClass, + activation?: AgentCandidateProtectedModelActivation, + memoryActivation?: { env: Readonly> }, +): Promise { + beginPreparedCandidateSettlement(prepared) + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const [memoryClose, settled] = await Promise.all([ + closeMemoryAccess(state, reason, cleanupDeadlineAtMs), + settleModelGrant(state, reason, cleanupDeadlineAtMs), + ]) + const protectedValues = protectedEnvironmentValues(activation, memoryActivation) + const safeReason = redactProtectedReason( + joinErrors(error, memoryClose.error, settled.error), + protectedValues, + ) + let finishFailed = false + let persistenceFailure: unknown + if (settled.settlement && memoryClose.closed) { + try { + const modelSettlement = await withinCandidateCleanupDeadline( + () => persistCandidateModelSettlement(state, settled.settlement!, outputArtifacts), + lease.expiresAtMs, + 'candidate pre-run model-settlement persistence', + ) + const failureEvidence = await withinCandidateCleanupDeadline( + () => persistFailureEvidence(state, safeReason, failureClass, undefined, outputArtifacts), + lease.expiresAtMs, + 'candidate pre-run failure-evidence persistence', + ) + finishFailed = !(await finishClaim( + claimStore, + lease, + { + schemaVersion: 1, + status: 'failed', + failureClass, + usage: settled.settlement.fixedUsage, + modelSettlement: modelSettlement.artifact, + failureEvidence, + }, + lease.expiresAtMs, + )) + } catch (persistenceError) { + finishFailed = true + persistenceFailure = persistenceError + } + } + consumePreparedCandidateExecution(prepared, 'failed') + return failedAgentCandidateRun( + state, + redactProtectedReason( + joinErrors( + safeReason, + persistenceFailure, + memoryClose.error, + settled.error, + !memoryClose.closed || !settled.settlement + ? new Error('candidate claim remains recoverable until protected cleanup is proven') + : undefined, + finishFailed + ? new Error('candidate execution terminal record could not be persisted') + : undefined, + ), + protectedValues, + ), + undefined, + settled.settlement?.usage ?? null, + ) +} + +async function closeMemoryAccess( + state: PreparedCandidateState, + reason: 'completed' | 'failed' | 'timeout' | 'replayed', + cleanupDeadlineAtMs: number, +): Promise<{ closed: true; error?: undefined } | { closed: false; error: unknown }> { + if (state.memory.mode === 'disabled') return { closed: true } + try { + const reservation = state.memoryReservation + if (!reservation) throw new Error('isolated memory reservation is missing') + const closed = await withinCandidateCleanupDeadline( + () => + state.ports.memory.close({ + executionId: state.executionId, + preparationId: reservation.preparationId, + accessDigest: reservation.accessDigest, + effectiveNamespace: reservation.effectiveNamespace, + reason, + }), + cleanupDeadlineAtMs, + 'isolated memory closure', + ) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('isolated memory access did not acknowledge closure') + } + return { closed: true } + } catch (error) { + return { closed: false, error } + } +} + +async function settleModelGrant( + state: PreparedCandidateState, + reason: 'completed' | 'failed' | 'timeout' | 'replayed', + cleanupDeadlineAtMs: number, +): Promise<{ + settlement?: SealedAgentCandidateModelSettlement + error?: unknown +}> { + try { + const value = await withinCandidateCleanupDeadline( + () => + state.ports.models.settleGrant({ + executionId: state.executionId, + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + resolved: state.resolvedModel, + reason, + }), + cleanupDeadlineAtMs, + 'protected model settlement', + ) + return { + settlement: sealAgentCandidateModelSettlement(value, { + preparationId: state.preparationId, + grantDigest: state.modelReservation.digest, + model: state.resolvedModel.model, + }), + } + } catch (error) { + return { error: new Error('protected model settlement failed', { cause: error }) } + } +} + +async function finishClaim( + store: AgentCandidateExecutionClaimStore, + lease: AgentCandidateExecutionLease, + terminal: AgentCandidateExecutionTerminalResult, + deadlineAtMs?: number, +): Promise { + try { + const staged = await withinCandidateCleanupDeadline( + () => store.stageTerminal(lease, terminal), + deadlineAtMs ?? lease.expiresAtMs, + 'candidate terminal staging', + ) + if (!staged.staged && !staged.exactReplay) return false + const result = await withinCandidateCleanupDeadline( + () => store.finish(lease, staged.terminal.terminalDigest), + deadlineAtMs ?? lease.expiresAtMs, + 'candidate terminal publication', + ) + return result.finished || result.exactReplay + } catch { + return false + } +} + +async function persistFailureEvidence( + state: PreparedCandidateState, + reason: string, + failureClass: AgentCandidateExecutionFailureClass, + termination: AgentCandidateTermination | undefined, + outputArtifacts: AgentCandidateOutputArtifactPort, +) { + const bytes = canonicalCandidateBytes({ + schemaVersion: 1, + kind: 'agent-candidate-execution-failure', + executionId: state.executionId, + bundleDigest: state.bundle.digest, + executionPlanDigest: state.executionPlan.value.digest, + failureClass, + reason, + ...(termination ? { termination } : {}), + }) + return await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'failure-evidence', + bytes, + }) +} + +function protectedEnvironmentValues( + activation?: AgentCandidateProtectedModelActivation, + memoryActivation?: { env: Readonly> }, +): string[] { + return [...Object.values(activation?.env ?? {}), ...Object.values(memoryActivation?.env ?? {})] +} + +function joinErrors(...errors: unknown[]): string { + return errors + .filter((error) => error !== undefined) + .map(errorMessage) + .join('; ') +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +class CandidateExecutionDeadlineError extends Error { + constructor(timeoutMs: number) { + super(`candidate execution reached its frozen ${timeoutMs}ms deadline`) + this.name = 'CandidateExecutionDeadlineError' + } +} diff --git a/src/candidate-execution/execution-window.ts b/src/candidate-execution/execution-window.ts new file mode 100644 index 00000000..960407d8 --- /dev/null +++ b/src/candidate-execution/execution-window.ts @@ -0,0 +1,41 @@ +import { MAX_CANDIDATE_TIMER_INTERVAL_MS } from './cleanup' + +export const CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS = 10_000 +const CANDIDATE_POST_RUN_CLEANUP_PHASES = 4 + +/** Process stop, access closure, model evidence, and task/result evidence. */ +export function candidatePostRunWindowMs( + cleanupTimeoutMs: number, + resultTimeoutMs: number, +): number { + const windowMs = + cleanupTimeoutMs * CANDIDATE_POST_RUN_CLEANUP_PHASES + + resultTimeoutMs + + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS + if (!Number.isSafeInteger(windowMs) || windowMs > MAX_CANDIDATE_TIMER_INTERVAL_MS) { + throw new Error('candidate post-run window exceeds the supported timer range') + } + return windowMs +} + +/** Maximum owner lifetime from claim through process stop, access closure, and terminal write. */ +export function candidateExecutionOwnerWindowMs( + timeoutMs: number, + cleanupTimeoutMs: number, + resultTimeoutMs: number, +): number { + const windowMs = timeoutMs + candidatePostRunWindowMs(cleanupTimeoutMs, resultTimeoutMs) + if (!Number.isSafeInteger(windowMs) || windowMs > MAX_CANDIDATE_TIMER_INTERVAL_MS) { + throw new Error('candidate execution and cleanup window exceeds the supported timer range') + } + return windowMs +} + +/** Time reserved after scoring for failure evidence plus terminal publication. */ +export function candidateTerminalWindowMs(cleanupTimeoutMs: number): number { + const windowMs = cleanupTimeoutMs + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS + if (!Number.isSafeInteger(windowMs) || windowMs > MAX_CANDIDATE_TIMER_INTERVAL_MS) { + throw new Error('candidate terminal window exceeds the supported timer range') + } + return windowMs +} diff --git a/src/candidate-execution/executor-capture.ts b/src/candidate-execution/executor-capture.ts new file mode 100644 index 00000000..d8a68cee --- /dev/null +++ b/src/candidate-execution/executor-capture.ts @@ -0,0 +1,90 @@ +import { + agentCandidateTerminationSchema, + agentCandidateWorkspaceManifestMaterialSchema, +} from '@tangle-network/agent-interface' +import { assertExactObjectKeys as assertExactKeys } from './exact-object' +import type { AgentCandidateExecutorFinalCapture, AgentCandidateProtectedRunCapture } from './types' + +/** Validate and detach the only candidate-authored fields accepted from execution. */ +export function sealAgentCandidateProtectedRunCapture( + value: unknown, +): AgentCandidateProtectedRunCapture { + const capture = requireRecord(value, 'candidate execution capture') + assertExactKeys(capture, ['executionId', 'termination'], 'candidate execution capture') + if (typeof capture.executionId !== 'string' || capture.executionId.length === 0) { + throw new Error('candidate execution capture has an invalid executionId') + } + return Object.freeze({ + executionId: capture.executionId, + termination: Object.freeze(agentCandidateTerminationSchema.parse(capture.termination)), + }) +} + +/** Validate, detach, and freeze evaluator-owned evidence captured after process death. */ +export function sealAgentCandidateExecutorFinalCapture( + value: unknown, +): AgentCandidateExecutorFinalCapture { + const capture = requireRecord(value, 'candidate final capture') + assertExactKeys(capture, ['stopped'], 'candidate final capture', ['taskOutcome', 'memoryAfter']) + if (capture.stopped !== true) { + throw new Error('candidate final capture does not prove process death') + } + + const taskOutcome = capture.taskOutcome ? sealTaskOutcomeCapture(capture.taskOutcome) : undefined + const memoryAfter = capture.memoryAfter ? sealMemoryCapture(capture.memoryAfter) : undefined + return Object.freeze({ + stopped: true, + ...(taskOutcome ? { taskOutcome } : {}), + ...(memoryAfter ? { memoryAfter: Object.freeze(memoryAfter) } : {}), + }) +} + +function sealMemoryCapture( + value: unknown, +): NonNullable { + const capture = requireRecord(value, 'candidate memory capture') + assertExactKeys(capture, ['afterState', 'archive'], 'candidate memory capture') + if (!(capture.archive instanceof Uint8Array)) { + throw new Error('candidate memory capture archive must be a byte array') + } + return Object.freeze({ + afterState: Object.freeze( + agentCandidateWorkspaceManifestMaterialSchema.parse(capture.afterState), + ), + archive: Uint8Array.from(capture.archive), + }) +} + +function sealTaskOutcomeCapture( + value: unknown, +): NonNullable { + const capture = requireRecord(value, 'candidate task capture') + assertExactKeys( + capture, + ['resultTree', 'afterState', 'archive', 'gitDiff'], + 'candidate task capture', + ) + if ( + typeof capture.resultTree !== 'string' || + !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(capture.resultTree) + ) { + throw new Error('candidate task capture resultTree is not a Git object id') + } + if (!(capture.archive instanceof Uint8Array) || !(capture.gitDiff instanceof Uint8Array)) { + throw new Error('candidate task capture archive and gitDiff must be byte arrays') + } + const afterState = agentCandidateWorkspaceManifestMaterialSchema.parse(capture.afterState) + return Object.freeze({ + resultTree: capture.resultTree, + afterState: Object.freeze(afterState), + archive: Uint8Array.from(capture.archive), + gitDiff: Uint8Array.from(capture.gitDiff), + }) +} + +function requireRecord(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + return value as Record +} diff --git a/src/candidate-execution/finalize.ts b/src/candidate-execution/finalize.ts new file mode 100644 index 00000000..85eee32c --- /dev/null +++ b/src/candidate-execution/finalize.ts @@ -0,0 +1,363 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { Span, TraceStore } from '@tangle-network/agent-eval' +import { isLlmSpan, REDACTION_VERSION } from '@tangle-network/agent-eval' +import type { + AgentCandidateBenchmarkResultEvidence, + AgentCandidateMemoryReceipt, + AgentCandidateModelSettlementEvidence, + AgentCandidateRunReceiptV2, + AgentCandidateSpend, + AgentCandidateTermination, +} from '@tangle-network/agent-interface' +import { + agentCandidateRunReceiptV2Schema, + agentCandidateWorkspaceSnapshotEvidenceSchema, +} from '@tangle-network/agent-interface' + +import { readMaterializedWorkspaceFiles } from './artifacts' +import { + canonicalCandidateBytes, + canonicalCandidateDocument, + embeddedCandidateArtifact, + sha256Bytes, +} from './digest' +import { + sealAgentCandidateExecutorFinalCapture, + sealAgentCandidateProtectedRunCapture, +} from './executor-capture' +import { + assertTraceMatchesModelSettlement, + type SealedAgentCandidateModelSettlement, + usdToNanos, +} from './model-settlement' +import { persistCandidateOutputArtifact } from './output-artifacts' +import type { PreparedCandidateState } from './prepared-state' +import { + assertNoProtectedBytes, + type ProtectedRedactionReport, + redactProtectedReason, + redactProtectedValue, +} from './protected-redaction' +import { + type AgentCandidateExecutorFinalCapture, + type AgentCandidateOutputArtifactPort, + type AgentCandidateProtectedRunCapture, + type AgentCandidateRunFinalization, + CANDIDATE_TRACE_TAGS, + type VerifiedAgentCandidateTaskOutcome, +} from './types' + +interface CandidateFinalizationEvidence { + finalCapture: AgentCandidateExecutorFinalCapture + modelSettlement: AgentCandidateModelSettlementEvidence & { + artifact: import('@tangle-network/agent-interface').AgentCandidateArtifactRef + } + taskOutcome: VerifiedAgentCandidateTaskOutcome + benchmarkResult: AgentCandidateBenchmarkResultEvidence & { + artifact: import('@tangle-network/agent-interface').AgentCandidateArtifactRef + } + outputArtifacts: AgentCandidateOutputArtifactPort +} + +/** Builds a candidate run receipt exclusively from protected trace and memory evidence. */ +export async function finalizeAgentCandidateRun( + state: PreparedCandidateState, + capture: AgentCandidateProtectedRunCapture, + traceStore: TraceStore, + settlement: SealedAgentCandidateModelSettlement, + evidence: CandidateFinalizationEvidence, + protectedValues: readonly string[], + writeRedactionReport?: ProtectedRedactionReport, + signal?: AbortSignal, +): Promise { + let termination: AgentCandidateTermination | undefined + try { + signal?.throwIfAborted() + const protectedCapture = sealAgentCandidateProtectedRunCapture(capture) + if (protectedCapture.executionId !== state.executionId) { + throw new Error('protected capture execution id does not match the prepared execution') + } + termination = protectedCapture.termination + if ( + termination.kind === 'timeout' && + termination.timeoutMs !== state.executionPlan.value.material.limits.timeoutMs + ) { + throw new Error('timeout termination does not match the frozen execution limit') + } + + const run = await traceStore.getRun(state.trace.runId) + if (!run) throw new Error(`protected trace run is missing: ${state.trace.runId}`) + if (run.status === 'running' || run.endedAt === undefined) { + throw new Error('protected trace run is not terminal') + } + assertTraceBindings(run.tags, state) + + const [spans, events, budget, artifacts] = await Promise.all([ + traceStore.spans({ runId: run.runId }), + traceStore.events({ runId: run.runId }), + traceStore.budget(run.runId), + traceStore.artifacts(run.runId), + ]) + const orderedSpans = [...spans].sort( + (a, b) => a.startedAt - b.startedAt || compareStrings(a.spanId, b.spanId), + ) + const orderedEvents = [...events].sort( + (a, b) => a.timestamp - b.timestamp || compareStrings(a.eventId, b.eventId), + ) + const orderedBudget = [...budget].sort( + (a, b) => a.timestamp - b.timestamp || compareStrings(a.dimension, b.dimension), + ) + const orderedArtifacts = [...artifacts].sort((a, b) => + compareStrings(a.artifactId, b.artifactId), + ) + + const modelSpans = orderedSpans.filter(isLlmSpan) + assertTraceMatchesModelSettlement(modelSpans, settlement) + const usage = settlement.usage + enforceLimits(state, run.startedAt, run.endedAt, orderedSpans, settlement) + const finalCapture = sealAgentCandidateExecutorFinalCapture(evidence.finalCapture) + const memory = await memoryReceipt( + state, + finalCapture, + evidence.outputArtifacts, + protectedValues, + signal, + ) + + const redacted = redactProtectedValue( + { + schemaVersion: 1, + run: { ...run, redactionVersion: REDACTION_VERSION }, + spans: orderedSpans, + events: orderedEvents, + budget: orderedBudget, + artifacts: orderedArtifacts, + }, + protectedValues, + ) + const combinedByRule = { ...(writeRedactionReport?.byRule ?? {}) } + for (const [rule, count] of Object.entries(redacted.report.byRule)) { + combinedByRule[rule] = (combinedByRule[rule] ?? 0) + count + } + const traceMaterial = { + ...(redacted.value as Record), + evaluatorLimits: { resultTimeoutMs: state.resultTimeoutMs }, + redaction: { + version: REDACTION_VERSION, + redactionCount: + (writeRedactionReport?.redactionCount ?? 0) + redacted.report.redactionCount, + byRule: combinedByRule, + }, + } + const traceBytes = canonicalCandidateBytes(traceMaterial) + assertNoProtectedBytes(traceBytes, protectedValues) + const traceArtifact = await persistCandidateOutputArtifact(evidence.outputArtifacts, { + executionId: state.executionId, + purpose: 'trace', + bytes: traceBytes, + signal, + }) + const trace = { + schemaVersion: 1 as const, + artifact: traceArtifact, + eventCount: + 1 + + orderedSpans.length + + orderedEvents.length + + orderedBudget.length + + orderedArtifacts.length, + modelCallCount: modelSpans.length, + } + const document = canonicalCandidateDocument({ + schemaVersion: 2, + kind: 'agent-candidate-run', + digestAlgorithm: 'rfc8785-sha256', + bundleDigest: state.bundle.digest, + materializationReceiptDigest: state.materializationReceipt.digest, + executionPlanDigest: state.executionPlan.value.digest, + memory, + usage, + modelUsage: { resolved: state.resolvedModel, usage }, + trace, + termination, + fixedUsage: settlement.fixedUsage, + modelSettlement: evidence.modelSettlement, + taskOutcome: evidence.taskOutcome.evidence, + benchmarkResult: evidence.benchmarkResult, + }) + agentCandidateRunReceiptV2Schema.parse(document.value) + const runReceipt = await persistCandidateOutputArtifact(evidence.outputArtifacts, { + executionId: state.executionId, + purpose: 'run-receipt', + bytes: document.bytes, + signal, + }) + return { + succeeded: true, + receipt: document, + artifacts: { + modelSettlement: evidence.modelSettlement.artifact, + taskOutcome: evidence.taskOutcome.evidence.artifact, + benchmarkResult: evidence.benchmarkResult.artifact, + runReceipt, + }, + } + } catch (error) { + return { + ...failedAgentCandidateRun( + state, + redactProtectedReason( + error instanceof Error ? error.message : String(error), + protectedValues, + ), + termination, + settlement.usage, + ), + } + } +} + +function assertTraceBindings( + tags: Record | undefined, + state: PreparedCandidateState, +): void { + const expected = state.trace.tags + for (const name of Object.values(CANDIDATE_TRACE_TAGS)) { + if (tags?.[name] !== expected[name]) { + throw new Error(`protected trace is not bound to prepared execution tag ${name}`) + } + } +} + +function enforceLimits( + state: PreparedCandidateState, + startedAt: number, + endedAt: number, + spans: Span[], + settlement: SealedAgentCandidateModelSettlement, +): void { + const limits = state.executionPlan.value.material.limits + const usage = settlement.usage + // The runtime-owned Date.now deadline decides when the process must stop. + // This separately rejects a receipt whose evaluator-owned trace claims a + // longer run; neither clock can make an over-limit execution admissible. + const wallMs = endedAt - startedAt + if (!Number.isFinite(wallMs) || wallMs < 0 || wallMs > limits.timeoutMs) { + throw new Error(`protected trace wall time ${wallMs} exceeds ${limits.timeoutMs}`) + } + const steps = spans.filter((span) => span.kind === 'tool').length + const checks: Array<[number, number, string]> = [ + [steps, limits.maxSteps, 'tool steps'], + [usage.modelCalls, limits.maxModelCalls, 'model calls'], + [usage.inputTokens, limits.maxInputTokens, 'input tokens'], + [usage.outputTokens, limits.maxOutputTokens, 'output tokens'], + ] + for (const [actual, limit, label] of checks) { + if (actual > limit) throw new Error(`protected ${label} ${actual} exceeds ${limit}`) + } + if (settlement.costUsdNanos > usdToNanos(limits.maxCostUsd, 'frozen maxCostUsd')) { + throw new Error(`protected cost USD ${usage.costUsd} exceeds ${limits.maxCostUsd}`) + } +} + +async function memoryReceipt( + state: PreparedCandidateState, + capture: AgentCandidateExecutorFinalCapture, + outputArtifacts: AgentCandidateOutputArtifactPort, + protectedValues: readonly string[], + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted() + if (state.memory.mode === 'disabled') { + if (capture.memoryAfter !== undefined) { + throw new Error('disabled memory cannot return an after-state') + } + return { mode: 'disabled' } + } + if (!capture.memoryAfter) throw new Error('isolated memory is missing its protected after-state') + const afterState = capture.memoryAfter.afterState + const archive = Uint8Array.from(capture.memoryAfter.archive) + if (archive.byteLength === 0) throw new Error('isolated memory archive cannot be empty') + const manifestBytes = canonicalCandidateBytes(afterState) + assertNoProtectedBytes(manifestBytes, protectedValues) + assertNoProtectedBytes(archive, protectedValues) + const provisionalSnapshot = agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), + material: afterState, + manifest: embeddedCandidateArtifact(manifestBytes), + archive: embeddedCandidateArtifact(archive), + }) + const root = await mkdtemp(join(tmpdir(), 'agent-candidate-memory-after-')) + try { + await state.ports.workspaces.materialize({ + role: 'memory', + snapshot: provisionalSnapshot, + archive: Uint8Array.from(archive), + destination: root, + }) + const files = await readMaterializedWorkspaceFiles(root, afterState) + for (const file of files) assertNoProtectedBytes(file.bytes, protectedValues) + signal?.throwIfAborted() + } finally { + await rm(root, { recursive: true, force: true }) + } + const [manifest, archiveRef] = await Promise.all([ + persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'memory-after-manifest', + bytes: manifestBytes, + signal, + }), + persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'memory-after-archive', + bytes: archive, + signal, + }), + ]) + const snapshot = agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), + material: afterState, + manifest, + archive: archiveRef, + }) + return { + mode: 'isolated', + scope: 'task', + effectiveNamespace: state.memory.effectiveNamespace, + resetEvidenceDigest: state.memory.reset.evidence.sha256, + beforeStateDigest: state.memory.beforeState.digest, + afterState: snapshot, + } +} + +export function failedAgentCandidateRun( + state: PreparedCandidateState, + reason: string, + termination?: AgentCandidateTermination, + usage: AgentCandidateSpend | null = null, +): AgentCandidateRunFinalization & { succeeded: false } { + return { + succeeded: false, + reason, + partial: { + executionId: state.executionId, + bundleDigest: state.bundle.digest, + executionPlanDigest: state.executionPlan.value.digest, + materializationReceiptDigest: state.materializationReceipt.digest, + ...(termination ? { termination } : {}), + }, + usage, + } +} + +function compareStrings(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0 +} diff --git a/src/candidate-execution/git-materialize.ts b/src/candidate-execution/git-materialize.ts new file mode 100644 index 00000000..a5505f17 --- /dev/null +++ b/src/candidate-execution/git-materialize.ts @@ -0,0 +1,429 @@ +import { spawn } from 'node:child_process' +import { mkdir, mkdtemp, readFile, realpath, rm, stat } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +import type { + AgentCandidateCode, + AgentCandidateGitHubRepository, + AgentCandidateGitHubResource, + AgentCandidateWorkspaceManifestMaterialV1, +} from '@tangle-network/agent-interface' + +import { verifyBytes } from './artifacts' +import { canonicalCandidateBytes, sha256Bytes } from './digest' +import type { AgentCandidateRepositoryPort } from './types' + +interface GitResult { + stdout: Buffer + stderr: Buffer +} + +export async function verifyCandidateCode( + code: AgentCandidateCode, + repositories: AgentCandidateRepositoryPort, + patchBytes?: Uint8Array, +): Promise { + if (code.kind === 'disabled') return undefined + const repositoryRoot = await verifiedRepositoryRoot(code.repository, repositories) + await assertCommitAndBaseTree(repositoryRoot, code.baseCommit, code.baseTree) + + if (code.kind === 'no-op') { + await assertSafeTree(repositoryRoot, code.baseTree) + return code.baseTree + } + if (!patchBytes) throw new Error('git-patch candidate is missing verified patch bytes') + + const temporary = await mkdtemp(join(tmpdir(), 'agent-candidate-git-')) + try { + const indexFile = join(temporary, 'index') + await git(repositoryRoot, ['read-tree', code.baseTree], undefined, { + GIT_INDEX_FILE: indexFile, + }) + await git( + repositoryRoot, + ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'], + patchBytes, + { GIT_INDEX_FILE: indexFile }, + ) + const candidateTree = ( + await git(repositoryRoot, ['write-tree'], undefined, { GIT_INDEX_FILE: indexFile }) + ).stdout + .toString('utf8') + .trim() + if (candidateTree !== code.candidateTree) { + throw new Error( + `git patch materialized tree ${candidateTree} does not match ${code.candidateTree}`, + ) + } + await assertSafeTree(repositoryRoot, candidateTree) + return candidateTree + } finally { + await rm(temporary, { recursive: true, force: true }) + } +} + +export async function readCandidateGitHubResource( + resource: AgentCandidateGitHubResource, + repositories: AgentCandidateRepositoryPort, +): Promise { + const repositoryRoot = await verifiedRepositoryRoot(resource.repository, repositories) + const commitType = (await git(repositoryRoot, ['cat-file', '-t', resource.commit])).stdout + .toString('utf8') + .trim() + if (commitType !== 'commit') { + throw new Error(`GitHub resource commit is not a commit object: ${resource.commit}`) + } + const listing = ( + await git(repositoryRoot, ['ls-tree', '-z', resource.commit, '--', resource.path]) + ).stdout + const entries = parseTreeEntries(listing) + if (entries.length !== 1 || entries[0]?.path !== resource.path) { + throw new Error(`GitHub resource path is not one exact file: ${resource.path}`) + } + const entry = entries[0] + if (!entry || entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) { + throw new Error(`GitHub resource path is not a regular Git blob: ${resource.path}`) + } + const bytes = (await git(repositoryRoot, ['cat-file', 'blob', entry.object])).stdout + verifyBytes(bytes, resource.sha256, resource.byteLength, `GitHub resource ${resource.path}`) + return Uint8Array.from(bytes) +} + +export async function verifyTaskCheckout( + taskRoot: string, + expected: { baseCommit: string; baseTree: string }, +): Promise { + const root = resolve(taskRoot) + const head = (await git(root, ['rev-parse', 'HEAD'])).stdout.toString('utf8').trim() + if (head !== expected.baseCommit) { + throw new Error(`task checkout HEAD ${head} does not match ${expected.baseCommit}`) + } + const tree = (await git(root, ['rev-parse', 'HEAD^{tree}'])).stdout.toString('utf8').trim() + if (tree !== expected.baseTree) { + throw new Error(`task checkout base tree ${tree} does not match ${expected.baseTree}`) + } +} + +/** + * Apply an evaluator-captured binary diff in a detached object database and + * prove its result tree exactly matches the captured after-state manifest. + */ +export async function verifyTaskOutcomePatch(input: { + repositoryRoot: string + baseCommit: string + baseTree: string + resultTree: string + patch: Uint8Array + afterState: AgentCandidateWorkspaceManifestMaterialV1 +}): Promise<{ resultTree: string; resultCommit: string }> { + const repositoryRoot = resolve(input.repositoryRoot) + await verifyTaskCheckout(repositoryRoot, input) + const gitDir = resolve( + (await git(repositoryRoot, ['rev-parse', '--absolute-git-dir'])).stdout.toString('utf8').trim(), + ) + if ((await realpath(gitDir)) !== gitDir || gitDir.includes(':')) { + throw new Error('task Git object store has an unsupported path') + } + await assertNoGitIndirection(repositoryRoot, gitDir, 'task repository') + + const temporary = await mkdtemp(join(tmpdir(), 'agent-candidate-task-outcome-')) + try { + const objectDirectory = join(temporary, 'objects') + const indexFile = join(temporary, 'index') + await mkdir(objectDirectory) + const gitEnvironment = { + GIT_INDEX_FILE: indexFile, + GIT_OBJECT_DIRECTORY: objectDirectory, + GIT_ALTERNATE_OBJECT_DIRECTORIES: join(gitDir, 'objects'), + } + await git(repositoryRoot, ['read-tree', input.baseTree], undefined, gitEnvironment) + if (input.patch.byteLength > 0) { + await git( + repositoryRoot, + ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'], + input.patch, + gitEnvironment, + ) + } + const resultTree = (await git(repositoryRoot, ['write-tree'], undefined, gitEnvironment)).stdout + .toString('utf8') + .trim() + if (resultTree !== input.resultTree) { + throw new Error( + `task outcome patch materialized tree ${resultTree} does not match ${input.resultTree}`, + ) + } + await assertSafeTree(repositoryRoot, resultTree, gitEnvironment) + const observed = await workspaceManifestFromGitTree(repositoryRoot, resultTree, gitEnvironment) + if ( + !Buffer.from(canonicalCandidateBytes(observed)).equals( + Buffer.from(canonicalCandidateBytes(input.afterState)), + ) + ) { + throw new Error('task outcome after-state does not match the materialized result tree') + } + const resultCommit = ( + await git( + repositoryRoot, + ['commit-tree', resultTree, '-p', input.baseCommit], + Buffer.from('candidate task outcome\n', 'utf8'), + { + ...gitEnvironment, + GIT_AUTHOR_NAME: 'Tangle Evaluator', + GIT_AUTHOR_EMAIL: 'evaluator@tangle.tools', + GIT_AUTHOR_DATE: '2000-01-01T00:00:00Z', + GIT_COMMITTER_NAME: 'Tangle Evaluator', + GIT_COMMITTER_EMAIL: 'evaluator@tangle.tools', + GIT_COMMITTER_DATE: '2000-01-01T00:00:00Z', + }, + ) + ).stdout + .toString('utf8') + .trim() + const committedTree = ( + await git(repositoryRoot, ['rev-parse', `${resultCommit}^{tree}`], undefined, gitEnvironment) + ).stdout + .toString('utf8') + .trim() + if (committedTree !== resultTree) { + throw new Error('task outcome evaluator commit does not bind the verified result tree') + } + return { resultTree, resultCommit } + } finally { + await rm(temporary, { recursive: true, force: true }) + } +} + +async function verifiedRepositoryRoot( + repository: AgentCandidateGitHubRepository, + repositories: AgentCandidateRepositoryPort, +): Promise { + const repositoryRoot = resolve(await repositories.resolve(repository)) + const rootStats = await stat(repositoryRoot) + if (!rootStats.isDirectory()) throw new Error('candidate repository path is not a directory') + + const origin = (await git(repositoryRoot, ['remote', 'get-url', 'origin'])).stdout + .toString('utf8') + .trim() + const actual = parseGitHubRemote(origin) + if (!actual || actual.owner !== repository.owner || actual.repo !== repository.repo) { + throw new Error( + `local repository origin ${origin || ''} does not match github.com/${repository.owner}/${repository.repo}`, + ) + } + + const gitDirText = (await git(repositoryRoot, ['rev-parse', '--absolute-git-dir'])).stdout + .toString('utf8') + .trim() + const gitDir = resolve(gitDirText) + await assertNoGitIndirection(repositoryRoot, gitDir, 'candidate repository') + return repositoryRoot +} + +async function assertNoGitIndirection( + repositoryRoot: string, + gitDir: string, + label: string, +): Promise { + const replacements = ( + await git(repositoryRoot, ['for-each-ref', '--format=%(refname)', 'refs/replace']) + ).stdout + .toString('utf8') + .trim() + if (replacements) throw new Error(`${label} contains Git replace refs`) + for (const name of ['alternates', 'http-alternates']) { + const path = join(gitDir, 'objects', 'info', name) + try { + const contents = await readFile(path, 'utf8') + if (contents.trim()) throw new Error(`${label} uses forbidden Git ${name}`) + } catch (error) { + if (!isNoEntry(error)) throw error + } + } +} + +async function assertCommitAndBaseTree( + repositoryRoot: string, + commit: string, + expectedTree: string, +): Promise { + const type = (await git(repositoryRoot, ['cat-file', '-t', commit])).stdout + .toString('utf8') + .trim() + if (type !== 'commit') throw new Error(`candidate base object is not a commit: ${commit}`) + const actualTree = (await git(repositoryRoot, ['rev-parse', `${commit}^{tree}`])).stdout + .toString('utf8') + .trim() + if (actualTree !== expectedTree) { + throw new Error(`candidate base commit tree ${actualTree} does not match ${expectedTree}`) + } +} + +async function assertSafeTree( + repositoryRoot: string, + tree: string, + environment: Record = {}, +): Promise { + const listing = ( + await git(repositoryRoot, ['ls-tree', '-rz', '--full-tree', tree], undefined, environment) + ).stdout + const entries = parseTreeEntries(listing) + if (entries.length === 0) throw new Error('candidate Git tree cannot be empty') + for (const entry of entries) { + if (entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) { + throw new Error( + `candidate Git tree contains a symlink, submodule, or non-blob: ${entry.path}`, + ) + } + assertSafeGitPath(entry.path) + } +} + +async function workspaceManifestFromGitTree( + repositoryRoot: string, + tree: string, + environment: Record, +): Promise { + const listing = ( + await git(repositoryRoot, ['ls-tree', '-rz', '--full-tree', tree], undefined, environment) + ).stdout + const entries = parseTreeEntries(listing) + const files = await Promise.all( + entries.map(async (entry) => { + if (entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) { + throw new Error(`task outcome tree contains a non-regular file: ${entry.path}`) + } + const bytes = ( + await git(repositoryRoot, ['cat-file', 'blob', entry.object], undefined, environment) + ).stdout + return { + path: entry.path, + mode: entry.mode === '100755' ? (0o755 as const) : (0o644 as const), + sha256: sha256Bytes(bytes), + byteLength: bytes.byteLength, + } + }), + ) + files.sort((left, right) => left.path.localeCompare(right.path)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files, + } +} + +function parseTreeEntries(bytes: Uint8Array): Array<{ + mode: string + type: string + object: string + path: string +}> { + const raw = Buffer.from(bytes) + const decoded = raw.toString('utf8') + if (!Buffer.from(decoded, 'utf8').equals(raw)) { + throw new Error('candidate Git tree contains a non-UTF-8 path') + } + const rows = decoded.split('\0').filter(Boolean) + return rows.map((row) => { + const tab = row.indexOf('\t') + const header = row.slice(0, tab).split(' ') + const path = row.slice(tab + 1) + const [mode, type, object] = header + if (tab < 1 || !mode || !type || !object || !path) { + throw new Error('malformed Git tree entry') + } + return { mode, type, object, path } + }) +} + +function assertSafeGitPath(path: string): void { + if ( + !path || + path.startsWith('/') || + path.includes('\\') || + path.includes('\0') || + hasControlCharacter(path) || + path.split('/').some((part) => !part || part === '.' || part === '..') || + path.split('/')[0]?.toLowerCase() === '.git' + ) { + throw new Error(`candidate Git tree contains an unsafe path: ${path}`) + } +} + +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code < 0x20 || code === 0x7f) return true + } + return false +} + +function parseGitHubRemote(value: string): { owner: string; repo: string } | undefined { + const match = value.match( + /^(?:https?:\/\/github\.com\/|ssh:\/\/git@github\.com\/|git@github\.com:)([^/]+)\/([^/]+?)(?:\.git)?$/, + ) + if (!match?.[1] || !match[2]) return undefined + return { owner: match[1], repo: match[2] } +} + +async function git( + repositoryRoot: string, + args: string[], + input?: Uint8Array, + extraEnv: Record = {}, +): Promise { + const env = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_')), + ) as Record + Object.assign(env, { + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_NO_REPLACE_OBJECTS: '1', + LC_ALL: 'C', + ...extraEnv, + }) + const fullArgs = [ + '-c', + 'core.hooksPath=/dev/null', + '-c', + 'protocol.file.allow=never', + '-C', + repositoryRoot, + ...args, + ] + return await new Promise((resolveResult, reject) => { + const child = spawn('git', fullArgs, { env, stdio: ['pipe', 'pipe', 'pipe'] }) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)) + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)) + child.on('error', reject) + child.on('close', (code, signal) => { + const out = Buffer.concat(stdout) + const err = Buffer.concat(stderr) + if (code !== 0) { + reject( + new Error( + `git ${args[0] ?? ''} failed (${signal ?? code}): ${err.toString('utf8').trim()}`, + ), + ) + return + } + resolveResult({ stdout: out, stderr: err }) + }) + if (input) child.stdin.end(input) + else child.stdin.end() + }) +} + +function isNoEntry(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code?: string }).code === 'ENOENT' + ) +} diff --git a/src/candidate-execution/index.ts b/src/candidate-execution/index.ts new file mode 100644 index 00000000..9627adad --- /dev/null +++ b/src/candidate-execution/index.ts @@ -0,0 +1,84 @@ +export { + type AgentCandidateExecutionAttemptRecord, + type AgentCandidateExecutionAttemptRef, + type AgentCandidateExecutionClaim, + type AgentCandidateExecutionClaimResult, + type AgentCandidateExecutionClaimStore, + type AgentCandidateExecutionCleanupHandles, + type AgentCandidateExecutionFailureClass, + type AgentCandidateExecutionFinishResult, + type AgentCandidateExecutionLease, + type AgentCandidateExecutionPhase, + type AgentCandidateExecutionPhaseResult, + type AgentCandidateExecutionRecoveryEvidence, + type AgentCandidateExecutionStageResult, + type AgentCandidateExecutionTerminalRecord, + type AgentCandidateExecutionTerminalResult, + type AgentCandidateExecutionUsage, + type AgentCandidateRetryRejection, + InMemoryAgentCandidateExecutionClaimStore, +} from './claim' +export { + FileAgentCandidateExecutionClaimStore, + type FileAgentCandidateExecutionClaimStoreOptions, +} from './claim-file-store' +export { candidateExecutionClaim } from './claim-plan' +export { + type DisposePreparedAgentCandidateOptions, + disposePreparedAgentCandidateExecution, +} from './dispose' +export { + type ExecutePreparedAgentCandidateOptions, + executePreparedAgentCandidate, +} from './execute' +export { persistCandidateOutputArtifact } from './output-artifacts' +export { + type PrepareAgentCandidateExecutionOptions, + prepareAgentCandidateExecution, +} from './prepare' +export { + type RecoverExpiredAgentCandidateOptions, + recoverExpiredAgentCandidateExecution, +} from './recover' +export { + type AgentCandidateArtifactPort, + type AgentCandidateBenchmarkGraderPort, + type AgentCandidateContainerPort, + type AgentCandidateExecutionPorts, + type AgentCandidateExecutorFinalCapture, + type AgentCandidateExecutorMemoryCapture, + type AgentCandidateExecutorPort, + type AgentCandidateExecutorProfileFile, + type AgentCandidateExecutorRequest, + type AgentCandidateExecutorStopRequest, + type AgentCandidateExecutorTaskOutcomeCapture, + type AgentCandidateExecutorWorkspaceFile, + type AgentCandidateExecutorWorkspaceInput, + type AgentCandidateMemoryPort, + type AgentCandidateMemoryResetResult, + type AgentCandidateModelLimits, + type AgentCandidateModelPort, + type AgentCandidateOutputArtifactPort, + type AgentCandidateOutputPurpose, + type AgentCandidateProtectedModelActivation, + type AgentCandidateProtectedModelCall, + type AgentCandidateProtectedModelReservation, + type AgentCandidateProtectedModelSettlement, + type AgentCandidateProtectedRunCapture, + type AgentCandidateRepositoryPort, + type AgentCandidateRunFinalization, + type AgentCandidateTaskExecution, + type AgentCandidateVerificationPorts, + type AgentCandidateWorkspacePort, + CANDIDATE_TRACE_ENV, + CANDIDATE_TRACE_TAGS, + type CanonicalCandidateDocument, + type PreparedAgentCandidateExecution, + type PreparedAgentCandidateInstruction, + type PreparedAgentCandidateLaunch, + type PreparedAgentCandidateTrace, + type ResolvedAgentCandidateContainer, + type VerifiedAgentCandidate, + type VerifiedAgentCandidateTaskOutcome, +} from './types' +export { verifyAgentCandidateBundle } from './verify' diff --git a/src/candidate-execution/model-settlement.ts b/src/candidate-execution/model-settlement.ts new file mode 100644 index 00000000..4a35f18e --- /dev/null +++ b/src/candidate-execution/model-settlement.ts @@ -0,0 +1,188 @@ +import type { LlmSpan } from '@tangle-network/agent-eval' +import type { AgentCandidateSpend } from '@tangle-network/agent-interface' +import type { AgentCandidateExecutionUsage } from './claim' +import { assertExactObjectKeys } from './exact-object' +import type { + AgentCandidateProtectedModelCall, + AgentCandidateProtectedModelSettlement, +} from './types' + +const USD_NANOS = 1_000_000_000 + +export interface SealedAgentCandidateModelSettlement { + readonly value: AgentCandidateProtectedModelSettlement + readonly usage: AgentCandidateSpend + readonly fixedUsage: AgentCandidateExecutionUsage + readonly costUsdNanos: number +} + +/** Validate and detach the evaluator gateway's terminal, revoked call ledger. */ +export function sealAgentCandidateModelSettlement( + settlement: AgentCandidateProtectedModelSettlement, + expected: { preparationId: string; grantDigest: string; model: string }, +): SealedAgentCandidateModelSettlement { + assertExactObjectKeys( + settlement, + ['preparationId', 'grantDigest', 'closed', 'calls'], + 'model settlement', + ) + if (settlement.closed !== true) throw new Error('protected model grant is not closed') + if (settlement.grantDigest !== expected.grantDigest) { + throw new Error('protected model settlement grant digest does not match the reservation') + } + if (settlement.preparationId !== expected.preparationId) { + throw new Error('protected model settlement preparation does not match the reservation') + } + if (!Array.isArray(settlement.calls)) { + throw new Error('protected model settlement calls must be an array') + } + + const callIds = new Set() + const spanIds = new Set() + let inputTokens = 0 + let outputTokens = 0 + let cachedInputTokens = 0 + let reasoningTokens = 0 + let hasCachedInput = false + let costUsdNanos = 0 + const calls = settlement.calls.map((source, index) => { + assertExactObjectKeys( + source, + ['callId', 'traceSpanId', 'model', 'inputTokens', 'outputTokens', 'costUsdNanos'], + `model settlement call ${index}`, + ['cachedInputTokens', 'reasoningTokens'], + ) + assertIdentifier(source.callId, `model settlement call ${index} callId`) + assertIdentifier(source.traceSpanId, `model settlement call ${index} traceSpanId`) + if (callIds.has(source.callId)) + throw new Error('protected model settlement has duplicate call ids') + if (spanIds.has(source.traceSpanId)) { + throw new Error('protected model settlement has duplicate trace span ids') + } + callIds.add(source.callId) + spanIds.add(source.traceSpanId) + if (source.model !== expected.model) { + throw new Error(`protected model settlement call ${index} has an unexpected model`) + } + assertCount(source.inputTokens, `model settlement call ${index} inputTokens`) + assertCount(source.outputTokens, `model settlement call ${index} outputTokens`) + if (source.cachedInputTokens !== undefined) { + assertCount(source.cachedInputTokens, `model settlement call ${index} cachedInputTokens`) + cachedInputTokens = safeAdd( + cachedInputTokens, + source.cachedInputTokens, + 'cached input token total', + ) + hasCachedInput = true + } + if (source.reasoningTokens !== undefined) { + assertCount(source.reasoningTokens, `model settlement call ${index} reasoningTokens`) + reasoningTokens = safeAdd(reasoningTokens, source.reasoningTokens, 'reasoning token total') + } + assertCount(source.costUsdNanos, `model settlement call ${index} costUsdNanos`) + inputTokens = safeAdd(inputTokens, source.inputTokens, 'input token total') + outputTokens = safeAdd(outputTokens, source.outputTokens, 'output token total') + costUsdNanos = safeAdd(costUsdNanos, source.costUsdNanos, 'cost total') + return Object.freeze({ ...source }) + }) + + const usage = Object.freeze({ + costUsd: costUsdNanos / USD_NANOS, + inputTokens, + outputTokens, + ...(hasCachedInput ? { cachedInputTokens } : {}), + modelCalls: calls.length, + }) + const fixedUsage = Object.freeze({ + costUsdNanos, + inputTokens, + outputTokens, + cachedInputTokens, + reasoningTokens, + modelCalls: calls.length, + }) + return Object.freeze({ + value: Object.freeze({ + preparationId: settlement.preparationId, + grantDigest: settlement.grantDigest, + closed: true as const, + calls: Object.freeze(calls), + }), + usage, + fixedUsage, + costUsdNanos, + }) +} + +/** Match every protected trace span one-for-one against gateway call evidence. */ +export function assertTraceMatchesModelSettlement( + spans: readonly LlmSpan[], + settlement: SealedAgentCandidateModelSettlement, +): void { + if (spans.length !== settlement.value.calls.length) { + throw new Error( + `protected trace model calls ${spans.length} do not match model ledger ${settlement.value.calls.length}`, + ) + } + const byId = new Map(spans.map((span) => [span.spanId, span])) + if (byId.size !== spans.length) throw new Error('protected trace has duplicate model span ids') + for (const call of settlement.value.calls) { + const span = byId.get(call.traceSpanId) + if (!span) { + throw new Error(`protected trace is missing model ledger span ${call.traceSpanId}`) + } + assertTraceCall(span, call) + } +} + +export function usdToNanos(value: number, label: string): number { + if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be nonnegative`) + const nanos = Math.round(value * USD_NANOS) + if (!Number.isSafeInteger(nanos)) throw new Error(`${label} exceeds fixed-point range`) + return nanos +} + +function assertTraceCall(span: LlmSpan, call: AgentCandidateProtectedModelCall): void { + if (span.model !== call.model) { + throw new Error(`protected trace span ${span.spanId} model does not match model ledger`) + } + for (const [name, traced, settled] of [ + ['inputTokens', span.inputTokens, call.inputTokens], + ['outputTokens', span.outputTokens, call.outputTokens], + ['cachedInputTokens', span.cachedTokens ?? 0, call.cachedInputTokens ?? 0], + ['reasoningTokens', span.reasoningTokens ?? 0, call.reasoningTokens ?? 0], + ] as const) { + if (traced === undefined || traced !== settled) { + throw new Error( + `protected trace span ${span.spanId} ${name} ${traced} does not match model ledger ${settled}`, + ) + } + } + if (span.costUsd === undefined) { + throw new Error(`protected trace span ${span.spanId} is missing costUsd`) + } + const tracedCost = usdToNanos(span.costUsd, `protected trace span ${span.spanId} costUsd`) + if (tracedCost !== call.costUsdNanos) { + throw new Error( + `protected trace span ${span.spanId} costUsdNanos ${tracedCost} does not match model ledger ${call.costUsdNanos}`, + ) + } +} + +function assertIdentifier(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || value.length === 0 || value.length > 256) { + throw new Error(`${label} must be a non-empty bounded string`) + } +} + +function assertCount(value: unknown, label: string): asserts value is number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error(`${label} must be a nonnegative safe integer`) + } +} + +function safeAdd(left: number, right: number, label: string): number { + const total = left + right + if (!Number.isSafeInteger(total)) throw new Error(`${label} exceeds safe integer range`) + return total +} diff --git a/src/candidate-execution/outcome-evidence.ts b/src/candidate-execution/outcome-evidence.ts new file mode 100644 index 00000000..c2f7c821 --- /dev/null +++ b/src/candidate-execution/outcome-evidence.ts @@ -0,0 +1,353 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { BenchmarkEvaluation } from '@tangle-network/agent-eval' +import { + type AgentCandidateArtifactRef, + type AgentCandidateBenchmarkResultEvidence, + type AgentCandidateModelSettlementEvidence, + type AgentCandidateResolvedModel, + type AgentCandidateTaskOutcomeEvidence, + type AgentCandidateTermination, + agentCandidateBenchmarkResultEvidenceSchema, + agentCandidateModelSettlementEvidenceSchema, + agentCandidateTaskOutcomeEvidenceSchema, + agentCandidateWorkspaceSnapshotEvidenceSchema, + type Sha256Digest, +} from '@tangle-network/agent-interface' + +import { readMaterializedWorkspaceFiles } from './artifacts' +import { runBoundCandidateBenchmarkGrader } from './benchmark-grader' +import { + canonicalCandidateBytes, + embeddedCandidateArtifact, + immutableCandidateValue, + sha256Bytes, +} from './digest' +import { verifyTaskOutcomePatch } from './git-materialize' +import type { SealedAgentCandidateModelSettlement } from './model-settlement' +import { persistCandidateOutputArtifact } from './output-artifacts' +import type { PreparedCandidateState } from './prepared-state' +import { assertNoProtectedBytes } from './protected-redaction' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutorTaskOutcomeCapture, + AgentCandidateOutputArtifactPort, + VerifiedAgentCandidateTaskOutcome, +} from './types' +import { verifiedTaskOutcomeBrand } from './types' + +export type PersistedAgentCandidateModelSettlement = AgentCandidateModelSettlementEvidence & { + artifact: AgentCandidateArtifactRef +} + +export type PersistedAgentCandidateBenchmarkResult = AgentCandidateBenchmarkResultEvidence & { + artifact: AgentCandidateArtifactRef +} + +/** Persist the closed evaluator model ledger as canonical V2 receipt evidence. */ +export async function persistCandidateModelSettlement( + state: PreparedCandidateState, + settlement: SealedAgentCandidateModelSettlement, + outputArtifacts: AgentCandidateOutputArtifactPort, +): Promise { + return await persistCandidateModelSettlementEvidence( + { + executionId: state.executionId, + executionPlanDigest: state.executionPlan.value.digest, + resolvedModel: state.resolvedModel, + }, + settlement, + outputArtifacts, + ) +} + +/** Persist a closed model ledger when only durable recovery identity remains. */ +export async function persistCandidateModelSettlementEvidence( + identity: { + executionId: string + executionPlanDigest: Sha256Digest + resolvedModel: AgentCandidateResolvedModel + }, + settlement: SealedAgentCandidateModelSettlement, + outputArtifacts: AgentCandidateOutputArtifactPort, +): Promise { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-model-settlement-material' as const, + executionPlanDigest: identity.executionPlanDigest, + preparationId: settlement.value.preparationId, + grantDigest: settlement.value.grantDigest, + closed: true as const, + resolved: identity.resolvedModel, + calls: settlement.value.calls.map((call) => ({ + callId: call.callId, + traceSpanId: call.traceSpanId, + model: call.model, + inputTokens: call.inputTokens, + outputTokens: call.outputTokens, + cachedInputTokens: call.cachedInputTokens ?? 0, + reasoningTokens: call.reasoningTokens ?? 0, + costUsdNanos: call.costUsdNanos, + })), + usage: settlement.fixedUsage, + } + const bytes = canonicalCandidateBytes(material) + const digest = sha256Bytes(bytes) + const artifact = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: identity.executionId, + purpose: 'model-settlement', + bytes, + }) + return immutableCandidateValue( + agentCandidateModelSettlementEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-model-settlement', + digest, + material, + artifact, + }), + ) as PersistedAgentCandidateModelSettlement +} + +/** Recompute the result tree from the patch, then persist its exact task evidence. */ +export async function persistVerifiedCandidateTaskOutcome( + state: PreparedCandidateState, + capture: AgentCandidateExecutorTaskOutcomeCapture, + outputArtifacts: AgentCandidateOutputArtifactPort, + protectedValues: readonly string[], + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted() + const patch = Uint8Array.from(capture.gitDiff) + const archive = Uint8Array.from(capture.archive) + if (archive.byteLength === 0) throw new Error('candidate task archive cannot be empty') + assertNoProtectedBytes(patch, protectedValues) + assertNoProtectedBytes(archive, protectedValues) + const afterState = immutableCandidateValue(capture.afterState) + const repository = state.executionPlan.value.material.task.repository + const verified = await verifyTaskOutcomePatch({ + repositoryRoot: state.roots.staging.taskRoot, + baseCommit: repository.baseCommit, + baseTree: repository.baseTree, + resultTree: capture.resultTree, + patch, + afterState, + }) + signal?.throwIfAborted() + const manifestBytes = canonicalCandidateBytes(afterState) + assertNoProtectedBytes(manifestBytes, protectedValues) + const provisionalSnapshot = agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), + material: afterState, + manifest: embeddedCandidateArtifact(manifestBytes), + archive: embeddedCandidateArtifact(archive), + }) + await verifyTaskOutcomeArchive(state, provisionalSnapshot, archive, protectedValues) + signal?.throwIfAborted() + const [manifest, archiveRef, gitDiff] = await Promise.all([ + persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'task-manifest', + bytes: manifestBytes, + signal, + }), + persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'task-archive', + bytes: archive, + signal, + }), + persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'task-patch', + bytes: patch, + signal, + }), + ]) + const snapshot = agentCandidateWorkspaceSnapshotEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: sha256Bytes(manifestBytes), + material: afterState, + manifest, + archive: archiveRef, + }) + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-task-outcome-material' as const, + executionPlanDigest: state.executionPlan.value.digest, + baseRepository: { + identity: repository.identity, + rootIdentity: repository.rootIdentity, + commit: repository.baseCommit, + tree: repository.baseTree, + }, + resultRepository: { + identity: repository.identity, + rootIdentity: repository.rootIdentity, + commit: verified.resultCommit, + tree: verified.resultTree, + }, + afterState: snapshot, + gitDiff: { + format: 'git-diff-binary' as const, + artifact: gitDiff, + }, + } + const bytes = canonicalCandidateBytes(material) + const digest = sha256Bytes(bytes) + const artifact = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'task-outcome', + bytes, + signal, + }) + const evidence = immutableCandidateValue( + agentCandidateTaskOutcomeEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-task-outcome', + digest, + material, + artifact, + }), + ) as AgentCandidateTaskOutcomeEvidence & { artifact: AgentCandidateArtifactRef } + const storedPatch = Uint8Array.from(patch) + return Object.freeze({ + evidence, + get patch(): Uint8Array { + return Uint8Array.from(storedPatch) + }, + [verifiedTaskOutcomeBrand]: true as const, + }) +} + +/** Grade only a runtime-verified outcome and persist both raw and normalized evidence. */ +export async function persistCandidateBenchmarkResult( + state: PreparedCandidateState, + termination: AgentCandidateTermination, + outcome: VerifiedAgentCandidateTaskOutcome, + grader: AgentCandidateBenchmarkGraderPort, + outputArtifacts: AgentCandidateOutputArtifactPort, + protectedValues: readonly string[], + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted() + const frozenTermination = immutableCandidateValue(termination) + const graded = await runBoundCandidateBenchmarkGrader({ + executionId: state.executionId, + termination: frozenTermination, + outcome, + grader, + artifacts: outputArtifacts, + signal, + }) + signal?.throwIfAborted() + const evaluation = normalizeEvaluation(graded.evaluation, frozenTermination) + const rawEvidence = Uint8Array.from(graded.evidence) + if (rawEvidence.byteLength === 0) throw new Error('candidate benchmark evidence cannot be empty') + assertNoProtectedBytes(rawEvidence, protectedValues) + const evidenceRef = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'grader-evidence', + bytes: rawEvidence, + signal, + }) + const task = state.executionPlan.value.material.task + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-benchmark-result-material' as const, + executionPlanDigest: state.executionPlan.value.digest, + taskOutcomeDigest: outcome.evidence.digest, + benchmark: { + name: task.benchmark, + version: task.benchmarkVersion, + taskId: task.taskId, + splitDigest: task.splitDigest, + }, + grader: { + name: graded.grader.name, + version: graded.grader.version, + artifact: graded.grader.artifact, + }, + evidence: evidenceRef, + score: evaluation.score, + passed: evaluation.passed, + dimensions: evaluation.dimensions, + } + const bytes = canonicalCandidateBytes(material) + const digest = sha256Bytes(bytes) + const artifact = await persistCandidateOutputArtifact(outputArtifacts, { + executionId: state.executionId, + purpose: 'benchmark-result', + bytes, + signal, + }) + return immutableCandidateValue( + agentCandidateBenchmarkResultEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-benchmark-result', + digest, + material, + artifact, + }), + ) as PersistedAgentCandidateBenchmarkResult +} + +async function verifyTaskOutcomeArchive( + state: PreparedCandidateState, + snapshot: ReturnType, + archive: Uint8Array, + protectedValues: readonly string[], +): Promise { + const root = await mkdtemp(join(tmpdir(), 'agent-candidate-task-archive-')) + try { + await state.ports.workspaces.materialize({ + role: 'task', + snapshot, + archive: Uint8Array.from(archive), + destination: root, + }) + const files = await readMaterializedWorkspaceFiles(root, snapshot.material) + for (const file of files) assertNoProtectedBytes(file.bytes, protectedValues) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +function normalizeEvaluation( + evaluation: BenchmarkEvaluation, + termination: AgentCandidateTermination, +): { score: number; passed: boolean; dimensions: Array<{ name: string; score: number }> } { + if (!evaluation || typeof evaluation !== 'object' || Array.isArray(evaluation)) { + throw new Error('candidate benchmark evaluation must be an object') + } + assertUnitScore(evaluation.score, 'candidate benchmark score') + if (evaluation.passed !== undefined && typeof evaluation.passed !== 'boolean') { + throw new Error('candidate benchmark passed must be boolean') + } + const cleanExit = termination.kind === 'exit' && termination.exitCode === 0 + const dimensions = Object.entries(evaluation.dimensions ?? {}) + .map(([name, score]) => { + if (!/^[a-z0-9]+(?:[._-][a-z0-9]+)*$/.test(name)) { + throw new Error(`candidate benchmark dimension is not normalized: ${name}`) + } + assertUnitScore(score, `candidate benchmark dimension ${name}`) + return { name, score: cleanExit ? score : 0 } + }) + .sort((left, right) => left.name.localeCompare(right.name)) + return { + score: cleanExit ? evaluation.score : 0, + passed: cleanExit && (evaluation.passed ?? evaluation.score > 0), + dimensions, + } +} + +function assertUnitScore(value: unknown, label: string): asserts value is number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`${label} must be finite and within [0, 1]`) + } +} diff --git a/src/candidate-execution/output-artifacts.ts b/src/candidate-execution/output-artifacts.ts new file mode 100644 index 00000000..af91e4b9 --- /dev/null +++ b/src/candidate-execution/output-artifacts.ts @@ -0,0 +1,39 @@ +import { + type AgentCandidateArtifactRef, + agentCandidateArtifactRefSchema, +} from '@tangle-network/agent-interface' + +import { verifyBytes } from './artifacts' +import { immutableCandidateValue, sha256Bytes } from './digest' +import type { AgentCandidateOutputArtifactPort, AgentCandidateOutputPurpose } from './types' + +/** Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. */ +export async function persistCandidateOutputArtifact( + port: AgentCandidateOutputArtifactPort, + input: { + executionId: string + purpose: AgentCandidateOutputPurpose + bytes: Uint8Array + signal?: AbortSignal + }, +): Promise { + input.signal?.throwIfAborted() + const bytes = Uint8Array.from(input.bytes) + const expectedDigest = sha256Bytes(bytes) + const ref = agentCandidateArtifactRefSchema.parse( + await port.put({ + executionId: input.executionId, + purpose: input.purpose, + bytes: Uint8Array.from(bytes), + ...(input.signal ? { signal: input.signal } : {}), + }), + ) + input.signal?.throwIfAborted() + if (ref.sha256 !== expectedDigest || ref.byteLength !== bytes.byteLength) { + throw new Error('candidate output locator does not identify the submitted bytes') + } + const stored = await port.read(ref) + input.signal?.throwIfAborted() + verifyBytes(stored, expectedDigest, bytes.byteLength, 'persisted candidate output') + return immutableCandidateValue(ref) +} diff --git a/src/candidate-execution/prepare.ts b/src/candidate-execution/prepare.ts new file mode 100644 index 00000000..0a19a929 --- /dev/null +++ b/src/candidate-execution/prepare.ts @@ -0,0 +1,960 @@ +import { randomBytes } from 'node:crypto' +import { lstat, readdir } from 'node:fs/promises' +import { isAbsolute, posix, relative, resolve as resolveHostPath } from 'node:path' + +import type { + AgentCandidateConfigValue, + AgentCandidateEffectiveMemory, + AgentCandidateExecutionLimits, + AgentCandidateExecutionPlanEvidence, + AgentCandidateExecutionPlanMaterialV1, + AgentCandidateMaterializationReceipt, + AgentCandidateResolvedModel, + HarnessType, +} from '@tangle-network/agent-interface' +import { + agentCandidateContainerSchema, + agentCandidateExecutionLimitsSchema, + agentCandidateExecutionPlanEvidenceSchema, + agentCandidateExecutionPlanMaterialSchema, + agentCandidateMaterializationReceiptSchema, + agentCandidateWorkspaceSnapshotEvidenceSchema, + sha256DigestSchema, +} from '@tangle-network/agent-interface' +import { + applyAgentCandidateWorkspacePlan, + type HarnessId, + materializeCandidateProfile, +} from '@tangle-network/agent-profile-materialize' + +import { + readMaterializedWorkspaceFiles, + readVerifiedArtifact, + verifyMaterializedProfileWorkspace, + verifyMaterializedWorkspace, + verifyWorkspaceSnapshotArtifacts, +} from './artifacts' +import { + candidateCleanupDeadline, + candidateCleanupTimeout, + candidateResultTimeout, + MAX_CANDIDATE_TIMER_INTERVAL_MS, + withinCandidateCleanupDeadline, +} from './cleanup' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + canonicalCandidateDocument, + embeddedCandidateArtifact, + sha256Bytes, +} from './digest' +import { candidateExecutionOwnerWindowMs } from './execution-window' +import { verifyTaskCheckout } from './git-materialize' +import { sealAgentCandidateModelSettlement, usdToNanos } from './model-settlement' +import { createPreparedCandidateExecution } from './prepared-state' +import { + type AgentCandidateExecutionPorts, + type AgentCandidateTaskExecution, + CANDIDATE_TRACE_ENV, + CANDIDATE_TRACE_TAGS, + type PreparedAgentCandidateExecution, + type ResolvedAgentCandidateContainer, + type VerifiedAgentCandidate, +} from './types' +import { + getVerifiedCandidateState, + verifiedArtifactBytes, + verifiedResourceTextByDigest, +} from './verify' + +const MATERIALIZER_HARNESSES = new Set([ + 'claude-code', + 'claude', + 'claudish', + 'nanoclaw', + 'codex', + 'opencode', + 'kimi-code', + 'kimi', + 'pi', + 'gemini', + 'hermes', + 'openclaw', +]) + +const MIN_RESERVATION_TTL_MS = 15 * 60_000 +const PREPARED_HOLD_MARGIN_MS = 5 * 60_000 + +export interface PrepareAgentCandidateExecutionOptions { + cleanupTimeoutMs?: number + /** Maximum time for task verification, executable grading, and receipt construction. */ + resultTimeoutMs?: number +} + +/** Materializes a verified candidate into one immutable evaluator-owned execution plan. */ +export async function prepareAgentCandidateExecution( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + ports: AgentCandidateExecutionPorts, + options: PrepareAgentCandidateExecutionOptions = {}, +): Promise { + const cleanupTimeoutMs = candidateCleanupTimeout(options.cleanupTimeoutMs) + const verifiedState = getVerifiedCandidateState(candidate) + assertSameVerificationPorts(verifiedState.ports, ports) + const bundle = candidate.bundle + const harness = materializerHarness(bundle.execution.harness) + assertTaskInput(task, bundle.execution.instructionDelivery) + const resultTimeoutMs = candidateResultTimeout(options.resultTimeoutMs, task.limits.timeoutMs) + const ownerWindowMs = candidateExecutionOwnerWindowMs( + task.limits.timeoutMs, + cleanupTimeoutMs, + resultTimeoutMs, + ) + const reservationWindowMs = ownerWindowMs + PREPARED_HOLD_MARGIN_MS + if (reservationWindowMs > MAX_CANDIDATE_TIMER_INTERVAL_MS) { + throw new Error('candidate reservation window exceeds the supported timer range') + } + assertDisjointHostStagingRoots(task) + + const instructionBytes = Buffer.from(task.instruction, 'utf8') + const instructionDigest = sha256Bytes(instructionBytes) + + const taskArtifacts = await verifyWorkspaceSnapshotArtifacts(task.workspace, ports.artifacts) + await ports.workspaces.materialize({ + role: 'task', + snapshot: task.workspace, + archive: taskArtifacts.archive, + destination: task.stagingRoots.taskRoot, + }) + await verifyMaterializedWorkspace(task.stagingRoots.taskRoot, task.workspace.material, { + ignoredProtectedRootEntries: ['.git', '.sidecar'], + }) + await verifyTaskCheckout(task.stagingRoots.taskRoot, task.repository) + const taskExecutorFiles = await readMaterializedWorkspaceFiles( + task.stagingRoots.taskRoot, + task.workspace.material, + { ignoredProtectedRootEntries: ['.git', '.sidecar'] }, + ) + + let candidateArchive: Uint8Array | undefined + let candidateExecutorFiles: + | ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> + | undefined + if (bundle.execution.workspace) { + if (!task.stagingRoots.candidateRoot || !task.executionRoots.candidateRoot) { + throw new Error('active candidate execution requires host and container candidate roots') + } + candidateArchive = await verifiedArtifactBytes(candidate, bundle.execution.workspace.archive) + await ports.workspaces.materialize({ + role: 'candidate', + snapshot: bundle.execution.workspace, + archive: candidateArchive, + destination: task.stagingRoots.candidateRoot, + }) + await verifyMaterializedWorkspace( + task.stagingRoots.candidateRoot, + bundle.execution.workspace.material, + ) + candidateExecutorFiles = await readMaterializedWorkspaceFiles( + task.stagingRoots.candidateRoot, + bundle.execution.workspace.material, + ) + } else if (task.stagingRoots.candidateRoot || task.executionRoots.candidateRoot) { + throw new Error('disabled code cannot receive a candidate workspace root') + } + + await assertEmptyDirectory(task.stagingRoots.profileRoot) + const profileWorkspacePlan = materializeCandidateProfile(bundle.profile, harness, { + resolvedResources: verifiedResourceTextByDigest(candidate), + }) + const profileApplication = applyAgentCandidateWorkspacePlan( + profileWorkspacePlan, + task.stagingRoots.profileRoot, + bundle.execution.cwd.workspace, + ) + await verifyMaterializedProfileWorkspace( + task.stagingRoots.profileRoot, + profileApplication.profilePlan.material, + ) + const profilePlanBytes = await readVerifiedArtifact( + profileApplication.profilePlan.artifact, + ports.artifacts, + ) + if ( + !Buffer.from(profilePlanBytes).equals( + Buffer.from(canonicalCandidateBytes(profileApplication.profilePlan.material)), + ) + ) { + throw new Error('profile materializer did not capture exact canonical plan bytes') + } + + const container = await resolveContainer(candidate, task, ports) + const resolvedModel = await resolveModel(candidate, task, ports) + const preparationId = `candidate-preparation-v1.${randomBytes(32).toString('base64url')}` + const reservationExpiresAtMs = Date.now() + Math.max(MIN_RESERVATION_TTL_MS, reservationWindowMs) + const modelReservation = await withinCandidateCleanupDeadline( + () => + ports.models.reserveGrant({ + executionId: task.executionId, + preparationId, + expiresAtMs: reservationExpiresAtMs, + attempt: task.attempt, + bundleDigest: bundle.digest, + resolved: resolvedModel, + limits: modelLimits(task.limits), + }), + candidateCleanupDeadline(cleanupTimeoutMs), + 'protected model reservation', + ) + let preparedMemory: Awaited> | undefined + try { + validateProtectedModelReservation( + modelReservation, + task.limits, + preparationId, + reservationExpiresAtMs, + ) + preparedMemory = await prepareMemory( + candidate, + task, + ports, + preparationId, + reservationExpiresAtMs, + cleanupTimeoutMs, + ) + const memory = preparedMemory.value + const knowledge = bundle.knowledge + ? { + snapshotId: bundle.knowledge.snapshotId, + manifestDigest: bundle.knowledge.manifest.sha256, + manifest: await verifiedArtifactBytes(candidate, bundle.knowledge.manifest), + } + : undefined + + const baseLaunch = buildLaunch(candidate, task, profileApplication.flags) + const publicEnv = mergePublicEnvironment( + bundle.execution.env ?? {}, + profileApplication.env, + bundle.execution.instructionDelivery.kind === 'utf8-file' + ? { + [bundle.execution.instructionDelivery.env]: { + kind: 'public', + value: bundle.execution.instructionDelivery.path, + }, + } + : {}, + ) + const routes = modelRoutes(bundle.profile, task.model.requested) + const executionMaterial: AgentCandidateExecutionPlanMaterialV1 = { + schemaVersion: 1, + kind: 'agent-candidate-execution-plan-material', + bundleDigest: bundle.digest, + executionId: task.executionId, + attempt: task.attempt, + task: { + benchmark: task.benchmark, + benchmarkVersion: task.benchmarkVersion, + taskId: task.taskId, + splitDigest: task.splitDigest, + instruction: { + encoding: 'utf8', + sha256: instructionDigest, + byteLength: instructionBytes.byteLength, + delivery: bundle.execution.instructionDelivery, + }, + repository: task.repository, + workspace: task.workspace, + }, + workspaces: { + taskRoot: task.executionRoots.taskRoot, + ...(task.executionRoots.candidateRoot + ? { candidateRoot: task.executionRoots.candidateRoot } + : {}), + }, + codeKind: bundle.code.kind, + ...(bundle.execution.workspace ? { candidateWorkspace: bundle.execution.workspace } : {}), + profile: profileApplication.application, + harness: bundle.execution.harness, + harnessVersion: bundle.execution.harnessVersion, + container, + model: { + policy: 'single', + resolved: resolvedModel, + access: { kind: 'evaluator-mediated', grantDigest: modelReservation.digest }, + routes, + }, + launch: { + executable: baseLaunch.executable, + args: baseLaunch.args, + env: publicEnv, + cwd: bundle.execution.cwd, + }, + ...(bundle.knowledge ? { knowledgeManifestDigest: bundle.knowledge.manifest.sha256 } : {}), + memory, + limits: task.limits, + network: { mode: 'disabled' }, + } + agentCandidateExecutionPlanMaterialSchema.parse(executionMaterial) + const executionBytes = canonicalCandidateBytes(executionMaterial) + const executionDigest = canonicalCandidateDigest(executionMaterial) + if (sha256Bytes(executionBytes) !== executionDigest) { + throw new Error('execution plan canonical serializers disagree') + } + const executionPlan: AgentCandidateExecutionPlanEvidence = + agentCandidateExecutionPlanEvidenceSchema.parse({ + schemaVersion: 1, + kind: 'agent-candidate-execution-plan', + digest: executionDigest, + material: executionMaterial, + artifact: embeddedCandidateArtifact(executionBytes), + }) + + const entrypoint = candidateEntrypointReceipt(candidate) + const materializationReceipt = canonicalCandidateDocument( + { + schemaVersion: 1, + kind: 'agent-candidate-materialization', + digestAlgorithm: 'rfc8785-sha256', + bundleDigest: bundle.digest, + profilePlan: profileApplication.profilePlan, + executionPlan, + ...(bundle.execution.workspace ? { candidateWorkspace: bundle.execution.workspace } : {}), + codeKind: bundle.code.kind, + ...(candidate.materializedTree ? { materializedTree: candidate.materializedTree } : {}), + harness: bundle.execution.harness, + harnessVersion: bundle.execution.harnessVersion, + container, + resolvedModel, + ...(bundle.knowledge ? { knowledgeManifestDigest: bundle.knowledge.manifest.sha256 } : {}), + ...(entrypoint ? { entrypoint } : {}), + }, + ) + agentCandidateMaterializationReceiptSchema.parse(materializationReceipt.value) + + const traceRunId = `${task.executionId}:attempt-${task.attempt.number}:${canonicalCandidateDigest({ preparationId }).slice(7, 23)}` + const traceTags = { + [CANDIDATE_TRACE_TAGS.executionId]: task.executionId, + [CANDIDATE_TRACE_TAGS.bundleDigest]: bundle.digest, + [CANDIDATE_TRACE_TAGS.executionPlanDigest]: executionPlan.digest, + [CANDIDATE_TRACE_TAGS.materializationReceiptDigest]: materializationReceipt.digest, + } + const traceEnv = { + [CANDIDATE_TRACE_ENV.executionId]: task.executionId, + [CANDIDATE_TRACE_ENV.bundleDigest]: bundle.digest, + [CANDIDATE_TRACE_ENV.executionPlanDigest]: executionPlan.digest, + [CANDIDATE_TRACE_ENV.materializationReceiptDigest]: materializationReceipt.digest, + [CANDIDATE_TRACE_ENV.traceRunId]: traceRunId, + } + assertEnvironmentDisjoint(publicEnv, traceEnv) + + return createPreparedCandidateExecution({ + ports, + bundle, + executionId: task.executionId, + roots: { + execution: { ...task.executionRoots }, + staging: { ...task.stagingRoots }, + }, + profilePlan: { + value: profileApplication.profilePlan, + bytes: profilePlanBytes, + written: [...profileApplication.application.mountPaths], + }, + executionPlan: { value: executionPlan, bytes: executionBytes }, + materializationReceipt, + launch: { + executable: baseLaunch.executable, + args: baseLaunch.args.map((value) => value.value), + env: unwrapPublicEnvironment(publicEnv), + flags: profileApplication.flags.map((value) => value.value), + cwd: absoluteExecutionCwd(bundle.execution.cwd, task.executionRoots), + }, + instruction: { + bytes: Uint8Array.from(instructionBytes), + delivery: bundle.execution.instructionDelivery, + }, + resolvedModel, + preparationId, + reservationExpiresAtMs, + cleanupTimeoutMs, + resultTimeoutMs, + modelReservation: { + preparationId: modelReservation.preparationId, + digest: modelReservation.digest, + expiresAtMs: modelReservation.expiresAtMs, + enforcedLimits: modelReservation.enforcedLimits, + }, + executorInputs: { + taskFiles: taskExecutorFiles, + ...(candidateExecutorFiles ? { candidateFiles: candidateExecutorFiles } : {}), + profileFiles: exactProfileExecutorFiles( + profileWorkspacePlan.files, + profileApplication.profilePlan.material.files, + ), + }, + ...(preparedMemory.accessDigest && preparedMemory.value.mode === 'isolated' + ? { + memoryReservation: { + preparationId, + accessDigest: preparedMemory.accessDigest, + expiresAtMs: reservationExpiresAtMs, + effectiveNamespace: preparedMemory.value.effectiveNamespace, + }, + } + : {}), + ...(knowledge ? { knowledge } : {}), + trace: { runId: traceRunId, tags: traceTags, env: traceEnv }, + memory, + }) + } catch (error) { + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const cleanup: Array> = [] + if (preparedMemory?.value.mode === 'isolated') { + const accessDigest = preparedMemory.accessDigest + const effectiveNamespace = preparedMemory.value.effectiveNamespace + if (!accessDigest) throw new Error('isolated memory preparation is missing access identity') + cleanup.push( + withinCandidateCleanupDeadline( + async () => { + const closed = await ports.memory.close({ + executionId: task.executionId, + preparationId, + accessDigest, + effectiveNamespace, + reason: 'preparation-failed', + }) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('failed preparation did not close isolated memory access') + } + }, + cleanupDeadlineAtMs, + 'failed preparation memory cleanup', + ), + ) + } + cleanup.push( + withinCandidateCleanupDeadline( + async () => { + const settlement = sealAgentCandidateModelSettlement( + await ports.models.settleGrant({ + executionId: task.executionId, + preparationId, + grantDigest: modelReservation.digest, + resolved: resolvedModel, + reason: 'preparation-failed', + }), + { + preparationId, + grantDigest: modelReservation.digest, + model: resolvedModel.model, + }, + ) + if (settlement.usage.modelCalls !== 0) { + throw new Error('failed preparation unexpectedly contains model calls') + } + }, + cleanupDeadlineAtMs, + 'failed preparation model cleanup', + ), + ) + const cleanupResults = await Promise.allSettled(cleanup) + const cleanupErrors = cleanupResults + .filter((result): result is PromiseRejectedResult => result.status === 'rejected') + .map((result) => result.reason) + if (cleanupErrors.length > 0) { + throw new Error( + `candidate preparation failed and protected access cleanup failed: ${cleanupErrors.map(errorMessage).join('; ')}`, + { cause: error }, + ) + } + throw error + } +} + +function assertSameVerificationPorts( + verified: AgentCandidateExecutionPorts | { artifacts: unknown; repositories: unknown }, + execution: AgentCandidateExecutionPorts, +): void { + if ( + verified.artifacts !== execution.artifacts || + verified.repositories !== execution.repositories + ) { + throw new Error( + 'prepare must use the same artifact and repository ports that verified the bundle', + ) + } +} + +function assertTaskInput( + task: AgentCandidateTaskExecution, + delivery: VerifiedAgentCandidate['bundle']['execution']['instructionDelivery'], +): void { + const requiredStrings: Array<[string, string]> = [ + ['executionId', task.executionId], + ['benchmark', task.benchmark], + ['benchmarkVersion', task.benchmarkVersion], + ['taskId', task.taskId], + ['repository identity', task.repository.identity], + ['repository root identity', task.repository.rootIdentity], + ] + for (const [name, value] of requiredStrings) { + if (!value.trim()) throw new Error(`${name} must be non-empty`) + } + if (!/^[A-Za-z0-9._:-]{1,200}$/.test(task.executionId)) { + throw new Error('executionId must be a stable filesystem-neutral identifier') + } + if (!task.instruction || !isWellFormedUnicode(task.instruction)) { + throw new Error('task instruction must be non-empty well-formed Unicode') + } + sha256DigestSchema.parse(task.splitDigest) + agentCandidateWorkspaceSnapshotEvidenceSchema.parse(task.workspace) + agentCandidateExecutionLimitsSchema.parse(task.limits) + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(task.repository.baseCommit)) { + throw new Error('task repository base commit is not a full Git object id') + } + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(task.repository.baseTree)) { + throw new Error('task repository base tree is not a full Git object id') + } + if (task.repository.baseCommit.length !== task.repository.baseTree.length) { + throw new Error('task repository Git object formats disagree') + } + for (const [name, root] of [ + ['execution task root', task.executionRoots.taskRoot], + ['execution candidate root', task.executionRoots.candidateRoot], + ['staging task root', task.stagingRoots.taskRoot], + ['staging candidate root', task.stagingRoots.candidateRoot], + ['staging profile root', task.stagingRoots.profileRoot], + ] as const) { + if (root === undefined) continue + const canonical = name.startsWith('execution') + ? posix.isAbsolute(root) && posix.normalize(root) === root + : isAbsolute(root) && resolveHostPath(root) === root + if (!canonical) throw new Error(`${name} must be a canonical absolute path`) + } + if ( + !Number.isInteger(task.attempt.number) || + !Number.isInteger(task.attempt.maxAttempts) || + task.attempt.number < 1 || + task.attempt.number > task.attempt.maxAttempts || + (task.attempt.retryPolicy === 'none' && task.attempt.maxAttempts !== 1) + ) { + throw new Error('task attempt policy is invalid') + } + const limits = task.limits + if ( + !Number.isInteger(limits.timeoutMs) || + limits.timeoutMs <= 0 || + limits.timeoutMs > MAX_CANDIDATE_TIMER_INTERVAL_MS || + !Number.isInteger(limits.maxSteps) || + limits.maxSteps <= 0 || + !Number.isInteger(limits.maxModelCalls) || + limits.maxModelCalls < 0 || + !Number.isInteger(limits.maxInputTokens) || + limits.maxInputTokens < 0 || + !Number.isInteger(limits.maxOutputTokens) || + limits.maxOutputTokens < 0 || + !Number.isFinite(limits.maxCostUsd) || + limits.maxCostUsd < 0 + ) { + throw new Error('task execution limits are invalid') + } + usdToNanos(limits.maxCostUsd, 'task maxCostUsd') + if (!task.model.requested.trim()) throw new Error('evaluator model request must be non-empty') + if (task.evaluatorTaskContainer) { + if ( + task.evaluatorTaskContainer.source !== 'evaluator-task-container' || + !task.evaluatorTaskContainer.image.trim() || + !task.evaluatorTaskContainer.platform.os.trim() || + !task.evaluatorTaskContainer.platform.architecture.trim() + ) { + throw new Error('evaluator task container evidence is incomplete') + } + sha256DigestSchema.parse(task.evaluatorTaskContainer.indexDigest) + sha256DigestSchema.parse(task.evaluatorTaskContainer.manifestDigest) + agentCandidateContainerSchema.parse({ + image: task.evaluatorTaskContainer.image, + indexDigest: task.evaluatorTaskContainer.indexDigest, + }) + } + if (delivery.kind === 'utf8-file') { + if ( + executionPathsOverlap(task.executionRoots.taskRoot, delivery.path) || + (task.executionRoots.candidateRoot !== undefined && + executionPathsOverlap(task.executionRoots.candidateRoot, delivery.path)) + ) { + throw new Error('task instruction file must remain outside execution workspaces') + } + } +} + +function executionPathsOverlap(left: string, right: string): boolean { + const a = posix.normalize(left) + const b = posix.normalize(right) + return ( + a === b || b.startsWith(a === '/' ? '/' : `${a}/`) || a.startsWith(b === '/' ? '/' : `${b}/`) + ) +} + +function assertDisjointHostStagingRoots(task: AgentCandidateTaskExecution): void { + const roots = [ + task.stagingRoots.taskRoot, + task.stagingRoots.candidateRoot, + task.stagingRoots.profileRoot, + ] + .filter((value): value is string => value !== undefined) + .map((value) => resolveHostPath(value)) + for (let left = 0; left < roots.length; left++) { + for (let right = left + 1; right < roots.length; right++) { + const a = roots[left] + const b = roots[right] + if (a && b && (a === b || isContainedPath(a, b) || isContainedPath(b, a))) { + throw new Error('host task, candidate, and profile staging roots must be disjoint') + } + } + } +} + +function isContainedPath(parent: string, child: string): boolean { + const path = relative(parent, child) + return path !== '' && !path.startsWith('..') && !isAbsolute(path) +} + +async function assertEmptyDirectory(path: string): Promise { + const stats = await lstat(path) + if (!stats.isDirectory() || stats.isSymbolicLink()) { + throw new Error('profile staging root must be a real directory') + } + if ((await readdir(path)).length !== 0) { + throw new Error('profile staging root must be empty before materialization') + } +} + +function materializerHarness(harness: HarnessType): HarnessId { + if (!MATERIALIZER_HARNESSES.has(harness)) { + throw new Error( + `sealed candidate profile materialization is unsupported for harness ${harness}`, + ) + } + return harness as HarnessId +} + +async function resolveContainer( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + ports: AgentCandidateExecutionPorts, +): Promise { + const environment = candidate.bundle.execution.environment + const pinned = environment.kind === 'pinned-container' ? environment.container : undefined + if (environment.kind === 'evaluator-task-container' && !task.evaluatorTaskContainer) { + throw new Error('evaluator-task-container candidate requires an evaluator-owned task image') + } + if (environment.kind === 'pinned-container' && task.evaluatorTaskContainer) { + throw new Error('pinned candidate containers cannot be replaced by a task image') + } + const resolved = await ports.containers.resolve({ + candidate: pinned, + evaluatorTaskContainer: task.evaluatorTaskContainer, + }) + if (resolved.source !== environment.kind) throw new Error('resolved container source drifted') + if (pinned && (resolved.image !== pinned.image || resolved.indexDigest !== pinned.indexDigest)) { + throw new Error('resolved pinned container does not match the candidate image index') + } + if ( + task.evaluatorTaskContainer && + JSON.stringify(resolved) !== JSON.stringify(task.evaluatorTaskContainer) + ) { + throw new Error('resolved task container does not match evaluator-owned image evidence') + } + return resolved +} + +async function resolveModel( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + ports: AgentCandidateExecutionPorts, +): Promise { + const hints = candidate.bundle.profile.model + if (hints?.default !== undefined && hints.default !== task.model.requested) { + throw new Error('candidate model preference conflicts with the evaluator-owned model') + } + if ( + hints?.reasoningEffort !== undefined && + hints.reasoningEffort !== task.model.reasoningEffort + ) { + throw new Error('candidate reasoning effort conflicts with the evaluator-owned effort') + } + const resolved = await ports.models.resolve({ + requested: task.model.requested, + harness: candidate.bundle.execution.harness, + reasoningEffort: task.model.reasoningEffort, + }) + if ( + resolved.requested !== task.model.requested || + resolved.reasoningEffort !== task.model.reasoningEffort + ) { + throw new Error('model resolver drifted from the evaluator-owned request') + } + return resolved +} + +async function prepareMemory( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + ports: AgentCandidateExecutionPorts, + preparationId: string, + expiresAtMs: number, + cleanupTimeoutMs: number, +): Promise<{ + value: AgentCandidateEffectiveMemory + accessDigest?: `sha256:${string}` +}> { + const policy = candidate.bundle.memory + if (policy.mode === 'disabled') return { value: { mode: 'disabled' } } + const seed = policy.seed ? await verifiedArtifactBytes(candidate, policy.seed) : undefined + const executionSegment = canonicalCandidateDigest({ executionId: task.executionId }).slice(7) + const taskSegment = canonicalCandidateDigest({ taskId: task.taskId }).slice(7) + const preparationSegment = canonicalCandidateDigest({ preparationId }).slice(7, 23) + const effectiveNamespace = `candidate/${candidate.bundle.digest.slice(7, 23)}/${executionSegment}/${taskSegment}/${preparationSegment}` + const reset = await withinCandidateCleanupDeadline( + () => + ports.memory.reset({ + executionId: task.executionId, + preparationId, + expiresAtMs, + effectiveNamespace, + ...(seed ? { seed } : {}), + ...(policy.seed ? { seedDigest: policy.seed.sha256 } : {}), + }), + candidateCleanupDeadline(cleanupTimeoutMs), + 'isolated memory reset', + ) + try { + if ( + reset.preparationId !== preparationId || + reset.expiresAtMs !== expiresAtMs || + !/^sha256:[a-f0-9]{64}$/.test(reset.accessDigest) + ) { + throw new Error('isolated memory reservation is not scoped to this preparation') + } + await readVerifiedArtifact(reset.evidence, ports.artifacts) + await verifyWorkspaceSnapshotArtifacts(reset.beforeState, ports.artifacts) + return { + value: { + mode: 'isolated', + scope: 'task', + effectiveNamespace, + reset: { + kind: 'fresh', + evidence: reset.evidence, + emptyStateDigest: reset.emptyStateDigest, + }, + beforeState: reset.beforeState, + ...(policy.seed ? { seedDigest: policy.seed.sha256 } : {}), + }, + accessDigest: reset.accessDigest, + } + } catch (error) { + try { + const closed = await withinCandidateCleanupDeadline( + () => + ports.memory.close({ + executionId: task.executionId, + preparationId, + accessDigest: reset.accessDigest, + effectiveNamespace, + reason: 'preparation-failed', + }), + candidateCleanupDeadline(cleanupTimeoutMs), + 'invalid isolated memory reset cleanup', + ) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('invalid isolated memory reset did not acknowledge closure') + } + } catch (closeError) { + throw new Error('isolated memory preparation and cleanup both failed', { + cause: new AggregateError([error, closeError]), + }) + } + throw error + } +} + +function buildLaunch( + candidate: VerifiedAgentCandidate, + task: AgentCandidateTaskExecution, + profileFlags: AgentCandidateConfigValue[], +): { executable: string; args: AgentCandidateConfigValue[] } { + const launch = candidate.bundle.execution.launch + if (launch.kind === 'container-command') { + return { executable: launch.executable, args: [...(launch.args ?? []), ...profileFlags] } + } + const candidateRoot = task.executionRoots.candidateRoot + if (!candidateRoot) throw new Error('candidate entrypoint requires a container candidate root') + const entrypoint = posix.join(candidateRoot, launch.entrypoint) + const candidateArgs = launch.args ?? [] + if (launch.interpreter) { + return { + executable: launch.interpreter, + args: [{ kind: 'public', value: entrypoint }, ...candidateArgs, ...profileFlags], + } + } + return { executable: entrypoint, args: [...candidateArgs, ...profileFlags] } +} + +function mergePublicEnvironment( + ...records: Array> +): Record { + const output: Record = {} + for (const record of records) { + for (const [name, value] of Object.entries(record)) { + const previous = output[name] + if (previous && previous.value !== value.value) { + throw new Error(`candidate and profile environment disagree on ${name}`) + } + output[name] = value + } + } + return output +} + +function unwrapPublicEnvironment( + values: Record, +): Record { + return Object.fromEntries(Object.entries(values).map(([name, value]) => [name, value.value])) +} + +function modelRoutes( + profile: VerifiedAgentCandidate['bundle']['profile'], + requested: string, +): AgentCandidateExecutionPlanMaterialV1['model']['routes'] { + const routes: AgentCandidateExecutionPlanMaterialV1['model']['routes'] = [ + { kind: 'primary', requested }, + ] + if (profile.model?.small) routes.push({ kind: 'small', requested }) + for (const name of Object.keys(profile.modes ?? {}).sort()) { + if (profile.modes?.[name]?.model) routes.push({ kind: 'mode', name, requested }) + } + for (const name of Object.keys(profile.subagents ?? {}).sort()) { + if (profile.subagents?.[name]?.model) routes.push({ kind: 'subagent', name, requested }) + } + return routes +} + +function candidateEntrypointReceipt( + candidate: VerifiedAgentCandidate, +): { path: string; sha256: `sha256:${string}`; byteLength: number } | undefined { + const launch = candidate.bundle.execution.launch + const workspace = candidate.bundle.execution.workspace + if (launch.kind !== 'candidate-entrypoint' || !workspace) return undefined + const file = workspace.material.files.find((entry) => entry.path === launch.entrypoint) + if (!file) throw new Error('candidate entrypoint is absent from the verified workspace') + return { path: file.path, sha256: file.sha256, byteLength: file.byteLength } +} + +function absoluteExecutionCwd( + cwd: VerifiedAgentCandidate['bundle']['execution']['cwd'], + roots: AgentCandidateTaskExecution['executionRoots'], +): string { + const root = cwd.workspace === 'task' ? roots.taskRoot : roots.candidateRoot + if (!root) throw new Error('candidate cwd is missing its execution workspace root') + const absolute = cwd.path === '.' ? root : posix.join(root, cwd.path) + if (absolute !== root && !absolute.startsWith(`${root}/`)) { + throw new Error('candidate cwd escapes its execution workspace') + } + return absolute +} + +function validateProtectedModelReservation( + reservation: { + preparationId: string + digest: string + expiresAtMs: number + enforcedLimits: { + maxModelCalls: number + maxInputTokens: number + maxOutputTokens: number + maxCostUsd: number + } + }, + expectedLimits: AgentCandidateExecutionLimits, + preparationId: string, + expiresAtMs: number, +): void { + if (!/^sha256:[a-f0-9]{64}$/.test(reservation.digest)) { + throw new Error('protected model reservation has an invalid identity digest') + } + if (reservation.preparationId !== preparationId || reservation.expiresAtMs !== expiresAtMs) { + throw new Error('protected model reservation is not scoped to this preparation') + } + const limits = modelLimits(expectedLimits) + if (canonicalCandidateDigest(reservation.enforcedLimits) !== canonicalCandidateDigest(limits)) { + throw new Error('protected model reservation does not enforce the frozen model limits') + } +} + +function assertEnvironmentDisjoint( + publicEnv: Record, + traceEnv: Record, +): void { + const seen = new Set(Object.keys(publicEnv)) + for (const name of Object.keys(traceEnv)) { + if (seen.has(name)) throw new Error(`evaluator environment binding collides with ${name}`) + seen.add(name) + } +} + +function modelLimits(limits: AgentCandidateExecutionLimits): { + maxModelCalls: number + maxInputTokens: number + maxOutputTokens: number + maxCostUsd: number +} { + return { + maxModelCalls: limits.maxModelCalls, + maxInputTokens: limits.maxInputTokens, + maxOutputTokens: limits.maxOutputTokens, + maxCostUsd: limits.maxCostUsd, + } +} + +function exactProfileExecutorFiles( + sourceFiles: ReadonlyArray<{ relPath: string; content: string; mode?: number }>, + expectedFiles: ReadonlyArray<{ relPath: string; mode: number; contentSha256: string }>, +): Array<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> { + const byPath = new Map(sourceFiles.map((file) => [file.relPath, file])) + if (byPath.size !== sourceFiles.length || sourceFiles.length !== expectedFiles.length) { + throw new Error('profile source files do not match the signed profile plan') + } + return expectedFiles.map((expected) => { + const source = byPath.get(expected.relPath) + const mode = source?.mode ?? 0o644 + const bytes = Buffer.from(source?.content ?? '', 'utf8') + if ( + !source || + (mode !== 0o644 && mode !== 0o755) || + mode !== expected.mode || + sha256Bytes(bytes) !== expected.contentSha256 + ) { + throw new Error('profile source files do not match the signed profile plan') + } + return { path: expected.relPath, mode, bytes: Uint8Array.from(bytes) } + }) +} + +function isWellFormedUnicode(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1) + if (!(next >= 0xdc00 && next <= 0xdfff)) return false + index++ + } else if (code >= 0xdc00 && code <= 0xdfff) { + return false + } + } + return true +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/src/candidate-execution/prepared-state.ts b/src/candidate-execution/prepared-state.ts new file mode 100644 index 00000000..d241ce2d --- /dev/null +++ b/src/candidate-execution/prepared-state.ts @@ -0,0 +1,606 @@ +import { + agentCandidateExecutionPlanEvidenceSchema, + agentCandidateMaterializationReceiptSchema, + agentCandidateProfilePlanEvidenceSchema, +} from '@tangle-network/agent-interface' + +import { verifyMaterializedProfileWorkspace, verifyMaterializedWorkspace } from './artifacts' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + canonicalCandidateDocument, + immutableCandidateValue, + omitTopLevelDigest, + sha256Bytes, +} from './digest' +import { verifyTaskCheckout } from './git-materialize' +import type { + AgentCandidateExecutionPorts, + AgentCandidateExecutorRequest, + AgentCandidateProtectedModelActivation, + AgentCandidateProtectedModelReservation, + PreparedAgentCandidateExecution, +} from './types' +import { CANDIDATE_TRACE_ENV, CANDIDATE_TRACE_TAGS, preparedCandidateBrand } from './types' + +export interface PreparedCandidateState { + ports: AgentCandidateExecutionPorts + bundle: PreparedAgentCandidateExecution['bundle'] + executionId: string + roots: PreparedAgentCandidateExecution['roots'] + profilePlan: PreparedAgentCandidateExecution['profilePlan'] + executionPlan: PreparedAgentCandidateExecution['executionPlan'] + materializationReceipt: PreparedAgentCandidateExecution['materializationReceipt'] + launch: PreparedAgentCandidateExecution['launch'] + instruction: PreparedAgentCandidateExecution['instruction'] + resolvedModel: PreparedAgentCandidateExecution['resolvedModel'] + preparationId: string + reservationExpiresAtMs: number + cleanupTimeoutMs: number + resultTimeoutMs: number + modelReservation: AgentCandidateProtectedModelReservation + executorInputs: { + taskFiles: ReadonlyArray<{ + path: string + mode: 0o644 | 0o755 + bytes: Uint8Array + }> + candidateFiles?: ReadonlyArray<{ + path: string + mode: 0o644 | 0o755 + bytes: Uint8Array + }> + profileFiles: ReadonlyArray<{ + path: string + mode: 0o644 | 0o755 + bytes: Uint8Array + }> + } + memoryReservation?: { + preparationId: string + accessDigest: `sha256:${string}` + expiresAtMs: number + effectiveNamespace: string + } + knowledge?: PreparedAgentCandidateExecution['knowledge'] + trace: PreparedAgentCandidateExecution['trace'] + memory: PreparedAgentCandidateExecution['memory'] +} + +const stateByExecution = new WeakMap() +const lifecycleByExecution = new WeakMap< + PreparedAgentCandidateExecution, + { + status: + | 'prepared' + | 'claiming' + | 'claimed' + | 'running' + | 'settling' + | 'disposing' + | 'succeeded' + | 'failed' + | 'disposal-failed' + | 'cleanup-failed' + | 'disposed' + } +>() + +export function createPreparedCandidateExecution( + input: PreparedCandidateState, +): PreparedAgentCandidateExecution { + const state = detachPreparedCandidateState(input) + assertPrivateCandidateIntegrity(state) + const prepared = Object.freeze({ + bundle: state.bundle, + executionId: state.executionId, + roots: state.roots, + profilePlan: evidenceView(state.profilePlan), + executionPlan: evidenceView(state.executionPlan), + materializationReceipt: state.materializationReceipt, + launch: state.launch, + instruction: bytesView(state.instruction, 'bytes'), + resolvedModel: state.resolvedModel, + ...(state.knowledge ? { knowledge: knowledgeView(state.knowledge) } : {}), + trace: state.trace, + memory: state.memory, + [preparedCandidateBrand]: true as const, + }) + stateByExecution.set(prepared, state) + lifecycleByExecution.set(prepared, { status: 'prepared' }) + return prepared +} + +export function getPreparedCandidateState( + prepared: PreparedAgentCandidateExecution, +): PreparedCandidateState { + const state = stateByExecution.get(prepared) + if (!state || prepared[preparedCandidateBrand] !== true) { + throw new Error('execution must come from prepareAgentCandidateExecution') + } + return state +} + +/** Revalidates the exact private bytes immediately before execution or finalization. */ +export function assertPreparedCandidateIntegrity( + prepared: PreparedAgentCandidateExecution, +): PreparedCandidateState { + const state = getPreparedCandidateState(prepared) + assertPrivateCandidateIntegrity(state) + return state +} + +/** Atomically reserves this in-memory prepared value before the first await. */ +export function beginPreparedCandidateClaim( + prepared: PreparedAgentCandidateExecution, +): PreparedCandidateState { + const state = assertPreparedCandidateIntegrity(prepared) + transitionLifecycle(prepared, ['prepared'], 'claiming') + return state +} + +/** Claim an unexecuted preparation for explicit resource disposal. */ +export function beginPreparedCandidateDisposal( + prepared: PreparedAgentCandidateExecution, +): PreparedCandidateState { + const state = assertPreparedCandidateIntegrity(prepared) + transitionLifecycle(prepared, ['prepared', 'disposal-failed', 'cleanup-failed'], 'disposing') + return state +} + +export function markPreparedCandidateClaimed(prepared: PreparedAgentCandidateExecution): void { + transitionLifecycle(prepared, ['claiming'], 'claimed') +} + +/** Reveal protected values only after the durable claim and workspace checks succeed. */ +export function beginPreparedCandidateRun( + prepared: PreparedAgentCandidateExecution, + modelAccess: AgentCandidateProtectedModelActivation, + memoryAccess?: { env: Readonly> }, +): { state: PreparedCandidateState; request: AgentCandidateExecutorRequest } { + const state = assertPreparedCandidateIntegrity(prepared) + assertProtectedEnvironment(state, modelAccess.env, memoryAccess?.env) + transitionLifecycle(prepared, ['claimed'], 'running') + const completeEnvironment = immutableCandidateValue({ + ...state.launch.env, + ...modelAccess.env, + ...(memoryAccess?.env ?? {}), + ...state.trace.env, + }) + const material = state.executionPlan.value.material + const request = Object.freeze({ + executionId: state.executionId, + inputs: Object.freeze({ + task: workspaceInputView(material.task.workspace, state.executorInputs.taskFiles), + ...(material.candidateWorkspace && state.executorInputs.candidateFiles + ? { + candidate: workspaceInputView( + material.candidateWorkspace, + state.executorInputs.candidateFiles, + ), + } + : {}), + profile: Object.freeze({ + files: Object.freeze( + state.executorInputs.profileFiles.map((file) => profileFileView(file)), + ), + }), + }), + roots: state.roots.execution, + profilePlan: evidenceView(state.profilePlan), + executionPlan: evidenceView(state.executionPlan), + materializationReceipt: state.materializationReceipt, + launch: immutableCandidateValue({ ...state.launch, env: completeEnvironment }), + instruction: bytesView(state.instruction, 'bytes'), + resolvedModel: state.resolvedModel, + hardLimits: Object.freeze({ timeoutMs: state.executionPlan.value.material.limits.timeoutMs }), + observedLimits: Object.freeze({ maxSteps: state.executionPlan.value.material.limits.maxSteps }), + ...(state.knowledge ? { knowledge: knowledgeView(state.knowledge) } : {}), + trace: state.trace, + memory: state.memory, + }) + return { state, request } +} + +export function consumePreparedCandidateExecution( + prepared: PreparedAgentCandidateExecution, + outcome: 'succeeded' | 'failed' | 'disposed' | 'disposal-failed' | 'cleanup-failed', +): void { + transitionLifecycle( + prepared, + outcome === 'disposed' || outcome === 'disposal-failed' ? ['disposing'] : ['settling'], + outcome, + ) +} + +export function beginPreparedCandidateSettlement(prepared: PreparedAgentCandidateExecution): void { + transitionLifecycle(prepared, ['claiming', 'claimed', 'running'], 'settling') +} + +/** Rechecks every mutable staging byte immediately before handing control to an executor. */ +export async function assertPreparedCandidateWorkspaces( + state: PreparedCandidateState, +): Promise { + const plan = state.executionPlan.value.material + await verifyMaterializedWorkspace(state.roots.staging.taskRoot, plan.task.workspace.material, { + ignoredProtectedRootEntries: ['.git', '.sidecar'], + }) + await verifyTaskCheckout(state.roots.staging.taskRoot, plan.task.repository) + await verifyMaterializedProfileWorkspace( + state.roots.staging.profileRoot, + state.profilePlan.value.material, + ) + if (plan.candidateWorkspace) { + const candidateRoot = state.roots.staging.candidateRoot + if (!candidateRoot) throw new Error('prepared candidate staging root is missing') + await verifyMaterializedWorkspace(candidateRoot, plan.candidateWorkspace.material) + } else if (state.roots.staging.candidateRoot !== undefined) { + throw new Error('disabled candidate unexpectedly has a staging root') + } +} + +function detachPreparedCandidateState(input: PreparedCandidateState): PreparedCandidateState { + const profilePlan = immutableCandidateValue( + agentCandidateProfilePlanEvidenceSchema.parse(input.profilePlan.value), + ) + const executionPlan = immutableCandidateValue( + agentCandidateExecutionPlanEvidenceSchema.parse(input.executionPlan.value), + ) + const receiptValue = immutableCandidateValue( + agentCandidateMaterializationReceiptSchema.parse(input.materializationReceipt.value), + ) + const materializationReceipt = canonicalCandidateDocument( + omitTopLevelDigest(receiptValue), + ) as PreparedAgentCandidateExecution['materializationReceipt'] + if (materializationReceipt.digest !== input.materializationReceipt.digest) { + throw new Error('materialization receipt digest changed while sealing prepared execution') + } + return Object.freeze({ + ports: input.ports, + bundle: input.bundle, + executionId: input.executionId, + roots: immutableCandidateValue(input.roots), + profilePlan: Object.freeze({ + value: profilePlan, + bytes: Uint8Array.from(input.profilePlan.bytes), + written: Object.freeze([...input.profilePlan.written]), + }), + executionPlan: Object.freeze({ + value: executionPlan, + bytes: Uint8Array.from(input.executionPlan.bytes), + }), + materializationReceipt, + launch: immutableCandidateValue(input.launch), + instruction: Object.freeze({ + bytes: Uint8Array.from(input.instruction.bytes), + delivery: immutableCandidateValue(input.instruction.delivery), + }), + resolvedModel: immutableCandidateValue(input.resolvedModel), + preparationId: input.preparationId, + reservationExpiresAtMs: input.reservationExpiresAtMs, + cleanupTimeoutMs: input.cleanupTimeoutMs, + resultTimeoutMs: input.resultTimeoutMs, + modelReservation: immutableCandidateValue(input.modelReservation), + executorInputs: Object.freeze({ + taskFiles: immutableExecutorFiles(input.executorInputs.taskFiles), + ...(input.executorInputs.candidateFiles + ? { candidateFiles: immutableExecutorFiles(input.executorInputs.candidateFiles) } + : {}), + profileFiles: Object.freeze( + input.executorInputs.profileFiles.map((file) => + Object.freeze({ ...file, bytes: Uint8Array.from(file.bytes) }), + ), + ), + }), + ...(input.memoryReservation + ? { memoryReservation: immutableCandidateValue(input.memoryReservation) } + : {}), + ...(input.knowledge + ? { + knowledge: Object.freeze({ + snapshotId: input.knowledge.snapshotId, + manifestDigest: input.knowledge.manifestDigest, + manifest: Uint8Array.from(input.knowledge.manifest), + }), + } + : {}), + trace: immutableCandidateValue(input.trace), + memory: immutableCandidateValue(input.memory), + }) +} + +function assertPrivateCandidateIntegrity(state: PreparedCandidateState): void { + const bundleMaterial = omitTopLevelDigest(state.bundle) + if (canonicalCandidateDigest(bundleMaterial) !== state.bundle.digest) { + throw new Error('prepared candidate bundle no longer matches its digest') + } + assertPlanEvidence(state.profilePlan.value, state.profilePlan.bytes, 'profile plan') + assertPlanEvidence(state.executionPlan.value, state.executionPlan.bytes, 'execution plan') + agentCandidateExecutionPlanEvidenceSchema.parse(state.executionPlan.value) + + const receipt = agentCandidateMaterializationReceiptSchema.parse( + state.materializationReceipt.value, + ) + const receiptBytes = canonicalCandidateBytes(omitTopLevelDigest(receipt)) + if ( + canonicalCandidateDigest(omitTopLevelDigest(receipt)) !== state.materializationReceipt.digest || + !Buffer.from(receiptBytes).equals(Buffer.from(state.materializationReceipt.bytes)) + ) { + throw new Error('prepared materialization receipt no longer matches its canonical bytes') + } + + const instruction = state.executionPlan.value.material.task.instruction + if ( + sha256Bytes(state.instruction.bytes) !== instruction.sha256 || + state.instruction.bytes.byteLength !== instruction.byteLength || + JSON.stringify(state.instruction.delivery) !== JSON.stringify(instruction.delivery) + ) { + throw new Error('prepared instruction no longer matches the signed execution plan') + } + if ( + !/^candidate-preparation-v1\.[A-Za-z0-9_-]{43}$/.test(state.preparationId) || + !Number.isSafeInteger(state.reservationExpiresAtMs) || + state.reservationExpiresAtMs <= 0 || + !Number.isSafeInteger(state.cleanupTimeoutMs) || + state.cleanupTimeoutMs <= 0 || + !Number.isSafeInteger(state.resultTimeoutMs) || + state.resultTimeoutMs <= 0 || + JSON.stringify(state.resolvedModel) !== + JSON.stringify(state.executionPlan.value.material.model.resolved) || + state.modelReservation.digest !== state.executionPlan.value.material.model.access.grantDigest || + state.modelReservation.preparationId !== state.preparationId || + state.modelReservation.expiresAtMs !== state.reservationExpiresAtMs || + canonicalCandidateDigest(state.modelReservation.enforcedLimits) !== + canonicalCandidateDigest(modelLimits(state.executionPlan.value.material.limits)) + ) { + throw new Error('prepared model access no longer matches the signed execution plan') + } + assertExecutorInputs(state) + if ( + (state.memory.mode === 'isolated' && !state.memoryReservation) || + (state.memory.mode === 'disabled' && state.memoryReservation) + ) { + throw new Error('prepared memory reservation does not match the signed execution plan') + } + if ( + state.memory.mode === 'isolated' && + state.memoryReservation && + (state.memoryReservation.preparationId !== state.preparationId || + state.memoryReservation.expiresAtMs !== state.reservationExpiresAtMs || + state.memoryReservation.effectiveNamespace !== state.memory.effectiveNamespace) + ) { + throw new Error('prepared memory reservation identity no longer matches the execution') + } + if (JSON.stringify(state.memory) !== JSON.stringify(state.executionPlan.value.material.memory)) { + throw new Error('prepared memory no longer matches the signed execution plan') + } + + const expectedTags = { + [CANDIDATE_TRACE_TAGS.executionId]: state.executionId, + [CANDIDATE_TRACE_TAGS.bundleDigest]: state.bundle.digest, + [CANDIDATE_TRACE_TAGS.executionPlanDigest]: state.executionPlan.value.digest, + [CANDIDATE_TRACE_TAGS.materializationReceiptDigest]: state.materializationReceipt.digest, + } + const expectedTraceEnvironment = { + [CANDIDATE_TRACE_ENV.executionId]: state.executionId, + [CANDIDATE_TRACE_ENV.bundleDigest]: state.bundle.digest, + [CANDIDATE_TRACE_ENV.executionPlanDigest]: state.executionPlan.value.digest, + [CANDIDATE_TRACE_ENV.materializationReceiptDigest]: state.materializationReceipt.digest, + [CANDIDATE_TRACE_ENV.traceRunId]: state.trace.runId, + } + if ( + !state.trace.runId.startsWith( + `${state.executionId}:attempt-${state.executionPlan.value.material.attempt.number}:`, + ) || + canonicalCandidateDigest(state.trace.tags) !== canonicalCandidateDigest(expectedTags) || + canonicalCandidateDigest(state.trace.env) !== canonicalCandidateDigest(expectedTraceEnvironment) + ) { + throw new Error('prepared trace identity no longer matches the signed execution') + } +} + +function assertPlanEvidence( + evidence: + | PreparedAgentCandidateExecution['profilePlan']['value'] + | PreparedAgentCandidateExecution['executionPlan']['value'], + bytes: Uint8Array, + label: string, +): void { + const expected = canonicalCandidateBytes(evidence.material) + if ( + sha256Bytes(expected) !== evidence.digest || + !Buffer.from(expected).equals(Buffer.from(bytes)) || + evidence.artifact.sha256 !== evidence.digest || + evidence.artifact.byteLength !== bytes.byteLength || + !('content' in evidence.artifact) || + !Buffer.from(evidence.artifact.content, 'base64').equals(Buffer.from(bytes)) + ) { + throw new Error(`prepared ${label} no longer matches its canonical bytes`) + } +} + +function evidenceView(evidence: T): T { + const bytes = Uint8Array.from(evidence.bytes) + return Object.freeze({ + ...evidence, + get bytes(): Uint8Array { + return Uint8Array.from(bytes) + }, + }) +} + +function bytesView(value: T, key: 'bytes'): T { + const bytes = Uint8Array.from(value.bytes) + return Object.freeze({ + ...value, + get [key](): Uint8Array { + return Uint8Array.from(bytes) + }, + }) +} + +function knowledgeView( + knowledge: NonNullable, +): NonNullable { + const manifest = Uint8Array.from(knowledge.manifest) + return Object.freeze({ + snapshotId: knowledge.snapshotId, + manifestDigest: knowledge.manifestDigest, + get manifest(): Uint8Array { + return Uint8Array.from(manifest) + }, + }) +} + +function workspaceInputView( + snapshot: AgentCandidateExecutorRequest['inputs']['task']['snapshot'], + sourceFiles: PreparedCandidateState['executorInputs']['taskFiles'], +): AgentCandidateExecutorRequest['inputs']['task'] { + return Object.freeze({ + snapshot, + files: Object.freeze(sourceFiles.map((file) => profileFileView(file))), + }) +} + +function profileFileView( + source: PreparedCandidateState['executorInputs']['profileFiles'][number], +): AgentCandidateExecutorRequest['inputs']['profile']['files'][number] { + const bytes = Uint8Array.from(source.bytes) + return Object.freeze({ + path: source.path, + mode: source.mode, + get bytes(): Uint8Array { + return Uint8Array.from(bytes) + }, + }) +} + +function assertExecutorInputs(state: PreparedCandidateState): void { + const material = state.executionPlan.value.material + assertWorkspaceExecutorFiles(state.executorInputs.taskFiles, material.task.workspace.material) + if (material.candidateWorkspace) { + if (!state.executorInputs.candidateFiles) { + throw new Error('prepared candidate executor files are missing') + } + assertWorkspaceExecutorFiles( + state.executorInputs.candidateFiles, + material.candidateWorkspace.material, + ) + } else if (state.executorInputs.candidateFiles) { + throw new Error('disabled candidate has executor files') + } + + const expectedFiles = state.profilePlan.value.material.files + if (state.executorInputs.profileFiles.length !== expectedFiles.length) { + throw new Error('prepared profile executor files do not match the signed profile plan') + } + for (let index = 0; index < expectedFiles.length; index++) { + const expected = expectedFiles[index] + const actual = state.executorInputs.profileFiles[index] + if ( + !expected || + !actual || + actual.path !== expected.relPath || + actual.mode !== expected.mode || + sha256Bytes(actual.bytes) !== expected.contentSha256 + ) { + throw new Error('prepared profile executor files do not match the signed profile plan') + } + } +} + +function assertWorkspaceExecutorFiles( + actualFiles: PreparedCandidateState['executorInputs']['taskFiles'], + expected: PreparedAgentCandidateExecution['executionPlan']['value']['material']['task']['workspace']['material'], +): void { + if (actualFiles.length !== expected.files.length) { + throw new Error('prepared workspace executor files do not match the signed manifest') + } + for (let index = 0; index < expected.files.length; index++) { + const actual = actualFiles[index] + const planned = expected.files[index] + if ( + !actual || + !planned || + actual.path !== planned.path || + actual.mode !== planned.mode || + actual.bytes.byteLength !== planned.byteLength || + sha256Bytes(actual.bytes) !== planned.sha256 + ) { + throw new Error('prepared workspace executor files do not match the signed manifest') + } + } +} + +function immutableExecutorFiles( + files: PreparedCandidateState['executorInputs']['taskFiles'], +): ReadonlyArray<{ path: string; mode: 0o644 | 0o755; bytes: Uint8Array }> { + return Object.freeze( + files.map((file) => Object.freeze({ ...file, bytes: Uint8Array.from(file.bytes) })), + ) +} + +function modelLimits( + limits: PreparedAgentCandidateExecution['executionPlan']['value']['material']['limits'], +): AgentCandidateProtectedModelReservation['enforcedLimits'] { + return { + maxModelCalls: limits.maxModelCalls, + maxInputTokens: limits.maxInputTokens, + maxOutputTokens: limits.maxOutputTokens, + maxCostUsd: limits.maxCostUsd, + } +} + +function transitionLifecycle( + prepared: PreparedAgentCandidateExecution, + expected: ReadonlyArray< + | 'prepared' + | 'claiming' + | 'claimed' + | 'running' + | 'settling' + | 'disposing' + | 'succeeded' + | 'failed' + | 'disposal-failed' + | 'cleanup-failed' + | 'disposed' + >, + next: + | 'claiming' + | 'claimed' + | 'running' + | 'settling' + | 'disposing' + | 'succeeded' + | 'failed' + | 'disposal-failed' + | 'cleanup-failed' + | 'disposed', +): void { + const lifecycle = lifecycleByExecution.get(prepared) + if (!lifecycle || !expected.includes(lifecycle.status)) { + throw new Error(`prepared candidate execution is already ${lifecycle?.status ?? 'unknown'}`) + } + lifecycle.status = next +} + +function assertProtectedEnvironment( + state: PreparedCandidateState, + modelEnvironment: Readonly>, + memoryEnvironment: Readonly> | undefined, +): void { + const seen = new Set([...Object.keys(state.launch.env), ...Object.keys(state.trace.env)]) + for (const [name, value] of [ + ...Object.entries(modelEnvironment), + ...Object.entries(memoryEnvironment ?? {}), + ]) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || typeof value !== 'string' || value.length < 8) { + throw new Error('protected model activation contains an invalid environment binding') + } + if (seen.has(name)) { + throw new Error(`protected model activation collides with environment binding ${name}`) + } + seen.add(name) + } +} diff --git a/src/candidate-execution/protected-redaction.ts b/src/candidate-execution/protected-redaction.ts new file mode 100644 index 00000000..7392e909 --- /dev/null +++ b/src/candidate-execution/protected-redaction.ts @@ -0,0 +1,176 @@ +import { + DEFAULT_REDACTION_RULES, + REDACTION_VERSION, + type RedactionRule, + redactString, +} from '@tangle-network/agent-eval' + +export interface ProtectedRedactionReport { + version: string + redactionCount: number + byRule: Record +} + +/** Redact protected values at the first persistence boundary, including object keys and bytes. */ +export function redactProtectedValue( + value: T, + protectedValues: readonly string[], +): { value: T; report: ProtectedRedactionReport } { + const report: ProtectedRedactionReport = { + version: REDACTION_VERSION, + redactionCount: 0, + byRule: {}, + } + const rules = protectedRedactionRules(protectedValues) + const redacted = redactNode(value, protectedValues, rules, report) as T + assertNoProtectedEvidence(redacted, protectedValues) + return { value: redacted, report } +} + +export function redactProtectedReason(reason: string, protectedValues: readonly string[]): string { + try { + return redactProtectedValue(reason, protectedValues).value + } catch { + return 'candidate execution failed with a protected error' + } +} + +export function assertNoProtectedEvidence( + value: unknown, + protectedValues: readonly string[], +): void { + if (value instanceof Uint8Array) { + assertNoProtectedBytes(value, protectedValues) + return + } + if (typeof value === 'string') { + for (const protectedValue of protectedValueVariants(protectedValues)) { + if (value.includes(protectedValue)) { + throw new Error('protected value survived candidate evidence redaction') + } + } + if (decodedBase64ContainsProtectedValue(value, protectedValues)) { + throw new Error('base64 protected value survived candidate evidence redaction') + } + return + } + if (Array.isArray(value)) { + for (const entry of value) assertNoProtectedEvidence(entry, protectedValues) + return + } + if (value && typeof value === 'object') { + for (const [key, entry] of Object.entries(value)) { + assertNoProtectedEvidence(key, protectedValues) + assertNoProtectedEvidence(entry, protectedValues) + } + } +} + +export function assertNoProtectedBytes( + bytes: Uint8Array, + protectedValues: readonly string[], +): void { + if (containsProtectedBytes(bytes, protectedValues)) { + throw new Error('protected value survived candidate evidence byte redaction') + } +} + +function redactNode( + value: unknown, + protectedValues: readonly string[], + rules: readonly RedactionRule[], + report: ProtectedRedactionReport, +): unknown { + if (value instanceof Uint8Array) { + if (containsProtectedBytes(value, protectedValues)) { + recordRedaction(report, 'candidate-access-binary', 1) + return Uint8Array.from(Buffer.from('[redacted:candidate-access-binary]', 'utf8')) + } + return Uint8Array.from(value) + } + if (typeof value === 'string') { + if (decodedBase64ContainsProtectedValue(value, protectedValues)) { + recordRedaction(report, 'candidate-access-binary', 1) + return '[redacted:candidate-access-binary]' + } + const redacted = redactString(value, [...rules]) + for (const [rule, count] of Object.entries(redacted.report.byRule)) { + recordRedaction(report, rule, count) + } + return redacted.output + } + if (Array.isArray(value)) { + return value.map((entry) => redactNode(entry, protectedValues, rules, report)) + } + if (value && typeof value === 'object') { + const entries: Array<[string, unknown]> = [] + const seenKeys = new Set() + for (const [key, entry] of Object.entries(value)) { + const redactedKey = redactNode(key, protectedValues, rules, report) + if (typeof redactedKey !== 'string' || seenKeys.has(redactedKey)) { + throw new Error('protected evidence redaction produced an ambiguous object key') + } + seenKeys.add(redactedKey) + entries.push([redactedKey, redactNode(entry, protectedValues, rules, report)]) + } + return Object.fromEntries(entries) + } + return value +} + +function protectedRedactionRules(protectedValues: readonly string[]): RedactionRule[] { + const exactRules = protectedValueVariants(protectedValues) + .sort((left, right) => right.length - left.length || left.localeCompare(right)) + .map((value, index) => ({ + id: `candidate-access-${index}`, + pattern: new RegExp(escapeRegularExpression(value), 'g'), + replacement: '[redacted:candidate-access]', + })) + return [...exactRules, ...DEFAULT_REDACTION_RULES] +} + +function recordRedaction(report: ProtectedRedactionReport, rule: string, count: number): void { + if (count <= 0) return + report.redactionCount += count + report.byRule[rule] = (report.byRule[rule] ?? 0) + count +} + +function containsProtectedBytes(bytes: Uint8Array, protectedValues: readonly string[]): boolean { + const source = Buffer.from(bytes) + return protectedValueVariants(protectedValues).some((value) => + source.includes(Buffer.from(value, 'utf8')), + ) +} + +function decodedBase64ContainsProtectedValue( + value: string, + protectedValues: readonly string[], +): boolean { + if (value.length < 4 || value.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(value)) { + return false + } + try { + return containsProtectedBytes(Buffer.from(value, 'base64'), protectedValues) + } catch { + return false + } +} + +function protectedValueVariants(protectedValues: readonly string[]): string[] { + const variants = new Set() + for (const value of normalizedProtectedValues(protectedValues)) { + variants.add(value) + variants.add(Buffer.from(value, 'utf8').toString('base64')) + variants.add(Buffer.from(value, 'utf8').toString('base64url')) + variants.add(encodeURIComponent(value)) + } + return [...variants] +} + +function normalizedProtectedValues(values: readonly string[]): string[] { + return [...new Set(values.filter((value) => value.length > 0))] +} + +function escapeRegularExpression(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} diff --git a/src/candidate-execution/protected-trace-store.ts b/src/candidate-execution/protected-trace-store.ts new file mode 100644 index 00000000..23ae3cde --- /dev/null +++ b/src/candidate-execution/protected-trace-store.ts @@ -0,0 +1,154 @@ +import { REDACTION_VERSION, type TraceStore } from '@tangle-network/agent-eval' + +import { type ProtectedRedactionReport, redactProtectedValue } from './protected-redaction' + +/** Trace-store proxy that removes live credentials before any write reaches durable storage. */ +export class ProtectedAgentCandidateTraceStore implements TraceStore { + private readonly aggregate: ProtectedRedactionReport = { + version: REDACTION_VERSION, + redactionCount: 0, + byRule: {}, + } + + constructor( + private readonly inner: TraceStore, + private readonly protectedValues: readonly string[], + ) {} + + report(): ProtectedRedactionReport { + return { + version: this.aggregate.version, + redactionCount: this.aggregate.redactionCount, + byRule: { ...this.aggregate.byRule }, + } + } + + async appendRun(run: Parameters[0]): Promise { + await this.inner.appendRun(this.redact(run)) + } + + async updateRun( + runId: Parameters[0], + patch: Parameters[1], + ): Promise { + await this.inner.updateRun(this.redact(runId), this.redact(patch)) + } + + async appendSpan(span: Parameters[0]): Promise { + await this.inner.appendSpan(this.redact(span)) + } + + async updateSpan( + spanId: Parameters[0], + patch: Parameters[1], + ): Promise { + await this.inner.updateSpan(this.redact(spanId), this.redact(patch)) + } + + async appendEvent(event: Parameters[0]): Promise { + await this.inner.appendEvent(this.redact(event)) + } + + async appendArtifact(artifact: Parameters[0]): Promise { + await this.inner.appendArtifact(this.redact(artifact)) + } + + async appendBudgetEntry(entry: Parameters[0]): Promise { + await this.inner.appendBudgetEntry(this.redact(entry)) + } + + getRun(...args: Parameters): ReturnType { + return this.inner.getRun(...args) + } + + listRuns(...args: Parameters): ReturnType { + return this.inner.listRuns(...args) + } + + spans(...args: Parameters): ReturnType { + return this.inner.spans(...args) + } + + events(...args: Parameters): ReturnType { + return this.inner.events(...args) + } + + budget(...args: Parameters): ReturnType { + return this.inner.budget(...args) + } + + artifacts(...args: Parameters): ReturnType { + return this.inner.artifacts(...args) + } + + private redact(value: T): T { + const redacted = redactProtectedValue(value, this.protectedValues) + this.aggregate.version = redacted.report.version + this.aggregate.redactionCount += redacted.report.redactionCount + for (const [rule, count] of Object.entries(redacted.report.byRule)) { + this.aggregate.byRule[rule] = (this.aggregate.byRule[rule] ?? 0) + count + } + return redacted.value + } +} + +/** Recovery can read existing trace state but must never accept new unredactable writes. */ +export class RecoveryAgentCandidateTraceStore implements TraceStore { + constructor(private readonly inner: TraceStore) {} + + appendRun(): Promise { + return this.rejectWrite() + } + + updateRun(): Promise { + return this.rejectWrite() + } + + appendSpan(): Promise { + return this.rejectWrite() + } + + updateSpan(): Promise { + return this.rejectWrite() + } + + appendEvent(): Promise { + return this.rejectWrite() + } + + appendArtifact(): Promise { + return this.rejectWrite() + } + + appendBudgetEntry(): Promise { + return this.rejectWrite() + } + + getRun(...args: Parameters): ReturnType { + return this.inner.getRun(...args) + } + + listRuns(...args: Parameters): ReturnType { + return this.inner.listRuns(...args) + } + + spans(...args: Parameters): ReturnType { + return this.inner.spans(...args) + } + + events(...args: Parameters): ReturnType { + return this.inner.events(...args) + } + + budget(...args: Parameters): ReturnType { + return this.inner.budget(...args) + } + + artifacts(...args: Parameters): ReturnType { + return this.inner.artifacts(...args) + } + + private rejectWrite(): Promise { + return Promise.reject(new Error('expired candidate recovery cannot append trace evidence')) + } +} diff --git a/src/candidate-execution/recover.ts b/src/candidate-execution/recover.ts new file mode 100644 index 00000000..cff11369 --- /dev/null +++ b/src/candidate-execution/recover.ts @@ -0,0 +1,175 @@ +import type { TraceStore } from '@tangle-network/agent-eval' + +import type { + AgentCandidateExecutionAttemptRef, + AgentCandidateExecutionClaimStore, + AgentCandidateExecutionFinishResult, +} from './claim' +import { + candidateCleanupDeadline, + candidateCleanupTimeout, + withinCandidateCleanupDeadline, +} from './cleanup' +import { sealAgentCandidateExecutorFinalCapture } from './executor-capture' +import { sealAgentCandidateModelSettlement } from './model-settlement' +import { persistCandidateModelSettlementEvidence } from './outcome-evidence' +import { RecoveryAgentCandidateTraceStore } from './protected-trace-store' +import type { + AgentCandidateExecutionPorts, + AgentCandidateExecutorPort, + AgentCandidateOutputArtifactPort, +} from './types' + +export interface RecoverExpiredAgentCandidateOptions { + attempt: AgentCandidateExecutionAttemptRef + claimStore: AgentCandidateExecutionClaimStore + executor: AgentCandidateExecutorPort + traceStore: TraceStore + ports: Pick + outputArtifacts: AgentCandidateOutputArtifactPort + cleanupTimeoutMs?: number + /** Evaluator clock; must be the same clock used by the claim store. */ + now?: () => number +} + +/** Close an expired crashed attempt from persisted non-secret handles, then record failure. */ +export async function recoverExpiredAgentCandidateExecution( + options: RecoverExpiredAgentCandidateOptions, +): Promise { + const record = await options.claimStore.getAttempt(options.attempt) + if (!record) throw new Error('candidate execution recovery attempt is missing') + if (record.terminal) { + return Object.freeze({ finished: false, terminal: record.terminal, exactReplay: true }) + } + const cleanupTimeoutMs = candidateCleanupTimeout( + options.cleanupTimeoutMs ?? record.claim.cleanup.cleanupTimeoutMs, + ) + if (cleanupTimeoutMs > record.claim.cleanup.cleanupTimeoutMs) { + throw new Error('recovery cleanup timeout exceeds the frozen preparation bound') + } + + const now = options.now ?? Date.now + if (now() < record.claim.leaseExpiresAtMs) { + throw new Error('candidate execution lease has not expired') + } + + const cleanupDeadlineAtMs = candidateCleanupDeadline(cleanupTimeoutMs) + const controller = new AbortController() + controller.abort(new Error('recovering an expired candidate execution')) + const recoveryTraceStore = new RecoveryAgentCandidateTraceStore(options.traceStore) + const processClosure = withinCandidateCleanupDeadline( + async () => { + const stopped = await options.executor.stopAndCapture( + { + executionId: record.claim.executionId, + executionPlanDigest: record.claim.executionPlanDigest, + }, + { + traceStore: recoveryTraceStore, + reason: 'failed', + signal: controller.signal, + deadlineAtMs: record.claim.leaseExpiresAtMs, + }, + ) + sealAgentCandidateExecutorFinalCapture(stopped) + return { stopped: true as const } + }, + cleanupDeadlineAtMs, + 'expired candidate process termination', + ) + + const modelClosure = withinCandidateCleanupDeadline( + async () => + sealAgentCandidateModelSettlement( + await options.ports.models.settleGrant({ + executionId: record.claim.executionId, + preparationId: record.claim.cleanup.preparationId, + grantDigest: record.claim.cleanup.modelGrantDigest, + resolved: record.claim.cleanup.resolvedModel, + reason: 'failed', + }), + { + preparationId: record.claim.cleanup.preparationId, + grantDigest: record.claim.cleanup.modelGrantDigest, + model: record.claim.cleanup.resolvedModel.model, + }, + ), + cleanupDeadlineAtMs, + 'expired candidate model settlement', + ) + + const memoryClosure = record.claim.cleanup.memory + ? withinCandidateCleanupDeadline( + async () => { + const memory = record.claim.cleanup.memory + if (!memory) throw new Error('expired candidate memory handle is missing') + const closed = await options.ports.memory.close({ + executionId: record.claim.executionId, + preparationId: record.claim.cleanup.preparationId, + accessDigest: memory.accessDigest, + effectiveNamespace: memory.effectiveNamespace, + reason: 'failed', + }) + if (closed.closed !== true || Object.keys(closed).some((key) => key !== 'closed')) { + throw new Error('expired candidate memory did not acknowledge closure') + } + return { closed: true as const } + }, + cleanupDeadlineAtMs, + 'expired candidate memory closure', + ) + : undefined + + const operations = [processClosure, modelClosure, ...(memoryClosure ? [memoryClosure] : [])] + const outcomes = await Promise.allSettled(operations) + const failures = outcomes + .filter((outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected') + .map((outcome) => outcome.reason) + if (failures.length > 0) { + throw new AggregateError(failures, 'expired candidate cleanup could not be proven') + } + + const model = await modelClosure + const modelSettlement = await withinCandidateCleanupDeadline( + () => + persistCandidateModelSettlementEvidence( + { + executionId: record.claim.executionId, + executionPlanDigest: record.claim.executionPlanDigest, + resolvedModel: record.claim.cleanup.resolvedModel, + }, + model, + options.outputArtifacts, + ), + cleanupDeadlineAtMs, + 'expired candidate model-settlement persistence', + ) + const memory = record.claim.cleanup.memory + return await options.claimStore.recoverExpired(options.attempt, { + failureClass: + record.phase === 'claimed' && model.fixedUsage.modelCalls === 0 + ? 'pre-model-infrastructure' + : 'unknown', + usage: model.fixedUsage, + modelSettlement: modelSettlement.artifact, + process: { + stopped: true, + executionPlanDigest: record.claim.executionPlanDigest, + }, + model: { + closed: true, + preparationId: record.claim.cleanup.preparationId, + grantDigest: record.claim.cleanup.modelGrantDigest, + }, + ...(memory + ? { + memory: { + closed: true, + preparationId: record.claim.cleanup.preparationId, + accessDigest: memory.accessDigest, + effectiveNamespace: memory.effectiveNamespace, + }, + } + : {}), + }) +} diff --git a/src/candidate-execution/types.ts b/src/candidate-execution/types.ts new file mode 100644 index 00000000..38fa6256 --- /dev/null +++ b/src/candidate-execution/types.ts @@ -0,0 +1,560 @@ +import type { BenchmarkEvaluation, TraceStore } from '@tangle-network/agent-eval' +import type { + AgentCandidateArtifactRef, + AgentCandidateAttemptPolicy, + AgentCandidateBundle, + AgentCandidateCapturedArtifact, + AgentCandidateContainer, + AgentCandidateEffectiveMemory, + AgentCandidateExecutionLimits, + AgentCandidateExecutionPlanEvidence, + AgentCandidateGitHubRepository, + AgentCandidateInstructionDelivery, + AgentCandidateMaterializationReceipt, + AgentCandidateMemoryReceipt, + AgentCandidateOciPlatform, + AgentCandidateProfilePlanEvidence, + AgentCandidateResolvedModel, + AgentCandidateRunReceiptV2, + AgentCandidateSpend, + AgentCandidateTaskOutcomeEvidence, + AgentCandidateTermination, + AgentCandidateWorkspaceManifestMaterialV1, + AgentCandidateWorkspaceSnapshotEvidence, + ReasoningEffort, + Sha256Digest, +} from '@tangle-network/agent-interface' + +export const verifiedCandidateBrand: unique symbol = Symbol('verifiedAgentCandidate') +export const preparedCandidateBrand: unique symbol = Symbol('preparedAgentCandidate') +export const verifiedTaskOutcomeBrand: unique symbol = Symbol('verifiedTaskOutcome') + +/** Reads one content-addressed object from the closed S3/IPFS locator set. */ +export interface AgentCandidateArtifactPort { + read(ref: AgentCandidateArtifactRef): Promise +} + +export type AgentCandidateOutputPurpose = + | 'task-manifest' + | 'task-archive' + | 'task-patch' + | 'task-outcome' + | 'memory-after-manifest' + | 'memory-after-archive' + | 'grader-evidence' + | 'benchmark-result' + | 'model-settlement' + | 'trace' + | 'run-receipt' + | 'failure-evidence' + +/** Durable content-addressed evidence store controlled only by the evaluator. */ +export interface AgentCandidateOutputArtifactPort extends AgentCandidateArtifactPort { + /** Must be idempotent for identical bytes and return only a durable S3/IPFS locator. */ + put(input: { + executionId: string + purpose: AgentCandidateOutputPurpose + bytes: Uint8Array + /** Abort must prevent durable publication when it happens before resolution. */ + signal?: AbortSignal + }): Promise +} + +/** Resolves a declared GitHub repository to an already-present local Git object store. */ +export interface AgentCandidateRepositoryPort { + resolve(repository: AgentCandidateGitHubRepository): Promise +} + +export interface AgentCandidateVerificationPorts { + artifacts: AgentCandidateArtifactPort + repositories: AgentCandidateRepositoryPort +} + +/** + * Materializes an already-verified workspace archive. + * + * The runtime independently scans every resulting byte, mode, and path against + * the signed manifest after this returns. Implementations may therefore unpack + * any archive encoding, or no-op when the exact workspace is already present. + */ +export interface AgentCandidateWorkspacePort { + materialize(input: { + role: 'task' | 'candidate' | 'memory' + snapshot: AgentCandidateWorkspaceSnapshotEvidence + archive: Uint8Array + destination: string + }): Promise +} + +export interface ResolvedAgentCandidateContainer { + source: 'pinned-container' | 'evaluator-task-container' + image: string + indexDigest: Sha256Digest + manifestDigest: Sha256Digest + platform: AgentCandidateOciPlatform +} + +export interface AgentCandidateContainerPort { + resolve(input: { + candidate: AgentCandidateContainer | undefined + evaluatorTaskContainer: ResolvedAgentCandidateContainer | undefined + }): Promise +} + +export interface AgentCandidateModelPort { + resolve(input: { + requested: string + harness: AgentCandidateBundle['execution']['harness'] + reasoningEffort: NonNullable['reasoningEffort'] + }): Promise + /** + * Reserve a stable access identity without creating a live credential. + * The reservation is scoped to `preparationId` and must automatically expire + * at `expiresAtMs`, even if this call returns ambiguously to the runtime. + */ + reserveGrant(input: { + executionId: string + preparationId: string + expiresAtMs: number + attempt: AgentCandidateAttemptPolicy + bundleDigest: Sha256Digest + resolved: AgentCandidateResolvedModel + limits: AgentCandidateModelLimits + }): Promise + /** Create the live scoped credential only after the execution attempt is durably claimed. */ + activateGrant(input: { + executionId: string + preparationId: string + grantDigest: Sha256Digest + resolved: AgentCandidateResolvedModel + deadlineAtMs: number + }): Promise + /** + * Atomically revoke the grant, drain in-flight calls, and return its immutable final ledger. + * This operation must be idempotent for the exact preparation and must also + * settle a reservation that was never activated. It must never affect a + * different preparation, even when both reservations report the same digest. + */ + settleGrant(input: { + executionId: string + preparationId: string + grantDigest: Sha256Digest + resolved: AgentCandidateResolvedModel + reason: 'completed' | 'failed' | 'timeout' | 'replayed' | 'preparation-failed' | 'abandoned' + }): Promise +} + +/** Limits mechanically enforced by the evaluator-owned model gateway. */ +export type AgentCandidateModelLimits = Pick< + AgentCandidateExecutionLimits, + 'maxModelCalls' | 'maxInputTokens' | 'maxOutputTokens' | 'maxCostUsd' +> + +export interface AgentCandidateProtectedModelReservation { + preparationId: string + digest: Sha256Digest + /** Evaluator service must expire and revoke this reservation at this epoch millisecond. */ + expiresAtMs: number + /** The gateway must stop calls before any one of these limits is exceeded. */ + enforcedLimits: AgentCandidateModelLimits +} + +export interface AgentCandidateProtectedModelActivation { + /** Injected only into the trusted executor after all pre-launch checks pass. */ + env: Readonly> +} + +/** One evaluator-gateway call in the final, revoked model-access ledger. */ +export interface AgentCandidateProtectedModelCall { + callId: string + /** Exact protected agent-eval LLM span produced for this call. */ + traceSpanId: string + model: string + inputTokens: number + outputTokens: number + cachedInputTokens?: number + reasoningTokens?: number + /** Integer billionths of one US dollar; avoids floating-point ledger drift. */ + costUsdNanos: number +} + +export interface AgentCandidateProtectedModelSettlement { + preparationId: string + grantDigest: Sha256Digest + closed: true + calls: readonly AgentCandidateProtectedModelCall[] +} + +export interface AgentCandidateMemoryResetResult { + preparationId: string + accessDigest: Sha256Digest + expiresAtMs: number + evidence: AgentCandidateCapturedArtifact + emptyStateDigest: Sha256Digest + beforeState: AgentCandidateWorkspaceSnapshotEvidence +} + +export interface AgentCandidateMemoryPort { + /** + * Reset and reserve exact task memory without returning live access. + * The service must scope the reservation to `preparationId`, automatically + * revoke it at `expiresAtMs`, and never reuse it for another preparation. + */ + reset(input: { + executionId: string + preparationId: string + expiresAtMs: number + effectiveNamespace: string + seed?: Uint8Array + seedDigest?: Sha256Digest + }): Promise + /** + * Create live scoped access only after the execution attempt is durably claimed. + * Activation must match the exact preparation/access pair and may not extend expiry. + */ + activate(input: { + executionId: string + preparationId: string + accessDigest: Sha256Digest + effectiveNamespace: string + deadlineAtMs: number + }): Promise<{ env: Readonly> }> + /** + * Revoke evaluator-owned access after process death or a failed preparation. + * Must be idempotent and concurrency-safe for the exact preparation/access + * pair and must never close a different preparation. + */ + close(input: { + executionId: string + preparationId: string + accessDigest: Sha256Digest + effectiveNamespace: string + reason: 'completed' | 'failed' | 'timeout' | 'replayed' | 'preparation-failed' | 'abandoned' + }): Promise<{ closed: true }> +} + +export interface AgentCandidateExecutionPorts extends AgentCandidateVerificationPorts { + workspaces: AgentCandidateWorkspacePort + containers: AgentCandidateContainerPort + models: AgentCandidateModelPort + memory: AgentCandidateMemoryPort +} + +export interface AgentCandidateTaskExecution { + executionId: string + benchmark: string + benchmarkVersion: string + taskId: string + splitDigest: Sha256Digest + /** Exact agent-visible task instruction. The runtime rejects malformed Unicode. */ + instruction: string + repository: { + identity: string + rootIdentity: string + baseCommit: string + baseTree: string + } + attempt: AgentCandidateAttemptPolicy + model: { + requested: string + reasoningEffort: ReasoningEffort + } + /** Absolute paths inside the evaluator-owned execution environment. */ + executionRoots: { + taskRoot: string + candidateRoot?: string + } + /** Host-side staging roots. These are verified but never signed as container paths. */ + stagingRoots: { + taskRoot: string + candidateRoot?: string + profileRoot: string + } + workspace: AgentCandidateWorkspaceSnapshotEvidence + evaluatorTaskContainer?: ResolvedAgentCandidateContainer + limits: AgentCandidateExecutionLimits +} + +export interface VerifiedAgentCandidate { + readonly bundle: AgentCandidateBundle + readonly materializedTree?: string + readonly [verifiedCandidateBrand]: true +} + +export interface CanonicalCandidateDocument { + readonly value: T + /** Canonical UTF-8 bytes of `value` with its top-level digest omitted. */ + readonly bytes: Uint8Array + readonly digest: Sha256Digest +} + +export interface PreparedAgentCandidateLaunch { + executable: string + /** Complete fixed argv, including profile materializer flags but excluding task delivery. */ + args: readonly string[] + env: Readonly> + /** Informational subset already present at the tail of `args`; executors must not append twice. */ + flags: readonly string[] + cwd: string +} + +export interface PreparedAgentCandidateInstruction { + bytes: Uint8Array + delivery: AgentCandidateInstructionDelivery +} + +export interface PreparedAgentCandidateTrace { + runId: string + tags: Readonly> + env: Readonly> +} + +export interface PreparedAgentCandidateExecution { + readonly bundle: AgentCandidateBundle + readonly executionId: string + readonly roots: { + execution: { + taskRoot: string + candidateRoot?: string + } + staging: { + taskRoot: string + candidateRoot?: string + profileRoot: string + } + } + readonly profilePlan: { + value: AgentCandidateProfilePlanEvidence + bytes: Uint8Array + written: readonly string[] + } + readonly executionPlan: { + value: AgentCandidateExecutionPlanEvidence + bytes: Uint8Array + } + readonly materializationReceipt: CanonicalCandidateDocument + readonly launch: PreparedAgentCandidateLaunch + readonly instruction: PreparedAgentCandidateInstruction + readonly resolvedModel: AgentCandidateResolvedModel + readonly knowledge?: { + snapshotId: string + manifestDigest: Sha256Digest + manifest: Uint8Array + } + readonly trace: PreparedAgentCandidateTrace + readonly memory: AgentCandidateEffectiveMemory + readonly [preparedCandidateBrand]: true +} + +export interface AgentCandidateProtectedRunCapture { + executionId: string + termination: AgentCandidateTermination +} + +/** Raw evaluator capture made only after the candidate process is dead. */ +export interface AgentCandidateExecutorTaskOutcomeCapture { + /** Claimed final tree. The runtime recomputes it independently from `gitDiff`. */ + resultTree: string + /** Complete evaluator-captured workspace description after candidate execution. */ + afterState: AgentCandidateWorkspaceManifestMaterialV1 + /** Reproducible workspace archive corresponding to `afterState`. */ + archive: Uint8Array + /** Exact binary patch from the signed task base to `afterState`. */ + gitDiff: Uint8Array +} + +/** Raw isolated-memory capture made only after access has been revoked. */ +export interface AgentCandidateExecutorMemoryCapture { + readonly afterState: AgentCandidateWorkspaceManifestMaterialV1 + readonly archive: Uint8Array +} + +/** Idempotent executor result after process death and trace drain. */ +export interface AgentCandidateExecutorFinalCapture { + readonly stopped: true + readonly taskOutcome?: AgentCandidateExecutorTaskOutcomeCapture + /** Required only when the prepared candidate uses isolated task memory. */ + readonly memoryAfter?: AgentCandidateExecutorMemoryCapture +} + +/** Branded task outcome that has survived independent patch and tree verification. */ +export interface VerifiedAgentCandidateTaskOutcome { + readonly evidence: AgentCandidateTaskOutcomeEvidence & { + readonly artifact: AgentCandidateArtifactRef + } + readonly patch: Uint8Array + readonly [verifiedTaskOutcomeBrand]: true +} + +/** + * Evaluator-owned executable grader, pinned by immutable implementation bytes. + * + * `run` is an isolation boundary, not an arbitrary scoring callback. The + * implementation admitted to that boundary is supplied by the runtime after + * artifact verification. Implementations must derive every returned binding + * digest from the bytes and task outcome they actually admitted, rather than + * copying an expected digest from ambient configuration. + */ +export interface AgentCandidateBenchmarkGraderPort { + readonly name: string + readonly version: string + readonly artifact: AgentCandidateArtifactRef + run(input: { + readonly executionId: string + readonly termination: AgentCandidateTermination + readonly outcome: VerifiedAgentCandidateTaskOutcome + /** Exact verified artifact bytes. Each read returns a detached copy. */ + readonly implementation: { + readonly byteLength: number + readonly bytes: Uint8Array + } + /** Frozen result deadline; runners must stop work and side effects when aborted. */ + readonly signal: AbortSignal + }): Promise<{ + readonly evaluation: BenchmarkEvaluation + /** Raw grader output needed to audit or reproduce the normalized result. */ + readonly evidence: Uint8Array + /** Runtime-checked binding between admitted code, task input, and raw output. */ + readonly binding: { + /** Digest computed from the implementation bytes admitted to execution. */ + readonly implementationDigest: Sha256Digest + /** Digest of the exact runtime-verified task outcome graded by this run. */ + readonly taskOutcomeDigest: Sha256Digest + /** Digest computed from `evidence` before it leaves the execution boundary. */ + readonly outputDigest: Sha256Digest + } + }> +} + +/** One detached request passed to the trusted environment-specific executor. */ +export interface AgentCandidateExecutorRequest { + readonly executionId: string + /** Immutable bytes from which the executor creates fresh isolated workspaces. */ + readonly inputs: { + readonly task: AgentCandidateExecutorWorkspaceInput + readonly candidate?: AgentCandidateExecutorWorkspaceInput + readonly profile: { + readonly files: readonly AgentCandidateExecutorProfileFile[] + } + } + readonly roots: PreparedAgentCandidateExecution['roots']['execution'] + readonly profilePlan: PreparedAgentCandidateExecution['profilePlan'] + readonly executionPlan: PreparedAgentCandidateExecution['executionPlan'] + readonly materializationReceipt: CanonicalCandidateDocument + readonly launch: PreparedAgentCandidateLaunch + readonly instruction: PreparedAgentCandidateInstruction + readonly resolvedModel: AgentCandidateResolvedModel + /** Mechanically enforced by the runtime plus executor process-death acknowledgement. */ + readonly hardLimits: Pick + /** Validity bound checked against protected traces; generic black-box executors cannot preempt it. */ + readonly observedLimits: Pick + readonly knowledge?: PreparedAgentCandidateExecution['knowledge'] + readonly trace: PreparedAgentCandidateTrace + readonly memory: AgentCandidateEffectiveMemory +} + +/** + * Executes one prepared request inside an evaluator-owned isolation boundary. + * + * `request.launch.env` is the complete allowlisted environment, including + * protected model, memory, and trace bindings. Implementations must not merge + * ambient host variables into it. The returned capture deliberately contains + * no candidate-authored usage or score fields. + */ +export interface AgentCandidateExecutorPort { + execute( + request: AgentCandidateExecutorRequest, + context: { + traceStore: TraceStore + /** Aborted by the runtime at the exact frozen wall-time deadline. */ + signal: AbortSignal + /** Absolute epoch-millisecond deadline owned by the runtime. */ + deadlineAtMs: number + }, + ): Promise + /** + * Kill any process/container still associated with the request, drain trace + * writes, and capture the final task workspace before teardown. + * The runtime calls this on success, failure, and timeout before model settlement. + * Implementations must be idempotent and concurrency-safe for this exact + * execution/plan pair because a fresh worker may repeat crash recovery. + */ + stopAndCapture( + request: AgentCandidateExecutorStopRequest, + context: { + traceStore: TraceStore + reason: 'completed' | 'failed' | 'timeout' + /** Aborted at the frozen execution deadline or evaluator cleanup deadline. */ + signal: AbortSignal + /** Absolute execution deadline; a later stop acknowledgement cannot produce success. */ + deadlineAtMs: number + }, + ): Promise +} + +/** Opaque process identity used for termination without re-exposing launch credentials. */ +export interface AgentCandidateExecutorStopRequest { + readonly executionId: string + readonly executionPlanDigest: Sha256Digest +} + +export interface AgentCandidateExecutorWorkspaceInput { + readonly snapshot: AgentCandidateWorkspaceSnapshotEvidence + readonly files: readonly AgentCandidateExecutorWorkspaceFile[] +} + +export interface AgentCandidateExecutorWorkspaceFile { + readonly path: string + readonly mode: 0o644 | 0o755 + readonly bytes: Uint8Array +} + +export interface AgentCandidateExecutorProfileFile { + readonly path: string + readonly mode: 0o644 | 0o755 + readonly bytes: Uint8Array +} + +export type AgentCandidateRunFinalization = + | { + succeeded: true + receipt: CanonicalCandidateDocument + artifacts: { + modelSettlement: AgentCandidateArtifactRef + taskOutcome: AgentCandidateArtifactRef + benchmarkResult: AgentCandidateArtifactRef + runReceipt: AgentCandidateArtifactRef + } + } + | { + succeeded: false + reason: string + partial: { + executionId: string + bundleDigest: Sha256Digest + executionPlanDigest: Sha256Digest + materializationReceiptDigest: Sha256Digest + termination?: AgentCandidateTermination + } + /** Independent evaluator-gateway usage, even when execution or trace capture failed. */ + usage: AgentCandidateSpend | null + } + +/** Protected trace tags that bind a run to one prepared candidate execution. */ +export const CANDIDATE_TRACE_TAGS = { + executionId: 'tangle.candidate.execution_id', + bundleDigest: 'tangle.candidate.bundle_digest', + executionPlanDigest: 'tangle.candidate.execution_plan_digest', + materializationReceiptDigest: 'tangle.candidate.materialization_receipt_digest', +} as const + +/** Environment keys used to propagate immutable candidate trace identity. */ +export const CANDIDATE_TRACE_ENV = { + executionId: 'TANGLE_CANDIDATE_EXECUTION_ID', + bundleDigest: 'TANGLE_CANDIDATE_BUNDLE_DIGEST', + executionPlanDigest: 'TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST', + materializationReceiptDigest: 'TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST', + traceRunId: 'TANGLE_TRACE_RUN_ID', +} as const + +export type PreparedMemoryReceipt = AgentCandidateMemoryReceipt diff --git a/src/candidate-execution/verify.ts b/src/candidate-execution/verify.ts new file mode 100644 index 00000000..befde8e2 --- /dev/null +++ b/src/candidate-execution/verify.ts @@ -0,0 +1,167 @@ +import type { + AgentCandidateBundle, + AgentCandidateCapturedArtifact, + AgentCandidateResourceRef, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { agentCandidateBundleSchema } from '@tangle-network/agent-interface' + +import { + artifactCacheKey, + readVerifiedArtifact, + verifyBytes, + verifyWorkspaceSnapshotArtifacts, +} from './artifacts' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + deepFreezeCandidate, + omitTopLevelDigest, +} from './digest' +import { readCandidateGitHubResource, verifyCandidateCode } from './git-materialize' +import { + type AgentCandidateVerificationPorts, + type VerifiedAgentCandidate, + verifiedCandidateBrand, +} from './types' + +interface VerifiedCandidateState { + ports: AgentCandidateVerificationPorts + artifactBytes: Map + resourceBytes: Map +} + +const verifiedCandidateState = new WeakMap() + +/** Verifies every digest, resource, workspace, and Git object in a candidate bundle. */ +export async function verifyAgentCandidateBundle( + input: unknown, + ports: AgentCandidateVerificationPorts, +): Promise { + const parsed = agentCandidateBundleSchema.parse(input) + const withoutDigest = omitTopLevelDigest(parsed) + const actualDigest = canonicalCandidateDigest(withoutDigest) + if (actualDigest !== parsed.digest) { + throw new Error(`candidate bundle digest ${parsed.digest} does not match ${actualDigest}`) + } + const canonicalBytes = canonicalCandidateBytes(withoutDigest) + verifyBytes(canonicalBytes, parsed.digest, canonicalBytes.byteLength, 'candidate bundle') + + const artifactBytes = new Map() + const readArtifact = async (artifact: AgentCandidateCapturedArtifact): Promise => { + const key = artifactCacheKey(artifact) + const existing = artifactBytes.get(key) + if (existing) return Uint8Array.from(existing) + const bytes = await readVerifiedArtifact(artifact, ports.artifacts) + artifactBytes.set(key, Uint8Array.from(bytes)) + return bytes + } + + let patchBytes: Uint8Array | undefined + if (parsed.code.kind === 'git-patch') { + patchBytes = await readArtifact(parsed.code.patch.artifact) + } + const materializedTree = await verifyCandidateCode(parsed.code, ports.repositories, patchBytes) + + const resourceBytes = new Map() + for (const resource of candidateResources(parsed)) { + const bytes = + resource.kind === 'inline' + ? Buffer.from(resource.content, 'utf8') + : await readCandidateGitHubResource(resource, ports.repositories) + verifyBytes( + bytes, + resource.sha256, + resource.byteLength, + `candidate resource ${resource.name ?? (resource.kind === 'github' ? resource.path : '')}`, + ) + resourceBytes.set(resourceKey(resource), Uint8Array.from(bytes)) + resourceBytes.set(resource.sha256, Uint8Array.from(bytes)) + } + + if (parsed.execution.workspace) { + const workspace = await verifyWorkspaceSnapshotArtifacts( + parsed.execution.workspace, + ports.artifacts, + ) + artifactBytes.set(artifactCacheKey(parsed.execution.workspace.manifest), workspace.manifest) + artifactBytes.set(artifactCacheKey(parsed.execution.workspace.archive), workspace.archive) + } + if (parsed.knowledge) await readArtifact(parsed.knowledge.manifest) + if (parsed.memory.mode === 'isolated' && parsed.memory.seed) + await readArtifact(parsed.memory.seed) + + deepFreezeCandidate(parsed) + const verified = Object.freeze({ + bundle: parsed, + ...(materializedTree === undefined ? {} : { materializedTree }), + [verifiedCandidateBrand]: true as const, + }) + verifiedCandidateState.set(verified, { ports, artifactBytes, resourceBytes }) + return verified +} + +export function getVerifiedCandidateState( + candidate: VerifiedAgentCandidate, +): VerifiedCandidateState { + const state = verifiedCandidateState.get(candidate) + if (!state || candidate[verifiedCandidateBrand] !== true) { + throw new Error('candidate must come from verifyAgentCandidateBundle') + } + return state +} + +export async function verifiedArtifactBytes( + candidate: VerifiedAgentCandidate, + artifact: AgentCandidateCapturedArtifact, +): Promise { + const state = getVerifiedCandidateState(candidate) + const key = artifactCacheKey(artifact) + const existing = state.artifactBytes.get(key) + if (existing) return Uint8Array.from(existing) + const bytes = await readVerifiedArtifact(artifact, state.ports.artifacts) + state.artifactBytes.set(key, Uint8Array.from(bytes)) + return bytes +} + +export function verifiedResourceBytes( + candidate: VerifiedAgentCandidate, + resource: AgentCandidateResourceRef, +): Uint8Array { + const bytes = getVerifiedCandidateState(candidate).resourceBytes.get(resourceKey(resource)) + if (!bytes) { + throw new Error( + `candidate resource was not verified: ${resource.name ?? (resource.kind === 'github' ? resource.path : '')}`, + ) + } + return Uint8Array.from(bytes) +} + +export function verifiedResourceTextByDigest( + candidate: VerifiedAgentCandidate, +): ReadonlyMap { + const output = new Map() + for (const resource of candidateResources(candidate.bundle)) { + const bytes = getVerifiedCandidateState(candidate).resourceBytes.get(resource.sha256) + if (!bytes) throw new Error(`candidate resource digest was not verified: ${resource.sha256}`) + output.set(resource.sha256, new TextDecoder('utf-8', { fatal: true }).decode(bytes)) + } + return output +} + +function candidateResources(bundle: AgentCandidateBundle): AgentCandidateResourceRef[] { + const resources = bundle.profile.resources + if (!resources) return [] + const output: AgentCandidateResourceRef[] = [] + for (const mount of resources.files ?? []) output.push(mount.resource) + output.push(...(resources.tools ?? [])) + output.push(...(resources.skills ?? [])) + output.push(...(resources.agents ?? [])) + output.push(...(resources.commands ?? [])) + if (typeof resources.instructions === 'object') output.push(resources.instructions) + return output +} + +function resourceKey(resource: AgentCandidateResourceRef): string { + return canonicalCandidateDigest(resource) +} diff --git a/src/index.ts b/src/index.ts index 2fde0664..a3410db3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,6 +28,11 @@ export { createOpenAICompatibleBackend, createSandboxPromptBackend, } from './backends' +// ── Immutable candidate execution ───────────────────────────────────── +// One verified bundle → one exact per-task plan → one protected run receipt. +// This composes the shared profile materializer and agent-eval trace store; +// benchmark adapters supply only environment-specific artifact/container ports. +export * from './candidate-execution' export type { AuthSource, BackendCallPolicy, diff --git a/tests/candidate-execution-claim.test.ts b/tests/candidate-execution-claim.test.ts new file mode 100644 index 00000000..0f139045 --- /dev/null +++ b/tests/candidate-execution-claim.test.ts @@ -0,0 +1,893 @@ +import { spawn } from 'node:child_process' +import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { resolve } from 'node:path' +import type { AgentCandidateArtifactRef } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { + type AgentCandidateExecutionClaim, + type AgentCandidateExecutionClaimStore, + type AgentCandidateExecutionFailureClass, + type AgentCandidateExecutionLease, + type AgentCandidateExecutionRecoveryEvidence, + type AgentCandidateExecutionTerminalResult, + type AgentCandidateExecutionUsage, + InMemoryAgentCandidateExecutionClaimStore, +} from '../src/candidate-execution/claim' +import { FileAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim-file-store' +import { candidateExecutionClaim } from '../src/candidate-execution/claim-plan' +import { candidateExecutionOwnerWindowMs } from '../src/candidate-execution/execution-window' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + cleanupCandidateFixtures, + createCandidateExecutionFixture, +} from './helpers/candidate-execution-fixture' + +const temporaryDirectories: string[] = [] +const FUTURE_EXPIRY_MS = Date.now() + 60 * 60 * 1_000 + +afterEach(async () => { + vi.restoreAllMocks() + cleanupCandidateFixtures() + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ) +}) + +describe('candidate execution claim lifecycle', () => { + it('leases only the frozen execution and cleanup owner window', async () => { + const fixture = createCandidateExecutionFixture() + const cleanupTimeoutMs = 25 + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs }, + ) + const claimedAtMs = Date.now() + vi.spyOn(Date, 'now').mockReturnValue(claimedAtMs) + + const requested = candidateExecutionClaim(prepared) + const ownerWindowMs = candidateExecutionOwnerWindowMs( + fixture.task.limits.timeoutMs, + cleanupTimeoutMs, + fixture.task.limits.timeoutMs, + fixture.task.limits.timeoutMs, + ) + + expect(requested.leaseExpiresAtMs).toBe(claimedAtMs + ownerWindowMs) + expect(requested.cleanup.cleanupTimeoutMs).toBe(cleanupTimeoutMs) + expect(requested.resultTimeoutMs).toBe(fixture.task.limits.timeoutMs) + expect(ownerWindowMs).toBeLessThan(15 * 60_000) + }) + + it('rejects a preparation too near reservation expiry for the owner window', async () => { + const fixture = createCandidateExecutionFixture() + const cleanupTimeoutMs = 25 + let reservationExpiresAtMs = 0 + const reserveGrant = fixture.ports.models.reserveGrant + fixture.ports.models.reserveGrant = async (input) => { + reservationExpiresAtMs = input.expiresAtMs + return await reserveGrant(input) + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs }, + ) + const ownerWindowMs = candidateExecutionOwnerWindowMs( + fixture.task.limits.timeoutMs, + cleanupTimeoutMs, + fixture.task.limits.timeoutMs, + fixture.task.limits.timeoutMs, + ) + vi.spyOn(Date, 'now').mockReturnValue(reservationExpiresAtMs - ownerWindowMs + 1) + + expect(() => candidateExecutionClaim(prepared)).toThrow( + /full execution and cleanup owner window/, + ) + }) + + it('lets exactly one in-process caller acquire an attempt', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const results = await Promise.all(Array.from({ length: 32 }, () => store.tryClaim(claim()))) + + expect(results.filter((result) => result.acquired)).toHaveLength(1) + expect(results.filter((result) => !result.acquired)).toHaveLength(31) + expect( + results + .filter((result) => !result.acquired && result.reason === 'already-claimed') + .every((result) => result.exactReplay), + ).toBe(true) + }) + + it('does not call a different preparation an exact replay', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + await acquire(store, claim()) + + expect( + await store.tryClaim( + claim({ cleanup: { ...cleanupHandles(), preparationId: preparationId('q') } }), + ), + ).toMatchObject({ acquired: false, reason: 'already-claimed', exactReplay: false }) + }) + + it('requires stage before finish and the exact staged terminal digest', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + const result = failed() + + await expect(store.finish(acquired.lease, sha256('f'))).rejects.toThrow('has not been staged') + const staged = await store.stageTerminal(acquired.lease, result) + if (!staged.staged) throw new Error('test setup did not stage') + await expect(store.finish(acquired.lease, sha256('f'))).rejects.toThrow( + 'does not match staged outbox', + ) + const finished = await store.finish(acquired.lease, staged.terminal.terminalDigest) + + expect(finished).toEqual({ finished: true, terminal: staged.terminal }) + expect(finished.terminal).toEqual(staged.terminal) + }) + + it('requires the winning lease for phase, stage, and finish', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + const invalid = { ...acquired.lease, token: leaseToken('x') } + + await expect(store.markCandidateMayRun(invalid)).rejects.toThrow('lease is invalid') + await expect(store.stageTerminal(invalid, failed())).rejects.toThrow('lease is invalid') + const staged = await store.stageTerminal(acquired.lease, failed()) + await expect(store.finish(invalid, staged.terminal.terminalDigest)).rejects.toThrow( + 'lease is invalid', + ) + }) + + it('stages exactly once with exact-replay and mismatch evidence', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + const first = await store.stageTerminal(acquired.lease, failed()) + const replay = await store.stageTerminal(acquired.lease, failed()) + const mismatch = await store.stageTerminal( + acquired.lease, + failed(0, 'unknown', { failureEvidence: artifact('9') }), + ) + + expect(first).toMatchObject({ staged: true }) + expect(replay).toMatchObject({ staged: false, exactReplay: true }) + expect(mismatch).toMatchObject({ staged: false, exactReplay: false }) + }) + + it('persists every fixed usage field and durable success artifact reference', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + await store.markCandidateMayRun(acquired.lease) + const result = succeeded() + const terminal = await complete(store, acquired.lease, result) + const observed = await store.getAttempt({ executionId: 'execution-1', attempt: 1 }) + + expect(terminal.usage).toEqual(result.usage) + expect(terminal).toMatchObject({ + schemaVersion: 1, + modelSettlement: result.modelSettlement, + taskOutcome: result.taskOutcome, + benchmarkResult: result.benchmarkResult, + runReceipt: result.runReceipt, + terminalDigest: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + }) + expect(observed).toMatchObject({ + phase: 'candidate-may-run', + staged: terminal, + terminal, + }) + }) + + it('persists the candidate-may-run phase monotonically before execution terminals', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + + expect(await store.getAttempt({ executionId: 'execution-1', attempt: 1 })).toMatchObject({ + phase: 'claimed', + }) + expect(await store.markCandidateMayRun(acquired.lease)).toEqual({ + marked: true, + phase: 'candidate-may-run', + }) + expect(await store.markCandidateMayRun(acquired.lease)).toEqual({ + marked: false, + phase: 'candidate-may-run', + }) + await expect(store.stageTerminal(acquired.lease, failed())).rejects.toThrow( + 'crossed candidate-may-run', + ) + await complete(store, acquired.lease, failed(0, 'execution')) + expect(await store.getAttempt({ executionId: 'execution-1', attempt: 1 })).toMatchObject({ + phase: 'candidate-may-run', + }) + }) + + it('refuses success unless candidate-may-run was persisted first', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, claim()) + + await expect(store.stageTerminal(acquired.lease, succeeded())).rejects.toThrow( + 'requires candidate-may-run phase', + ) + }) + + it('keeps exact stage and finish replay idempotent after lease expiry', async () => { + let now = 1_000 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const acquired = await acquire(store, claim({ leaseExpiresAtMs: 1_100 })) + const staged = await store.stageTerminal(acquired.lease, failed()) + const finished = await store.finish(acquired.lease, staged.terminal.terminalDigest) + expect(finished.finished).toBe(true) + now = 1_100 + + expect(await store.stageTerminal(acquired.lease, failed())).toMatchObject({ + staged: false, + exactReplay: true, + }) + expect(await store.finish(acquired.lease, staged.terminal.terminalDigest)).toMatchObject({ + finished: false, + exactReplay: true, + }) + }) + + it('rejects new phase, stage, and finish writes after lease expiry', async () => { + let now = 2_000 + const phaseStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const phaseLease = await acquire(phaseStore, claim({ leaseExpiresAtMs: 2_100 })) + const finishStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const finishLease = await acquire(finishStore, claim({ leaseExpiresAtMs: 2_100 })) + const staged = await finishStore.stageTerminal(finishLease.lease, failed()) + now = 2_100 + + await expect(phaseStore.markCandidateMayRun(phaseLease.lease)).rejects.toThrow( + 'lease has expired', + ) + await expect(phaseStore.stageTerminal(phaseLease.lease, failed())).rejects.toThrow( + 'lease has expired', + ) + await expect( + finishStore.finish(finishLease.lease, staged.terminal.terminalDigest), + ).rejects.toThrow('lease has expired') + }) + + it.each([ + 'phase', + 'stage', + ] as const)('refuses file-backed %s publication when the lease expires during durable write preparation and lets recovery win', async (operation) => { + const directory = await tempDirectory() + const expiresAtMs = 2_600 + let now = 2_500 + let expireAfterNextRead = false + const store = new FileAgentCandidateExecutionClaimStore({ + directory, + now: () => { + const observed = now + if (expireAfterNextRead) { + expireAfterNextRead = false + now = expiresAtMs + } + return observed + }, + }) + const requested = retryClaim({ leaseExpiresAtMs: expiresAtMs }) + const acquired = await acquire(store, requested) + now = expiresAtMs - 1 + expireAfterNextRead = true + + await expect( + operation === 'phase' + ? store.markCandidateMayRun(acquired.lease) + : store.stageTerminal(acquired.lease, failed()), + ).rejects.toThrow('lease has expired') + const beforeRecovery = await store.getAttempt({ + executionId: requested.executionId, + attempt: requested.attempt, + }) + expect(beforeRecovery?.phase).toBe('claimed') + expect(beforeRecovery?.staged).toBeUndefined() + expect(beforeRecovery?.terminal).toBeUndefined() + + const recovered = await store.recoverExpired( + { executionId: requested.executionId, attempt: requested.attempt }, + recoveryEvidence(requested), + ) + expect(recovered).toMatchObject({ + finished: true, + terminal: { status: 'failed', failureClass: 'pre-model-infrastructure' }, + }) + }) + + it('refuses file-backed finish publication when the lease expires during durable write preparation and lets recovery win', async () => { + const directory = await tempDirectory() + const expiresAtMs = 2_900 + let now = 2_800 + let expireAfterNextRead = false + const store = new FileAgentCandidateExecutionClaimStore({ + directory, + now: () => { + const observed = now + if (expireAfterNextRead) { + expireAfterNextRead = false + now = expiresAtMs + } + return observed + }, + }) + const requested = retryClaim({ leaseExpiresAtMs: expiresAtMs }) + const acquired = await acquire(store, requested) + const staged = await store.stageTerminal(acquired.lease, failed()) + now = expiresAtMs - 1 + expireAfterNextRead = true + + await expect(store.finish(acquired.lease, staged.terminal.terminalDigest)).rejects.toThrow( + 'lease has expired', + ) + const beforeRecovery = await store.getAttempt({ + executionId: requested.executionId, + attempt: requested.attempt, + }) + expect(beforeRecovery?.staged).toEqual(staged.terminal) + expect(beforeRecovery?.terminal).toBeUndefined() + + const recovered = await store.recoverExpired( + { executionId: requested.executionId, attempt: requested.attempt }, + recoveryEvidence(requested), + ) + expect(recovered).toEqual({ finished: true, terminal: staged.terminal }) + }) + + it('recovers zero-call pre-model work only when candidate-may-run was never crossed', async () => { + let now = 3_000 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const requested = retryClaim({ leaseExpiresAtMs: 3_100, cleanup: cleanupHandles(true) }) + await acquire(store, requested) + now = 3_100 + + const recovered = await store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested), + ) + + expect(recovered).toMatchObject({ + finished: true, + terminal: { + status: 'failed', + failureClass: 'pre-model-infrastructure', + usage: { modelCalls: 0 }, + }, + }) + expect((await store.tryClaim(retryClaim({ attempt: 2 }))).acquired).toBe(true) + }) + + it('normalizes recovered pre-model evidence to unknown after candidate-may-run', async () => { + let now = 3_500 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const requested = retryClaim({ leaseExpiresAtMs: 3_600 }) + const acquired = await acquire(store, requested) + await store.markCandidateMayRun(acquired.lease) + now = 3_600 + + const recovered = await store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested), + ) + + expect(recovered).toMatchObject({ + terminal: { failureClass: 'unknown', usage: { modelCalls: 0 } }, + }) + expect(await store.tryClaim(retryClaim({ attempt: 2 }))).toMatchObject({ + acquired: false, + detail: 'prior-attempt-not-pre-model-infrastructure', + }) + }) + + it('promotes an exact staged terminal after a crash instead of replacing it', async () => { + let now = 4_000 + const directory = await tempDirectory() + const requested = claim({ leaseExpiresAtMs: 4_100 }) + const owner = new FileAgentCandidateExecutionClaimStore({ directory, now: () => now }) + const acquired = await acquire(owner, requested) + await owner.markCandidateMayRun(acquired.lease) + const staged = await owner.stageTerminal(acquired.lease, succeeded()) + expect( + (await owner.getAttempt({ executionId: 'execution-1', attempt: 1 }))?.terminal, + ).toBeUndefined() + now = 4_100 + + const recovery = new FileAgentCandidateExecutionClaimStore({ directory, now: () => now }) + const recovered = await recovery.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested, { + failureClass: 'execution', + usage: staged.terminal.usage, + }), + ) + + expect(recovered).toEqual({ finished: true, terminal: staged.terminal }) + expect( + await recovery.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested, { + failureClass: 'execution', + usage: staged.terminal.usage, + }), + ), + ).toMatchObject({ finished: false, exactReplay: true }) + }) + + it('refuses recovery whose model evidence differs from the staged outbox', async () => { + let now = 4_500 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const requested = claim({ leaseExpiresAtMs: 4_600 }) + const acquired = await acquire(store, requested) + await store.stageTerminal(acquired.lease, failed()) + now = 4_600 + + await expect( + store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + recoveryEvidence(requested, { modelSettlement: artifact('8') }), + ), + ).rejects.toThrow('does not match staged model evidence') + }) + + it('requires exact process, model, and memory closure before recovery', async () => { + let now = 5_000 + const store = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const requested = claim({ leaseExpiresAtMs: 5_100, cleanup: cleanupHandles(true) }) + await acquire(store, requested) + const evidence = recoveryEvidence(requested) + await expect( + store.recoverExpired({ executionId: requested.executionId, attempt: 1 }, evidence), + ).rejects.toThrow('lease has not expired') + now = 5_100 + const { memory: _memory, ...withoutMemory } = evidence + + await expect( + store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + { ...evidence, process: { ...evidence.process, stopped: false as true } }, + ), + ).rejects.toThrow('does not prove the claimed process stopped') + await expect( + store.recoverExpired( + { executionId: requested.executionId, attempt: 1 }, + { ...evidence, model: { ...evidence.model, grantDigest: sha256('f') } }, + ), + ).rejects.toThrow('does not prove the claimed model grant closed') + await expect( + store.recoverExpired({ executionId: requested.executionId, attempt: 1 }, withoutMemory), + ).rejects.toThrow('missing memory closure evidence') + }) + + it('unlocks retry only after a zero-call pre-model infrastructure terminal', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const first = await acquire(store, retryClaim()) + expect(await store.tryClaim(retryClaim({ attempt: 2 }))).toMatchObject({ + acquired: false, + detail: 'prior-attempt-running', + }) + await complete(store, first.lease, failed()) + expect((await store.tryClaim(retryClaim({ attempt: 2 }))).acquired).toBe(true) + }) + + it.each([ + 'execution', + 'post-model-infrastructure', + 'unknown', + ] as const)('rejects a zero-call %s terminal as non-retryable', async (failureClass) => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const first = await acquire(store, retryClaim()) + if (failureClass !== 'unknown') await store.markCandidateMayRun(first.lease) + await complete(store, first.lease, failed(0, failureClass)) + + expect(await store.tryClaim(retryClaim({ attempt: 2 }))).toMatchObject({ + acquired: false, + detail: 'prior-attempt-not-pre-model-infrastructure', + }) + }) + + it('rejects a pre-model terminal with paid calls', async () => { + const store = new InMemoryAgentCandidateExecutionClaimStore() + const acquired = await acquire(store, retryClaim()) + + await expect(store.stageTerminal(acquired.lease, failed(1))).rejects.toThrow( + 'pre-model infrastructure failure cannot contain model calls', + ) + }) + + it('rejects changed retry lineage and a missing prior attempt', async () => { + const missing = new InMemoryAgentCandidateExecutionClaimStore() + expect(await missing.tryClaim(retryClaim({ attempt: 2 }))).toMatchObject({ + acquired: false, + detail: 'prior-attempt-missing', + }) + const changed = new InMemoryAgentCandidateExecutionClaimStore() + const first = await acquire(changed, retryClaim()) + await complete(changed, first.lease, failed()) + + expect( + await changed.tryClaim(retryClaim({ attempt: 2, retryLineageDigest: sha256('f') })), + ).toMatchObject({ acquired: false, detail: 'retry-lineage-mismatch' }) + }) + + it('persists claim7, pending1, terminal3, phase, staged, and full usage across stores', async () => { + const directory = await tempDirectory() + const store = new FileAgentCandidateExecutionClaimStore({ directory }) + const acquired = await acquire(store, claim()) + await store.markCandidateMayRun(acquired.lease) + const terminal = await complete(store, acquired.lease, failed(2, 'execution')) + const reopened = new FileAgentCandidateExecutionClaimStore({ directory }) + const observed = await reopened.getAttempt({ executionId: 'execution-1', attempt: 1 }) + const files = await Promise.all( + (await readdir(directory)).map(async (name) => ({ + name, + text: await readFile(resolve(directory, name), 'utf8'), + })), + ) + + expect(observed).toMatchObject({ + phase: 'candidate-may-run', + staged: terminal, + terminal, + }) + expect(files.find(({ name }) => name.endsWith('.claim.json'))?.text).toContain('"version":7') + expect(files.find(({ name }) => name.includes('transition-2'))?.text).toContain('"version":1') + expect(files.find(({ name }) => name.endsWith('.terminal.json'))?.text).toContain('"version":3') + expect(files.map(({ text }) => text).join('\n')).not.toContain(acquired.lease.token) + }) + + it('finishes a staged file outbox after reopening without recomputing content', async () => { + const directory = await tempDirectory() + const owner = new FileAgentCandidateExecutionClaimStore({ directory }) + const acquired = await acquire(owner, claim()) + const staged = await owner.stageTerminal(acquired.lease, failed()) + const reopened = new FileAgentCandidateExecutionClaimStore({ directory }) + + const observed = await reopened.getAttempt({ executionId: 'execution-1', attempt: 1 }) + expect(observed).toMatchObject({ + staged: staged.terminal, + }) + expect(observed?.terminal).toBeUndefined() + expect(await reopened.finish(acquired.lease, staged.terminal.terminalDigest)).toEqual({ + finished: true, + terminal: staged.terminal, + }) + }) + + it('lets exactly one independent process acquire the same attempt', async () => { + const directory = await tempDirectory() + const results = await Promise.all( + Array.from({ length: 8 }, () => claimFromChildProcess(directory, claim())), + ) + + expect(results.filter((result) => result === 'acquired')).toHaveLength(1) + expect(results.filter((result) => result === 'already-claimed')).toHaveLength(7) + }) + + it('linearizes candidate-may-run across independent processes', async () => { + const directory = await tempDirectory() + const store = new FileAgentCandidateExecutionClaimStore({ directory }) + const acquired = await acquire(store, claim()) + const results = await Promise.all( + Array.from({ length: 8 }, () => markFromChildProcess(directory, acquired.lease)), + ) + + expect(results.filter((result) => result === 'marked')).toHaveLength(1) + expect(results.filter((result) => result === 'already-marked')).toHaveLength(7) + }) + + it('linearizes staging and finishing across independent processes', async () => { + const directory = await tempDirectory() + const store = new FileAgentCandidateExecutionClaimStore({ directory }) + const acquired = await acquire(store, retryClaim()) + const stageResults = await Promise.all( + Array.from({ length: 8 }, () => stageFromChildProcess(directory, acquired.lease, failed())), + ) + const record = await store.getAttempt({ executionId: 'execution-1', attempt: 1 }) + if (!record?.staged) throw new Error('cross-process stage is missing') + const finishResults = await Promise.all( + Array.from({ length: 8 }, () => + finishFromChildProcess( + directory, + acquired.lease, + record.staged?.terminalDigest ?? sha256('f'), + ), + ), + ) + + expect(stageResults.filter((result) => result === 'staged')).toHaveLength(1) + expect(stageResults.filter((result) => result === 'already-staged:exact')).toHaveLength(7) + expect(finishResults.filter((result) => result === 'finished')).toHaveLength(1) + expect(finishResults.filter((result) => result === 'already-finished:exact')).toHaveLength(7) + }, 15_000) + + it('linearizes expired recovery across independent processes', async () => { + const directory = await tempDirectory() + const requested = retryClaim({ leaseExpiresAtMs: 6_000 }) + const store = new FileAgentCandidateExecutionClaimStore({ directory, now: () => 5_900 }) + await acquire(store, requested) + const evidence = recoveryEvidence(requested) + const results = await Promise.all( + Array.from({ length: 8 }, () => + recoverFromChildProcess( + directory, + { executionId: requested.executionId, attempt: 1 }, + evidence, + 6_000, + ), + ), + ) + + expect(results.filter((result) => result === 'recovered')).toHaveLength(1) + expect(results.filter((result) => result === 'already-recovered:exact')).toHaveLength(7) + }) +}) + +function claim( + overrides: Partial = {}, +): AgentCandidateExecutionClaim { + return { + executionId: 'execution-1', + attempt: 1, + maxAttempts: 1, + retryPolicy: 'none', + bundleDigest: sha256('a'), + executionPlanDigest: sha256('b'), + retryLineageDigest: sha256('c'), + leaseExpiresAtMs: FUTURE_EXPIRY_MS, + resultTimeoutMs: 60_000, + cleanup: cleanupHandles(), + ...overrides, + } +} + +function retryClaim( + overrides: Partial = {}, +): AgentCandidateExecutionClaim { + return claim({ maxAttempts: 3, retryPolicy: 'pre-model-infrastructure-only', ...overrides }) +} + +function usage( + modelCalls = 0, + overrides: Partial = {}, +): AgentCandidateExecutionUsage { + return { + costUsdNanos: modelCalls * 123_456, + inputTokens: modelCalls * 101, + outputTokens: modelCalls * 29, + cachedInputTokens: modelCalls * 17, + reasoningTokens: modelCalls * 11, + modelCalls, + ...overrides, + } +} + +function failed( + modelCalls = 0, + failureClass: AgentCandidateExecutionFailureClass = 'pre-model-infrastructure', + overrides: Partial> = {}, +): Extract { + return { + schemaVersion: 1, + status: 'failed', + failureClass, + usage: usage(modelCalls), + modelSettlement: artifact('1'), + ...overrides, + } +} + +function succeeded(): Extract { + return { + schemaVersion: 1, + status: 'succeeded', + usage: usage(2), + modelSettlement: artifact('1'), + taskOutcome: artifact('2'), + benchmarkResult: artifact('3'), + runReceipt: artifact('4'), + } +} + +function artifact(character: string): AgentCandidateArtifactRef { + return { + locator: { kind: 's3', bucket: 'candidate-evidence', key: `${character}/artifact.json` }, + sha256: sha256(character), + byteLength: character.charCodeAt(0), + } +} + +function cleanupHandles(memory = false): AgentCandidateExecutionClaim['cleanup'] { + return { + preparationId: preparationId('p'), + modelGrantDigest: sha256('d'), + resolvedModel: { + requested: 'provider/model', + provider: 'provider', + model: 'model-snapshot', + snapshot: 'model-snapshot-2026-07-01', + reasoningEffort: 'high', + }, + traceRunId: 'execution-1:attempt-1:trace', + cleanupTimeoutMs: 30_000, + ...(memory + ? { + memory: { + accessDigest: sha256('e'), + effectiveNamespace: 'candidate/bundle/execution/task/preparation', + }, + } + : {}), + } +} + +function preparationId(character: string): string { + return `candidate-preparation-v1.${character.repeat(43)}` +} + +function recoveryEvidence( + requested: AgentCandidateExecutionClaim, + overrides: { + failureClass?: AgentCandidateExecutionFailureClass + usage?: AgentCandidateExecutionUsage + modelSettlement?: AgentCandidateArtifactRef + } = {}, +): AgentCandidateExecutionRecoveryEvidence { + return { + failureClass: overrides.failureClass ?? 'pre-model-infrastructure', + usage: overrides.usage ?? usage(), + modelSettlement: overrides.modelSettlement ?? artifact('1'), + process: { stopped: true, executionPlanDigest: requested.executionPlanDigest }, + model: { + closed: true, + preparationId: requested.cleanup.preparationId, + grantDigest: requested.cleanup.modelGrantDigest, + }, + ...(requested.cleanup.memory + ? { + memory: { + closed: true as const, + preparationId: requested.cleanup.preparationId, + accessDigest: requested.cleanup.memory.accessDigest, + effectiveNamespace: requested.cleanup.memory.effectiveNamespace, + }, + } + : {}), + } +} + +function sha256(character: string): `sha256:${string}` { + return `sha256:${character.repeat(64)}` +} + +function leaseToken(character: string): string { + return `candidate-execution-lease-v1.${character.repeat(43)}` +} + +async function acquire( + store: Pick, + requested: AgentCandidateExecutionClaim, +) { + const result = await store.tryClaim(requested) + if (!result.acquired) throw new Error(`test setup failed to acquire: ${result.reason}`) + return result +} + +async function complete( + store: Pick, + lease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, +) { + const staged = await store.stageTerminal(lease, result) + return (await store.finish(lease, staged.terminal.terminalDigest)).terminal +} + +async function tempDirectory(): Promise { + const directory = await mkdtemp(resolve(tmpdir(), 'candidate-execution-claim-')) + temporaryDirectories.push(directory) + return directory +} + +async function claimFromChildProcess( + directory: string, + requested: AgentCandidateExecutionClaim, +): Promise { + return runChild([ + storeSource(directory), + `const result = await store.tryClaim(${JSON.stringify(requested)})`, + "process.stdout.write(result.acquired ? 'acquired' : result.reason)", + ]) +} + +async function markFromChildProcess( + directory: string, + lease: AgentCandidateExecutionLease, +): Promise { + return runChild([ + storeSource(directory), + `const result = await store.markCandidateMayRun(${JSON.stringify(lease)})`, + "process.stdout.write(result.marked ? 'marked' : 'already-marked')", + ]) +} + +async function stageFromChildProcess( + directory: string, + lease: AgentCandidateExecutionLease, + result: AgentCandidateExecutionTerminalResult, +): Promise { + return runChild([ + storeSource(directory), + `const result = await store.stageTerminal(${JSON.stringify(lease)}, ${JSON.stringify(result)})`, + "process.stdout.write(result.staged ? 'staged' : 'already-staged:' + (result.exactReplay ? 'exact' : 'mismatch'))", + ]) +} + +async function finishFromChildProcess( + directory: string, + lease: AgentCandidateExecutionLease, + terminalDigest: string, +): Promise { + return runChild([ + storeSource(directory), + `const result = await store.finish(${JSON.stringify(lease)}, ${JSON.stringify(terminalDigest)})`, + "process.stdout.write(result.finished ? 'finished' : 'already-finished:' + (result.exactReplay ? 'exact' : 'mismatch'))", + ]) +} + +async function recoverFromChildProcess( + directory: string, + attempt: { executionId: string; attempt: number }, + evidence: AgentCandidateExecutionRecoveryEvidence, + nowMs: number, +): Promise { + return runChild([ + storeSource(directory, nowMs), + `const result = await store.recoverExpired(${JSON.stringify(attempt)}, ${JSON.stringify(evidence)})`, + "process.stdout.write(result.finished ? 'recovered' : 'already-recovered:' + (result.exactReplay ? 'exact' : 'mismatch'))", + ]) +} + +function storeSource(directory: string, nowMs?: number): string { + return `const store = new FileAgentCandidateExecutionClaimStore({ directory: ${JSON.stringify(directory)}${nowMs === undefined ? '' : `, now: () => ${nowMs}`} })` +} + +async function runChild(lines: readonly string[]): Promise { + const moduleUrl = new URL('../src/candidate-execution/claim-file-store.ts', import.meta.url).href + const source = [ + `import { FileAgentCandidateExecutionClaimStore } from ${JSON.stringify(moduleUrl)}`, + ...lines, + ].join('\n') + + return await new Promise((resolveChild, rejectChild) => { + const child = spawn( + process.execPath, + ['--import', 'tsx', '--input-type=module', '--eval', source], + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ) + let stdout = '' + let stderr = '' + child.stdout.setEncoding('utf8').on('data', (chunk: string) => { + stdout += chunk + }) + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + stderr += chunk + }) + child.once('error', rejectChild) + child.once('close', (code) => { + if (code !== 0) { + rejectChild(new Error(`claim child exited ${code}: ${stderr}`)) + return + } + resolveChild(stdout) + }) + }) +} diff --git a/tests/candidate-execution-cleanup.test.ts b/tests/candidate-execution-cleanup.test.ts new file mode 100644 index 00000000..36b431c6 --- /dev/null +++ b/tests/candidate-execution-cleanup.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { + CandidateCleanupTimeoutError, + CandidateResultTimeoutError, + candidateCleanupTimeout, + MAX_CANDIDATE_TIMER_INTERVAL_MS, + withinCandidateCleanupDeadline, + withinCandidateResultDeadline, +} from '../src/candidate-execution/cleanup' +import { + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS, + candidateExecutionOwnerWindowMs, + candidatePostRunWindowMs, +} from '../src/candidate-execution/execution-window' + +describe('candidate cleanup timer bounds', () => { + afterEach(() => vi.useRealTimers()) + + it('accepts the Node timer boundary and rejects values that would clamp', () => { + expect(candidateCleanupTimeout(MAX_CANDIDATE_TIMER_INTERVAL_MS)).toBe( + MAX_CANDIDATE_TIMER_INTERVAL_MS, + ) + expect(() => candidateCleanupTimeout(MAX_CANDIDATE_TIMER_INTERVAL_MS + 1)).toThrow( + /supported timer range/, + ) + expect(() => candidateCleanupTimeout(Number.MAX_SAFE_INTEGER)).toThrow(/supported timer range/) + }) + + it('budgets every sequential post-run phase plus terminal persistence', () => { + expect(candidatePostRunWindowMs(25, 500)).toBe( + 25 * 4 + 500 + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS, + ) + expect(candidateExecutionOwnerWindowMs(1_000, 25, 500)).toBe( + 1_000 + 25 * 4 + 500 + CANDIDATE_TERMINAL_PERSISTENCE_MARGIN_MS, + ) + expect(() => candidatePostRunWindowMs(MAX_CANDIDATE_TIMER_INTERVAL_MS, 1)).toThrow( + /post-run window/, + ) + }) + + it('rejects results observed exactly at their frozen deadline', async () => { + vi.useFakeTimers({ now: 100 }) + await expect( + withinCandidateCleanupDeadline( + async () => { + vi.setSystemTime(110) + return 'ambiguous' + }, + 110, + 'cleanup', + ), + ).rejects.toBeInstanceOf(CandidateCleanupTimeoutError) + + let observedSignal: AbortSignal | undefined + await expect( + withinCandidateResultDeadline( + async (signal) => { + observedSignal = signal + vi.setSystemTime(120) + return 'ambiguous' + }, + 120, + 'result', + ), + ).rejects.toBeInstanceOf(CandidateResultTimeoutError) + expect(observedSignal?.aborted).toBe(true) + }) +}) diff --git a/tests/candidate-execution-core.test.ts b/tests/candidate-execution-core.test.ts new file mode 100644 index 00000000..5b3cb361 --- /dev/null +++ b/tests/candidate-execution-core.test.ts @@ -0,0 +1,244 @@ +import { execFileSync } from 'node:child_process' +import { + chmodSync, + linkSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { + AgentCandidateGitPatch, + AgentCandidateWorkspaceManifestMaterialV1, +} from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' + +import { + readVerifiedArtifact, + verifyMaterializedWorkspace, + verifyWorkspaceSnapshotArtifacts, +} from '../src/candidate-execution/artifacts' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + embeddedCandidateArtifact, +} from '../src/candidate-execution/digest' +import { verifyCandidateCode } from '../src/candidate-execution/git-materialize' + +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + roots.push(root) + return root +} + +function git(root: string, args: string[]): string { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim() +} + +function repositoryFixture(): { + root: string + baseCommit: string + baseTree: string + candidateTree: string + patch: Uint8Array +} { + const root = temporaryRoot('candidate-git-') + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'test@example.com']) + git(root, ['config', 'user.name', 'Test']) + git(root, ['config', 'core.hooksPath', '/dev/null']) + git(root, ['remote', 'add', 'origin', 'git@github.com:owner/repo.git']) + writeFileSync(join(root, 'index.ts'), 'export const value = 1\n') + git(root, ['add', 'index.ts']) + git(root, ['commit', '-m', 'base']) + const baseCommit = git(root, ['rev-parse', 'HEAD']) + const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) + writeFileSync(join(root, 'index.ts'), 'export const value = 2\n') + const patch = Buffer.from(execFileSync('git', ['diff', '--binary'], { cwd: root })) + git(root, ['add', 'index.ts']) + const candidateTree = git(root, ['write-tree']) + git(root, ['reset', '--hard', baseCommit]) + return { root, baseCommit, baseTree, candidateTree, patch } +} + +function codeFixture(fixture: ReturnType): AgentCandidateGitPatch { + return { + kind: 'git-patch', + repository: { kind: 'github', owner: 'owner', repo: 'repo' }, + baseCommit: fixture.baseCommit, + baseTree: fixture.baseTree, + candidateTree: fixture.candidateTree, + patch: { format: 'git-diff-binary', artifact: embeddedCandidateArtifact(fixture.patch) }, + } +} + +describe('candidate canonical bytes and artifacts', () => { + it('uses the existing stable content address for exact canonical bytes', () => { + const first = { z: [3, { b: true, a: 'x' }], a: -0 } + const second = { a: 0, z: [3, { a: 'x', b: true }] } + expect(canonicalCandidateBytes(first)).toEqual(canonicalCandidateBytes(second)) + expect(canonicalCandidateDigest(first)).toBe(canonicalCandidateDigest(second)) + }) + + it('rejects an artifact whose claimed hash does not match its bytes', async () => { + const artifact = embeddedCandidateArtifact(Buffer.from('actual')) + await expect( + readVerifiedArtifact( + { ...artifact, content: Buffer.from('forged').toString('base64') }, + { read: async () => new Uint8Array() }, + ), + ).rejects.toThrow(/digest|byte length/) + }) + + it('requires workspace manifest bytes to equal canonical material, not only a locator', async () => { + const material: AgentCandidateWorkspaceManifestMaterialV1 = { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files: [], + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + const archive = embeddedCandidateArtifact(Buffer.from('archive')) + await expect( + verifyWorkspaceSnapshotArtifacts( + { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive, + }, + { read: async () => new Uint8Array() }, + ), + ).resolves.toMatchObject({ archive: expect.any(Uint8Array) }) + await expect( + verifyWorkspaceSnapshotArtifacts( + { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest: embeddedCandidateArtifact(Buffer.from('{}')), + archive, + }, + { read: async () => new Uint8Array() }, + ), + ).rejects.toThrow(/canonical manifest/) + }) +}) + +describe('candidate Git identity', () => { + it('replays a binary-safe patch against the exact base tree', async () => { + const fixture = repositoryFixture() + await expect( + verifyCandidateCode( + codeFixture(fixture), + { resolve: async () => fixture.root }, + fixture.patch, + ), + ).resolves.toBe(fixture.candidateTree) + }) + + it('rejects candidate-tree drift and the wrong declared repository', async () => { + const fixture = repositoryFixture() + const code = codeFixture(fixture) + await expect( + verifyCandidateCode( + { ...code, candidateTree: fixture.baseTree }, + { resolve: async () => fixture.root }, + fixture.patch, + ), + ).rejects.toThrow(/does not match/) + await expect( + verifyCandidateCode( + { ...code, repository: { kind: 'github', owner: 'other', repo: 'repo' } }, + { resolve: async () => fixture.root }, + fixture.patch, + ), + ).rejects.toThrow(/does not match github/) + }) + + it('rejects a candidate Git tree containing symlinks', async () => { + const fixture = repositoryFixture() + symlinkSync('index.ts', join(fixture.root, 'linked.ts')) + git(fixture.root, ['add', 'linked.ts']) + const candidateTree = git(fixture.root, ['write-tree']) + const patch = Buffer.from( + execFileSync('git', ['diff', '--cached', '--binary', fixture.baseCommit], { + cwd: fixture.root, + }), + ) + const code = { + ...codeFixture(fixture), + candidateTree, + patch: { format: 'git-diff-binary' as const, artifact: embeddedCandidateArtifact(patch) }, + } + await expect( + verifyCandidateCode(code, { resolve: async () => fixture.root }, patch), + ).rejects.toThrow(/symlink|non-blob/) + }) +}) + +describe('materialized workspace identity', () => { + it('accepts only the exact regular files, modes, and bytes in the manifest', async () => { + const root = temporaryRoot('candidate-workspace-') + mkdirSync(join(root, '.git')) + writeFileSync(join(root, '.git', 'ignored'), 'Git internals are not uploaded') + writeFileSync(join(root, 'run.js'), 'ok\n', { mode: 0o755 }) + const bytes = Buffer.from('ok\n') + const material: AgentCandidateWorkspaceManifestMaterialV1 = { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files: [ + { + path: 'run.js', + mode: 0o755, + sha256: embeddedCandidateArtifact(bytes).sha256, + byteLength: bytes.byteLength, + }, + ], + } + await expect( + verifyMaterializedWorkspace(root, material, { ignoredProtectedRootEntries: ['.git'] }), + ).resolves.toBeUndefined() + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow( + /do not match|unsupported mode/, + ) + rmSync(join(root, '.git'), { recursive: true }) + mkdirSync(join(root, '.sidecar')) + writeFileSync(join(root, '.sidecar', 'unsigned'), 'hidden executable bytes') + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow( + /do not match|unsupported mode/, + ) + rmSync(join(root, '.sidecar'), { recursive: true }) + writeFileSync(join(root, 'extra.txt'), 'extra') + chmodSync(join(root, 'extra.txt'), 0o644) + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow(/do not match/) + }) + + it('rejects symlink and hard-link ambiguity', async () => { + const root = temporaryRoot('candidate-workspace-links-') + writeFileSync(join(root, 'source'), 'x') + chmodSync(join(root, 'source'), 0o644) + symlinkSync('source', join(root, 'symlink')) + const material: AgentCandidateWorkspaceManifestMaterialV1 = { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files: [], + } + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow(/symlink/) + rmSync(join(root, 'symlink')) + linkSync(join(root, 'source'), join(root, 'hardlink')) + await expect(verifyMaterializedWorkspace(root, material)).rejects.toThrow(/hard-linked/) + }) +}) diff --git a/tests/candidate-execution-dispose.test.ts b/tests/candidate-execution-dispose.test.ts new file mode 100644 index 00000000..89f7fe46 --- /dev/null +++ b/tests/candidate-execution-dispose.test.ts @@ -0,0 +1,108 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { InMemoryAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim' +import { disposePreparedAgentCandidateExecution } from '../src/candidate-execution/dispose' +import { executePreparedAgentCandidate } from '../src/candidate-execution/execute' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +describe('prepared candidate disposal', () => { + it('settles an unexecuted reservation exactly once and consumes the preparation', async () => { + const fixture = createCandidateExecutionFixture() + const reasons: string[] = [] + fixture.ports.models.settleGrant = async ({ preparationId, reason }) => { + reasons.push(`${preparationId}:${reason}`) + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + + await expect(disposePreparedAgentCandidateExecution(prepared)).resolves.toEqual({ + disposed: true, + }) + expect(reasons).toHaveLength(1) + expect(reasons[0]).toMatch(/candidate-preparation-v1\..+:abandoned/) + await expect( + executePreparedAgentCandidate(prepared, { + traceStore: {} as never, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + executor: {} as never, + ...createCandidateOutputFixture(), + }), + ).resolves.toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/already disposed/), + }) + }) + + it('retries exact idempotent cleanup when disposal transiently fails', async () => { + const fixture = createCandidateExecutionFixture() + let settlements = 0 + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlements++ + if (settlements === 1) throw new Error('gateway unavailable') + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + + await expect( + disposePreparedAgentCandidateExecution(prepared, { cleanupTimeoutMs: 25 }), + ).rejects.toThrow(/disposal failed/) + await expect(disposePreparedAgentCandidateExecution(prepared)).resolves.toEqual({ + disposed: true, + }) + expect(settlements).toBe(2) + }) + + it('allows only one concurrent disposal attempt', async () => { + const fixture = createCandidateExecutionFixture() + let release!: () => void + const wait = new Promise((resolve) => { + release = resolve + }) + fixture.ports.models.settleGrant = async ({ preparationId }) => { + await wait + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const first = disposePreparedAgentCandidateExecution(prepared) + await expect(disposePreparedAgentCandidateExecution(prepared)).rejects.toThrow( + /already disposing/, + ) + release() + await expect(first).resolves.toEqual({ disposed: true }) + }) +}) diff --git a/tests/candidate-execution-execute.test.ts b/tests/candidate-execution-execute.test.ts new file mode 100644 index 00000000..8498e989 --- /dev/null +++ b/tests/candidate-execution-execute.test.ts @@ -0,0 +1,1432 @@ +import { chmodSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { gzipSync } from 'node:zlib' + +import { InMemoryTraceStore, type TraceStore } from '@tangle-network/agent-eval' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { InMemoryAgentCandidateExecutionClaimStore } from '../src/candidate-execution/claim' +import { sha256Bytes } from '../src/candidate-execution/digest' +import { disposePreparedAgentCandidateExecution } from '../src/candidate-execution/dispose' +import { executePreparedAgentCandidate } from '../src/candidate-execution/execute' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import type { + AgentCandidateExecutorPort, + AgentCandidateExecutorRequest, +} from '../src/candidate-execution/types' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + candidateBundle, + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, + emptyCandidateSnapshot, + redigestCandidateBundle, + unchangedTaskOutcomeCapture, +} from './helpers/candidate-execution-fixture' + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + cleanupCandidateFixtures() +}) + +async function terminalTrace( + request: AgentCandidateExecutorRequest, + store: TraceStore, + endedAt = 200, +) { + await store.appendRun({ + runId: request.trace.runId, + scenarioId: 'candidate-execution', + startedAt: 100, + endedAt, + status: 'completed', + tags: { ...request.trace.tags }, + }) +} + +function options( + executor: AgentCandidateExecutorPort, + traceStore = new InMemoryTraceStore(), + claimStore = new InMemoryAgentCandidateExecutionClaimStore(), +) { + const outputs = createCandidateOutputFixture() + let request: AgentCandidateExecutorRequest | undefined + return { + executor: { + execute: async (...args: Parameters) => { + request = args[0] + return await executor.execute(...args) + }, + stopAndCapture: async (...args: Parameters) => { + const capture = await executor.stopAndCapture(...args) + if (capture.taskOutcome || !request) return capture + return { + ...capture, + taskOutcome: { + resultTree: request.executionPlan.value.material.task.repository.baseTree, + afterState: request.executionPlan.value.material.task.workspace.material, + archive: Buffer.from('fixture unchanged task archive', 'utf8'), + gitDiff: Buffer.alloc(0), + }, + } + }, + }, + traceStore, + claimStore, + ...outputs, + } +} + +describe('atomic prepared candidate execution', () => { + it('reveals credentials only to one trusted executor and returns a durable receipt', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = candidateBundle({ + env: { PUBLIC_MODE: { kind: 'public', value: 'fixture' } }, + }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + expect(JSON.stringify(prepared)).not.toContain('MODEL_GATEWAY_TOKEN') + expect(JSON.stringify(prepared)).not.toContain('protected') + + const traceStore = new InMemoryTraceStore() + let observed: AgentCandidateExecutorRequest | undefined + let stopped = 0 + const executor: AgentCandidateExecutorPort = { + execute: async (request, context) => { + observed = request + expect(context.traceStore).not.toBe(traceStore) + expect(context.signal.aborted).toBe(false) + expect(context.deadlineAtMs).toBeGreaterThan(Date.now()) + expect(request.launch.env).toMatchObject({ + PUBLIC_MODE: 'fixture', + MODEL_GATEWAY_TOKEN: 'protected', + TANGLE_CANDIDATE_EXECUTION_ID: prepared.executionId, + }) + expect(request.launch.env.PATH).toBeUndefined() + expect(request.hardLimits).toEqual({ timeoutMs: fixture.task.limits.timeoutMs }) + expect(request.observedLimits).toEqual({ maxSteps: fixture.task.limits.maxSteps }) + expect(Buffer.from(request.inputs.task.files[0]?.bytes ?? []).toString('utf8')).toBe( + 'export const value = 1\n', + ) + await terminalTrace(request, traceStore) + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: 0 }, + } + }, + stopAndCapture: async () => { + stopped++ + return { stopped: true } + }, + } + + const executionOptions = options(executor, traceStore) + const result = await executePreparedAgentCandidate(prepared, executionOptions) + expect(observed).toBeDefined() + expect(stopped).toBe(1) + if (!result.succeeded) throw new Error(result.reason) + expect(result).toMatchObject({ + succeeded: true, + receipt: { + value: { + schemaVersion: 2, + usage: { modelCalls: 0, costUsd: 0 }, + fixedUsage: { modelCalls: 0, costUsdNanos: 0 }, + benchmarkResult: { + material: { + score: 1, + passed: true, + grader: { name: 'fixture-executable-grader', version: '1.0.0' }, + }, + }, + }, + }, + }) + await expect( + executionOptions.outputArtifacts.read(result.artifacts.runReceipt), + ).resolves.toEqual(result.receipt.bytes) + }) + + it('rejects pre-launch staging mutation before activation or executor access', async () => { + const fixture = createCandidateExecutionFixture(true) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + if (!fixture.candidateRoot) throw new Error('active fixture root is missing') + writeFileSync(join(fixture.candidateRoot, 'run.js'), 'tampered-after-prepare\n') + let activations = 0 + let executions = 0 + let settlements = 0 + fixture.ports.models.activateGrant = async () => { + activations++ + return { env: { MODEL_GATEWAY_TOKEN: 'protected' } } + } + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlements++ + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + const result = await executePreparedAgentCandidate( + prepared, + options({ + execute: async () => { + executions++ + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/do not match the signed manifest/), + usage: { modelCalls: 0 }, + }) + expect({ activations, executions, settlements }).toEqual({ + activations: 0, + executions: 0, + settlements: 1, + }) + }) + + it('refuses activation when slow claim persistence consumed the execution window', async () => { + let now = Date.now() + vi.spyOn(Date, 'now').mockImplementation(() => now) + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const baseStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + let activations = 0 + fixture.ports.models.activateGrant = async () => { + activations++ + return { env: { MODEL_GATEWAY_TOKEN: 'protected' } } + } + const claimStore = { + tryClaim: async (claim: Parameters[0]) => { + const result = await baseStore.tryClaim(claim) + now += fixture.task.limits.timeoutMs + 1 + return result + }, + getAttempt: (attempt: Parameters[0]) => + baseStore.getAttempt(attempt), + markCandidateMayRun: (lease: Parameters[0]) => + baseStore.markCandidateMayRun(lease), + stageTerminal: ( + lease: Parameters[0], + result: Parameters[1], + ) => baseStore.stageTerminal(lease, result), + finish: ( + lease: Parameters[0], + terminalDigest: Parameters[1], + ) => baseStore.finish(lease, terminalDigest), + recoverExpired: ( + attempt: Parameters[0], + evidence: Parameters[1], + ) => baseStore.recoverExpired(attempt, evidence), + } + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async () => { + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + new InMemoryTraceStore(), + claimStore, + ), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/no longer covers/), + usage: { modelCalls: 0 }, + }) + expect(activations).toBe(0) + }) + + it('does not launch when phase persistence consumes the frozen execution window', async () => { + let now = 1_000_000 + vi.spyOn(Date, 'now').mockImplementation(() => now) + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 100 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs: 25, resultTimeoutMs: 100 }, + ) + const baseStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claimStore = { + tryClaim: (claim: Parameters[0]) => baseStore.tryClaim(claim), + getAttempt: (attempt: Parameters[0]) => + baseStore.getAttempt(attempt), + markCandidateMayRun: async (lease: Parameters[0]) => { + const result = await baseStore.markCandidateMayRun(lease) + now += fixture.task.limits.timeoutMs + return result + }, + stageTerminal: ( + lease: Parameters[0], + terminal: Parameters[1], + ) => baseStore.stageTerminal(lease, terminal), + finish: ( + lease: Parameters[0], + terminalDigest: Parameters[1], + ) => baseStore.finish(lease, terminalDigest), + recoverExpired: ( + attempt: Parameters[0], + evidence: Parameters[1], + ) => baseStore.recoverExpired(attempt, evidence), + } + let executions = 0 + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async () => { + executions++ + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + new InMemoryTraceStore(), + claimStore, + ), + ) + + expect(executions).toBe(0) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/deadline elapsed while persisting its launch phase/), + usage: { modelCalls: 0 }, + }) + await expect( + baseStore.getAttempt({ executionId: prepared.executionId, attempt: 1 }), + ).resolves.toMatchObject({ terminal: { status: 'failed', failureClass: 'unknown' } }) + }) + + it('publishes after every post-run phase consumes nearly its full reserved interval', async () => { + let now = 1_000_000 + vi.spyOn(Date, 'now').mockImplementation(() => now) + const cleanupTimeoutMs = 100 + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + fixture.ports.models.settleGrant = async ({ preparationId }) => { + now += cleanupTimeoutMs - 1 + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs }, + ) + const traceStore = new InMemoryTraceStore() + const baseStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claimStore = { + tryClaim: (claim: Parameters[0]) => baseStore.tryClaim(claim), + getAttempt: (attempt: Parameters[0]) => + baseStore.getAttempt(attempt), + markCandidateMayRun: (lease: Parameters[0]) => + baseStore.markCandidateMayRun(lease), + stageTerminal: async ( + lease: Parameters[0], + terminal: Parameters[1], + ) => { + now += 5_000 + return await baseStore.stageTerminal(lease, terminal) + }, + finish: async ( + lease: Parameters[0], + terminalDigest: Parameters[1], + ) => { + now += 4_999 + return await baseStore.finish(lease, terminalDigest) + }, + recoverExpired: ( + attempt: Parameters[0], + evidence: Parameters[1], + ) => baseStore.recoverExpired(attempt, evidence), + } + const outputs = createCandidateOutputFixture() + let advancedModelEvidence = false + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + if (input.purpose === 'model-settlement' && !advancedModelEvidence) { + advancedModelEvidence = true + now += cleanupTimeoutMs - 1 + } + return await outputs.outputArtifacts.put(input) + }, + } + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request, context) => { + now = context.deadlineAtMs - 1 + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => { + now += cleanupTimeoutMs - 1 + return { stopped: true, taskOutcome: unchangedTaskOutcomeCapture(fixture) } + }, + }, + grader: { + ...outputs.grader, + run: async (input) => { + now += cleanupTimeoutMs - 1 + return await outputs.grader.run(input) + }, + }, + outputArtifacts, + traceStore, + claimStore, + cleanupTimeoutMs, + }) + + expect(result.succeeded).toBe(true) + const attempt = await baseStore.getAttempt({ executionId: prepared.executionId, attempt: 1 }) + expect(attempt?.terminal?.status).toBe('succeeded') + expect(now).toBeLessThanOrEqual(attempt?.claim.leaseExpiresAtMs ?? 0) + }) + + it('allows executable grading to exceed cleanup time inside its frozen result budget', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs: 10, resultTimeoutMs: 100 }, + ) + const traceStore = new InMemoryTraceStore() + const outputs = createCandidateOutputFixture() + vi.useFakeTimers({ now: Date.now() }) + let markGraderStarted!: () => void + const graderStarted = new Promise((resolve) => { + markGraderStarted = resolve + }) + const pending = executePreparedAgentCandidate(prepared, { + ...options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + }), + }, + traceStore, + ), + grader: { + ...outputs.grader, + run: async (input) => { + markGraderStarted() + await new Promise((resolve) => setTimeout(resolve, 50)) + return await outputs.grader.run(input) + }, + }, + outputArtifacts: outputs.outputArtifacts, + cleanupTimeoutMs: 10, + resultTimeoutMs: 100, + }) + + await graderStarted + await vi.advanceTimersByTimeAsync(50) + await expect(pending).resolves.toMatchObject({ succeeded: true }) + }) + + it('cancels over-budget grading without any output appearing after terminal failure', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 1_000 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + { cleanupTimeoutMs: 25, resultTimeoutMs: 50 }, + ) + const traceStore = new InMemoryTraceStore() + const outputs = createCandidateOutputFixture() + const persistedPurposes: string[] = [] + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + input.signal?.throwIfAborted() + persistedPurposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + } + vi.useFakeTimers({ now: Date.now() }) + let markGraderStarted!: () => void + const graderStarted = new Promise((resolve) => { + markGraderStarted = resolve + }) + const pending = executePreparedAgentCandidate(prepared, { + ...options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + }), + }, + traceStore, + ), + grader: { + ...outputs.grader, + run: async (input) => { + markGraderStarted() + await new Promise((resolve) => setTimeout(resolve, 100)) + return await outputs.grader.run(input) + }, + }, + outputArtifacts, + cleanupTimeoutMs: 25, + resultTimeoutMs: 50, + }) + + await graderStarted + await vi.advanceTimersByTimeAsync(50) + const result = await pending + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/result deadline/), + }) + const purposesAtTerminalFailure = [...persistedPurposes] + await vi.advanceTimersByTimeAsync(100) + await Promise.resolve() + expect(persistedPurposes).toEqual(purposesAtTerminalFailure) + expect(persistedPurposes).not.toContain('grader-evidence') + expect(persistedPurposes).not.toContain('benchmark-result') + expect(persistedPurposes).not.toContain('trace') + expect(persistedPurposes).not.toContain('run-receipt') + }) + + it('allows disposal to retry an unproven pre-claim cleanup', async () => { + const fixture = createCandidateExecutionFixture(true) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + if (!fixture.candidateRoot) throw new Error('active fixture root is missing') + writeFileSync(join(fixture.candidateRoot, 'run.js'), 'tampered-after-prepare\n') + let settlements = 0 + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlements++ + if (settlements === 1) throw new Error('transient gateway failure') + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + const result = await executePreparedAgentCandidate( + prepared, + options({ + execute: async () => { + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/cleanup remains incomplete/), + }) + await expect(disposePreparedAgentCandidateExecution(prepared)).resolves.toEqual({ + disposed: true, + }) + expect(settlements).toBe(2) + }) + + it('executes detached original bytes when staging changes after its final check', async () => { + const fixture = createCandidateExecutionFixture(true) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + if (!fixture.candidateRoot) throw new Error('active fixture root is missing') + fixture.ports.models.activateGrant = async () => { + writeFileSync(join(fixture.candidateRoot as string, 'run.js'), 'concurrent-replacement\n') + return { env: { MODEL_GATEWAY_TOKEN: 'protected' } } + } + const traceStore = new InMemoryTraceStore() + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request) => { + expect( + Buffer.from(request.inputs.candidate?.files[0]?.bytes ?? []).toString('utf8'), + ).toBe('#!/usr/bin/env node\n') + const detached = request.inputs.candidate?.files[0]?.bytes + detached?.fill(0) + expect( + Buffer.from(request.inputs.candidate?.files[0]?.bytes ?? []).toString('utf8'), + ).toBe('#!/usr/bin/env node\n') + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + ), + ) + expect(result.succeeded).toBe(true) + }) + + it('withholds a receipt when process death has no captured task outcome', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + ...createCandidateOutputFixture(), + }) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/without a captured task outcome/), + usage: { modelCalls: 0, costUsd: 0 }, + }) + }) + + it('rejects candidate-authored score fields in the final capture before grading', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + let graderCalls = 0 + const outputs = createCandidateOutputFixture() + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => + ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + score: 1, + }) as never, + }, + grader: { + ...outputs.grader, + run: async (input) => { + graderCalls++ + return await outputs.grader.run(input) + }, + }, + outputArtifacts: outputs.outputArtifacts, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/unknown field score/), + usage: { modelCalls: 0, costUsd: 0 }, + }) + expect(graderCalls).toBe(0) + }) + + it('rejects candidate-authored score fields in the execution capture before grading', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + let graderCalls = 0 + const outputs = createCandidateOutputFixture() + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: 0 }, + score: 1, + } as never + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + }), + }, + grader: { + ...outputs.grader, + run: async (input) => { + graderCalls++ + return await outputs.grader.run(input) + }, + }, + outputArtifacts: outputs.outputArtifacts, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/unknown field score/), + usage: { modelCalls: 0, costUsd: 0 }, + }) + expect(graderCalls).toBe(0) + }) + + it('allows exactly one concurrent invocation of the same prepared value', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + let executions = 0 + let release!: () => void + const wait = new Promise((resolve) => { + release = resolve + }) + let started!: () => void + const didStart = new Promise((resolve) => { + started = resolve + }) + const executor: AgentCandidateExecutorPort = { + execute: async (request) => { + executions++ + started() + await wait + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + } + const first = executePreparedAgentCandidate(prepared, options(executor, traceStore, claimStore)) + await didStart + const second = await executePreparedAgentCandidate( + prepared, + options(executor, traceStore, claimStore), + ) + release() + const firstResult = await first + expect(firstResult.succeeded).toBe(true) + expect(second).toMatchObject({ succeeded: false, reason: expect.stringMatching(/already/) }) + expect(executions).toBe(1) + }) + + it('keeps independently prepared reservations isolated when one loses the durable claim', async () => { + const fixture = createCandidateExecutionFixture() + const openPreparations = new Set() + const closedPreparations = new Set() + let activePreparation: string | undefined + fixture.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => { + openPreparations.add(preparationId) + return { + preparationId, + digest: candidateSha('c'), + expiresAtMs, + enforcedLimits: limits, + } + } + fixture.ports.models.activateGrant = async ({ preparationId }) => { + if (!openPreparations.has(preparationId) || closedPreparations.has(preparationId)) { + throw new Error('activation used a closed preparation') + } + activePreparation = preparationId + return { env: { MODEL_GATEWAY_TOKEN: `protected-${preparationId}` } } + } + fixture.ports.models.settleGrant = async ({ preparationId }) => { + if (!openPreparations.has(preparationId)) throw new Error('unknown preparation') + closedPreparations.add(preparationId) + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + } + } + + const verified = await verifyAgentCandidateBundle(fixture.bundle, fixture.ports) + const firstPrepared = await prepareAgentCandidateExecution( + verified, + fixture.task, + fixture.ports, + ) + rmSync(fixture.task.stagingRoots.profileRoot, { recursive: true }) + mkdirSync(fixture.task.stagingRoots.profileRoot) + const secondPrepared = await prepareAgentCandidateExecution( + verified, + fixture.task, + fixture.ports, + ) + expect(firstPrepared.trace.runId).not.toBe(secondPrepared.trace.runId) + + const traceStore = new InMemoryTraceStore() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + let release!: () => void + const wait = new Promise((resolve) => { + release = resolve + }) + let started!: () => void + const didStart = new Promise((resolve) => { + started = resolve + }) + const executor: AgentCandidateExecutorPort = { + execute: async (request) => { + started() + await wait + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + } + + const first = executePreparedAgentCandidate( + firstPrepared, + options(executor, traceStore, claimStore), + ) + await didStart + const loser = await executePreparedAgentCandidate( + secondPrepared, + options(executor, traceStore, claimStore), + ) + expect(loser).toMatchObject({ succeeded: false, reason: expect.stringMatching(/claimed/) }) + expect(activePreparation).toBeDefined() + expect(closedPreparations.has(activePreparation as string)).toBe(false) + expect(closedPreparations.size).toBe(1) + + release() + expect((await first).succeeded).toBe(true) + expect(closedPreparations.size).toBe(2) + }) + + it('settles and records paid usage when the executor fails', async () => { + const fixture = createCandidateExecutionFixture() + const secret = 'sk-runtime-secret-123456789' + let closed = false + let settlementReads = 0 + const serveLateCall = () => { + if (closed) throw new Error('grant is closed') + } + fixture.ports.models.activateGrant = async () => ({ + env: { MODEL_GATEWAY_TOKEN: secret }, + }) + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlementReads++ + closed = true + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [ + { + callId: 'call-paid-1', + traceSpanId: 'llm-paid-1', + model: 'model-snapshot', + inputTokens: 10, + outputTokens: 5, + costUsdNanos: 10_000_000, + }, + ], + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const result = await executePreparedAgentCandidate( + prepared, + options({ + execute: async () => Promise.reject(new Error(`container failed with ${secret}`)), + stopAndCapture: async () => ({ stopped: true }), + }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.not.stringContaining(secret), + usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5, costUsd: 0.01 }, + }) + expect(result.reason).toContain('[redacted:candidate-access]') + expect({ closed, settlementReads }).toEqual({ closed: true, settlementReads: 1 }) + expect(serveLateCall).toThrow(/closed/) + }) + + it('owns the deadline, aborts, waits for process death, and then settles', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 15 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + let observedSignal: AbortSignal | undefined + let timeoutRequest: AgentCandidateExecutorRequest | undefined + let stoppedAfterAbort = false + let settledAfterStop = false + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settledAfterStop = stoppedAfterAbort + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + vi.useFakeTimers({ now: Date.now() }) + const startedAt = Date.now() + let markExecutorStarted!: () => void + const executorStarted = new Promise((resolve) => { + markExecutorStarted = resolve + }) + const pending = executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request, context) => { + timeoutRequest = request + observedSignal = context.signal + markExecutorStarted() + return await new Promise((_resolve, reject) => { + context.signal.addEventListener('abort', () => reject(context.signal.reason), { + once: true, + }) + }) + }, + stopAndCapture: async (_request, context) => { + expect(context.reason).toBe('timeout') + expect(context.signal).toBe(observedSignal) + expect(context.deadlineAtMs).toBe(startedAt + 15) + stoppedAfterAbort = observedSignal?.aborted === true + if (!timeoutRequest) throw new Error('timeout executor never received its request') + await terminalTrace(timeoutRequest, traceStore, 115) + return { stopped: true } + }, + }, + traceStore, + ), + ) + await executorStarted + await vi.advanceTimersByTimeAsync(15) + const result = await pending + expect(Date.now()).toBe(startedAt + 15) + expect({ stoppedAfterAbort, settledAfterStop }).toEqual({ + stoppedAfterAbort: true, + settledAfterStop: true, + }) + expect(result).toMatchObject({ + succeeded: true, + receipt: { value: { termination: { kind: 'timeout', timeoutMs: 15 } } }, + }) + }) + + it('cannot turn a stop acknowledgement after the frozen deadline into an exit success', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = { ...fixture.task.limits, timeoutMs: 15 } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + vi.useFakeTimers({ now: Date.now() }) + let markStopStarted!: () => void + const stopStarted = new Promise((resolve) => { + markStopStarted = resolve + }) + const pending = executePreparedAgentCandidate(prepared, { + ...options( + { + execute: async (request) => { + await terminalTrace(request, traceStore, 105) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => { + markStopStarted() + await new Promise((resolve) => setTimeout(resolve, 30)) + return { stopped: true } + }, + }, + traceStore, + ), + cleanupTimeoutMs: 100, + }) + await stopStarted + await vi.advanceTimersByTimeAsync(30) + const result = await pending + expect(result).toMatchObject({ + succeeded: true, + receipt: { value: { termination: { kind: 'timeout', timeoutMs: 15 } } }, + }) + }) + + it('bounds a hanging process stop and leaves the claim for explicit recovery', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + const startedAt = Date.now() + const result = await executePreparedAgentCandidate(prepared, { + ...options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => await new Promise(() => undefined), + }, + traceStore, + claimStore, + ), + cleanupTimeoutMs: 20, + }) + expect(Date.now() - startedAt).toBeLessThan(250) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/termination is not proven/), + usage: { modelCalls: 0 }, + }) + }) + + it('withholds a receipt but still revokes model access when process death is unproven', async () => { + const fixture = createCandidateExecutionFixture() + let settlements = 0 + fixture.ports.models.settleGrant = async ({ preparationId }) => { + settlements++ + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => { + throw new Error('container death not observed') + }, + }, + traceStore, + ), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/death not observed/), + usage: { modelCalls: 0 }, + }) + expect(settlements).toBe(1) + }) + + it('leaves unknown settlement unfinished so a retry cannot guess zero spend', async () => { + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + const first = createCandidateExecutionFixture() + first.task.attempt = { + number: 1, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + first.ports.models.settleGrant = async () => { + throw new Error('gateway settlement unavailable') + } + const firstPrepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(first.bundle, first.ports), + first.task, + first.ports, + ) + const firstResult = await executePreparedAgentCandidate( + firstPrepared, + options( + { + execute: async () => { + throw new Error('infrastructure failed') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + new InMemoryTraceStore(), + claimStore, + ), + ) + expect(firstResult).toMatchObject({ succeeded: false, usage: null }) + + rmSync(first.task.stagingRoots.profileRoot, { recursive: true }) + mkdirSync(first.task.stagingRoots.profileRoot) + first.task.attempt = { + number: 2, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const retryPrepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(first.bundle, first.ports), + first.task, + first.ports, + ) + let retryExecutions = 0 + const retryResult = await executePreparedAgentCandidate( + retryPrepared, + options( + { + execute: async () => { + retryExecutions++ + throw new Error('retry must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + new InMemoryTraceStore(), + claimStore, + ), + ) + expect(retryResult).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/prior-attempt-running/), + }) + expect(retryExecutions).toBe(0) + }) + + it('admits a zero-call retry only for an explicit pre-model infrastructure failure', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.attempt = { + number: 1, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const claimStore = new InMemoryAgentCandidateExecutionClaimStore() + let activations = 0 + fixture.ports.models.activateGrant = async () => { + activations++ + if (activations === 1) throw new Error('model gateway unavailable before launch') + return { env: { MODEL_GATEWAY_TOKEN: 'protected' } } + } + const firstPrepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const firstTraceStore = new InMemoryTraceStore() + const first = await executePreparedAgentCandidate( + firstPrepared, + options( + { + execute: async () => { + throw new Error('executor must not run before model activation') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + firstTraceStore, + claimStore, + ), + ) + expect(first).toMatchObject({ succeeded: false, usage: { modelCalls: 0 } }) + + rmSync(fixture.task.stagingRoots.profileRoot, { recursive: true }) + mkdirSync(fixture.task.stagingRoots.profileRoot) + fixture.task.attempt = { + number: 2, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const retryPrepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const retry = await executePreparedAgentCandidate( + retryPrepared, + options( + { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + claimStore, + ), + ) + expect(retry.succeeded).toBe(true) + }) + + it('closes isolated memory after process death and before model settlement', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }) + const before = emptyCandidateSnapshot('before') + const after = emptyCandidateSnapshot('after') + const order: string[] = [] + fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: candidateSha('8'), + expiresAtMs, + evidence: before.manifest, + emptyStateDigest: before.digest, + beforeState: before, + }) + fixture.ports.memory.activate = async ({ effectiveNamespace }) => ({ + env: { TANGLE_MEMORY_NAMESPACE: effectiveNamespace }, + }) + fixture.ports.memory.close = async () => { + order.push('memory-close') + return { closed: true } + } + fixture.ports.models.settleGrant = async ({ preparationId }) => { + order.push('model-settle') + return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + if (prepared.memory.mode !== 'isolated') throw new Error('isolated memory was not prepared') + const traceStore = new InMemoryTraceStore() + const result = await executePreparedAgentCandidate( + prepared, + options( + { + execute: async (request) => { + expect(request.launch.env.TANGLE_MEMORY_NAMESPACE).toBe( + prepared.memory.effectiveNamespace, + ) + await terminalTrace(request, traceStore) + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: 0 }, + } + }, + stopAndCapture: async () => { + order.push('process-stop') + if (!('content' in after.archive)) + throw new Error('fixture memory archive is not embedded') + return { + stopped: true, + memoryAfter: { + afterState: after.material, + archive: Buffer.from(after.archive.content, 'base64'), + }, + } + }, + }, + traceStore, + ), + ) + expect(result.succeeded).toBe(true) + expect(order).toEqual(['process-stop', 'memory-close', 'model-settle']) + }) + + it('rejects a compressed protected value in isolated memory before persisting memory evidence', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }) + const before = emptyCandidateSnapshot('secret-memory-before') + fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: candidateSha('8'), + expiresAtMs, + evidence: before.manifest, + emptyStateDigest: before.digest, + beforeState: before, + }) + fixture.ports.memory.activate = async () => ({ env: { TANGLE_MEMORY_ACCESS: 'memory-access' } }) + fixture.ports.memory.close = async () => ({ closed: true }) + const secret = 'sk-memory-secret-123456789' + fixture.ports.models.activateGrant = async () => ({ env: { MODEL_GATEWAY_TOKEN: secret } }) + const originalMaterialize = fixture.ports.workspaces.materialize + fixture.ports.workspaces.materialize = async (input) => { + if (input.role !== 'memory') return await originalMaterialize(input) + const output = join(input.destination, 'notes.txt') + writeFileSync(output, Buffer.from(secret, 'utf8')) + chmodSync(output, 0o644) + } + const memoryBytes = Buffer.from(secret, 'utf8') + const afterState = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: [ + { + path: 'notes.txt', + mode: 0o644 as const, + sha256: sha256Bytes(memoryBytes), + byteLength: memoryBytes.byteLength, + }, + ], + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const outputs = createCandidateOutputFixture() + const purposes: string[] = [] + const result = await executePreparedAgentCandidate(prepared, { + executor: { + execute: async (request) => { + await terminalTrace(request, traceStore) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ + stopped: true, + taskOutcome: unchangedTaskOutcomeCapture(fixture), + memoryAfter: { afterState, archive: gzipSync(memoryBytes) }, + }), + }, + grader: outputs.grader, + outputArtifacts: { + read: outputs.outputArtifacts.read, + put: async (input) => { + purposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + }, + traceStore, + claimStore: new InMemoryAgentCandidateExecutionClaimStore(), + }) + + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/protected value/), + }) + expect(purposes).not.toContain('memory-after-manifest') + expect(purposes).not.toContain('memory-after-archive') + }) + + it('redacts protected values from spans, events, artifacts, and receipt bytes', async () => { + const fixture = createCandidateExecutionFixture() + const secret = 'sk-redaction-proof-123456789' + fixture.ports.models.activateGrant = async () => ({ + env: { MODEL_GATEWAY_TOKEN: secret }, + }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + const traceStore = new InMemoryTraceStore() + const executionOptions = options( + { + execute: async (request, context) => { + await terminalTrace(request, context.traceStore) + await context.traceStore.appendSpan({ + runId: request.trace.runId, + spanId: 'tool-secret', + kind: 'tool', + name: 'tool', + toolName: 'exec', + args: { + token: secret, + [secret]: Buffer.from(secret, 'utf8').toString('base64url'), + }, + result: secret, + startedAt: 110, + endedAt: 120, + status: 'ok', + }) + await context.traceStore.appendEvent({ + runId: request.trace.runId, + eventId: 'event-secret', + kind: 'log', + timestamp: 130, + payload: { message: secret }, + }) + await context.traceStore.appendArtifact({ + runId: request.trace.runId, + artifactId: 'artifact-secret', + contentType: 'text/plain', + sizeBytes: secret.length, + hash: '0'.repeat(64), + inlineContent: secret, + }) + return { executionId: request.executionId, termination: { kind: 'exit', exitCode: 0 } } + }, + stopAndCapture: async () => ({ stopped: true }), + }, + traceStore, + ) + const result = await executePreparedAgentCandidate(prepared, executionOptions) + if (!result.succeeded) throw new Error(result.reason) + const traceJson = Buffer.from( + await executionOptions.outputArtifacts.read(result.receipt.value.trace.artifact), + ).toString('utf8') + expect(traceJson).not.toContain(secret) + expect(traceJson).not.toContain(Buffer.from(secret, 'utf8').toString('base64url')) + expect(traceJson).toContain('[redacted:candidate-access]') + expect(Buffer.from(result.receipt.bytes).toString('utf8')).not.toContain(secret) + expect( + JSON.stringify({ + spans: await traceStore.spans({ runId: prepared.trace.runId }), + events: await traceStore.events({ runId: prepared.trace.runId }), + artifacts: await traceStore.artifacts(prepared.trace.runId), + }), + ).not.toContain(secret) + }) + + it('rejects protected environment collisions before executor access', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = candidateBundle({ + env: { PUBLIC_MODE: { kind: 'public', value: 'fixture' } }, + }) + fixture.ports.models.activateGrant = async () => ({ env: { PUBLIC_MODE: 'secret-value' } }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let executions = 0 + const result = await executePreparedAgentCandidate( + prepared, + options({ + execute: async () => { + executions++ + throw new Error('must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }), + ) + expect(result).toMatchObject({ succeeded: false, reason: expect.stringMatching(/collides/) }) + expect(executions).toBe(0) + }) +}) diff --git a/tests/candidate-execution-finalize.test.ts b/tests/candidate-execution-finalize.test.ts new file mode 100644 index 00000000..64c2846d --- /dev/null +++ b/tests/candidate-execution-finalize.test.ts @@ -0,0 +1,434 @@ +import { InMemoryTraceStore } from '@tangle-network/agent-eval' +import { afterEach, describe, expect, it } from 'vitest' + +import { finalizeAgentCandidateRun } from '../src/candidate-execution/finalize' +import { + type SealedAgentCandidateModelSettlement, + sealAgentCandidateModelSettlement, +} from '../src/candidate-execution/model-settlement' +import { + persistCandidateBenchmarkResult, + persistCandidateModelSettlement, + persistVerifiedCandidateTaskOutcome, +} from '../src/candidate-execution/outcome-evidence' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { assertPreparedCandidateIntegrity } from '../src/candidate-execution/prepared-state' +import { + CANDIDATE_TRACE_TAGS, + type PreparedAgentCandidateExecution, +} from '../src/candidate-execution/types' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +async function prepared( + limits: { + timeoutMs: number + maxSteps: number + maxModelCalls: number + maxInputTokens: number + maxOutputTokens: number + maxCostUsd: number + } = { + timeoutMs: 1_000, + maxSteps: 2, + maxModelCalls: 1, + maxInputTokens: 20, + maxOutputTokens: 10, + maxCostUsd: 0.1, + }, +): Promise { + const fixture = createCandidateExecutionFixture() + fixture.task.limits = limits + return await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) +} + +async function finalizePrepared( + execution: PreparedAgentCandidateExecution, + capture: Parameters[1], + store: InMemoryTraceStore, + overrides: { + settlement?: SealedAgentCandidateModelSettlement + outputs?: ReturnType + } = {}, +) { + const state = assertPreparedCandidateIntegrity(execution) + const settlement = + overrides.settlement ?? + sealAgentCandidateModelSettlement( + { + preparationId: state.preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [ + { + callId: 'call-1', + traceSpanId: 'llm-1', + model: execution.resolvedModel.model, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + costUsdNanos: 10_000_000, + }, + ], + }, + { + preparationId: state.preparationId, + grantDigest: candidateSha('c'), + model: execution.resolvedModel.model, + }, + ) + const outputs = overrides.outputs ?? createCandidateOutputFixture() + const finalCapture = { + stopped: true as const, + taskOutcome: { + resultTree: state.executionPlan.value.material.task.repository.baseTree, + afterState: state.executionPlan.value.material.task.workspace.material, + archive: Buffer.from('fixture unchanged task archive', 'utf8'), + gitDiff: Buffer.alloc(0), + }, + } + const [modelSettlement, taskOutcome] = await Promise.all([ + persistCandidateModelSettlement(state, settlement, outputs.outputArtifacts), + persistVerifiedCandidateTaskOutcome( + state, + finalCapture.taskOutcome, + outputs.outputArtifacts, + [], + ), + ]) + const benchmarkResult = await persistCandidateBenchmarkResult( + state, + capture.termination, + taskOutcome, + outputs.grader, + outputs.outputArtifacts, + [], + ) + return await finalizeAgentCandidateRun( + state, + capture, + store, + settlement, + { + finalCapture, + modelSettlement, + taskOutcome, + benchmarkResult, + outputArtifacts: outputs.outputArtifacts, + }, + [], + ) +} + +async function traceStore( + execution: PreparedAgentCandidateExecution, + overrides: { + tags?: Record + model?: string + omitUsage?: boolean + omitModelSpan?: boolean + toolSteps?: number + endedAt?: number + } = {}, +): Promise { + const store = new InMemoryTraceStore() + await store.appendRun({ + runId: execution.trace.runId, + scenarioId: 'task', + startedAt: 100, + endedAt: overrides.endedAt ?? 500, + status: 'completed', + tags: overrides.tags ?? execution.trace.tags, + }) + if (!overrides.omitModelSpan) { + await store.appendSpan({ + runId: execution.trace.runId, + spanId: 'llm-1', + kind: 'llm', + name: 'model call', + model: overrides.model ?? execution.resolvedModel.model, + messages: [], + startedAt: 120, + endedAt: 200, + status: 'ok', + ...(overrides.omitUsage + ? {} + : { inputTokens: 10, outputTokens: 5, cachedTokens: 2, costUsd: 0.01 }), + }) + } + for (let index = 0; index < (overrides.toolSteps ?? 1); index++) { + await store.appendSpan({ + runId: execution.trace.runId, + spanId: `tool-${index}`, + kind: 'tool', + name: 'tool', + toolName: 'exec', + args: {}, + startedAt: 210 + index, + endedAt: 220 + index, + status: 'ok', + }) + } + return store +} + +describe('protected candidate run finalization', () => { + it('derives exact V2 evidence and durable trace bytes from tied agent-eval spans', async () => { + const execution = await prepared() + const store = await traceStore(execution) + const outputs = createCandidateOutputFixture() + const result = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + store, + { outputs }, + ) + expect(result.succeeded).toBe(true) + if (!result.succeeded) return + expect(result.receipt.value.usage).toEqual({ + costUsd: 0.01, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + modelCalls: 1, + }) + expect(result.receipt.value.trace).toMatchObject({ eventCount: 3, modelCallCount: 1 }) + expect(result.receipt.bytes.byteLength).toBeGreaterThan(0) + const task = assertPreparedCandidateIntegrity(execution).executionPlan.value.material.task + expect(result.receipt.value).toMatchObject({ + schemaVersion: 2, + fixedUsage: { + costUsdNanos: 10_000_000, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + modelCalls: 1, + }, + taskOutcome: { + artifact: result.artifacts.taskOutcome, + material: { + executionPlanDigest: execution.executionPlan.value.digest, + baseRepository: { + identity: task.repository.identity, + rootIdentity: task.repository.rootIdentity, + commit: task.repository.baseCommit, + tree: task.repository.baseTree, + }, + resultRepository: { + identity: task.repository.identity, + rootIdentity: task.repository.rootIdentity, + commit: expect.stringMatching(/^[0-9a-f]{40}$/), + tree: task.repository.baseTree, + }, + }, + }, + benchmarkResult: { + artifact: result.artifacts.benchmarkResult, + material: { + taskOutcomeDigest: result.receipt.value.taskOutcome.digest, + score: 1, + passed: true, + }, + }, + modelSettlement: { + artifact: result.artifacts.modelSettlement, + material: { + executionPlanDigest: execution.executionPlan.value.digest, + closed: true, + usage: { + costUsdNanos: 10_000_000, + inputTokens: 10, + outputTokens: 5, + cachedInputTokens: 2, + modelCalls: 1, + }, + }, + }, + }) + await Promise.all([ + expect(outputs.outputArtifacts.read(result.artifacts.runReceipt)).resolves.toEqual( + result.receipt.bytes, + ), + expect( + outputs.outputArtifacts.read(result.receipt.value.taskOutcome.material.gitDiff.artifact), + ).resolves.toEqual(new Uint8Array()), + expect( + outputs.outputArtifacts.read(result.receipt.value.benchmarkResult.material.evidence), + ).resolves.toEqual(Uint8Array.from(Buffer.from('{"reward":1}\n', 'utf8'))), + expect( + outputs.outputArtifacts.read(result.receipt.value.benchmarkResult.material.grader.artifact), + ).resolves.toEqual(Uint8Array.from(Buffer.from('fixture executable grader', 'utf8'))), + ]) + }) + + it('returns invalid capture instead of minting zero usage when provider usage is missing', async () => { + const execution = await prepared() + const result = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { omitUsage: true }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/model ledger/), + }) + }) + + it('rejects a dropped model span against the protected gateway ledger', async () => { + const execution = await prepared() + const result = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { omitModelSpan: true }), + ) + expect(result).toMatchObject({ + succeeded: false, + reason: expect.stringMatching(/model ledger/), + }) + }) + + it('rejects trace-binding and resolved-model drift', async () => { + const execution = await prepared() + const badTags = { + ...execution.trace.tags, + [CANDIDATE_TRACE_TAGS.bundleDigest]: candidateSha('9'), + } + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { tags: badTags }), + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/not bound/) }) + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { model: 'other-model' }), + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/does not match/) }) + }) + + it('enforces tool-step and wall-time limits from protected spans', async () => { + const execution = await prepared() + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { toolSteps: 3 }), + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/tool steps/) }) + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + await traceStore(execution, { endedAt: 1_101 }), + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/wall time/) }) + }) + + it('preserves timeout usage but requires the exact frozen timeout', async () => { + const execution = await prepared() + const store = await traceStore(execution) + const valid = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'timeout', timeoutMs: 1_000 } }, + store, + ) + expect(valid.succeeded).toBe(true) + await expect( + finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'timeout', timeoutMs: 999 } }, + store, + ), + ).resolves.toMatchObject({ succeeded: false, reason: expect.stringMatching(/frozen/) }) + }) + + it('reconciles decimal costs in fixed-point units', async () => { + const execution = await prepared({ + timeoutMs: 1_000, + maxSteps: 2, + maxModelCalls: 2, + maxInputTokens: 20, + maxOutputTokens: 10, + maxCostUsd: 0.3, + }) + const store = new InMemoryTraceStore() + await store.appendRun({ + runId: execution.trace.runId, + scenarioId: 'decimal-cost', + startedAt: 100, + endedAt: 500, + status: 'completed', + tags: execution.trace.tags, + }) + for (const [index, costUsd] of [0.1, 0.2].entries()) { + await store.appendSpan({ + runId: execution.trace.runId, + spanId: `llm-${index + 1}`, + kind: 'llm', + name: 'model call', + model: execution.resolvedModel.model, + messages: [], + inputTokens: 5, + outputTokens: 2, + costUsd, + startedAt: 120 + index * 100, + endedAt: 180 + index * 100, + status: 'ok', + }) + } + const settlement = sealAgentCandidateModelSettlement( + { + preparationId: assertPreparedCandidateIntegrity(execution).preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [ + { + callId: 'call-1', + traceSpanId: 'llm-1', + model: execution.resolvedModel.model, + inputTokens: 5, + outputTokens: 2, + costUsdNanos: 100_000_000, + }, + { + callId: 'call-2', + traceSpanId: 'llm-2', + model: execution.resolvedModel.model, + inputTokens: 5, + outputTokens: 2, + costUsdNanos: 200_000_000, + }, + ], + }, + { + preparationId: assertPreparedCandidateIntegrity(execution).preparationId, + grantDigest: candidateSha('c'), + model: execution.resolvedModel.model, + }, + ) + const result = await finalizePrepared( + execution, + { executionId: execution.executionId, termination: { kind: 'exit', exitCode: 0 } }, + store, + { settlement }, + ) + expect(result).toMatchObject({ + succeeded: true, + receipt: { value: { usage: { costUsd: 0.3, modelCalls: 2 } } }, + }) + }) +}) diff --git a/tests/candidate-execution-outcome-evidence.test.ts b/tests/candidate-execution-outcome-evidence.test.ts new file mode 100644 index 00000000..73fa0a41 --- /dev/null +++ b/tests/candidate-execution-outcome-evidence.test.ts @@ -0,0 +1,274 @@ +import { chmodSync, mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { gunzipSync, gzipSync } from 'node:zlib' + +import { afterEach, describe, expect, it } from 'vitest' +import { sha256Bytes } from '../src/candidate-execution/digest' +import { + persistCandidateBenchmarkResult, + persistVerifiedCandidateTaskOutcome, +} from '../src/candidate-execution/outcome-evidence' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { assertPreparedCandidateIntegrity } from '../src/candidate-execution/prepared-state' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + cleanupCandidateFixtures, + createCandidateExecutionFixture, + createCandidateOutputFixture, + unchangedTaskOutcomeCapture, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +describe('candidate outcome evidence', () => { + it('rejects protected values in the canonical task manifest before any output write', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + let puts = 0 + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + puts++ + return await outputs.outputArtifacts.put(input) + }, + } + + await expect( + persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputArtifacts, + ['source.ts'], + ), + ).rejects.toThrow(/protected value/) + expect(puts).toBe(0) + }) + + it('rejects a compressed invalid archive before any candidate output is persisted', async () => { + const fixture = createCandidateExecutionFixture() + fixture.ports.workspaces.materialize = async ({ archive, destination }) => { + if (destination === fixture.task.stagingRoots.taskRoot) return + mkdirSync(destination, { recursive: true }) + const output = join(destination, 'source.ts') + writeFileSync(output, gunzipSync(archive)) + chmodSync(output, 0o644) + } + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + let puts = 0 + const outputArtifacts = { + read: outputs.outputArtifacts.read, + put: async (input: Parameters[0]) => { + puts++ + return await outputs.outputArtifacts.put(input) + }, + } + const secret = 'sk-compressed-before-put-123456789' + + await expect( + persistVerifiedCandidateTaskOutcome( + state, + { + ...unchangedTaskOutcomeCapture(fixture), + archive: gzipSync(Buffer.from(secret, 'utf8')), + }, + outputArtifacts, + [secret], + ), + ).rejects.toThrow(/do not match the signed manifest/) + expect(puts).toBe(0) + }) + + it('rejects protected access values in task archives and raw grader output', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const secret = 'sk-outcome-secret-123456789' + const capture = unchangedTaskOutcomeCapture(fixture) + await expect( + persistVerifiedCandidateTaskOutcome( + state, + { ...capture, archive: Buffer.from(secret, 'utf8') }, + outputs.outputArtifacts, + [secret], + ), + ).rejects.toThrow(/protected value/) + + const outcome = await persistVerifiedCandidateTaskOutcome( + state, + capture, + outputs.outputArtifacts, + [secret], + ) + await expect( + persistCandidateBenchmarkResult( + state, + { kind: 'exit', exitCode: 0 }, + outcome, + { + ...outputs.grader, + run: async ({ implementation, outcome }) => { + const evidence = Buffer.from(secret, 'utf8') + return { + evaluation: { score: 1, passed: true, raw: {} }, + evidence, + binding: { + implementationDigest: sha256Bytes(implementation.bytes), + taskOutcomeDigest: outcome.evidence.digest, + outputDigest: sha256Bytes(evidence), + }, + } + }, + }, + outputs.outputArtifacts, + [secret], + ), + ).rejects.toThrow(/protected value/) + }) + + it('retains raw executable-grade evidence but cannot pass a nonzero exit', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const outcome = await persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputs.outputArtifacts, + [], + ) + const result = await persistCandidateBenchmarkResult( + state, + { kind: 'exit', exitCode: 1 }, + outcome, + outputs.grader, + outputs.outputArtifacts, + [], + ) + + expect(result.material).toMatchObject({ + score: 0, + passed: false, + dimensions: [{ name: 'tests', score: 0 }], + }) + expect(Buffer.from(await outputs.outputArtifacts.read(result.material.evidence))).toEqual( + Buffer.from('{"reward":1}\n', 'utf8'), + ) + expect(result.material.evidence.sha256).not.toBe(result.material.grader.artifact.sha256) + }) + + it('cannot persist a result when the runner binding does not match executed bytes', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const outcome = await persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputs.outputArtifacts, + [], + ) + const persistedPurposes: string[] = [] + const observingArtifacts = { + ...outputs.outputArtifacts, + put: async (input: Parameters[0]) => { + persistedPurposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + } + let admittedImplementation = Buffer.alloc(0) + + await expect( + persistCandidateBenchmarkResult( + state, + { kind: 'exit', exitCode: 0 }, + outcome, + { + ...outputs.grader, + run: async ({ implementation, outcome: gradedOutcome }) => { + admittedImplementation = Buffer.from(implementation.bytes) + const evidence = Buffer.from('{"reward":1}\n', 'utf8') + return { + evaluation: { score: 1, passed: true, raw: {} }, + evidence, + binding: { + implementationDigest: sha256Bytes(Buffer.from('different grader', 'utf8')), + taskOutcomeDigest: gradedOutcome.evidence.digest, + outputDigest: sha256Bytes(evidence), + }, + } + }, + }, + observingArtifacts, + [], + ), + ).rejects.toThrow(/executed implementation digest does not match/) + + expect(admittedImplementation).toEqual(Buffer.from('fixture executable grader', 'utf8')) + expect(persistedPurposes).toEqual([]) + }) + + it('cannot persist a result bound to another task outcome or raw output', async () => { + const fixture = createCandidateExecutionFixture() + const state = await preparedState(fixture) + const outputs = createCandidateOutputFixture() + const outcome = await persistVerifiedCandidateTaskOutcome( + state, + unchangedTaskOutcomeCapture(fixture), + outputs.outputArtifacts, + [], + ) + const persistedPurposes: string[] = [] + const observingArtifacts = { + ...outputs.outputArtifacts, + put: async (input: Parameters[0]) => { + persistedPurposes.push(input.purpose) + return await outputs.outputArtifacts.put(input) + }, + } + + for (const mismatch of ['task', 'output'] as const) { + await expect( + persistCandidateBenchmarkResult( + state, + { kind: 'exit', exitCode: 0 }, + outcome, + { + ...outputs.grader, + run: async ({ implementation, outcome: gradedOutcome }) => { + const evidence = Buffer.from('{"reward":1}\n', 'utf8') + return { + evaluation: { score: 1, passed: true, raw: {} }, + evidence, + binding: { + implementationDigest: sha256Bytes(implementation.bytes), + taskOutcomeDigest: + mismatch === 'task' + ? sha256Bytes(Buffer.from('other task', 'utf8')) + : gradedOutcome.evidence.digest, + outputDigest: + mismatch === 'output' + ? sha256Bytes(Buffer.from('other output', 'utf8')) + : sha256Bytes(evidence), + }, + } + }, + }, + observingArtifacts, + [], + ), + ).rejects.toThrow(/does not match/) + } + + expect(persistedPurposes).toEqual([]) + }) +}) + +async function preparedState(fixture: ReturnType) { + return assertPreparedCandidateIntegrity( + await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ), + ) +} diff --git a/tests/candidate-execution-output-artifacts.test.ts b/tests/candidate-execution-output-artifacts.test.ts new file mode 100644 index 00000000..f138bcb4 --- /dev/null +++ b/tests/candidate-execution-output-artifacts.test.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto' + +import { describe, expect, it } from 'vitest' + +import { persistCandidateOutputArtifact } from '../src/candidate-execution/output-artifacts' +import type { AgentCandidateOutputArtifactPort } from '../src/candidate-execution/types' +import { candidateSha } from './helpers/candidate-execution-fixture' + +describe('candidate output artifact persistence', () => { + it('does not call the durable store after result cancellation', async () => { + const controller = new AbortController() + controller.abort(new Error('result deadline')) + let puts = 0 + const port: AgentCandidateOutputArtifactPort = { + put: async () => { + puts++ + throw new Error('must not write') + }, + read: async () => new Uint8Array(), + } + + await expect( + persistCandidateOutputArtifact(port, { + executionId: 'execution-1', + purpose: 'trace', + bytes: Buffer.from('late'), + signal: controller.signal, + }), + ).rejects.toThrow(/result deadline/) + expect(puts).toBe(0) + }) + + it('reads back exact durable bytes before returning the locator', async () => { + const bytes = Buffer.from('durable outcome') + const stored = new Map() + const port: AgentCandidateOutputArtifactPort = { + put: async ({ bytes: value }) => { + const ref = { + locator: { kind: 's3' as const, bucket: 'candidate-results', key: 'outcome.bin' }, + sha256: `sha256:${createHash('sha256').update(value).digest('hex')}` as const, + byteLength: value.byteLength, + } + stored.set(ref.sha256, Uint8Array.from(value)) + return ref + }, + read: async (ref) => stored.get(ref.sha256) ?? new Uint8Array(), + } + + await expect( + persistCandidateOutputArtifact(port, { + executionId: 'execution-1', + purpose: 'task-patch', + bytes, + }), + ).resolves.toMatchObject({ byteLength: bytes.byteLength }) + }) + + it('rejects a locator that lies about the submitted content', async () => { + const port: AgentCandidateOutputArtifactPort = { + put: async () => ({ + locator: { kind: 's3', bucket: 'candidate-results', key: 'wrong.bin' }, + sha256: candidateSha('f'), + byteLength: 1, + }), + read: async () => Uint8Array.from([0]), + } + await expect( + persistCandidateOutputArtifact(port, { + executionId: 'execution-1', + purpose: 'trace', + bytes: Buffer.from('expected'), + }), + ).rejects.toThrow(/does not identify/) + }) +}) diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts new file mode 100644 index 00000000..dce98dc7 --- /dev/null +++ b/tests/candidate-execution-prepare.test.ts @@ -0,0 +1,587 @@ +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { + AgentCandidateBundle, + AgentCandidateExecution, + AgentCandidateWorkspaceSnapshotEvidence, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' +import { MAX_CANDIDATE_TIMER_INTERVAL_MS } from '../src/candidate-execution/cleanup' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + embeddedCandidateArtifact, +} from '../src/candidate-execution/digest' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import type { + AgentCandidateExecutionPorts, + AgentCandidateTaskExecution, + ResolvedAgentCandidateContainer, +} from '../src/candidate-execution/types' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' + +const roots: string[] = [] +const sha = (character: string): Sha256Digest => `sha256:${character.repeat(64)}` + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + roots.push(root) + return root +} + +function git(root: string, args: string[]): string { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim() +} + +function taskRepository(): { root: string; commit: string; tree: string } { + const root = temporaryRoot('candidate-task-') + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'test@example.com']) + git(root, ['config', 'user.name', 'Test']) + git(root, ['config', 'core.hooksPath', '/dev/null']) + git(root, ['remote', 'add', 'origin', 'git@github.com:owner/repo.git']) + writeFileSync(join(root, 'source.ts'), 'export const value = 1\n') + chmodSync(join(root, 'source.ts'), 0o644) + git(root, ['add', 'source.ts']) + git(root, ['commit', '-m', 'base']) + return { + root, + commit: git(root, ['rev-parse', 'HEAD']), + tree: git(root, ['rev-parse', 'HEAD^{tree}']), + } +} + +function snapshot( + root: string, + files: Array<{ path: string; mode: 0o644 | 0o755 }>, +): AgentCandidateWorkspaceSnapshotEvidence { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: files + .map((file) => { + const bytes = execFileSync('node', [ + '-e', + `process.stdout.write(require('fs').readFileSync(${JSON.stringify(join(root, file.path))}))`, + ]) + return { + path: file.path, + mode: file.mode, + sha256: embeddedCandidateArtifact(bytes).sha256, + byteLength: bytes.byteLength, + } + }) + .sort((a, b) => a.path.localeCompare(b.path)), + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive: embeddedCandidateArtifact(Buffer.from(`archive:${manifest.sha256}`)), + } +} + +function bundle( + execution: Partial = {}, + active?: { commit: string; tree: string; workspace: AgentCandidateWorkspaceSnapshotEvidence }, +): AgentCandidateBundle { + const value = { + schemaVersion: 1 as const, + kind: 'agent-candidate-bundle' as const, + digestAlgorithm: 'rfc8785-sha256' as const, + profile: { + name: 'candidate', + prompt: { instructions: ['Inspect the repository, implement the fix, and run tests.'] }, + model: { default: 'provider/model', reasoningEffort: 'high' as const }, + harness: 'codex' as const, + resources: { failOnError: true as const }, + }, + code: active + ? { + kind: 'no-op' as const, + reason: 'proposer-no-change' as const, + repository: { kind: 'github' as const, owner: 'owner', repo: 'repo' }, + baseCommit: active.commit, + baseTree: active.tree, + } + : { kind: 'disabled' as const, reason: 'control' as const }, + execution: { + harness: 'codex' as const, + harnessVersion: '1.2.3', + launch: active + ? { + kind: 'candidate-entrypoint' as const, + entrypoint: 'run.js', + interpreter: 'node' as const, + } + : { kind: 'container-command' as const, executable: 'codex' }, + instructionDelivery: { kind: 'stdin-utf8' as const }, + cwd: { workspace: 'task' as const, path: '.' }, + environment: { kind: 'evaluator-task-container' as const }, + ...(active ? { workspace: active.workspace } : {}), + isolation: { + network: 'disabled' as const, + remoteIntegrations: 'disabled' as const, + candidateSecrets: 'disabled' as const, + }, + ...execution, + }, + memory: { mode: 'disabled' as const }, + lineage: active + ? { + source: 'optimizer' as const, + parentDigests: [sha('e')], + runIds: ['optimizer-run-1'], + benchmark: { name: 'development', version: '1', splitDigest: sha('f') }, + spend: { + proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + }, + } + : { source: 'human' as const }, + } + return { ...value, digest: canonicalCandidateDigest(value) } +} + +function redigestBundle( + source: AgentCandidateBundle, + overrides: Partial>, +): AgentCandidateBundle { + const { digest: _digest, ...withoutDigest } = source + const value = { ...withoutDigest, ...overrides } + return { ...value, digest: canonicalCandidateDigest(value) } +} + +function emptySnapshot(label: string): AgentCandidateWorkspaceSnapshotEvidence { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: [], + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive: embeddedCandidateArtifact(Buffer.from(`empty:${label}`)), + } +} + +function fixture(active = false): { + bundle: AgentCandidateBundle + task: AgentCandidateTaskExecution + ports: AgentCandidateExecutionPorts + candidateRoot?: string +} { + const repository = taskRepository() + const taskWorkspace = snapshot(repository.root, [{ path: 'source.ts', mode: 0o644 }]) + let candidateRoot: string | undefined + let candidateWorkspace: AgentCandidateWorkspaceSnapshotEvidence | undefined + if (active) { + candidateRoot = temporaryRoot('candidate-built-') + writeFileSync(join(candidateRoot, 'run.js'), '#!/usr/bin/env node\n') + chmodSync(join(candidateRoot, 'run.js'), 0o755) + candidateWorkspace = snapshot(candidateRoot, [{ path: 'run.js', mode: 0o755 }]) + } + const profileRoot = temporaryRoot('candidate-profile-') + const selectedContainer: ResolvedAgentCandidateContainer = { + source: 'evaluator-task-container', + image: 'ghcr.io/example/task', + indexDigest: sha('a'), + manifestDigest: sha('b'), + platform: { os: 'linux', architecture: 'amd64' }, + } + const ports: AgentCandidateExecutionPorts = { + artifacts: { + read: async () => { + throw new Error('all fixture artifacts are embedded') + }, + }, + repositories: { resolve: async () => repository.root }, + workspaces: { materialize: async () => undefined }, + containers: { resolve: async () => selectedContainer }, + models: { + resolve: async ({ requested, reasoningEffort }) => ({ + requested, + provider: 'provider', + model: 'model-snapshot', + snapshot: 'model-snapshot-2026-07-01', + reasoningEffort, + }), + reserveGrant: async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: limits, + }), + activateGrant: async () => ({ env: { MODEL_GATEWAY_TOKEN: 'protected' } }), + settleGrant: async ({ preparationId }) => ({ + preparationId, + grantDigest: sha('c'), + closed: true, + calls: [], + }), + }, + memory: { + reset: async () => { + throw new Error('disabled memory must not reset') + }, + activate: async () => { + throw new Error('disabled memory must not activate') + }, + close: async () => { + throw new Error('disabled memory must not close') + }, + }, + } + const task: AgentCandidateTaskExecution = { + executionId: 'execution-1', + benchmark: 'repository-disjoint-smoke', + benchmarkVersion: '1', + taskId: 'owner-repo-1', + splitDigest: sha('d'), + instruction: 'Fix the failing behavior without changing the public API.', + repository: { + identity: 'github.com/owner/repo', + rootIdentity: 'owner/repo', + baseCommit: repository.commit, + baseTree: repository.tree, + }, + attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, + model: { requested: 'provider/model', reasoningEffort: 'high' }, + executionRoots: { + taskRoot: '/workspace/task', + ...(active ? { candidateRoot: '/opt/candidate' } : {}), + }, + stagingRoots: { + taskRoot: repository.root, + ...(candidateRoot ? { candidateRoot } : {}), + profileRoot, + }, + workspace: taskWorkspace, + evaluatorTaskContainer: selectedContainer, + limits: { + timeoutMs: 60_000, + maxSteps: 100, + maxModelCalls: 50, + maxInputTokens: 100_000, + maxOutputTokens: 50_000, + maxCostUsd: 5, + }, + } + return { + bundle: bundle( + {}, + active && candidateWorkspace + ? { commit: repository.commit, tree: repository.tree, workspace: candidateWorkspace } + : undefined, + ), + task, + ports, + ...(candidateRoot ? { candidateRoot } : {}), + } +} + +describe('candidate execution preparation', () => { + it('binds exact instruction, repository, profile, model, image, roots, and limits', async () => { + const value = fixture() + value.bundle = bundle({ + instructionDelivery: { + kind: 'utf8-file', + env: 'TANGLE_CANDIDATE_TASK_PATH', + path: '/tangle/input/task.txt', + }, + }) + const verified = await verifyAgentCandidateBundle(value.bundle, value.ports) + const prepared = await prepareAgentCandidateExecution(verified, value.task, value.ports) + const plan = prepared.executionPlan.value.material + expect(plan.task.instruction).toEqual({ + encoding: 'utf8', + sha256: embeddedCandidateArtifact(Buffer.from(value.task.instruction)).sha256, + byteLength: Buffer.byteLength(value.task.instruction), + delivery: value.bundle.execution.instructionDelivery, + }) + expect(plan.task.repository).toEqual(value.task.repository) + expect(plan.profile).toEqual({ + planDigest: prepared.profilePlan.value.digest, + targetWorkspace: 'task', + mountPaths: ['AGENTS.md'], + }) + expect(plan.launch.env.TANGLE_CANDIDATE_TASK_PATH?.value).toBe('/tangle/input/task.txt') + expect(Buffer.from(prepared.instruction.bytes)).toEqual(Buffer.from(value.task.instruction)) + expect(Buffer.from(prepared.executionPlan.bytes)).toEqual( + Buffer.from(prepared.executionPlan.value.artifact.content, 'base64'), + ) + expect(prepared.materializationReceipt.bytes.byteLength).toBeGreaterThan(0) + expect(JSON.stringify(plan)).not.toContain(value.task.instruction) + expect(JSON.stringify(prepared)).not.toContain('MODEL_GATEWAY_TOKEN') + expect(JSON.stringify(prepared)).not.toContain('protected') + }) + + it('keeps argv task bytes out of fixed args and exposes deterministic delivery separately', async () => { + const value = fixture() + value.bundle = bundle({ instructionDelivery: { kind: 'argv-append' } }) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + expect(prepared.launch.args).not.toContain(value.task.instruction) + expect(prepared.instruction).toMatchObject({ delivery: { kind: 'argv-append' } }) + }) + + it('materializes and binds an active candidate entrypoint without task-root overlap', async () => { + const value = fixture(true) + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + expect(prepared.launch).toMatchObject({ + executable: 'node', + args: ['/opt/candidate/run.js'], + cwd: '/workspace/task', + }) + expect(prepared.materializationReceipt.value.entrypoint).toMatchObject({ path: 'run.js' }) + expect(prepared.executionPlan.value.material.candidateWorkspace?.digest).toBe( + value.bundle.execution.workspace?.digest, + ) + }) + + it('rejects task Git drift, dirty profile staging, and unenforced model limits', async () => { + const gitDrift = fixture() + gitDrift.task.repository.baseTree = '0'.repeat(40) + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(gitDrift.bundle, gitDrift.ports), + gitDrift.task, + gitDrift.ports, + ), + ).rejects.toThrow(/base tree/) + + const profileDrift = fixture() + writeFileSync(join(profileDrift.task.stagingRoots.profileRoot, 'stale'), 'stale') + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(profileDrift.bundle, profileDrift.ports), + profileDrift.task, + profileDrift.ports, + ), + ).rejects.toThrow(/must be empty/) + + const unenforced = fixture() + let settledReservations = 0 + unenforced.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: { ...limits, maxCostUsd: limits.maxCostUsd + 1 }, + }) + unenforced.ports.models.settleGrant = async ({ preparationId }) => { + settledReservations++ + return { preparationId, grantDigest: sha('c'), closed: true, calls: [] } + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(unenforced.bundle, unenforced.ports), + unenforced.task, + unenforced.ports, + ), + ).rejects.toThrow(/does not enforce/) + expect(settledReservations).toBe(1) + }) + + it('bounds failed-preparation cleanup and rejects another preparation settlement', async () => { + const hanging = fixture() + hanging.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: { ...limits, maxModelCalls: limits.maxModelCalls + 1 }, + }) + hanging.ports.models.settleGrant = async () => await new Promise(() => undefined) + const startedAt = Date.now() + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(hanging.bundle, hanging.ports), + hanging.task, + hanging.ports, + { cleanupTimeoutMs: 20 }, + ), + ).rejects.toThrow(/cleanup failed/) + expect(Date.now() - startedAt).toBeLessThan(250) + + const mismatched = fixture() + mismatched.ports.models.reserveGrant = async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: sha('c'), + expiresAtMs, + enforcedLimits: { ...limits, maxModelCalls: limits.maxModelCalls + 1 }, + }) + mismatched.ports.models.settleGrant = async () => ({ + preparationId: `candidate-preparation-v1.${'A'.repeat(43)}`, + grantDigest: sha('c'), + closed: true, + calls: [], + }) + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(mismatched.bundle, mismatched.ports), + mismatched.task, + mismatched.ports, + ), + ).rejects.toThrow(/cleanup failed/) + }) + + it('rejects a cost limit that cannot be represented by the protected integer ledger', async () => { + const value = fixture() + value.task.limits = { ...value.task.limits, maxCostUsd: 10 ** 20 } + let reservations = 0 + value.ports.models.reserveGrant = async () => { + reservations++ + throw new Error('must not reserve') + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/fixed-point range/) + expect(reservations).toBe(0) + }) + + it('rejects a wall-time limit that Node would clamp to an immediate timer', async () => { + const value = fixture() + value.task.limits = { + ...value.task.limits, + timeoutMs: MAX_CANDIDATE_TIMER_INTERVAL_MS + 1, + } + let reservations = 0 + value.ports.models.reserveGrant = async () => { + reservations++ + throw new Error('must not reserve') + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/limits are invalid/) + expect(reservations).toBe(0) + }) + + it('rejects unsigned hidden bytes in an active candidate workspace', async () => { + const value = fixture(true) + if (!value.candidateRoot) throw new Error('fixture missing candidate root') + mkdirSync(join(value.candidateRoot, '.sidecar')) + writeFileSync(join(value.candidateRoot, '.sidecar', 'hidden.js'), 'malicious') + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/do not match|unsupported mode/) + }) + + it('resets task memory into an execution-scoped namespace and carries verified knowledge', async () => { + const value = fixture() + value.task.executionId = '..' + const seedBytes = Buffer.from('seed-memory') + const knowledgeBytes = Buffer.from('{"documents":[]}') + const seed = { + locator: { kind: 's3' as const, bucket: 'test-artifacts', key: 'memory/seed.bin' }, + sha256: embeddedCandidateArtifact(seedBytes).sha256, + byteLength: seedBytes.byteLength, + } + const knowledge = { + locator: { kind: 's3' as const, bucket: 'test-artifacts', key: 'knowledge/manifest.json' }, + sha256: embeddedCandidateArtifact(knowledgeBytes).sha256, + byteLength: knowledgeBytes.byteLength, + } + value.bundle = redigestBundle(value.bundle, { + memory: { mode: 'isolated', scope: 'task', seed }, + knowledge: { snapshotId: 'knowledge-1', manifest: knowledge }, + }) + value.ports.artifacts.read = async (ref) => { + if (ref.sha256 === seed.sha256) return seedBytes + if (ref.sha256 === knowledge.sha256) return knowledgeBytes + throw new Error(`unexpected artifact ${ref.sha256}`) + } + let resetInput: Parameters[0] | undefined + value.ports.memory.reset = async (input) => { + resetInput = input + return { + preparationId: input.preparationId, + accessDigest: sha('8'), + expiresAtMs: input.expiresAtMs, + evidence: embeddedCandidateArtifact(Buffer.from('fresh-reset')), + emptyStateDigest: sha('7'), + beforeState: emptySnapshot('before'), + } + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ) + expect(Buffer.from(resetInput?.seed ?? [])).toEqual(seedBytes) + expect(resetInput?.effectiveNamespace).not.toContain('..') + expect(resetInput?.effectiveNamespace.split('/')).toHaveLength(5) + expect(prepared.memory).toMatchObject({ + mode: 'isolated', + seedDigest: seed.sha256, + }) + expect(JSON.stringify(prepared)).not.toContain('TANGLE_MEMORY_NAMESPACE') + expect(prepared.knowledge).toMatchObject({ + snapshotId: 'knowledge-1', + manifestDigest: knowledge.sha256, + }) + expect(Buffer.from(prepared.knowledge?.manifest ?? [])).toEqual(knowledgeBytes) + }) + + it('closes memory immediately when reset evidence fails before preparation can retain it', async () => { + const value = fixture() + value.bundle = redigestBundle(value.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }) + const before = emptySnapshot('malformed-reset') + const closed: string[] = [] + value.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: 'sha256:not-a-digest' as `sha256:${string}`, + expiresAtMs, + evidence: before.manifest, + emptyStateDigest: before.digest, + beforeState: before, + }) + value.ports.memory.close = async ({ preparationId, reason }) => { + closed.push(`${preparationId}:${reason}`) + return { closed: true } + } + await expect( + prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(value.bundle, value.ports), + value.task, + value.ports, + ), + ).rejects.toThrow(/not scoped/) + expect(closed).toHaveLength(1) + expect(closed[0]).toMatch(/candidate-preparation-v1\..+:preparation-failed/) + }) +}) diff --git a/tests/candidate-execution-recover.test.ts b/tests/candidate-execution-recover.test.ts new file mode 100644 index 00000000..4e3e380b --- /dev/null +++ b/tests/candidate-execution-recover.test.ts @@ -0,0 +1,342 @@ +import { createHash } from 'node:crypto' +import { mkdirSync, rmSync } from 'node:fs' + +import { InMemoryTraceStore } from '@tangle-network/agent-eval' +import { agentCandidateModelSettlementMaterialSchema } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' + +import { + candidateExecutionClaim, + InMemoryAgentCandidateExecutionClaimStore, +} from '../src/candidate-execution/claim' +import { prepareAgentCandidateExecution } from '../src/candidate-execution/prepare' +import { recoverExpiredAgentCandidateExecution } from '../src/candidate-execution/recover' +import type { AgentCandidateOutputArtifactPort } from '../src/candidate-execution/types' +import { verifyAgentCandidateBundle } from '../src/candidate-execution/verify' +import { + candidateSha, + cleanupCandidateFixtures, + createCandidateExecutionFixture, + emptyCandidateSnapshot, + redigestCandidateBundle, +} from './helpers/candidate-execution-fixture' + +afterEach(cleanupCandidateFixtures) + +describe('expired candidate recovery', () => { + it('uses persisted handles to prove cleanup and terminally fail a crashed attempt', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claim = candidateExecutionClaim(prepared) + const acquired = await claimStore.tryClaim(claim) + expect(acquired.acquired).toBe(true) + now = claim.leaseExpiresAtMs + let stops = 0 + let settlements = 0 + const originalSettle = fixture.ports.models.settleGrant + fixture.ports.models.settleGrant = async (input) => { + settlements++ + return await originalSettle(input) + } + const outputArtifacts = outputArtifactPort() + + const result = await recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts, + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async (request, context) => { + stops++ + expect(request).toEqual({ + executionId: claim.executionId, + executionPlanDigest: claim.executionPlanDigest, + }) + expect(context.signal.aborted).toBe(true) + expect(context.deadlineAtMs).toBe(claim.leaseExpiresAtMs) + return { stopped: true } + }, + }, + }) + expect(result).toMatchObject({ + finished: true, + terminal: { + status: 'failed', + failureClass: 'pre-model-infrastructure', + usage: { modelCalls: 0 }, + }, + }) + const recoveredSettlement = agentCandidateModelSettlementMaterialSchema.parse( + JSON.parse( + Buffer.from(await outputArtifacts.read(result.terminal.modelSettlement)).toString('utf8'), + ), + ) + expect(recoveredSettlement).toMatchObject({ + kind: 'agent-candidate-model-settlement-material', + executionPlanDigest: claim.executionPlanDigest, + preparationId: claim.cleanup.preparationId, + resolved: claim.cleanup.resolvedModel, + usage: { modelCalls: 0, costUsdNanos: 0 }, + }) + + const replay = await recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts, + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async () => { + throw new Error('terminal recovery must not stop twice') + }, + }, + }) + expect(replay).toMatchObject({ finished: false, exactReplay: true }) + expect({ stops, settlements }).toEqual({ stops: 1, settlements: 1 }) + }) + + it('does no outward cleanup before expiry and records nothing when cleanup is unproven', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claim = candidateExecutionClaim(prepared) + await claimStore.tryClaim(claim) + let stops = 0 + const outputArtifacts = outputArtifactPort() + const recover = () => + recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts, + now: () => now, + cleanupTimeoutMs: 25, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async () => { + stops++ + throw new Error('process still live') + }, + }, + }) + + await expect(recover()).rejects.toThrow(/has not expired/) + expect(stops).toBe(0) + now = claim.leaseExpiresAtMs + await expect(recover()).rejects.toThrow(/cleanup could not be proven/) + expect(stops).toBe(1) + expect( + await claimStore.getAttempt({ executionId: claim.executionId, attempt: 1 }), + ).not.toHaveProperty('terminal') + }) + + it('unlocks attempt two only when a recovered pre-run crash used zero model calls', async () => { + const fixture = createCandidateExecutionFixture() + fixture.task.attempt = { + number: 1, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const first = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const firstClaim = candidateExecutionClaim(first) + await claimStore.tryClaim(firstClaim) + now = firstClaim.leaseExpiresAtMs + const recovered = await recoverExpiredAgentCandidateExecution({ + attempt: { executionId: firstClaim.executionId, attempt: 1 }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts: outputArtifactPort(), + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async () => ({ stopped: true }), + }, + }) + expect(recovered).toMatchObject({ + finished: true, + terminal: { failureClass: 'pre-model-infrastructure', usage: { modelCalls: 0 } }, + }) + + rmSync(fixture.task.stagingRoots.profileRoot, { recursive: true, force: true }) + mkdirSync(fixture.task.stagingRoots.profileRoot) + fixture.task.attempt = { + number: 2, + maxAttempts: 2, + retryPolicy: 'pre-model-infrastructure-only', + } + const second = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + await expect(claimStore.tryClaim(candidateExecutionClaim(second))).resolves.toMatchObject({ + acquired: true, + }) + }) + + it('repeats idempotent cleanup after a partial recovery failure', async () => { + const fixture = createCandidateExecutionFixture() + fixture.bundle = redigestCandidateBundle(fixture.bundle, { + memory: { mode: 'isolated', scope: 'task' }, + }) + const before = emptyCandidateSnapshot('recovery-before') + fixture.ports.memory.reset = async ({ preparationId, expiresAtMs }) => ({ + preparationId, + accessDigest: candidateSha('8'), + expiresAtMs, + evidence: before.manifest, + emptyStateDigest: before.digest, + beforeState: before, + }) + let memoryCloses = 0 + fixture.ports.memory.close = async () => { + memoryCloses++ + if (memoryCloses === 1) throw new Error('transient memory service failure') + return { closed: true } + } + let settlements = 0 + const originalSettle = fixture.ports.models.settleGrant + fixture.ports.models.settleGrant = async (input) => { + settlements++ + return await originalSettle(input) + } + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claim = candidateExecutionClaim(prepared) + await claimStore.tryClaim(claim) + now = claim.leaseExpiresAtMs + let stops = 0 + const outputArtifacts = outputArtifactPort() + const recover = () => + recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore: new InMemoryTraceStore(), + ports: fixture.ports, + outputArtifacts, + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async () => { + stops++ + return { stopped: true } + }, + }, + }) + + await expect(recover()).rejects.toThrow(/cleanup could not be proven/) + expect( + await claimStore.getAttempt({ executionId: claim.executionId, attempt: 1 }), + ).not.toHaveProperty('terminal') + await expect(recover()).resolves.toMatchObject({ finished: true }) + expect({ stops, settlements, memoryCloses }).toEqual({ + stops: 2, + settlements: 2, + memoryCloses: 2, + }) + }) + + it('rejects late trace writes during recovery instead of persisting unknown credentials', async () => { + const fixture = createCandidateExecutionFixture() + const prepared = await prepareAgentCandidateExecution( + await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), + fixture.task, + fixture.ports, + ) + let now = Date.now() + const claimStore = new InMemoryAgentCandidateExecutionClaimStore({ now: () => now }) + const claim = candidateExecutionClaim(prepared) + await claimStore.tryClaim(claim) + now = claim.leaseExpiresAtMs + const traceStore = new InMemoryTraceStore() + + await expect( + recoverExpiredAgentCandidateExecution({ + attempt: { executionId: claim.executionId, attempt: claim.attempt }, + claimStore, + traceStore, + ports: fixture.ports, + outputArtifacts: outputArtifactPort(), + now: () => now, + executor: { + execute: async () => { + throw new Error('recovery must not execute') + }, + stopAndCapture: async (_request, context) => { + await context.traceStore.appendEvent({ + runId: claim.cleanup.traceRunId, + eventId: 'late-secret', + kind: 'log', + timestamp: now, + payload: { token: 'expired-model-token' }, + }) + return { stopped: true } + }, + }, + }), + ).rejects.toThrow(/cleanup could not be proven/) + await expect(traceStore.events()).resolves.toEqual([]) + await expect( + claimStore.getAttempt({ executionId: claim.executionId, attempt: claim.attempt }), + ).resolves.not.toHaveProperty('terminal') + }) +}) + +function outputArtifactPort(): AgentCandidateOutputArtifactPort { + const stored = new Map() + return { + put: async ({ purpose, bytes }) => { + const sha256 = `sha256:${createHash('sha256').update(bytes).digest('hex')}` as const + stored.set(sha256, Uint8Array.from(bytes)) + return { + locator: { + kind: 's3', + bucket: 'candidate-recovery-test', + key: `${purpose}/${sha256}.json`, + }, + sha256, + byteLength: bytes.byteLength, + } + }, + read: async (ref) => Uint8Array.from(stored.get(ref.sha256) ?? []), + } +} diff --git a/tests/candidate-execution-redaction.test.ts b/tests/candidate-execution-redaction.test.ts new file mode 100644 index 00000000..7234ee20 --- /dev/null +++ b/tests/candidate-execution-redaction.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest' + +import { redactProtectedValue } from '../src/candidate-execution/protected-redaction' + +describe('candidate protected-value redaction', () => { + it('redacts overlapping credentials longest-first without retaining a suffix', () => { + const redacted = redactProtectedValue({ short: 'secret123', long: 'secret123456' }, [ + 'secret123', + 'secret123456', + ]) + expect(redacted.value).toEqual({ + short: '[redacted:candidate-access]', + long: '[redacted:candidate-access]', + }) + }) +}) diff --git a/tests/candidate-execution-task-outcome.test.ts b/tests/candidate-execution-task-outcome.test.ts new file mode 100644 index 00000000..4b907464 --- /dev/null +++ b/tests/candidate-execution-task-outcome.test.ts @@ -0,0 +1,115 @@ +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, describe, expect, it } from 'vitest' + +import { verifyTaskOutcomePatch } from '../src/candidate-execution/git-materialize' +import { + cleanupCandidateFixtures, + createCandidateExecutionFixture, +} from './helpers/candidate-execution-fixture' + +const temporaryDirectories: string[] = [] + +afterEach(() => { + cleanupCandidateFixtures() + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function git(root: string, args: string[], input?: Uint8Array, env: Record = {}) { + return execFileSync('git', args, { + cwd: root, + encoding: 'utf8', + input, + env: { ...process.env, ...env }, + }).trim() +} + +function changedOutcome(root: string, baseTree: string) { + const patch = Buffer.from( + [ + 'diff --git a/source.ts b/source.ts', + '--- a/source.ts', + '+++ b/source.ts', + '@@ -1 +1 @@', + '-export const value = 1', + '+export const value = 2', + '', + ].join('\n'), + ) + const temporary = mkdtempSync(join(tmpdir(), 'candidate-outcome-test-')) + temporaryDirectories.push(temporary) + const environment = { GIT_INDEX_FILE: join(temporary, 'index') } + git(root, ['read-tree', baseTree], undefined, environment) + git(root, ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'], patch, environment) + const resultTree = git(root, ['write-tree'], undefined, environment) + const resultBytes = Buffer.from('export const value = 2\n') + const afterState = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: [ + { + path: 'source.ts', + mode: 0o644 as const, + sha256: `sha256:${createHash('sha256').update(resultBytes).digest('hex')}` as const, + byteLength: resultBytes.byteLength, + }, + ], + } + return { patch, resultTree, afterState } +} + +describe('candidate task outcome verification', () => { + it('proves a captured binary patch and after-state against the signed base tree', async () => { + const fixture = createCandidateExecutionFixture() + const outcome = changedOutcome( + fixture.task.stagingRoots.taskRoot, + fixture.task.repository.baseTree, + ) + await expect( + verifyTaskOutcomePatch({ + repositoryRoot: fixture.task.stagingRoots.taskRoot, + baseCommit: fixture.task.repository.baseCommit, + baseTree: fixture.task.repository.baseTree, + ...outcome, + }), + ).resolves.toMatchObject({ + resultTree: outcome.resultTree, + resultCommit: expect.stringMatching(/^[a-f0-9]{40}$/), + }) + }) + + it('rejects a claimed result tree or after-state that does not match the patch', async () => { + const fixture = createCandidateExecutionFixture() + const outcome = changedOutcome( + fixture.task.stagingRoots.taskRoot, + fixture.task.repository.baseTree, + ) + await expect( + verifyTaskOutcomePatch({ + repositoryRoot: fixture.task.stagingRoots.taskRoot, + baseCommit: fixture.task.repository.baseCommit, + baseTree: fixture.task.repository.baseTree, + ...outcome, + resultTree: '0'.repeat(fixture.task.repository.baseTree.length), + }), + ).rejects.toThrow(/materialized tree/) + await expect( + verifyTaskOutcomePatch({ + repositoryRoot: fixture.task.stagingRoots.taskRoot, + baseCommit: fixture.task.repository.baseCommit, + baseTree: fixture.task.repository.baseTree, + ...outcome, + afterState: { + ...outcome.afterState, + files: [{ ...outcome.afterState.files[0]!, byteLength: 999 }], + }, + }), + ).rejects.toThrow(/after-state/) + }) +}) diff --git a/tests/helpers/candidate-execution-fixture.ts b/tests/helpers/candidate-execution-fixture.ts new file mode 100644 index 00000000..7c2202e2 --- /dev/null +++ b/tests/helpers/candidate-execution-fixture.ts @@ -0,0 +1,387 @@ +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' + +import type { + AgentCandidateArtifactRef, + AgentCandidateBundle, + AgentCandidateExecution, + AgentCandidateWorkspaceSnapshotEvidence, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { + canonicalCandidateBytes, + canonicalCandidateDigest, + embeddedCandidateArtifact, + sha256Bytes, +} from '../../src/candidate-execution/digest' +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutionPorts, + AgentCandidateExecutorTaskOutcomeCapture, + AgentCandidateOutputArtifactPort, + AgentCandidateTaskExecution, + ResolvedAgentCandidateContainer, +} from '../../src/candidate-execution/types' + +const roots: string[] = [] + +export const candidateSha = (character: string): Sha256Digest => `sha256:${character.repeat(64)}` + +export function cleanupCandidateFixtures(): void { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +} + +function temporaryRoot(prefix: string): string { + const root = mkdtempSync(join(tmpdir(), prefix)) + roots.push(root) + return root +} + +function git(root: string, args: string[]): string { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim() +} + +function taskRepository(): { root: string; commit: string; tree: string } { + const root = temporaryRoot('candidate-task-') + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'test@example.com']) + git(root, ['config', 'user.name', 'Test']) + git(root, ['config', 'core.hooksPath', '/dev/null']) + git(root, ['remote', 'add', 'origin', 'git@github.com:owner/repo.git']) + writeFileSync(join(root, 'source.ts'), 'export const value = 1\n') + chmodSync(join(root, 'source.ts'), 0o644) + git(root, ['add', 'source.ts']) + git(root, ['commit', '-m', 'base']) + return { + root, + commit: git(root, ['rev-parse', 'HEAD']), + tree: git(root, ['rev-parse', 'HEAD^{tree}']), + } +} + +function snapshot( + root: string, + files: Array<{ path: string; mode: 0o644 | 0o755 }>, +): AgentCandidateWorkspaceSnapshotEvidence { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: files + .map((file) => { + const bytes = execFileSync('node', [ + '-e', + `process.stdout.write(require('fs').readFileSync(${JSON.stringify(join(root, file.path))}))`, + ]) + return { + path: file.path, + mode: file.mode, + sha256: embeddedCandidateArtifact(bytes).sha256, + byteLength: bytes.byteLength, + } + }) + .sort((a, b) => a.path.localeCompare(b.path)), + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive: embeddedCandidateArtifact(Buffer.from(`archive:${manifest.sha256}`)), + } +} + +export function candidateBundle( + execution: Partial = {}, + active?: { + commit: string + tree: string + workspace: AgentCandidateWorkspaceSnapshotEvidence + }, +): AgentCandidateBundle { + const value = { + schemaVersion: 1 as const, + kind: 'agent-candidate-bundle' as const, + digestAlgorithm: 'rfc8785-sha256' as const, + profile: { + name: 'candidate', + prompt: { instructions: ['Inspect the repository, implement the fix, and run tests.'] }, + model: { default: 'provider/model', reasoningEffort: 'high' as const }, + harness: 'codex' as const, + resources: { failOnError: true as const }, + }, + code: active + ? { + kind: 'no-op' as const, + reason: 'proposer-no-change' as const, + repository: { kind: 'github' as const, owner: 'owner', repo: 'repo' }, + baseCommit: active.commit, + baseTree: active.tree, + } + : { kind: 'disabled' as const, reason: 'control' as const }, + execution: { + harness: 'codex' as const, + harnessVersion: '1.2.3', + launch: active + ? { + kind: 'candidate-entrypoint' as const, + entrypoint: 'run.js', + interpreter: 'node' as const, + } + : { kind: 'container-command' as const, executable: 'codex' }, + instructionDelivery: { kind: 'stdin-utf8' as const }, + cwd: { workspace: 'task' as const, path: '.' }, + environment: { kind: 'evaluator-task-container' as const }, + ...(active ? { workspace: active.workspace } : {}), + isolation: { + network: 'disabled' as const, + remoteIntegrations: 'disabled' as const, + candidateSecrets: 'disabled' as const, + }, + ...execution, + }, + memory: { mode: 'disabled' as const }, + lineage: active + ? { + source: 'optimizer' as const, + parentDigests: [candidateSha('e')], + runIds: ['optimizer-run-1'], + benchmark: { + name: 'development', + version: '1', + splitDigest: candidateSha('f'), + }, + spend: { + proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + }, + } + : { source: 'human' as const }, + } + return { ...value, digest: canonicalCandidateDigest(value) } +} + +export function redigestCandidateBundle( + source: AgentCandidateBundle, + overrides: Partial>, +): AgentCandidateBundle { + const { digest: _digest, ...withoutDigest } = source + const value = { ...withoutDigest, ...overrides } + return { ...value, digest: canonicalCandidateDigest(value) } +} + +export function emptyCandidateSnapshot(label: string): AgentCandidateWorkspaceSnapshotEvidence { + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files: [], + } + const manifest = embeddedCandidateArtifact(canonicalCandidateBytes(material)) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: manifest.sha256, + material, + manifest, + archive: embeddedCandidateArtifact(Buffer.from(`empty:${label}`)), + } +} + +export interface CandidateExecutionFixture { + bundle: AgentCandidateBundle + task: AgentCandidateTaskExecution + ports: AgentCandidateExecutionPorts + candidateRoot?: string +} + +export function unchangedTaskOutcomeCapture( + fixture: CandidateExecutionFixture, +): AgentCandidateExecutorTaskOutcomeCapture { + return { + resultTree: fixture.task.repository.baseTree, + afterState: fixture.task.workspace.material, + archive: Buffer.from(`task-archive:${fixture.task.workspace.digest}`, 'utf8'), + gitDiff: Buffer.alloc(0), + } +} + +export function createCandidateOutputFixture(): { + outputArtifacts: AgentCandidateOutputArtifactPort + grader: AgentCandidateBenchmarkGraderPort +} { + const stored = new Map() + const put = (bytes: Uint8Array, purpose: string): AgentCandidateArtifactRef => { + const detached = Uint8Array.from(bytes) + const sha256 = sha256Bytes(detached) + const ref: AgentCandidateArtifactRef = { + locator: { + kind: 's3', + bucket: 'candidate-test-artifacts', + key: `${purpose}/${sha256.slice('sha256:'.length)}`, + }, + sha256, + byteLength: detached.byteLength, + } + stored.set(sha256, detached) + return ref + } + const graderArtifact = put(Buffer.from('fixture executable grader', 'utf8'), 'grader') + return { + outputArtifacts: { + put: async ({ bytes, purpose }) => put(bytes, purpose), + read: async (ref) => { + const bytes = stored.get(ref.sha256) + if (!bytes) throw new Error(`missing candidate test artifact ${ref.sha256}`) + return Uint8Array.from(bytes) + }, + }, + grader: { + name: 'fixture-executable-grader', + version: '1.0.0', + artifact: graderArtifact, + run: async ({ implementation, outcome }) => { + const evidence = Buffer.from('{"reward":1}\n', 'utf8') + return { + evaluation: { score: 1, passed: true, dimensions: { tests: 1 }, raw: {} }, + evidence, + binding: { + implementationDigest: sha256Bytes(implementation.bytes), + taskOutcomeDigest: outcome.evidence.digest, + outputDigest: sha256Bytes(evidence), + }, + } + }, + }, + } +} + +export function createCandidateExecutionFixture(active = false): CandidateExecutionFixture { + const repository = taskRepository() + const taskWorkspace = snapshot(repository.root, [{ path: 'source.ts', mode: 0o644 }]) + let candidateRoot: string | undefined + let candidateWorkspace: AgentCandidateWorkspaceSnapshotEvidence | undefined + if (active) { + candidateRoot = temporaryRoot('candidate-built-') + writeFileSync(join(candidateRoot, 'run.js'), '#!/usr/bin/env node\n') + chmodSync(join(candidateRoot, 'run.js'), 0o755) + candidateWorkspace = snapshot(candidateRoot, [{ path: 'run.js', mode: 0o755 }]) + } + const profileRoot = temporaryRoot('candidate-profile-') + const selectedContainer: ResolvedAgentCandidateContainer = { + source: 'evaluator-task-container', + image: 'ghcr.io/example/task', + indexDigest: candidateSha('a'), + manifestDigest: candidateSha('b'), + platform: { os: 'linux', architecture: 'amd64' }, + } + const ports: AgentCandidateExecutionPorts = { + artifacts: { + read: async () => { + throw new Error('all fixture artifacts are embedded') + }, + }, + repositories: { resolve: async () => repository.root }, + workspaces: { + materialize: async ({ role, snapshot: workspace, destination }) => { + if (role === 'memory') { + if (workspace.material.files.length !== 0) { + throw new Error('fixture memory materializer supports only an empty state') + } + return + } + const source = role === 'task' ? repository.root : candidateRoot + if (!source) throw new Error(`fixture ${role} workspace source is missing`) + if (source === destination) return + for (const file of workspace.material.files) { + const output = join(destination, file.path) + mkdirSync(dirname(output), { recursive: true }) + writeFileSync(output, readFileSync(join(source, file.path))) + chmodSync(output, file.mode) + } + }, + }, + containers: { resolve: async () => selectedContainer }, + models: { + resolve: async ({ requested, reasoningEffort }) => ({ + requested, + provider: 'provider', + model: 'model-snapshot', + snapshot: 'model-snapshot-2026-07-01', + reasoningEffort, + }), + reserveGrant: async ({ preparationId, expiresAtMs, limits }) => ({ + preparationId, + digest: candidateSha('c'), + expiresAtMs, + enforcedLimits: limits, + }), + activateGrant: async () => ({ env: { MODEL_GATEWAY_TOKEN: 'protected' } }), + settleGrant: async ({ preparationId }) => ({ + preparationId, + grantDigest: candidateSha('c'), + closed: true, + calls: [], + }), + }, + memory: { + reset: async () => { + throw new Error('disabled memory must not reset') + }, + activate: async () => { + throw new Error('disabled memory must not activate') + }, + close: async () => { + throw new Error('disabled memory must not close') + }, + }, + } + const task: AgentCandidateTaskExecution = { + executionId: 'execution-1', + benchmark: 'repository-disjoint-smoke', + benchmarkVersion: '1', + taskId: 'owner-repo-1', + splitDigest: candidateSha('d'), + instruction: 'Fix the failing behavior without changing the public API.', + repository: { + identity: 'github.com/owner/repo', + rootIdentity: 'owner/repo', + baseCommit: repository.commit, + baseTree: repository.tree, + }, + attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, + model: { requested: 'provider/model', reasoningEffort: 'high' }, + executionRoots: { + taskRoot: '/workspace/task', + ...(active ? { candidateRoot: '/opt/candidate' } : {}), + }, + stagingRoots: { + taskRoot: repository.root, + ...(candidateRoot ? { candidateRoot } : {}), + profileRoot, + }, + workspace: taskWorkspace, + evaluatorTaskContainer: selectedContainer, + limits: { + timeoutMs: 60_000, + maxSteps: 100, + maxModelCalls: 50, + maxInputTokens: 100_000, + maxOutputTokens: 50_000, + maxCostUsd: 5, + }, + } + return { + bundle: candidateBundle( + {}, + active && candidateWorkspace + ? { commit: repository.commit, tree: repository.tree, workspace: candidateWorkspace } + : undefined, + ), + task, + ports, + ...(candidateRoot ? { candidateRoot } : {}), + } +}