diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d52a7b..dd87902 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,13 +58,13 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: 531468c59c7a2dc32f5cbe92cfabf89805d27f6a + ref: 99a4d9ad5256f7330e75b06d48f340cc7239a40d path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-spec - ref: 2e33036563cb966b07124197db672159b4b7e1f4 + ref: ee38796d5e38e67e350a06548fd50faa530cbb12 path: .pinned/determa-state-spec - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/AGENTS.md b/AGENTS.md index 55e8237..e3bfdb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,19 +11,22 @@ package so it can coexist with the umbrella `determa` launcher. The implementation is conformant only when it passes the language-neutral suite. The implementation uses these immutable inputs: -- specification: `2e33036563cb966b07124197db672159b4b7e1f4`; -- conformance: `531468c59c7a2dc32f5cbe92cfabf89805d27f6a` (114 core cases, - 108 persistence vectors, 12 persistence-profile steps, 99 execution-checkpoint - vectors, 106 version-2 vectors, and 101 closed-code registry entries). +- specification: `ee38796d5e38e67e350a06548fd50faa530cbb12`; +- conformance: `99a4d9ad5256f7330e75b06d48f340cc7239a40d` (98 format-1 + core cases, 162 version-2 vectors, 138 durable-host vectors, and 381 generated + version-2 artifacts). The package metadata remains `0.2.0`. ## Boundaries -The pure public API remains `load_bundle`, `create`, and `dispatch`, plus validation -and error types exported by `determa.state`. The optional synchronous `ExecutionHost` -and execution-store APIs wrap that core without changing its exact `format: 1` -grammar. Do not restore abandoned draft field names or compatibility aliases. +The pure public API is `load_bundle`, `create`, `admit`, and `step`, plus validation +and error types exported by `determa.state`. Creation, admission, and stepping operate +only on queue-bearing aggregate state; the mailbox-free engine helpers are private. +Portable artifacts and checkpoints support +schema version 2 only. The optional synchronous `ExecutionHost` and execution-store +APIs wrap that core without changing its exact `format: 1` machine grammar. Do not +restore abandoned draft field names, artifact version 1 paths, or compatibility aliases. The core is a pure foreground transform over one root ownership aggregate. It has no hidden queues, timers, stores, or standardized execution CLI. Portable aggregate diff --git a/README.md b/README.md index dd55f17..6d41571 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,12 @@ Python implementation of [Determa State](https://github.com/fruwehq/determa-stat a language-agnostic statechart engine with a shared normative conformance suite. This implementation supports Determa State `format: 1` at specification -commit `2e33036563cb966b07124197db672159b4b7e1f4`. Correctness is determined by -the 114-case core suite, 108 persistence vectors, 12 persistence-profile steps, -99 execution-checkpoint vectors, 106 version-2 vectors, and 101 closed-code registry -entries at conformance commit `531468c59c7a2dc32f5cbe92cfabf89805d27f6a`. +commit `ee38796d5e38e67e350a06548fd50faa530cbb12`. Correctness is determined by +98 format-1 core cases, 162 version-2 vectors, 138 durable-host vectors, and 381 generated version-2 artifacts +at conformance commit `99a4d9ad5256f7330e75b06d48f340cc7239a40d`. -The package metadata remains `0.2.0`. The implementation includes the portable -execution-checkpoint host, selected legacy artifact decoding, and authoritative portable -code sets. +The package metadata remains `0.2.0`. Artifact and checkpoint schema version 2 is the +only supported portable artifact format. Machine YAML remains `format: 1`. ## Install @@ -73,8 +71,9 @@ names are not accepted. ## Use The Library -`create` and `dispatch` are pure foreground operations. They do not retain hidden -machine state or call queues, timers, databases, or remote services. +`create`, `admit`, and `step` are pure foreground operations over explicit, +queue-bearing aggregate state. They do not retain hidden machine state or call +databases or remote services. ```python from pathlib import Path @@ -91,38 +90,55 @@ created = ds.create( ) state = created["state"] +resolver = ds.MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + target = { "root": { "root_instance_id": state["root_instance_id"], "root_runtime_id": state["root_runtime_id"], } } -result = ds.dispatch( - bundle, +envelope = ds.portable_envelope( + "increment", + "counter-42:increment:1", + target, + {"amount": 2}, +) +delivery = { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": ds.delivery_request_digest( + "counter-42", "input", envelope + ), +} +admitted = ds.admit( state, - { - "input": { - "event": "increment", - "event_id": "counter-42:increment:1", - "target": target, - "payload": {"amount": 2}, - } - }, + [delivery], + resolver, ) +result = ds.step(admitted["state"], state["root_runtime_id"], resolver) assert result["status"] == "running" assert result["disposition"] == "handled" state = result["state"] -root = state["runtimes"][state["root_runtime_id"]] -assert root["scopes"]["root"]["count"] == 2 +root = next( + runtime + for runtime in state["runtimes"] + if runtime["runtime_id"] == state["root_runtime_id"] +) +count = next( + variable + for variable in root["variables"] + if variable["variable_declaration_pointer"].endswith("/count") +) +assert count["value"] == ["integer", "2"] ``` -Both calls return all result fields: `status`, `disposition`, `state`, `emissions`, -`fault`, and `rejection` (`create` has a null disposition). The caller owns delivery: -the core processes at most one supplied envelope and does not place it in an internal -queue. Successful processing returns a new JSON-compatible logical aggregate while -leaving the supplied prior state unchanged. Rejections and unhandled deliveries return -the exact supplied state object. +`create` initializes the aggregate, `admit` atomically appends accepted deliveries to +its runtime mailboxes, and `step` processes at most the selected runtime's ready head. +Ready and deferred mailboxes are literal portable aggregate state, so queued work +survives serialization and restoration. Each operation returns a new JSON-compatible +logical aggregate while leaving the supplied prior state unchanged. `load_bundle` also accepts a native Python mapping through the same structural and semantic validation path. Native values must satisfy the same portable Unicode and @@ -134,7 +150,7 @@ definitions. ## Persist And Migrate -`serialize_aggregate` produces the canonical ยง16 aggregate artifact. Restoration +`serialize_aggregate` produces the canonical schema-v2 aggregate artifact. Restoration resolves its exact validated definition by fingerprint and fails closed when the definition is absent or untrusted: @@ -145,11 +161,9 @@ restored = ds.restore_aggregate(encoded, resolver) ``` `restore_aggregate_package` verifies a self-contained transport package and seeds a -mutable resolver without replacing existing content. `migrate_aggregate` applies an -exact trusted descriptor route as a pure operation. `migrate_and_dispatch` returns one -commit-ready migration, audit, dispatch, aggregate, and outbox-intent boundary. Failed -migrations return a deterministic `MigrationFailure` and do not mutate the supplied -artifact or resolver. +mutable resolver without replacing existing content. `migrate_aggregate_v2` applies an +exact trusted descriptor route as a pure operation. Failed migrations do not mutate the +supplied artifact or resolver. Definition and descriptor resolvers are protocols, so applications can back them with an immutable registry or a transaction-local cache. @@ -167,7 +181,7 @@ store.setup_schema() # always explicit resolver = ds.MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) host = ds.ExecutionHost(store, resolver) -created = host.create( +created = host.create_v2( bundle, machine_id="counter", root_instance_id="counter-42", @@ -241,9 +255,9 @@ configuration. Root checkpoint deletion is unsupported. failure propagation, and cleanup cascades; - atomic RTC rollback, deterministic identities/counters, pure inspection, and incompatible or malformed prior-state rejection; -- canonical aggregate serialization/restoration, portable typed values, package +- schema-v2 aggregate serialization/restoration, portable typed values, package attachments, exact definition resolution, trusted lazy migration, deterministic - audits, resource limits, and atomic migrate-and-dispatch results. + audits, and resource limits; - strict portable execution-checkpoint parsing, canonical digests, semantic validation, synchronous transaction/CAS/replay orchestration, receipts, pending delivery, outbox lifecycle, replay retention, and root tombstones; diff --git a/conformance/durable_host.py b/conformance/durable_host.py new file mode 100644 index 0000000..6720de4 --- /dev/null +++ b/conformance/durable_host.py @@ -0,0 +1,654 @@ +"""Closed driver for the pinned durable-host conformance profiles.""" + +from __future__ import annotations + +import copy +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml +from jsonschema import Draft202012Validator + +import determa.state.checkpoint_v2 as checkpoint_module +from determa.state import ( + ArtifactError, + ExecutionHost, + ExecutionHostError, + ExecutionStore, + ExecutionStoreError, + ExecutionStoreRegistry, + MemoryArtifactResolver, + MemoryExecutionStore, + load_bundle, + restore_execution_checkpoint_v2, + validate_host_profile, +) +from determa.state.queueing import _runtime_id_for_target + +from .harness import conformance_root +from .version2 import _json, _pointer, _resolver + +_CHECKPOINT_BROKER_MANAGED_ROOTS = frozenset({"checkpoint-lifecycle-root"}) + + +@dataclass(frozen=True) +class DurableHostVector: + path: Path + vector: dict[str, Any] + + @property + def name(self) -> str: + return f"{self.path.name}/{self.vector['name']}" + + +def durable_host_vectors() -> list[DurableHostVector]: + result = [] + root = conformance_root() / "conformance" / "profiles" + for test_path in sorted(root.glob("**/test.yaml")): + document = yaml.safe_load(test_path.read_text(encoding="utf-8")) or {} + result.extend( + DurableHostVector(test_path.parent, vector) + for vector in document.get("durable_host_vectors", []) + ) + return result + + +def _request(item: DurableHostVector) -> dict[str, Any]: + reference = item.vector.get("request") + if reference is None: + return copy.deepcopy(item.vector["raw_admission_request"]) + return copy.deepcopy( + _pointer(_json(item.path / reference["file"]), reference["pointer"]) + ) + + +def _fault_injector(boundary: str | None) -> Any: + def inject(actual: str) -> None: + if actual == boundary == "before_commit": + raise ExecutionHostError("injected_pre_commit_failure") + if actual == boundary == "after_commit_before_response": + raise ExecutionHostError("response_lost_after_commit") + + return inject + + +class _StaticStore(ExecutionStore): + def __init__(self, capabilities: list[str], retention_mode: str = "permanent"): + self._capabilities = frozenset(capabilities) + self._retention_mode = retention_mode + + @property + def capabilities(self) -> frozenset[str]: + return self._capabilities + + @property + def checkpoint_retention_mode(self) -> str: + return self._retention_mode + + def transaction(self, root_instance_id: str) -> Any: + del root_instance_id + raise AssertionError("profile-only store must not open a transaction") + + def setup_schema(self) -> None: + return None + + def health(self) -> dict[str, Any]: + return {"healthy": True} + + +def _store_bytes(store: MemoryExecutionStore, root_instance_id: str) -> bytes | None: + with store.transaction(root_instance_id) as transaction: + return transaction.load() + + +def _checkpoint_source(item: DurableHostVector) -> tuple[str, bytes | None]: + vector = item.vector + name = vector.get("stored_checkpoint_before", vector.get("checkpoint_before")) + if name is None: + request = _request(item) + return request["root_instance_id"], None + document = _json(item.path / name) + return document["root_instance_id"], (item.path / name).read_bytes() + + +def _deliveries(request: dict[str, Any]) -> list[Any]: + if "envelopes" in request: + return request["envelopes"] + sources = request["ordered_member_sources"] + result = [] + for source in sources: + if "json_value" in source: + result.append(source["json_value"]) + else: + result.append(source["utf8_json"]) + return result + + +def _checkpoint_replayed( + item: DurableHostVector, + request: dict[str, Any], + response: dict[str, Any], + source: bytes | None, +) -> bool: + operation = item.vector["operation"] + if operation == "checkpoint_create_v2": + return source is not None + if operation == "checkpoint_admit_v2": + if response.get("result") == "replay": + return True + return response.get("result") == "batch" and all( + member["disposition"] == "replay" for member in response["members"] + ) + if source is None: + return False + checkpoint = json.loads(source) + if operation == "checkpoint_prune_v2": + retention = checkpoint["replay_retention"] + return ( + retention["mode"] == request["target_mode"] + and retention["policy_identifier"] == request["policy_identifier"] + and retention["pruned_through_receipt_sequence"] + == request["cutoff_receipt_sequence"] + ) + if operation == "checkpoint_tombstone_v2": + return checkpoint["root_record"]["status"] == "tombstone" + if operation == "checkpoint_update_outbox_v2": + desired = {"status": request["target_disposition"]} + if request["outcome"]["reason_code"] is not None: + desired["reason_code"] = request["outcome"]["reason_code"] + return any( + record["intent"]["effect_id"] == request["effect_id"] + and record["delivery_state"] == desired + for record in checkpoint["pending_outbox_intents"] + ) + if operation == "checkpoint_terminalize_outbox_v2": + return any( + record["intent"]["effect_id"] == request["effect_id"] + for record in checkpoint["terminal_outbox_records"] + ) or any( + record["effect_id"] == request["effect_id"] + for record in checkpoint["outbox_effect_tombstones"] + ) + if operation == "checkpoint_compact_outbox_v2": + return any( + record["effect_id"] == request["effect_id"] + for record in checkpoint["outbox_effect_tombstones"] + ) + return response.get("result") == "not_committed" + + +def _replay_evidence(checkpoint: dict[str, Any], event_id: str) -> dict[str, Any]: + for runtime in checkpoint["root_record"].get("aggregate_state", {}).get( + "runtimes", [] + ): + for mailbox, location in ( + ("ready_mailbox", "ready"), + ("deferred_mailbox", "deferred"), + ): + for entry in runtime[mailbox]: + if entry["envelope"]["event_id"] == event_id: + return { + "result": "replay", + "event_id": event_id, + "acceptance_sequence": entry["acceptance_sequence"], + "location": location, + } + acceptance = next( + ( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["operation_kind"] == "acceptance" + and receipt["event_id"] == event_id + ), + None, + ) + terminal = next( + ( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["operation_kind"] == "event_terminal" + and receipt["event_id"] == event_id + ), + None, + ) + if terminal is not None: + return { + "result": "replay", + "acceptance_receipt_sequence": ( + acceptance["receipt_sequence"] if acceptance is not None else "0" + ), + "terminal_receipt_sequence": terminal["receipt_sequence"], + } + tombstone = next( + ( + record + for record in checkpoint["event_identity_tombstones"] + if record["event_id"] == event_id + ), + None, + ) + assert tombstone is not None + return { + "result": "replay", + "terminal_receipt_sequence": tombstone["terminal_receipt_sequence"], + "terminal_disposition": tombstone["terminal_disposition"], + } + + +def _validate_public_response( + operation: str, + request: dict[str, Any], + response: dict[str, Any], + stored: bytes | None, +) -> None: + assert type(response) is dict + checkpoint = None if stored is None else json.loads(stored) + if operation == "checkpoint_create_v2": + assert checkpoint is not None + assert response == { + "result": "committed", + "receipt": checkpoint["operation_receipts"][0], + } + return + if operation in {"checkpoint_step_v2", "checkpoint_prune_v2"}: + assert checkpoint is not None + assert response == checkpoint + return + if operation == "checkpoint_admit_v2": + assert checkpoint is not None + if response.get("execution_checkpoint_schema_version") == 2: + assert response == checkpoint + return + if response.get("result") == "replay": + deliveries = _deliveries(request) + assert len(deliveries) == 1 + event_id = deliveries[0]["envelope"]["event_id"] + assert response == _replay_evidence(checkpoint, event_id) + return + assert set(response) == {"result", "checkpoint", "members"} + assert response["result"] == "batch" + assert response["checkpoint"] == checkpoint + event_ids = [ + delivery["envelope"]["event_id"] for delivery in _deliveries(request) + ] + assert [member.get("event_id") for member in response["members"]] == event_ids + for member in response["members"]: + if member.get("disposition") == "accepted": + assert set(member) == { + "event_id", + "disposition", + "acceptance_sequence", + "queue_sequence", + } + assert any( + entry["envelope"]["event_id"] == member["event_id"] + and entry["acceptance_sequence"] == member["acceptance_sequence"] + and entry["queue_sequence"] == member["queue_sequence"] + for runtime in checkpoint["root_record"]["aggregate_state"][ + "runtimes" + ] + for mailbox in ("ready_mailbox", "deferred_mailbox") + for entry in runtime[mailbox] + ) + else: + assert set(member) == {"event_id", "disposition", "evidence"} + assert member["disposition"] == "replay" + assert member["evidence"] == _replay_evidence( + checkpoint, member["event_id"] + ) + return + if operation == "checkpoint_tombstone_v2": + assert set(response) == {"result", "tombstone"} + assert response["result"] == "tombstoned" + assert checkpoint is not None + assert response["tombstone"] == checkpoint["root_record"] + return + assert checkpoint is not None + if operation == "checkpoint_update_outbox_v2": + records = checkpoint["pending_outbox_intents"] + elif operation == "checkpoint_terminalize_outbox_v2": + records = [ + *checkpoint["terminal_outbox_records"], + *checkpoint["outbox_effect_tombstones"], + ] + elif operation == "checkpoint_compact_outbox_v2": + records = checkpoint["outbox_effect_tombstones"] + else: + raise AssertionError(f"unvalidated production response: {operation}") + assert set(response) == {"result", "record"} + assert response["result"] == "committed" + assert response["record"] in records + record = response["record"] + effect_id = record.get("effect_id", record.get("intent", {}).get("effect_id")) + assert effect_id == request["effect_id"] + if operation == "checkpoint_update_outbox_v2": + desired = {"status": request["target_disposition"]} + if request["outcome"]["reason_code"] is not None: + desired["reason_code"] = request["outcome"]["reason_code"] + assert record["delivery_state"] == desired + elif operation == "checkpoint_terminalize_outbox_v2": + desired = {"status": request["target_disposition"]} + if request["outcome"]["reason_code"] is not None: + desired["reason_code"] = request["outcome"]["reason_code"] + assert record["outcome"] == desired + + +def _request_root_instance_id(request: dict[str, Any]) -> str | None: + if isinstance(request.get("root_instance_id"), str): + return request["root_instance_id"] + expected = request.get("expected_checkpoint") + if isinstance(expected, dict) and isinstance(expected.get("root_instance_id"), str): + return expected["root_instance_id"] + return None + + +class _BrokerAcknowledgementAdapter: + """Exercise host-owned broker acknowledgement for explicitly managed roots.""" + + def __init__(self, managed_root_instance_ids: frozenset[str]) -> None: + self._managed_root_instance_ids = managed_root_instance_ids + self._acknowledged_request_ids: set[str] = set() + + def acknowledge(self, request: dict[str, Any]) -> None: + if _request_root_instance_id(request) in self._managed_root_instance_ids: + self._acknowledged_request_ids.add(request["request_id"]) + + def acknowledged(self, request: dict[str, Any]) -> bool: + return request["request_id"] in self._acknowledged_request_ids + + +def _invoke_checkpoint( + item: DurableHostVector, request: dict[str, Any], observation: dict[str, Any] +) -> tuple[dict[str, Any], bytes | None, int]: + vector = item.vector + operation = vector["operation"] + root_instance_id, source = _checkpoint_source(item) + initial = {} if source is None else {root_instance_id: source} + store = MemoryExecutionStore(initial) + observation["initial_source"] = source + observation["store"] = store + observation["root_instance_id"] = root_instance_id + host = ExecutionHost( + store, + _resolver(item.path, request), + fault_injector=_fault_injector(vector.get("failure_boundary")), + ) + expected_checkpoint = request.get("expected_checkpoint", {}) + expected = { + "expected_revision": expected_checkpoint.get("revision", ""), + "expected_checkpoint_digest": expected_checkpoint.get("digest", ""), + } + core_calls = 0 + originals = ( + checkpoint_module.create_aggregate_v2, + checkpoint_module.admit_aggregate_v2, + checkpoint_module.step_aggregate_v2, + ) + + def observe(index: int) -> Any: + def wrapped(*args: Any, **kwargs: Any) -> Any: + nonlocal core_calls + core_calls += 1 + observation["core_calls"] = core_calls + return originals[index](*args, **kwargs) + + return wrapped + + checkpoint_module.create_aggregate_v2 = observe(0) + checkpoint_module.admit_aggregate_v2 = observe(1) + checkpoint_module.step_aggregate_v2 = observe(2) + try: + if operation == "checkpoint_create_v2": + bundle = load_bundle( + (item.path / request["bundle"]["file"]).read_text(encoding="utf-8") + ) + response = host.create_v2( + bundle, + request["machine"]["machine_id"], + request["root_instance_id"], + request["creation_id"], + request["bindings"], + ) + elif operation == "checkpoint_admit_v2": + response = host.admit_v2( + root_instance_id, _deliveries(request), **expected + ) + elif operation == "checkpoint_step_v2": + response = host.process_ready_v2( + root_instance_id, + _runtime_id_for_target(request["target"]), + **expected, + ) + elif operation == "checkpoint_prune_v2": + response = host.prune_v2( + root_instance_id, + request["cutoff_receipt_sequence"], + target_mode=request["target_mode"], + policy_identifier=request["policy_identifier"], + **expected, + ) + elif operation == "checkpoint_tombstone_v2": + response = host.tombstone_root_v2( + root_instance_id, + request["tombstone_operation_id"], + **expected, + ) + elif operation == "checkpoint_update_outbox_v2": + state = {"status": request["target_disposition"]} + if request["outcome"]["reason_code"] is not None: + state["reason_code"] = request["outcome"]["reason_code"] + response = host.update_pending_outbox( + root_instance_id, request["effect_id"], state, **expected + ) + elif operation == "checkpoint_terminalize_outbox_v2": + outcome = {"status": request["target_disposition"]} + if request["outcome"]["reason_code"] is not None: + outcome["reason_code"] = request["outcome"]["reason_code"] + response = host.terminalize_outbox( + root_instance_id, request["effect_id"], outcome, **expected + ) + elif operation == "checkpoint_compact_outbox_v2": + response = host.compact_outbox( + root_instance_id, request["effect_id"], **expected + ) + elif operation == "checkpoint_delete_retained_record_v2": + raise ExecutionHostError("invalid_execution_checkpoint") + else: + raise AssertionError(f"unsupported checkpoint operation: {operation}") + finally: + ( + checkpoint_module.create_aggregate_v2, + checkpoint_module.admit_aggregate_v2, + checkpoint_module.step_aggregate_v2, + ) = originals + return response, _store_bytes(store, root_instance_id), core_calls + + +def _register_declared( + registry: ExecutionStoreRegistry, registrations: list[dict[str, Any]] +) -> None: + for registration in registrations: + capabilities = registration["capabilities"] + schema = registration["configuration_schema"] + + def factory( + uri: str, + configuration: dict[str, Any], + *, + declared_capabilities: list[str] = capabilities, + declared_schema: dict[str, Any] = schema, + ) -> ExecutionStore: + del uri + errors = list(Draft202012Validator(declared_schema).iter_errors(configuration)) + if errors: + raise ExecutionStoreError("invalid_adapter_configuration") + return _StaticStore(declared_capabilities) + + registry.register(registration["uri_scheme"], factory) + + +def _validated_result() -> dict[str, Any]: + return { + "result": "validated", + "mutation": "none", + "core_calls": 0, + "broker_acknowledged": False, + } + + +def _invoke_contract(item: DurableHostVector, request: dict[str, Any]) -> dict[str, Any]: + operation = item.vector["operation"] + if operation == "checkpoint_inject_store_v2": + ExecutionHost( + _StaticStore(request["capabilities"]), MemoryArtifactResolver() + ) + return _validated_result() + if operation == "checkpoint_register_adapter_v2": + registry = ExecutionStoreRegistry() + _register_declared(registry, request["existing_registrations"]) + _register_declared(registry, [request["registration"]]) + return _validated_result() + if operation == "checkpoint_resolve_adapter_v2": + registry = ExecutionStoreRegistry() + _register_declared(registry, request["registrations"]) + registry.resolve( + request["uri"], + configuration=request["configuration"], + required_capabilities=frozenset(request["requested_capabilities"]), + ) + return _validated_result() + if operation == "checkpoint_validate_capabilities_v2": + validate_host_profile( + _StaticStore(request["store_capabilities"], request["retention_mode"]), + request["host_profile"], + host_features=frozenset(request["host_guarantees"]), + ) + return _validated_result() + if operation == "checkpoint_scope_operation_v2": + scope = request["scope"] + matches = [ + record + for record in request["store_records"] + if record["scope_id"] == scope["scope_id"] + and record["ownership_binding"] == scope["ownership_binding"] + and record["portable_identity"] == request["portable_identity"] + and record["effect_id"] == request["effect_id"] + ] + if scope["authorization"] != "authorized" or len(matches) != 1: + raise ExecutionHostError("invalid_store_scope") + return _validated_result() + if operation == "checkpoint_backup_restore_v2": + source_name = item.vector.get("checkpoint_before") + if source_name is None: + raise ExecutionHostError("invalid_execution_checkpoint") + source = (item.path / source_name).read_bytes() + checkpoint = restore_execution_checkpoint_v2( + source, _resolver(item.path, request) + ) + root_instance_id = checkpoint.document["root_instance_id"] + store = MemoryExecutionStore({root_instance_id: source}) + host = ExecutionHost(store, _resolver(item.path, request)) + return host.validate_backup_restore_v2( + action=request["action"], + checkpoint_members=[source], + checkpoint_digests=request["checkpoint_digests"], + trusted_artifact_digests=request["trusted_artifact_digests"], + adapter_metadata_digest=request["adapter_metadata_digest"], + retention_mode=request["retention_mode"], + consistency_point={ + "scope_id": request["scope"]["scope_id"], + "root_instance_ids": [root_instance_id], + }, + ) + raise AssertionError(f"unsupported durable-host contract operation: {operation}") + + +def run_durable_host_vector(item: DurableHostVector) -> None: + request = _request(item) + broker = _BrokerAcknowledgementAdapter(_CHECKPOINT_BROKER_MANAGED_ROOTS) + before_name = item.vector.get("checkpoint_before") + before_bytes = None if before_name is None else (item.path / before_name).read_bytes() + observation: dict[str, Any] = {"core_calls": 0} + try: + if item.vector["operation"].startswith("persistence_"): + from .persistence_host import run_persistence_vector + + actual, stored = run_persistence_vector(item, request) + elif item.vector["operation"] in { + "checkpoint_backup_restore_v2", + "checkpoint_inject_store_v2", + "checkpoint_register_adapter_v2", + "checkpoint_resolve_adapter_v2", + "checkpoint_scope_operation_v2", + "checkpoint_validate_capabilities_v2", + }: + actual = _invoke_contract(item, request) + stored = before_bytes + else: + response, stored, core_calls = _invoke_checkpoint(item, request, observation) + _validate_public_response( + item.vector["operation"], request, response, stored + ) + initial_source = observation.get("initial_source", before_bytes) + replayed = _checkpoint_replayed(item, request, response, initial_source) + result = "replayed" if replayed else "committed" + if result in {"committed", "replayed"}: + broker.acknowledge(request) + actual = { + "result": result, + "mutation": ( + "none" if _same_document(stored, initial_source) else "atomic" + ), + "core_calls": core_calls, + "broker_acknowledged": broker.acknowledged(request), + } + except (ArtifactError, ExecutionHostError, ExecutionStoreError) as error: + code = error.code + store = observation.get("store") + root_instance_id = observation.get("root_instance_id") + stored = ( + _store_bytes(store, root_instance_id) + if isinstance(store, MemoryExecutionStore) + and isinstance(root_instance_id, str) + else before_bytes + ) + core_calls = getattr(error, "core_calls", observation["core_calls"]) + if hasattr(error, "stored"): + stored = error.stored + result = "crashed" if code in { + "injected_pre_commit_failure", + "response_lost_after_commit", + } else "rejected" + actual = { + "result": result, + "mutation": ( + "atomic" if code == "response_lost_after_commit" else "none" + ), + "core_calls": core_calls, + "broker_acknowledged": False, + "code": code, + } + result_reference = item.vector["result"] + expected_result = _pointer( + _json(item.path / result_reference["file"]), result_reference["pointer"] + ) + assert actual == expected_result, (actual, expected_result) + if item.vector["operation"].startswith("persistence_"): + expected_bytes = (item.path / item.vector["store_after"]).read_bytes() + assert _same_document(stored, expected_bytes) + return + after_name = item.vector.get("checkpoint_after") + if after_name is not None: + expected_bytes = (item.path / after_name).read_bytes() + assert _same_document(stored, expected_bytes) + if expected_result["mutation"] == "none": + initial = observation.get("initial_source", before_bytes) + assert stored == initial + + +def _same_document(left: bytes | None, right: bytes | None) -> bool: + if left is None or right is None: + return left is right + return json.loads(left) == json.loads(right) diff --git a/conformance/execution_checkpoint.py b/conformance/execution_checkpoint.py deleted file mode 100644 index 415c407..0000000 --- a/conformance/execution_checkpoint.py +++ /dev/null @@ -1,751 +0,0 @@ -"""Driver for the optional execution-checkpoint host profile.""" - -from __future__ import annotations - -import copy -import hashlib -import json -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import yaml - -import determa.state.host as host_module -from determa.state import ( - ArtifactError, - ExecutionHost, - ExecutionHostError, - ExecutionStore, - ExecutionStoreError, - ExecutionStoreRegistry, - ExecutionStoreTransaction, - MemoryArtifactResolver, - MemoryExecutionStore, - load_bundle, - register_bundled_execution_stores, - restore_execution_checkpoint, - serialize_execution_checkpoint, -) -from determa.state.wire import hash_value - -from .harness import conformance_root - -PROFILE_DIR = ( - conformance_root() / "conformance" / "profiles" / "execution-checkpoint" -) - - -@dataclass(frozen=True) -class ExecutionCheckpointCase: - name: str - path: Path - - @property - def test(self) -> dict[str, Any]: - return yaml.safe_load((self.path / "test.yaml").read_text(encoding="utf-8")) - - -@dataclass(frozen=True) -class ExecutionCheckpointVector: - case: ExecutionCheckpointCase - vector: dict[str, Any] - - @property - def name(self) -> str: - return f"{self.case.name}/{self.vector['name']}" - - -def execution_checkpoint_cases() -> list[ExecutionCheckpointCase]: - if not PROFILE_DIR.exists(): - return [] - return [ - ExecutionCheckpointCase(path.name, path) - for path in sorted(PROFILE_DIR.iterdir()) - if path.is_dir() and (path / "test.yaml").exists() - ] - - -def execution_checkpoint_vectors() -> list[ExecutionCheckpointVector]: - return [ - ExecutionCheckpointVector(case, vector) - for case in execution_checkpoint_cases() - for vector in case.test.get("execution_checkpoint_profile", {}).get( - "vectors", [] - ) - ] - - -def _json(path: Path) -> Any: - return json.loads(path.read_text(encoding="utf-8")) - - -def _pointer(document: Any, pointer: str) -> Any: - current = document - for part in pointer.removeprefix("/").split("/"): - current = current[part.replace("~1", "/").replace("~0", "~")] - return current - - -def _resolver(case: ExecutionCheckpointCase) -> MemoryArtifactResolver: - definitions = {} - descriptors = {} - for path in case.path.parent.glob("**/*.yaml"): - try: - bundle = load_bundle(path.read_text(encoding="utf-8")) - except Exception: - continue - definitions[bundle.fingerprint] = bundle - from determa.state.wire import migration_descriptor_digest - - for path in case.path.glob("*migration-descriptor*.json"): - descriptor = _json(path) - descriptors[migration_descriptor_digest(descriptor)] = descriptor - return MemoryArtifactResolver( - definitions=definitions, migration_descriptors=descriptors - ) - - -def _checkpoint_root(path: Path) -> str: - return str(_json(path)["root_instance_id"]) - - -def _delivery_candidate(request: dict[str, Any]) -> dict[str, Any] | None: - if "candidate" in request: - return request["candidate"] - return { - name: copy.deepcopy(request[name]) - for name in ( - "root_instance_id", - "delivery_mode", - "origin", - "envelope", - "envelope_digest", - ) - if name in request - } - - -def _fault_injector(boundary: str | None) -> Any: - def inject(actual: str) -> None: - if actual == boundary == "before_commit": - raise ExecutionHostError("injected_pre_commit_failure") - if actual == boundary == "after_commit_before_response": - raise ExecutionHostError("response_lost_after_commit") - - return inject - - -def _invoke_host( - host: ExecutionHost, - operation: str, - root_instance_id: str, - request: dict[str, Any], -) -> dict[str, Any]: - expected = { - "expected_revision": request.get("expected_revision", ""), - "expected_checkpoint_digest": request.get( - "expected_checkpoint_digest", "" - ), - } - if operation == "create": - bundle = load_bundle( - (Path(request["_case_path"]) / request["bundle_file"]).read_text( - encoding="utf-8" - ) - ) - return host.create( - bundle, - request["machine_id"], - request["root_instance_id"], - request["creation_id"], - request["bindings"], - ) - if operation == "accept_delivery": - return host.accept_delivery( - root_instance_id, _delivery_candidate(request), **expected - ) - if operation == "process_pending_delivery": - return host.process_pending_delivery( - root_instance_id, _delivery_candidate(request), **expected - ) - if operation == "foreground_process_delivery": - return host.foreground_process_delivery( - root_instance_id, _delivery_candidate(request), **expected - ) - if operation == "maintenance_migration": - return host.maintenance_migration( - root_instance_id, - request["operation_id"], - request["target_validated_bundle_fingerprint"], - request["migration_descriptor_digest_route"], - source_aggregate_state_digest=request[ - "source_aggregate_state_digest" - ], - maintenance_mode=request["maintenance_mode"], - **expected, - ) - if operation == "update_pending_outbox": - return host.update_pending_outbox( - root_instance_id, - request["effect_id"], - request["desired_pending_state"], - **expected, - ) - if operation == "terminalize_outbox": - return host.terminalize_outbox( - root_instance_id, - request["effect_id"], - request["terminal_outcome"], - **expected, - ) - if operation == "compact_outbox": - return host.compact_outbox( - root_instance_id, request["effect_id"], **expected - ) - if operation == "delete_outbox_record": - return host.delete_outbox_record( - root_instance_id, request["effect_id"], **expected - ) - if operation == "update_replay_retention": - return host.update_replay_retention( - root_instance_id, - request["target_replay_retention"], - **expected, - ) - if operation == "tombstone_root": - return host.tombstone_root( - root_instance_id, request["operation_id"], **expected - ) - if operation == "delete_checkpoint": - return host.delete_checkpoint(root_instance_id, **expected) - raise AssertionError(f"unsupported host operation {operation}") - - -class _StaticStore(ExecutionStore): - def __init__( - self, capabilities: list[str], checkpoint_retention_mode: str = "permanent" - ) -> None: - self._capabilities = frozenset(capabilities) - self._checkpoint_retention_mode = checkpoint_retention_mode - - @property - def capabilities(self) -> frozenset[str]: - return self._capabilities - - @property - def checkpoint_retention_mode(self) -> str: - return self._checkpoint_retention_mode - - def transaction( - self, - root_instance_id: str, - ) -> Any: - del root_instance_id - raise AssertionError("profile-only store must not process roots") - - def setup_schema(self) -> None: - return None - - def health(self) -> dict[str, Any]: - return {"healthy": True} - - -class _ObservedTransaction(ExecutionStoreTransaction): - def __init__( - self, transaction: ExecutionStoreTransaction, calls: list[str] - ) -> None: - self._transaction = transaction - self._calls = calls - - @property - def root_instance_id(self) -> str: - return self._transaction.root_instance_id - - def load(self) -> bytes | None: - self._calls.append("load_checkpoint") - return self._transaction.load() - - def insert(self, checkpoint: bytes) -> bool: - self._calls.append("insert_checkpoint") - return self._transaction.insert(checkpoint) - - def replace( - self, - expected_revision: str, - expected_checkpoint_digest: str, - checkpoint: bytes, - ) -> bool: - self._calls.append("compare_and_swap_checkpoint") - return self._transaction.replace( - expected_revision, expected_checkpoint_digest, checkpoint - ) - - -class _ObservedMemoryStore(MemoryExecutionStore): - def __init__(self, initial: dict[str, bytes], calls: list[str]) -> None: - super().__init__(initial) - self._calls = calls - - @property - def root_instance_ids(self) -> frozenset[str]: - return frozenset(self._records) - - @contextmanager - def transaction( - self, root_instance_id: str - ) -> Iterator[ExecutionStoreTransaction]: - with super().transaction(root_instance_id) as transaction: - yield _ObservedTransaction(transaction, self._calls) - - -def _adapter_operation(vector: dict[str, Any]) -> dict[str, Any]: - operation = vector["operation"] - if operation == "inject_execution_store": - ExecutionHost(MemoryExecutionStore(), MemoryArtifactResolver()) - return {"result": "accepted"} - capabilities = vector.get("advertised_capabilities", []) - requested = set(vector.get("requested_capabilities", [])) - if operation == "validate_host_profile": - ExecutionHost( - _StaticStore(capabilities, vector["checkpoint_retention_mode"]), - MemoryArtifactResolver(), - required_capabilities=requested, - profile=vector["host_profile"], - host_features=frozenset(vector["host_features"]), - ) - return {"result": "accepted"} - - registry = ExecutionStoreRegistry() - identifier = vector["adapter_identifier"] - - def static_factory(uri: str, configuration: dict[str, Any]) -> ExecutionStore: - del uri - if not vector["configuration_valid"] or configuration: - raise ExecutionStoreError("invalid_adapter_configuration") - return _StaticStore(capabilities) - - if operation == "register_adapter": - if identifier == "memory": - register_bundled_execution_stores(registry) - store = registry.resolve( - "memory:", required_capabilities=frozenset(requested) - ) - assert store.capabilities == frozenset(capabilities) - else: - registry.register(identifier, static_factory) - registry.register(identifier, static_factory) - return {"result": "accepted"} - - uri = { - "memory": "memory:", - "file": "file:///tmp/unused", - "sqlite": "sqlite:///tmp/unused.sqlite", - "postgresql": "postgresql://unused", - }.get(identifier, f"{identifier}:") - if vector["registration_source"] == "bundled": - register_bundled_execution_stores(registry) - elif identifier != "absent-store": - registry.register(identifier, static_factory) - store = registry.resolve(uri, required_capabilities=frozenset(requested)) - actual_capabilities = store.capabilities - expected_capabilities = frozenset(capabilities) - if ( - vector["registration_source"] == "bundled" - and identifier in {"sqlite", "postgresql"} - ): - # An uninitialized database adapter must not claim persisted guarantees. - assert requested.issubset(actual_capabilities) - assert actual_capabilities.issubset(expected_capabilities) - else: - assert actual_capabilities == expected_capabilities - return {"result": "accepted"} - - -def _expected_response( - vector: dict[str, Any], - checkpoint: dict[str, Any] | None, - request: dict[str, Any], -) -> dict[str, Any]: - expected = vector["expect"] - result = expected["result"] - if result in {"failure", "response_lost", "not_accepted", "unsupported"}: - return {"result": result, "failure": {"code": expected["code"]}} - if result == "accepted": - return {"result": "accepted"} - assert checkpoint is not None - if result == "pending": - pending = next( - item - for item in checkpoint["pending_deliveries"] - if item["delivery_sequence"] == expected["delivery_sequence"] - ) - return { - "result": "pending", - "event_id": pending["envelope"]["event_id"], - "delivery_sequence": pending["delivery_sequence"], - "accepted_revision": pending["accepted_revision"], - } - if result == "tombstoned": - return { - "result": "tombstoned", - "tombstone": copy.deepcopy(checkpoint["root_record"]), - } - assert result == "committed" - operation = vector["operation"] - if "receipt_sequence" in expected: - receipt = next( - item - for item in checkpoint["operation_receipts"] - if item["receipt_sequence"] == expected["receipt_sequence"] - ) - return {"result": "committed", "receipt": copy.deepcopy(receipt)} - if operation == "update_pending_outbox": - record = next( - item - for item in checkpoint["pending_outbox_intents"] - if item["intent"]["effect_id"] == request["effect_id"] - ) - return {"result": "committed", "record": copy.deepcopy(record)} - if operation == "terminalize_outbox": - records = [ - *checkpoint["terminal_outbox_records"], - *checkpoint["outbox_effect_tombstones"], - ] - record = next( - item - for item in records - if ( - item["intent"]["effect_id"] - if "intent" in item - else item["effect_id"] - ) - == request["effect_id"] - ) - return {"result": "committed", "record": copy.deepcopy(record)} - if operation == "compact_outbox": - record = next( - item - for item in checkpoint["outbox_effect_tombstones"] - if item["effect_id"] == request["effect_id"] - ) - return {"result": "committed", "record": copy.deepcopy(record)} - if operation == "update_replay_retention": - return { - "result": "committed", - "replay_retention": copy.deepcopy(checkpoint["replay_retention"]), - } - if operation == "delete_outbox_record": - return {"result": "committed"} - raise AssertionError(f"no exact response projection for {operation}") - - -def _scope_state( - case: ExecutionCheckpointCase, reference: dict[str, str] -) -> dict[str, Any]: - return copy.deepcopy( - _pointer(_json(case.path / reference["file"]), reference["pointer"]) - ) - - -def _outbox_records(checkpoint: dict[str, Any]) -> list[dict[str, str]]: - records = [ - ("pending", record) for record in checkpoint["pending_outbox_intents"] - ] - records.extend( - ("terminal", record) for record in checkpoint["terminal_outbox_records"] - ) - records.extend( - ("tombstone", record) for record in checkpoint["outbox_effect_tombstones"] - ) - return [ - { - "effect_id": ( - record["effect_id"] - if kind == "tombstone" - else record["intent"]["effect_id"] - ), - "record_kind": kind, - "source_digest": hash_value(record), - } - for kind, record in records - ] - - -def _expected_scope_maps( - case: ExecutionCheckpointCase, - expected: dict[str, Any], -) -> dict[str, dict[str, Any]]: - result = {} - for scope in expected["scopes"]: - checkpoints = {} - outbox_records = {} - for root_instance_id, binding in scope["checkpoints"].items(): - expected_checkpoint = _json(case.path / binding["file"]) - canonical = serialize_execution_checkpoint(expected_checkpoint) - assert ( - f"sha256:{hashlib.sha256(canonical).hexdigest()}" - == binding["serialization_digest"] - ) - assert ( - expected_checkpoint["execution_checkpoint_digest"] - == binding["execution_checkpoint_digest"] - ) - checkpoints[root_instance_id] = canonical - outbox_records[root_instance_id] = _outbox_records(expected_checkpoint) - assert [ - record - for root_records in outbox_records.values() - for record in root_records - ] == scope["outbox_records"] - result[scope["logical_scope_id"]] = { - "checkpoints": checkpoints, - "outbox_records": outbox_records, - } - return result - - -def _actual_scope_maps( - hosts: dict[str, ExecutionHost], - stores: dict[str, _ObservedMemoryStore], -) -> dict[str, dict[str, Any]]: - result = {} - for scope_id, store in stores.items(): - checkpoints = {} - outbox_records = {} - for root_instance_id in sorted(store.root_instance_ids): - restored = hosts[scope_id].read_checkpoint(root_instance_id) - assert restored is not None - checkpoints[root_instance_id] = restored.canonical_bytes - outbox_records[root_instance_id] = _outbox_records(restored.document) - result[scope_id] = { - "checkpoints": checkpoints, - "outbox_records": outbox_records, - } - return result - - -def _assert_complete_scope_maps( - actual: dict[str, dict[str, Any]], - expected: dict[str, dict[str, Any]], -) -> None: - assert actual == expected - - -def _scope_hosts( - case: ExecutionCheckpointCase, - state: dict[str, Any], - store_calls: list[str], -) -> tuple[dict[str, ExecutionHost], dict[str, _ObservedMemoryStore]]: - hosts = {} - stores = {} - for scope in state["scopes"]: - initial = { - root_instance_id: (case.path / binding["file"]).read_bytes() - for root_instance_id, binding in scope["checkpoints"].items() - } - scope_id = scope["logical_scope_id"] - store = _ObservedMemoryStore(initial, store_calls) - stores[scope_id] = store - hosts[scope_id] = ExecutionHost(store, _resolver(case)) - return hosts, stores - - -def _run_scope_vector(item: ExecutionCheckpointVector) -> None: - case = item.case - vector = item.vector - expected = vector["expect"] - before = _scope_state(case, vector["scope_state_before"]) - after = _scope_state(case, vector["scope_state_after"]) - store_calls: list[str] = [] - hosts, stores = _scope_hosts(case, before, store_calls) - - resolver_calls = ["resolve_execution_store_scope"] - selection = vector["scope_selection"] - requested = selection["requested_scope_id"] - candidates = selection["candidates"] - selected = ( - requested - if requested is not None - and len(candidates) == 1 - and candidates[0]["logical_scope_id"] == requested - and candidates[0]["authorized"] - else None - ) - host_calls = [] - if selected is not None: - host_calls.append("update_pending_outbox") - selected_scope = next( - scope for scope in before["scopes"] if scope["logical_scope_id"] == selected - ) - root_instance_id, binding = next(iter(selected_scope["checkpoints"].items())) - checkpoint = _json(case.path / binding["file"]) - hosts[selected].update_pending_outbox( - root_instance_id, - vector["effect_id"], - vector["desired_pending_state"], - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - - assert expected["selection_result"] == ( - "selected" if selected is not None else "rejected" - ) - assert expected["selected_scope_id"] == selected - assert expected["calls"] == { - "resolver": resolver_calls, - "execution_host": host_calls, - "store": store_calls, - "core": [], - } - _assert_complete_scope_maps( - _actual_scope_maps(hosts, stores), - _expected_scope_maps(case, after), - ) - - -def run_execution_checkpoint_vector(item: ExecutionCheckpointVector) -> None: - case = item.case - vector = item.vector - expected = vector["expect"] - if "scope_state_before" in vector: - _run_scope_vector(item) - return - before_name = vector.get("checkpoint_before") - after_name = expected["checkpoint_after"] - if vector["operation"] in { - "inject_execution_store", - "register_adapter", - "resolve_adapter", - "validate_host_profile", - }: - try: - response = _adapter_operation(vector) - except (ExecutionHostError, ExecutionStoreError) as exc: - response = {"result": "failure", "failure": {"code": exc.code}} - assert response == _expected_response(vector, None, {}) - return - - initial = {} - if before_name is not None: - before_path = case.path / before_name - root_instance_id = _checkpoint_root(before_path) - initial[root_instance_id] = before_path.read_bytes() - else: - request_reference = vector.get("input") - assert request_reference is not None - request_document = _json(case.path / request_reference["file"]) - request = copy.deepcopy( - _pointer(request_document, request_reference["pointer"]) - ) - root_instance_id = request["root_instance_id"] - store = MemoryExecutionStore(initial) - host = ExecutionHost( - store, - _resolver(case), - fault_injector=_fault_injector(vector.get("failure_boundary")), - ) - request_reference = vector.get("input") - request = ( - {} - if request_reference is None - else copy.deepcopy( - _pointer( - _json(case.path / request_reference["file"]), - request_reference["pointer"], - ) - ) - ) - request["_case_path"] = str(case.path) - - calls: list[str] = [] - originals = ( - host_module.core_create, - host_module.core_dispatch, - host_module.migrate_aggregate, - ) - - def observed_create(*args: Any, **kwargs: Any) -> Any: - calls.append("create") - return originals[0](*args, **kwargs) - - def observed_dispatch(*args: Any, **kwargs: Any) -> Any: - calls.append("dispatch") - return originals[1](*args, **kwargs) - - def observed_migrate(*args: Any, **kwargs: Any) -> Any: - calls.append("migrate") - return originals[2](*args, **kwargs) - - host_module.core_create = observed_create - host_module.core_dispatch = observed_dispatch - host_module.migrate_aggregate = observed_migrate - try: - try: - response = _invoke_host( - host, vector["operation"], root_instance_id, request - ) - except ExecutionHostError as exc: - response = { - "result": ( - "response_lost" - if exc.code == "response_lost_after_commit" - else "failure" - ), - "failure": {"code": exc.code}, - } - finally: - ( - host_module.core_create, - host_module.core_dispatch, - host_module.migrate_aggregate, - ) = originals - restored = host.read_checkpoint(root_instance_id) - actual_checkpoint = None if restored is None else restored.document - expected_checkpoint = ( - None if after_name is None else _json(case.path / after_name) - ) - assert response == _expected_response(vector, expected_checkpoint, request) - assert calls == ([] if expected["core_call"] == "none" else [expected["core_call"]]) - assert actual_checkpoint == expected_checkpoint - - -def validate_execution_checkpoint_artifact( - case: ExecutionCheckpointCase, artifact: dict[str, Any] -) -> None: - path = case.path / artifact["file"] - resolver = _resolver(case) - if artifact.get("canonical_of"): - expected = _json(case.path / artifact["canonical_of"]) - assert path.read_bytes() == serialize_execution_checkpoint(expected) - if artifact.get("semantic_probe") == "compact_intent_digest": - source_path = case.path / artifact["semantic_source"] - root_instance_id = _checkpoint_root(source_path) - request = _pointer( - _json(case.path / artifact["semantic_input_file"]), - artifact["semantic_input_pointer"], - ) - host = ExecutionHost( - MemoryExecutionStore({root_instance_id: source_path.read_bytes()}), - resolver, - ) - host.compact_outbox( - root_instance_id, - request["effect_id"], - expected_revision=request["expected_revision"], - expected_checkpoint_digest=request["expected_checkpoint_digest"], - ) - actual = host.read_checkpoint(root_instance_id) - assert actual is not None - assert actual.document == _json(case.path / artifact["semantic_expected"]) - assert actual.document != _json(path) - return - try: - restore_execution_checkpoint(path.read_bytes(), resolver) - code = None - except ArtifactError as exc: - code = exc.code - expected = None if artifact["valid"] else artifact["error"] - assert code == expected diff --git a/conformance/harness.py b/conformance/harness.py index 9aa69ef..905be34 100644 --- a/conformance/harness.py +++ b/conformance/harness.py @@ -11,7 +11,9 @@ import yaml -from determa.state import ValidationError, create, dispatch, load_bundle +from determa.state import ValidationError, load_bundle +from determa.state.engine import _create as create +from determa.state.engine import _dispatch as dispatch from .pins import CONFORMANCE_CACHE diff --git a/conformance/persistence.py b/conformance/persistence.py deleted file mode 100644 index 65f569d..0000000 --- a/conformance/persistence.py +++ /dev/null @@ -1,297 +0,0 @@ -"""Driver for portable aggregate and migration conformance vectors.""" - -from __future__ import annotations - -import copy -from pathlib import Path -from typing import Any - -import yaml - -from determa.state import ( - ArtifactError, - MemoryArtifactResolver, - MigrationLimits, - aggregate_envelope, - create, - dispatch, - load_bundle, - migrate_aggregate, - migrate_and_dispatch, - restore_aggregate, - restore_aggregate_package, - serialize_aggregate, -) -from determa.state.wire import ( - canonical_bytes, - load_json_artifact, - migration_descriptor_digest, - strict_json, -) - -from .harness import CoreCase - - -def persistence_vector_cases(cases: list[CoreCase]) -> list[CoreCase]: - return [ - case - for case in cases - if (_load_yaml(case.test_file).get("persistence_vectors") or []) - ] - - -def run_persistence_vectors(case: CoreCase) -> None: - test = _load_yaml(case.test_file) - for vector in test.get("persistence_vectors") or []: - _run_vector(case.path, vector) - - -def _load_yaml(path: Path) -> dict[str, Any]: - return yaml.safe_load(path.read_text(encoding="utf-8")) or {} - - -def _json(path: Path) -> Any: - value, _ = strict_json(path.read_bytes()) - return value - - -def _resolver(path: Path, vector: dict[str, Any]) -> MemoryArtifactResolver: - if "artifact_resolver" in vector: - fixture = _json(path / vector["artifact_resolver"]) - definitions = { - entry["validated_bundle_fingerprint"]: (path / entry["bundle_file"]).read_text( - encoding="utf-8" - ) - for entry in fixture["definitions"] - } - descriptors = { - entry["migration_descriptor_digest"]: ( - path / entry["descriptor_file"] - ).read_bytes() - for entry in fixture["migration_descriptors"] - } - return MemoryArtifactResolver( - definitions=definitions, - migration_descriptors=descriptors, - trusted_definitions=[ - entry["validated_bundle_fingerprint"] - for entry in fixture["definitions"] - if entry["trusted"] - ], - trusted_migration_descriptors=[ - entry["migration_descriptor_digest"] - for entry in fixture["migration_descriptors"] - if entry["trusted"] - ], - ) - definitions: dict[str, Any] = {} - for filename in vector.get("definitions") or []: - bundle = load_bundle((path / filename).read_text(encoding="utf-8")) - definitions[bundle.fingerprint] = bundle - descriptors: dict[str, Any] = {} - for filename in vector.get("migration_descriptors") or []: - document = _json(path / filename) - digest = document.get("migration_descriptor_digest") - descriptors[ - digest if isinstance(digest, str) else migration_descriptor_digest(document) - ] = document - return MemoryArtifactResolver( - definitions=definitions, migration_descriptors=descriptors - ) - - -def _limits(path: Path, vector: dict[str, Any]) -> MigrationLimits | None: - filename = vector.get("resource_limits") - return MigrationLimits.from_mapping(_json(path / filename)) if filename else None - - -def _run_vector(path: Path, vector: dict[str, Any]) -> None: - operation = vector["operation"] - expected = vector["expect"] - descriptor_source = ( - (path / vector["migration_descriptor"]).read_bytes() - if operation == "decode_selected_migration_descriptor" - else None - ) - descriptor_snapshot = ( - bytes(descriptor_source) if descriptor_source is not None else None - ) - source_key = ( - "aggregate_state_package" - if operation.startswith("restore_package") - else "aggregate_state" - ) - source = (path / vector[source_key]).read_bytes() if source_key in vector else None - source_snapshot = bytes(source) if source is not None else None - resolver = _resolver(path, vector) - try: - if operation == "decode_selected_migration_descriptor": - assert descriptor_source is not None - load_json_artifact(descriptor_source, "migration_descriptor") - assert expected == {"result": "success"}, vector["name"] - assert descriptor_source == descriptor_snapshot - return - if operation == "serialize_created_aggregate": - bundle = load_bundle((path / vector["source_bundle"]).read_text(encoding="utf-8")) - result = create(bundle, **vector["creation"]) - assert result["state"] is not None - document = aggregate_envelope(bundle, result["state"]) - encoded = serialize_aggregate(bundle, result["state"]) - audits: tuple[dict[str, Any], ...] = () - emissions: tuple[dict[str, Any], ...] = () - disposition = None - elif operation == "restore_and_serialize": - assert source is not None - restored = restore_aggregate(source, resolver) - document = aggregate_envelope(restored.bundle, restored.state) - encoded = serialize_aggregate(restored.bundle, restored.state) - audits = () - emissions = () - disposition = None - elif operation == "restore_and_dispatch": - assert source is not None - restored = restore_aggregate(source, resolver) - core = dispatch( - restored.bundle, - restored.state, - {"input": _json(path / vector["input_envelope"])}, - ) - assert core["state"] is not None - document = aggregate_envelope(restored.bundle, core["state"]) - encoded = canonical_bytes(document) - audits = () - emissions = tuple(core["emissions"]) - disposition = core["disposition"] - elif operation == "restore_package": - assert source is not None - package = restore_aggregate_package(source, resolver) - document = package.aggregate.aggregate_envelope - encoded = package.aggregate.canonical_bytes - audits = () - emissions = () - disposition = None - elif operation == "restore_package_and_migrate": - assert source is not None - package = restore_aggregate_package(source, resolver) - migrated = migrate_aggregate( - package.aggregate.canonical_bytes, - vector["target_validated_bundle_fingerprint"], - vector["migration_route"], - resolver, - maintenance_mode=vector["maintenance_mode"], - resource_limits=_limits(path, vector), - ) - if migrated.failure is not None: - raise ArtifactError(migrated.failure.code) - assert migrated.aggregate_envelope is not None - assert migrated.aggregate_bytes is not None - document = migrated.aggregate_envelope - encoded = migrated.aggregate_bytes - audits = migrated.audit_records - emissions = () - disposition = None - elif operation == "migrate_aggregate": - assert source is not None - request = { - "target_validated_bundle_fingerprint": vector.get( - "target_validated_bundle_fingerprint" - ), - "migration_route": vector.get("migration_route"), - "maintenance_mode": vector.get("maintenance_mode"), - } - if "migration_request" in vector: - request_value = vector["migration_request"] - request = ( - _json(path / request_value) - if isinstance(request_value, str) - else copy.deepcopy(request_value) - ) - migrated = migrate_aggregate( - source, - request.get("target_validated_bundle_fingerprint"), - request.get("migration_route"), - resolver, - maintenance_mode=request.get("maintenance_mode"), - resource_limits=_limits(path, vector), - ) - if migrated.failure is not None: - raise ArtifactError(migrated.failure.code) - assert migrated.aggregate_envelope is not None - assert migrated.aggregate_bytes is not None - document = migrated.aggregate_envelope - encoded = migrated.aggregate_bytes - audits = migrated.audit_records - emissions = () - disposition = None - elif operation == "migrate_and_dispatch": - assert source is not None - migrated_dispatch = migrate_and_dispatch( - source, - vector["target_validated_bundle_fingerprint"], - vector["migration_route"], - resolver, - {"input": _json(path / vector["input_envelope"])}, - maintenance_mode=vector["maintenance_mode"], - resource_limits=_limits(path, vector), - ) - if migrated_dispatch.failure is not None: - raise ArtifactError(migrated_dispatch.failure.code) - assert migrated_dispatch.aggregate_envelope is not None - assert migrated_dispatch.aggregate_bytes is not None - document = migrated_dispatch.aggregate_envelope - encoded = migrated_dispatch.aggregate_bytes - audits = migrated_dispatch.audit_records - emissions = migrated_dispatch.emissions - disposition = migrated_dispatch.disposition - else: - raise AssertionError(f"unsupported persistence operation: {operation}") - except ArtifactError as error: - assert expected["result"] == "failure", (vector["name"], error.code) - assert error.code == expected["code"], (vector["name"], error.code) - if descriptor_source is not None: - assert descriptor_source == descriptor_snapshot - if expected.get("caller_still_owns_aggregate"): - assert source == source_snapshot - if vector.get("repeat_count", 1) > 1: - repeated = copy.deepcopy(vector) - repeated["repeat_count"] -= 1 - _run_vector(path, repeated) - return - assert expected["result"] == "success", vector["name"] - assert document == _json(path / expected["aggregate_state_file"]) - assert encoded == (path / expected["exact_bytes_file"]).read_bytes() - if "migration_audit_file" in expected: - assert list(audits) == _json(path / expected["migration_audit_file"]) - if "emissions_file" in expected: - assert list(emissions) == _json(path / expected["emissions_file"]) - if "disposition" in expected: - assert disposition == expected["disposition"] - if "artifact_resolver_file" in expected: - _assert_resolver(path, resolver, expected["artifact_resolver_file"]) - if vector.get("repeat_count", 1) > 1: - repeated = copy.deepcopy(vector) - repeated["repeat_count"] -= 1 - _run_vector(path, repeated) - - -def _assert_resolver( - path: Path, resolver: MemoryArtifactResolver, fixture_name: str -) -> None: - fixture = _json(path / fixture_name) - assert resolver.snapshot() == { - "definitions": sorted( - entry["validated_bundle_fingerprint"] for entry in fixture["definitions"] - ), - "migration_descriptors": sorted( - entry["migration_descriptor_digest"] - for entry in fixture["migration_descriptors"] - ), - } - for entry in fixture["definitions"]: - assert resolver.definition_is_trusted(entry["validated_bundle_fingerprint"]) is entry[ - "trusted" - ] - for entry in fixture["migration_descriptors"]: - assert resolver.migration_descriptor_is_trusted( - entry["migration_descriptor_digest"] - ) is entry["trusted"] diff --git a/conformance/persistence_host.py b/conformance/persistence_host.py new file mode 100644 index 0000000..2e5e38e --- /dev/null +++ b/conformance/persistence_host.py @@ -0,0 +1,58 @@ +"""Production-backed driver for the optional durable persistence profile.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path +from typing import Any + +from determa.state import ExecutionHostError +from determa.state.persistence import PersistenceHost, SQLiteDurableHostStore + +from .version2 import _json, _resolver + + +def _call_log(item: Any) -> list[str]: + return _json(item.path / item.vector["call_log"])["calls"] + + +def run_persistence_vector( + item: Any, request: dict[str, Any] +) -> tuple[dict[str, Any], bytes]: + before = (item.path / item.vector["store_before"]).read_bytes() + root_instance_id = _json(item.path / item.vector["store_before"])["checkpoint"][ + "root_instance_id" + ] + with tempfile.TemporaryDirectory(prefix="determa-persistence-") as directory: + store = SQLiteDurableHostStore(Path(directory) / "host.sqlite") + store.setup_schema() + store.seed(root_instance_id, before) + host = PersistenceHost(store, _resolver(item.path, request)) + try: + if item.vector["operation"] == "persistence_release_quarantine_v2": + response = host.release_quarantine_v2(request) + else: + response = host.process_v2(request) + except ExecutionHostError as error: + response = { + "result": ( + "crashed" + if error.code + in { + "injected_pre_commit_failure", + "response_lost_after_commit", + } + else "rejected" + ), + "mutation": ( + "atomic" + if error.code == "response_lost_after_commit" + else "none" + ), + "core_calls": host.core_calls, + "broker_acknowledged": False, + "code": error.code, + } + stored = store.snapshot(root_instance_id) + assert host.calls == _call_log(item) + return response, stored diff --git a/conformance/persistence_profiles.py b/conformance/persistence_profiles.py deleted file mode 100644 index 11b527e..0000000 --- a/conformance/persistence_profiles.py +++ /dev/null @@ -1,189 +0,0 @@ -"""In-memory transactional driver for the optional persistence profile.""" - -from __future__ import annotations - -import copy -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import yaml - -from determa.state import MemoryArtifactResolver, load_bundle, migrate_and_dispatch -from determa.state.wire import migration_descriptor_digest, strict_json - -from .harness import conformance_root - -PROFILE_DIR = conformance_root() / "conformance" / "profiles" / "persistence" - -_RESOLVED = [ - "resolve_target_definition", - "resolve_route", - "resolve_descriptors", - "verify_trust", -] -_TRANSACTION = ["begin_transaction", "lock_aggregate", "read_inbox"] -_COMMITTED = [ - *_RESOLVED, - *_TRANSACTION, - "migrate_aggregate", - "dispatch_once", - "stage_aggregate", - "stage_inbox", - "stage_outbox", - "stage_audit", - "commit", -] -_ROLLED_BACK = [*_RESOLVED, *_TRANSACTION, "migrate_aggregate", "rollback"] -_REPLAYED = [*_RESOLVED, *_TRANSACTION, "return_recorded_outcome", "commit"] -_QUARANTINED = [ - *_RESOLVED, - *_TRANSACTION, - "migration_failed_permanently", - "stage_blocked_inbox", - "stage_quarantine", - "stage_failure_audit", - "commit", -] - - -@dataclass(frozen=True) -class PersistenceProfileCase: - name: str - path: Path - - -def persistence_profile_cases() -> list[PersistenceProfileCase]: - if not PROFILE_DIR.exists(): - return [] - return [ - PersistenceProfileCase(path.name, path) - for path in sorted(PROFILE_DIR.iterdir()) - if path.is_dir() and (path / "test.yaml").exists() - ] - - -def run_persistence_profile(case: PersistenceProfileCase) -> None: - test = yaml.safe_load((case.path / "test.yaml").read_text(encoding="utf-8")) - profile = test["persistence_profile"] - store = _json(case.path / profile["initial_store"]) - resolver = _profile_resolver(case.path) - for step in profile["steps"]: - envelope = _json(case.path / step["input_envelope"]) - event_id = envelope["event_id"] - operation = step["operation"] - if operation == "replay": - assert any(item["event_id"] == event_id for item in store["inbox"]) - call_log = list(_REPLAYED) - if event_id not in store["acknowledged_event_ids"]: - store["acknowledged_event_ids"].append(event_id) - call_log.append("acknowledge_input") - elif ( - operation == "inject_failure" - and step["failure_class"] == "permanent" - ): - call_log = list(_QUARANTINED) - store["inbox"].append({"event_id": event_id, "status": "blocked"}) - store["quarantine"].append( - { - "event_id": event_id, - "code": "migration_totality_failure", - "aggregate_state_digest": store["aggregate_state"][ - "aggregate_state_digest" - ], - } - ) - else: - candidate = _process( - store, - envelope, - resolver, - step["target_validated_bundle_fingerprint"], - step["migration_route"], - ) - boundary = step.get("failure_boundary") - if operation == "inject_failure" and boundary != ( - "after_commit_before_acknowledgement" - ): - call_log = list(_ROLLED_BACK) - else: - store = candidate - call_log = list(_COMMITTED) - if boundary != "after_commit_before_acknowledgement": - store["acknowledged_event_ids"].append(event_id) - call_log.append("acknowledge_input") - assert store == _json(case.path / step["expect_store"]), step - assert call_log == _json(case.path / step["expect_call_log"]), step - - -def _process( - store: dict[str, Any], - envelope: dict[str, Any], - resolver: MemoryArtifactResolver, - target_fingerprint: str, - route: list[str], -) -> dict[str, Any]: - target_bundle = resolver.resolve_definition(target_fingerprint) - assert target_bundle is not None - bundle = target_bundle if hasattr(target_bundle, "raw") else load_bundle(target_bundle) - event_declaration = (bundle.raw.get("events") or {}).get(envelope["event"]) - delivery_kind = ( - "input" - if isinstance(event_declaration, dict) - and event_declaration.get("direction") == "input" - else "internal" - ) - result = migrate_and_dispatch( - store["aggregate_state"], - target_fingerprint, - route, - resolver, - {delivery_kind: copy.deepcopy(envelope)}, - maintenance_mode=False, - ) - assert result.failure is None - assert result.aggregate_envelope is not None - disposition = ( - "unhandled" - if result.disposition == "rejected" - and result.rejection == {"code": "invalid_event"} - else result.disposition - ) - candidate = copy.deepcopy(store) - candidate["aggregate_state"] = result.aggregate_envelope - candidate["inbox"] = [ - item for item in candidate["inbox"] if item["event_id"] != envelope["event_id"] - ] - candidate["inbox"].append( - { - "event_id": envelope["event_id"], - "disposition": disposition, - "status": "committed", - } - ) - candidate["outbox"].extend(copy.deepcopy(result.emissions)) - candidate["migration_audit"].extend(copy.deepcopy(result.audit_records)) - candidate["quarantine"] = [ - item - for item in candidate["quarantine"] - if item["event_id"] != envelope["event_id"] - ] - return candidate - - -def _profile_resolver(path: Path) -> MemoryArtifactResolver: - definitions = {} - for filename in ("machine.yaml", "target.yaml"): - bundle = load_bundle((path / filename).read_text(encoding="utf-8")) - definitions[bundle.fingerprint] = bundle - descriptor = _json(path / "migration-descriptor.json") - digest = migration_descriptor_digest(descriptor) - return MemoryArtifactResolver( - definitions=definitions, - migration_descriptors={digest: descriptor}, - ) - - -def _json(path: Path) -> Any: - value, _ = strict_json(path.read_bytes()) - return value diff --git a/conformance/pins.py b/conformance/pins.py index 046de39..41235f1 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,8 +4,8 @@ from pathlib import Path -CONFORMANCE_COMMIT = "531468c59c7a2dc32f5cbe92cfabf89805d27f6a" -SPEC_COMMIT = "2e33036563cb966b07124197db672159b4b7e1f4" +CONFORMANCE_COMMIT = "99a4d9ad5256f7330e75b06d48f340cc7239a40d" +SPEC_COMMIT = "ee38796d5e38e67e350a06548fd50faa530cbb12" ROOT = Path(__file__).resolve().parent.parent CONFORMANCE_CACHE = ROOT / ".cache" / f"determa-state-conformance-{CONFORMANCE_COMMIT[:12]}" diff --git a/conformance/test_conformance.py b/conformance/test_conformance.py index a828df3..999ef1f 100644 --- a/conformance/test_conformance.py +++ b/conformance/test_conformance.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import json import os from pathlib import Path @@ -9,37 +10,93 @@ import pytest import yaml from jsonschema import Draft202012Validator - -from determa.state import PORTABLE_CODE_SETS, load_bundle +from referencing import Resource + +from determa.state import ( + PORTABLE_CODE_SETS, + ExecutionHost, + ExecutionHostError, + MemoryArtifactResolver, + MemoryExecutionStore, + load_bundle, +) +from determa.state.host import ( + _required_backup_artifact_digests, + _required_backup_artifacts, + _validate_required_backup_artifacts, +) from determa.state.validator import schema as bundled_schema -from determa.state.wire import artifact_schema - -from .execution_checkpoint import ( - _actual_scope_maps, - _assert_complete_scope_maps, - _expected_scope_maps, - _scope_hosts, - _scope_state, - execution_checkpoint_cases, - execution_checkpoint_vectors, - run_execution_checkpoint_vector, - validate_execution_checkpoint_artifact, +from determa.state.wire import ( + _schema_registry, + artifact_schema, + hash_value, + migration_descriptor_digest, ) + +from .durable_host import durable_host_vectors, run_durable_host_vector from .harness import CORE_DIR, CoreCase, conformance_root, core_cases, run_case -from .persistence import persistence_vector_cases, run_persistence_vectors -from .persistence_profiles import ( - persistence_profile_cases, - run_persistence_profile, -) from .version2 import ( + _assert_checkpoint_unchanged, + _resolver, run_version2_vector, validate_version2_artifact, version2_vectors, ) +_PORTABLE_ARTIFACT_KINDS = { + "aggregate_state_v2", + "migration_descriptor_v2", + "aggregate_state_package_v2", + "execution_checkpoint_v2", + "core_step_result_v2", +} +_CONFORMANCE_ARTIFACT_SCHEMAS = { + "durable_host_call_log_v2": "durable-host-call-log-v2.schema.json", + "durable_host_inputs_v2": "durable-host-inputs-v2.schema.json", + "durable_host_results_v2": "durable-host-results-v2.schema.json", + "durable_host_store_v2": "durable-host-store-v2.schema.json", + "version2_operation_inputs": "version2-operation-inputs.schema.json", + "version2_operation_result": "version2-operation-result.schema.json", +} + + +def _manifest_artifacts() -> list[tuple[Path, dict]]: + root = conformance_root() / "conformance" + return [ + (test_path.parent, artifact) + for test_path in sorted(root.glob("**/test.yaml")) + for artifact in ( + yaml.safe_load(test_path.read_text(encoding="utf-8")) or {} + ).get("artifacts", {}).get("documents", []) + ] -def _load_case(case: CoreCase) -> dict: - return yaml.safe_load(case.test_file.read_text(encoding="utf-8")) or {} + +def _version2_artifacts() -> list[tuple[Path, dict]]: + return [ + (path, artifact) + for path, artifact in _manifest_artifacts() + if artifact["kind"] in _PORTABLE_ARTIFACT_KINDS + ] + + +def _validate_manifest_artifact(path: Path, artifact: dict) -> None: + if artifact["kind"] in _PORTABLE_ARTIFACT_KINDS: + validate_version2_artifact(path, artifact) + return + schema_name = _CONFORMANCE_ARTIFACT_SCHEMAS[artifact["kind"]] + schema_path = conformance_root() / "scripts" / "schemas" / schema_name + schema = json.loads(schema_path.read_text(encoding="utf-8")) + registry = _schema_registry().with_resource( + schema["$id"], Resource.from_contents(schema) + ) + validator = Draft202012Validator(schema, registry=registry) + try: + document = json.loads((path / artifact["file"]).read_text(encoding="utf-8")) + except (json.JSONDecodeError, UnicodeError): + valid = False + else: + valid = validator.is_valid(document) + assert valid is artifact["valid"] def _spec_schema() -> dict | None: @@ -60,9 +117,266 @@ def _spec_root() -> Path | None: def test_suite_present() -> None: assert CORE_DIR.exists(), "pinned conformance suite is unavailable" - assert len(core_cases()) == 114 - assert len(execution_checkpoint_vectors()) == 99 - assert len(version2_vectors()) == 106 + assert len(core_cases()) == 98 + assert len(version2_vectors()) == 162 + assert len(durable_host_vectors()) == 138 + assert len(_manifest_artifacts()) == 381 + + +@pytest.mark.parametrize("stored", [b"mutated", None]) +def test_v2_maintenance_failure_requires_exact_unchanged_checkpoint( + stored: bytes | None, +) -> None: + item = next( + item + for item in version2_vectors() + if item.vector["name"] == "native_v2_maintenance_stale_writer" + ) + before = json.loads( + (item.path / item.vector["checkpoint_before"]).read_text(encoding="utf-8") + ) + initial = {} if stored is None else {before["root_instance_id"]: stored} + observation = { + "store": MemoryExecutionStore(initial), + "root_instance_id": before["root_instance_id"], + } + + with pytest.raises(AssertionError): + _assert_checkpoint_unchanged(item, observation) + + +def test_valid_artifact_manifest_does_not_accept_forged_digest(tmp_path: Path) -> None: + case, artifact = next( + (case, artifact) + for case, artifact in _version2_artifacts() + if artifact["kind"] == "aggregate_state_v2" and artifact["valid"] + ) + document = json.loads((case / artifact["file"]).read_text(encoding="utf-8")) + document["aggregate_state_digest"] = "sha256:" + "0" * 64 + forged = tmp_path / "forged-aggregate.json" + forged.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(AssertionError): + validate_version2_artifact( + tmp_path, + {"file": forged.name, "kind": "aggregate_state_v2", "valid": True}, + ) + + +def test_backup_manifest_requires_migration_recovery_route() -> None: + path = ( + conformance_root() + / "conformance" + / "profiles" + / "execution-checkpoint" + / "checkpoint-04-version2-mailboxes" + ) + source = (path / "maintenance-one-hop-checkpoint-v2.json").read_bytes() + checkpoint = json.loads(source) + required = _required_backup_artifact_digests(checkpoint) + audit = checkpoint["migration_audit_records"][0] + expected = { + audit["source_validated_bundle_fingerprint"], + audit["target_validated_bundle_fingerprint"], + audit["migration_descriptor_digest"], + } + assert expected.issubset(required) + root_instance_id = checkpoint["root_instance_id"] + host = ExecutionHost( + MemoryExecutionStore({root_instance_id: source}), _resolver(path) + ) + arguments = { + "action": "backup", + "checkpoint_members": [source], + "checkpoint_digests": [checkpoint["execution_checkpoint_digest"]], + "adapter_metadata_digest": hash_value( + ["determa-backup-adapter-metadata-2", "migration-backup"] + ), + "retention_mode": checkpoint["replay_retention"]["mode"], + "consistency_point": { + "scope_id": "migration-backup", + "root_instance_ids": [root_instance_id], + }, + } + host.validate_backup_restore_v2( + **arguments, trusted_artifact_digests=sorted(required) + ) + for omitted in expected: + with pytest.raises(ExecutionHostError) as error: + host.validate_backup_restore_v2( + **arguments, + trusted_artifact_digests=sorted(required - {omitted}), + ) + assert error.value.code == "invalid_execution_checkpoint" + + +@pytest.mark.parametrize( + ("artifact_kind", "failure"), + [ + ("definition", "missing"), + ("definition", "untrusted"), + ("definition", "mismatched"), + ("descriptor", "missing"), + ("descriptor", "untrusted"), + ("descriptor", "mismatched"), + ], +) +def test_backup_manifest_requires_restorable_artifacts( + artifact_kind: str, failure: str +) -> None: + path = ( + conformance_root() + / "conformance" + / "profiles" + / "execution-checkpoint" + / "checkpoint-04-version2-mailboxes" + ) + source = (path / "maintenance-one-hop-checkpoint-v2.json").read_bytes() + checkpoint = json.loads(source) + definitions, descriptors = _required_backup_artifacts(checkpoint) + bundles = {} + for name in ["maintenance-source.yaml", "maintenance-target-one.yaml"]: + text = (path / name).read_text(encoding="utf-8") + bundles[load_bundle(text).fingerprint] = text + descriptor_documents = { + migration_descriptor_digest(document): document + for document in [ + json.loads((path / "maintenance-descriptor-one.json").read_text()), + json.loads((path / "maintenance-descriptor-two.json").read_text()), + ] + } + required_definition = next(iter(definitions)) + required_descriptor = next(iter(descriptors)) + resolver_definitions = dict(bundles) + resolver_descriptors = dict(descriptor_documents) + trusted_definitions = list(resolver_definitions) + trusted_descriptors = list(resolver_descriptors) + + if artifact_kind == "definition": + if failure == "missing": + resolver_definitions.pop(required_definition) + elif failure == "untrusted": + trusted_definitions.remove(required_definition) + else: + resolver_definitions[required_definition] = next( + bundle + for fingerprint, bundle in bundles.items() + if fingerprint != required_definition + ) + elif failure == "missing": + resolver_descriptors.pop(required_descriptor) + elif failure == "untrusted": + trusted_descriptors.remove(required_descriptor) + else: + resolver_descriptors[required_descriptor] = next( + descriptor + for digest, descriptor in descriptor_documents.items() + if digest != required_descriptor + ) + + root_instance_id = checkpoint["root_instance_id"] + host = ExecutionHost( + MemoryExecutionStore({root_instance_id: source}), + MemoryArtifactResolver( + definitions=resolver_definitions, + migration_descriptors=resolver_descriptors, + trusted_definitions=trusted_definitions, + trusted_migration_descriptors=trusted_descriptors, + ), + ) + if artifact_kind == "definition": + with pytest.raises(ExecutionHostError) as error: + _validate_required_backup_artifacts( + host.artifact_resolver, + set(definitions | descriptors), + checkpoint, + ) + assert error.value.code == "invalid_execution_checkpoint" + return + + with pytest.raises(ExecutionHostError) as error: + host.validate_backup_restore_v2( + action="backup", + checkpoint_members=[source], + checkpoint_digests=[checkpoint["execution_checkpoint_digest"]], + trusted_artifact_digests=sorted(definitions | descriptors), + adapter_metadata_digest=hash_value( + ["determa-backup-adapter-metadata-2", "migration-backup"] + ), + retention_mode=checkpoint["replay_retention"]["mode"], + consistency_point={ + "scope_id": "migration-backup", + "root_instance_ids": [root_instance_id], + }, + ) + assert error.value.code == "invalid_execution_checkpoint" + + +def test_backup_manifest_includes_historical_fault_definition() -> None: + path = ( + conformance_root() + / "conformance" + / "core" + / "118-version2-persistence" + ) + result = json.loads((path / "migration-historical-fault-result.json").read_text()) + runtime = result["aggregate_state"]["runtimes"][0] + definitions, descriptors = _required_backup_artifacts(result) + required = definitions | descriptors + + assert runtime["fault"]["definition_fingerprint"] in required + assert ( + runtime["identity_origin"]["definition"]["validated_bundle_fingerprint"] + in required + ) + assert runtime["current_definition"]["validated_bundle_fingerprint"] in required + assert result["audit_records"][0]["migration_descriptor_digest"] in required + + resolver = _resolver(path) + _validate_required_backup_artifacts(resolver, set(required), result) + origin = runtime["identity_origin"]["definition"][ + "validated_bundle_fingerprint" + ] + with pytest.raises(ExecutionHostError) as error: + _validate_required_backup_artifacts( + MemoryArtifactResolver( + definitions={ + fingerprint: resolver.resolve_definition(fingerprint) + for fingerprint in definitions - {origin} + }, + migration_descriptors={ + digest: resolver.resolve_migration_descriptor(digest) + for digest in descriptors + }, + ), + set(required), + result, + ) + assert error.value.code == "invalid_execution_checkpoint" + + +@pytest.mark.parametrize("malformation", ["extra_field", "wrong_record"]) +def test_durable_host_rejects_malformed_outbox_response( + monkeypatch: pytest.MonkeyPatch, malformation: str +) -> None: + item = next( + item + for item in durable_host_vectors() + if item.vector["name"] == "outbox_retryable_failure" + ) + original = ExecutionHost.update_pending_outbox + + def malformed_response(*args, **kwargs): + response = original(*args, **kwargs) + if malformation == "extra_field": + return {**response, "unexpected": True} + forged = copy.deepcopy(response) + forged["record"]["delivery_state"] = {"status": "ambiguous"} + return forged + + monkeypatch.setattr(ExecutionHost, "update_pending_outbox", malformed_response) + with pytest.raises(AssertionError): + run_durable_host_vector(item) def test_portable_code_sets_match_authoritative_registry() -> None: @@ -96,70 +410,6 @@ def test_portable_code_sets_match_authoritative_registry() -> None: ) -def _scope_map_pair() -> tuple[dict, dict]: - item = next( - item - for item in execution_checkpoint_vectors() - if item.vector["name"] == "pending_outbox_update_in_scope_a" - ) - before = _scope_state(item.case, item.vector["scope_state_before"]) - hosts, stores = _scope_hosts(item.case, before, []) - selected_scope = next( - scope - for scope in before["scopes"] - if scope["logical_scope_id"] == "scope-a" - ) - root_instance_id, binding = next(iter(selected_scope["checkpoints"].items())) - checkpoint = json.loads( - (item.case.path / binding["file"]).read_text(encoding="utf-8") - ) - hosts["scope-a"].update_pending_outbox( - root_instance_id, - item.vector["effect_id"], - item.vector["desired_pending_state"], - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - after = _scope_state(item.case, item.vector["scope_state_after"]) - return ( - _actual_scope_maps(hosts, stores), - _expected_scope_maps(item.case, after), - ) - - -def test_scope_map_rejects_omitted_unchanged_scope_records() -> None: - actual, expected = _scope_map_pair() - root_instance_id = next(iter(expected["scope-b"]["checkpoints"])) - expected["scope-b"]["checkpoints"].pop(root_instance_id) - expected["scope-b"]["outbox_records"].pop(root_instance_id) - - with pytest.raises(AssertionError): - _assert_complete_scope_maps(actual, expected) - - -def test_scope_map_rejects_unexpected_checkpoint() -> None: - actual, expected = _scope_map_pair() - actual["scope-a"]["checkpoints"]["unexpected-root"] = b"unexpected" - - with pytest.raises(AssertionError): - _assert_complete_scope_maps(actual, expected) - - -def test_scope_map_rejects_unexpected_outbox_record() -> None: - actual, expected = _scope_map_pair() - root_instance_id = next(iter(actual["scope-a"]["outbox_records"])) - actual["scope-a"]["outbox_records"][root_instance_id].append( - { - "effect_id": "unexpected-effect", - "record_kind": "pending", - "source_digest": "sha256:unexpected", - } - ) - - with pytest.raises(AssertionError): - _assert_complete_scope_maps(actual, expected) - - def test_bundled_schema_matches_pinned_spec() -> None: upstream = _spec_schema() assert upstream is not None, "pinned specification is unavailable" @@ -169,10 +419,6 @@ def test_bundled_schema_matches_pinned_spec() -> None: @pytest.mark.parametrize( ("name", "kind"), [ - ("aggregate-state.schema.json", "aggregate_state"), - ("migration-descriptor.schema.json", "migration_descriptor"), - ("aggregate-state-package.schema.json", "aggregate_state_package"), - ("execution-checkpoint.schema.json", "execution_checkpoint"), ("aggregate-state-v2.schema.json", "aggregate_state_v2"), ("migration-descriptor-v2.schema.json", "migration_descriptor_v2"), ("aggregate-state-package-v2.schema.json", "aggregate_state_package_v2"), @@ -190,10 +436,6 @@ def test_bundled_artifact_schemas_match_pinned_spec(name: str, kind: str) -> Non @pytest.mark.parametrize( "kind", [ - "aggregate_state", - "migration_descriptor", - "aggregate_state_package", - "execution_checkpoint", "aggregate_state_v2", "migration_descriptor_v2", "aggregate_state_package_v2", @@ -224,81 +466,20 @@ def test_core_case(case: CoreCase) -> None: run_case(case) -@pytest.mark.parametrize( - "case", persistence_vector_cases(core_cases()), ids=lambda case: case.name -) -def test_persistence_vectors(case: CoreCase) -> None: - run_persistence_vectors(case) - - -@pytest.mark.parametrize( - "case", persistence_profile_cases(), ids=lambda case: case.name -) -def test_persistence_profile(case) -> None: - run_persistence_profile(case) - - -@pytest.mark.parametrize( - "item", execution_checkpoint_vectors(), ids=lambda item: item.name -) -def test_execution_checkpoint_profile(item) -> None: - run_execution_checkpoint_vector(item) - - @pytest.mark.parametrize("item", version2_vectors(), ids=lambda item: item.name) def test_version2_vector(item) -> None: run_version2_vector(item) -@pytest.mark.parametrize( - ("case", "artifact"), - [ - (case, artifact) - for case in execution_checkpoint_cases() - for artifact in case.test["artifacts"]["documents"] - if artifact["kind"] == "execution_checkpoint" - ], - ids=lambda value: ( - value.name - if hasattr(value, "name") - else value["file"] - if isinstance(value, dict) - else None - ), -) -def test_execution_checkpoint_artifact(case, artifact) -> None: - validate_execution_checkpoint_artifact(case, artifact) +@pytest.mark.parametrize("item", durable_host_vectors(), ids=lambda item: item.name) +def test_durable_host_vector(item) -> None: + run_durable_host_vector(item) @pytest.mark.parametrize( ("case", "artifact"), - [ - (case, artifact) - for case in core_cases() - for artifact in (_load_case(case).get("artifacts", {}).get("documents", [])) - if artifact["kind"] - in { - "aggregate_state_v2", - "migration_descriptor_v2", - "aggregate_state_package_v2", - "execution_checkpoint_v2", - "core_step_result_v2", - } - ] - + [ - (case.path, artifact) - for case in execution_checkpoint_cases() - for artifact in case.test.get("artifacts", {}).get("documents", []) - if artifact["kind"] - in { - "aggregate_state_v2", - "migration_descriptor_v2", - "aggregate_state_package_v2", - "execution_checkpoint_v2", - "core_step_result_v2", - } - ], + _manifest_artifacts(), ) def test_version2_artifact(case, artifact) -> None: path = case.path if isinstance(case, CoreCase) else case - validate_version2_artifact(path, artifact) + _validate_manifest_artifact(path, artifact) diff --git a/conformance/version2.py b/conformance/version2.py index e3137fe..f8f5623 100644 --- a/conformance/version2.py +++ b/conformance/version2.py @@ -14,10 +14,13 @@ from determa.state import ( ArtifactError, ExecutionHost, + ExecutionHostError, MemoryArtifactResolver, MemoryExecutionStore, + MigrationLimits, load_bundle, restore_aggregate_package, + restore_migration_descriptor_v2, serialize_execution_checkpoint, ) from determa.state.checkpoint_v2 import ( @@ -25,18 +28,19 @@ prune_checkpoint_v2, restore_execution_checkpoint_v2, step_checkpoint_v2, - upgrade_checkpoint_v1_to_v2, ) from determa.state.queueing import ( admit_aggregate_v2, create_aggregate_v2, - downgrade_aggregate_v2_to_v1, migrate_aggregate_v2, restore_aggregate_v2, step_aggregate_v2, - upgrade_aggregate_v1_to_v2, ) -from determa.state.wire import load_json_artifact, migration_descriptor_digest +from determa.state.wire import ( + canonical_bytes, + load_json_artifact, + migration_descriptor_digest, +) from .harness import conformance_root @@ -90,24 +94,61 @@ def _conformance_definitions() -> dict[str, Any]: return definitions -def _resolver(path: Path) -> MemoryArtifactResolver: - descriptors = {} - for candidate in path.glob("*descriptor*.json"): - document = _json(candidate) - try: - descriptors[migration_descriptor_digest(document)] = document - except ArtifactError: - continue +def _resolver( + path: Path, request: dict[str, Any] | None = None +) -> MemoryArtifactResolver: + request = request or {} + specification = request.get("artifact_resolver") or request.get("definition_resolver") + if specification is None: + definitions = dict(_conformance_definitions()) + for candidate in path.glob("*machine*.yaml"): + bundle = load_bundle(candidate.read_text(encoding="utf-8")) + definitions[bundle.fingerprint] = bundle + descriptors = {} + for candidate in path.glob("*descriptor*.json"): + document = _json(candidate) + try: + descriptors[migration_descriptor_digest(document)] = document + except ArtifactError: + continue + return MemoryArtifactResolver( + definitions=definitions, migration_descriptors=descriptors + ) + definitions = { + item["validated_bundle_fingerprint"]: (path / item["bundle_file"]).read_text( + encoding="utf-8" + ) + for item in specification.get("definitions", []) + } + descriptors = { + item["migration_descriptor_digest"]: _json(path / item["descriptor_file"]) + for item in specification.get("migration_descriptors", []) + } return MemoryArtifactResolver( - definitions=_conformance_definitions(), migration_descriptors=descriptors + definitions=definitions, + migration_descriptors=descriptors, + trusted_definitions=[ + item["validated_bundle_fingerprint"] + for item in specification.get("definitions", []) + if item.get("trusted") + ], + trusted_migration_descriptors=[ + item["migration_descriptor_digest"] + for item in specification.get("migration_descriptors", []) + if item.get("trusted") + ], ) -def _invoke(item: Version2Vector, request: dict[str, Any]) -> Any: +def _invoke( + item: Version2Vector, + request: dict[str, Any], + observation: dict[str, Any] | None = None, +) -> Any: path = item.path vector = item.vector operation = vector["operation"] - resolver = _resolver(path) + resolver = _resolver(path, request) before_name = vector.get("state_before") or vector.get("checkpoint_before") before = _json(path / before_name) if before_name else None bundle = ( @@ -128,20 +169,77 @@ def _invoke(item: Version2Vector, request: dict[str, Any]) -> Any: return admit_aggregate_v2(before, request["deliveries"], resolver) if operation == "step_v2": return step_aggregate_v2(before, request["target_runtime_id"], resolver) - if operation == "upgrade_aggregate_v1_to_v2": - return upgrade_aggregate_v1_to_v2(before, resolver) - if operation == "downgrade_aggregate_v2_to_v1": - return downgrade_aggregate_v2_to_v1(before, resolver) + if operation == "round_trip_aggregate_v2": + return restore_aggregate_v2(before, resolver).aggregate_envelope + if operation == "restore_package_v2": + package = _json(path / vector["package_file"]) + restored_package = restore_aggregate_package(package, resolver) + if request["intent"] == "restore_aggregate": + return restored_package.aggregate.aggregate_envelope + if request["intent"] != "restore_and_apply_migration_route": + raise AssertionError("unsupported restore-package intent") + descriptor = resolver.resolve_migration_descriptor( + restored_package.migration_route[-1] + ) + assert descriptor is not None + target, _ = load_json_artifact(descriptor, "migration_descriptor_v2") + return migrate_aggregate_v2( + restored_package.aggregate.aggregate_envelope, + target["target_validated_bundle_fingerprint"], + restored_package.migration_route, + resolver, + maintenance_mode=request["maintenance_mode"], + ) if operation == "migrate_aggregate_v2": return migrate_aggregate_v2( + before, + request.get("target_bundle", {}).get("validated_bundle_fingerprint"), + request.get("migration_descriptor_digest_route"), + resolver, + maintenance_mode=request.get("maintenance_mode"), + resource_limits=( + MigrationLimits.from_mapping(request["resource_limits"]) + if "resource_limits" in request + else None + ), + ) + if operation == "migrate_then_process_v2": + migrated = migrate_aggregate_v2( before, request["target_bundle"]["validated_bundle_fingerprint"], request["migration_descriptor_digest_route"], resolver, maintenance_mode=request["maintenance_mode"], ) - if operation == "upgrade_checkpoint_v1_to_v2": - return upgrade_checkpoint_v1_to_v2(before, resolver) + admitted = admit_aggregate_v2( + migrated["aggregate_state"], [request["delivery"]], resolver + ) + if admitted["result"] != "accepted": + processing = { + "core_step_result_format": "determa.core_step_result", + "core_step_result_schema_version": 2, + "status": admitted["status"], + "disposition": "rejected", + "state": admitted["state"], + "emissions": [], + "lifecycle_dispositions": [], + "fault": None, + "rejection": admitted["rejection"], + } + else: + target = request["delivery"]["envelope"]["target"] + if "root" in target: + runtime_id = target["root"]["root_runtime_id"] + elif "component" in target: + runtime_id = target["component"]["component_runtime_id"] + else: + runtime_id = target["spawned_instance"]["instance_id"] + processing = step_aggregate_v2(admitted["state"], runtime_id, resolver) + return { + "result": "migrated_and_processed", + "migration_audit_records": migrated["audit_records"], + "processing": processing, + } if operation == "checkpoint_admit_v2": return admit_checkpoint_v2( before, @@ -166,49 +264,56 @@ def _invoke(item: Version2Vector, request: dict[str, Any]) -> Any: expected_revision=request["expected_revision"], expected_checkpoint_digest=request["expected_checkpoint_digest"], ) - if operation == "checkpoint_v1_accept": - assert bundle is not None - delivery = request["deliveries"][0] - envelope = { - key: copy.deepcopy(value) - for key, value in delivery["envelope"].items() - if key not in {"cause_id", "source"} - } - candidate = { - "root_instance_id": before["root_instance_id"], - "delivery_mode": delivery["delivery_mode"], - "origin": {"kind": "host_input"}, - "envelope": envelope, - } - host = ExecutionHost( - MemoryExecutionStore( - {before["root_instance_id"]: serialize_execution_checkpoint(before)} - ), - resolver, + if operation == "checkpoint_migrate_v2": + store = MemoryExecutionStore( + {before["root_instance_id"]: serialize_execution_checkpoint(before)} ) - result = host.accept_delivery( + if observation is not None: + observation["store"] = store + observation["root_instance_id"] = before["root_instance_id"] + host = ExecutionHost(store, resolver) + result = host.maintenance_migration_v2( before["root_instance_id"], - candidate, + request["operation_id"], + request["target_bundle"]["validated_bundle_fingerprint"], + request["migration_descriptor_digest_route"], expected_revision=request["expected_revision"], expected_checkpoint_digest=request["expected_checkpoint_digest"], - selected_bundle=bundle, + maintenance_mode=request["maintenance_mode"], ) - if result["result"] == "not_accepted": - raise ArtifactError(result["failure"]["code"]) + expected_after = vector.get("checkpoint_after") + if expected_after is not None: + restored = host.read_checkpoint(before["root_instance_id"]) + assert restored is not None + assert restored.document == _json(path / expected_after) return result raise AssertionError(f"unsupported version-2 operation: {operation}") +def _assert_checkpoint_unchanged( + item: Version2Vector, observation: dict[str, Any] +) -> None: + store = observation.get("store") + root_instance_id = observation.get("root_instance_id") + assert isinstance(store, MemoryExecutionStore) + assert isinstance(root_instance_id, str) + with store.transaction(root_instance_id) as transaction: + actual = transaction.load() + unchanged_file = item.vector["expect"]["unchanged_file"] + assert actual == (item.path / unchanged_file).read_bytes() + + def run_version2_vector(item: Version2Vector) -> None: vector = item.vector request = copy.deepcopy( _pointer(_json(item.path / vector["request_file"]), vector["request_pointer"]) ) request_snapshot = copy.deepcopy(request) + observation: dict[str, Any] = {} try: - actual = _invoke(item, request) + actual = _invoke(item, request, observation) code = None - except ArtifactError as error: + except (ArtifactError, ExecutionHostError) as error: actual = None code = error.code expected = vector["expect"] @@ -217,6 +322,10 @@ def run_version2_vector(item: Version2Vector) -> None: actual = None if expected["result"] == "failure": assert code == expected["code"] + if vector["operation"] == "checkpoint_migrate_v2" and expected.get( + "unchanged_file" + ): + _assert_checkpoint_unchanged(item, observation) else: assert code is None assert actual == _json(item.path / expected["exact_result_file"]) @@ -228,22 +337,60 @@ def validate_version2_artifact(path: Path, artifact: dict[str, Any]) -> None: resolver = _resolver(path) source = (path / artifact["file"]).read_bytes() try: - if artifact["kind"] == "aggregate_state_v2": - restored = restore_aggregate_v2(source, resolver) + if artifact["valid"]: + document, _ = load_json_artifact(source, artifact["kind"]) + if artifact.get("verify_digest", True): + if artifact["kind"] == "aggregate_state_v2": + restore_aggregate_v2(source, resolver) + elif artifact["kind"] == "execution_checkpoint_v2": + restore_execution_checkpoint_v2(source, resolver) + elif artifact["kind"] == "migration_descriptor_v2": + restore_migration_descriptor_v2(source, resolver) + elif artifact["kind"] == "aggregate_state_package_v2": + restore_aggregate_package(source, resolver) if artifact.get("canonical_of"): - assert restored.canonical_bytes == source + assert canonical_bytes(document) == source + elif artifact["kind"] == "aggregate_state_v2": + restore_aggregate_v2(source, resolver) elif artifact["kind"] == "execution_checkpoint_v2": restore_execution_checkpoint_v2(source, resolver) elif artifact["kind"] == "aggregate_state_package_v2": restore_aggregate_package(source, resolver) - elif artifact["kind"] in { - "migration_descriptor_v2", - "core_step_result_v2", - }: + elif artifact["kind"] == "migration_descriptor_v2": + document, _ = load_json_artifact(source, artifact["kind"]) + if ( + migration_descriptor_digest(document) + != document["migration_descriptor_digest"] + ): + raise ArtifactError("migration_descriptor_digest_mismatch") + elif artifact["kind"] == "core_step_result_v2": load_json_artifact(source, artifact["kind"]) else: return code = None except ArtifactError as error: + if artifact["valid"] and _is_declared_semantic_rejection( + path, artifact["file"], error.code + ): + return code = error.code assert code == (None if artifact["valid"] else artifact["error"]) + + +def _is_declared_semantic_rejection(path: Path, filename: str, code: str) -> bool: + """Recognize a schema-valid fixture exercised as a contextual negative.""" + reference_fields = { + "state_before", + "descriptor_file", + "checkpoint_before", + "package_file", + } + for item in version2_vectors(): + if item.path != path: + continue + if not any(item.vector.get(field) == filename for field in reference_fields): + continue + expected = item.vector.get("expect", {}) + if expected.get("result") == "failure" and expected.get("code") == code: + return True + return False diff --git a/pyproject.toml b/pyproject.toml index 203a007..86dbc86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,13 +63,9 @@ packages = ["src/determa"] [tool.hatch.build.targets.wheel.force-include] "src/determa/state/data/machine.schema.json" = "determa/state/data/machine.schema.json" -"src/determa/state/data/aggregate-state.schema.json" = "determa/state/data/aggregate-state.schema.json" "src/determa/state/data/aggregate-state-v2.schema.json" = "determa/state/data/aggregate-state-v2.schema.json" -"src/determa/state/data/migration-descriptor.schema.json" = "determa/state/data/migration-descriptor.schema.json" "src/determa/state/data/migration-descriptor-v2.schema.json" = "determa/state/data/migration-descriptor-v2.schema.json" -"src/determa/state/data/aggregate-state-package.schema.json" = "determa/state/data/aggregate-state-package.schema.json" "src/determa/state/data/aggregate-state-package-v2.schema.json" = "determa/state/data/aggregate-state-package-v2.schema.json" -"src/determa/state/data/execution-checkpoint.schema.json" = "determa/state/data/execution-checkpoint.schema.json" "src/determa/state/data/execution-checkpoint-v2.schema.json" = "determa/state/data/execution-checkpoint-v2.schema.json" "src/determa/state/data/core-step-result-v2.schema.json" = "determa/state/data/core-step-result-v2.schema.json" diff --git a/scripts/sync_schema.py b/scripts/sync_schema.py index 0209e12..b5ecac7 100644 --- a/scripts/sync_schema.py +++ b/scripts/sync_schema.py @@ -21,17 +21,13 @@ DEST = ROOT / "src" / "determa" / "state" / "data" SCHEMAS = ( "machine.schema.json", - "aggregate-state.schema.json", "aggregate-state-v2.schema.json", - "migration-descriptor.schema.json", "migration-descriptor-v2.schema.json", - "aggregate-state-package.schema.json", "aggregate-state-package-v2.schema.json", - "execution-checkpoint.schema.json", "execution-checkpoint-v2.schema.json", "core-step-result-v2.schema.json", ) -SPEC_COMMIT = "2e33036563cb966b07124197db672159b4b7e1f4" +SPEC_COMMIT = "ee38796d5e38e67e350a06548fd50faa530cbb12" def _fetch(name: str) -> str: diff --git a/src/determa/state/__init__.py b/src/determa/state/__init__.py index 096fdc8..1b6a115 100644 --- a/src/determa/state/__init__.py +++ b/src/determa/state/__init__.py @@ -6,22 +6,19 @@ from .__about__ import __version__ from .checkpoint import ( - RestoredExecutionCheckpoint, execution_checkpoint_digest, - restore_execution_checkpoint, seal_execution_checkpoint, serialize_execution_checkpoint, validate_execution_checkpoint_member, - validate_execution_checkpoint_semantics, ) from .checkpoint_v2 import ( - RestoredExecutionCheckpointV2, + RestoredExecutionCheckpoint, admit_checkpoint_v2, create_checkpoint_v2, + process_delivery_checkpoint_v2, prune_checkpoint_v2, restore_execution_checkpoint_v2, step_checkpoint_v2, - upgrade_checkpoint_v1_to_v2, ) from .codes import ( PORTABLE_CODE_SETS, @@ -37,7 +34,6 @@ PersistenceFailureCode, ) from .definition import Bundle, BundleSource, load_bundle -from .engine import Delivery, Result, create, dispatch from .errors import ( ArtifactError, CelError, @@ -59,22 +55,28 @@ validate_host_profile, ) from .migration import ( - MigrationDispatchResult, - MigrationFailure, MigrationLimits, - MigrationResult, - migrate_aggregate, - migrate_and_dispatch, + restore_migration_descriptor_v2, +) +from .persistence import ( + DurableHostStore, + MemoryDurableHostStore, + PersistenceHost, + SQLiteDurableHostStore, +) +from .queueing import ( + admit_aggregate_v2 as admit, +) +from .queueing import ( + create_aggregate_v2 as create, ) from .queueing import ( - admit_aggregate_v2, - create_aggregate_v2, - downgrade_aggregate_v2_to_v1, migrate_aggregate_v2, restore_aggregate_v2, seal_aggregate_v2, - step_aggregate_v2, - upgrade_aggregate_v1_to_v2, +) +from .queueing import ( + step_aggregate_v2 as step, ) from .stores import ( COMPACT_EFFECT_IDENTITY_RETENTION, @@ -121,7 +123,7 @@ __all__ = [ "ArtifactError", "ArtifactResolver", - "admit_aggregate_v2", + "admit", "admit_checkpoint_v2", "Bundle", "BundleSource", @@ -132,14 +134,12 @@ "COMPACT_EFFECT_IDENTITY_RETENTION", "DURABLE_CONCURRENT", "DURABLE_SINGLE_WRITER", + "DurableHostStore", "DetermaError", "DefinitionResolver", - "Delivery", "CreationRejectionCode", - "create_aggregate_v2", "create_checkpoint_v2", "DispatchRejectionCode", - "downgrade_aggregate_v2_to_v1", "DispositionCode", "ErrorRecord", "EPHEMERAL", @@ -154,31 +154,29 @@ "ExecutionStoreAdapterFailureCode", "FileExecutionStore", "MemoryArtifactResolver", + "MemoryDurableHostStore", "MemoryExecutionStore", "MachineLoadFailureCode", "MigrationDescriptorResolver", - "MigrationDispatchResult", - "MigrationFailure", "MigrationLimits", - "MigrationResult", "migrate_aggregate_v2", "PERMANENT_OUTBOX_TERMINAL_RETENTION", "PERMANENT_RECEIPT_RETENTION", "PostgreSQLExecutionStore", "PORTABLE_CODE_SETS", "PersistenceFailureCode", + "PersistenceHost", "RESTART_PERSISTENT", "ROOT_IDENTITY_RETENTION", - "Result", "RestoredAggregate", "RestoredAggregatePackage", "RestoredExecutionCheckpoint", - "RestoredExecutionCheckpointV2", "SHARED_APPLICATION_TRANSACTION", "STANDARD_CAPABILITIES", "SchemaError", "SharedExecutionTransaction", "SQLiteExecutionStore", + "SQLiteDurableHostStore", "StagedExecutionResult", "ValidationError", "__version__", @@ -189,37 +187,32 @@ "creation_request_digest", "bundled_execution_store_registry", "delivery_request_digest", - "dispatch", "execution_checkpoint_digest", "file_execution_store_factory", "load_bundle", - "migrate_aggregate", - "migrate_and_dispatch", "memory_execution_store_factory", "maintenance_migration_request_digest", "outbox_intent_digest", "portable_envelope", "postgresql_execution_store_factory", "prune_checkpoint_v2", + "process_delivery_checkpoint_v2", "register_bundled_execution_stores", "restore_aggregate", "restore_aggregate_v2", "restore_aggregate_package", - "restore_execution_checkpoint", "restore_execution_checkpoint_v2", + "restore_migration_descriptor_v2", "seal_aggregate_v2", "seal_execution_checkpoint", "serialize_aggregate", "serialize_execution_checkpoint", - "step_aggregate_v2", + "step", "step_checkpoint_v2", "sqlite_execution_store_factory", "validate", "validate_execution_checkpoint_member", - "validate_execution_checkpoint_semantics", "validate_host_profile", - "upgrade_aggregate_v1_to_v2", - "upgrade_checkpoint_v1_to_v2", ] logging.getLogger("determa.state").addHandler(logging.NullHandler()) diff --git a/src/determa/state/checkpoint.py b/src/determa/state/checkpoint.py index 4b1c317..68bae4d 100644 --- a/src/determa/state/checkpoint.py +++ b/src/determa/state/checkpoint.py @@ -1,57 +1,20 @@ -"""Portable execution-checkpoint artifacts and semantic validation.""" +"""Canonical helpers for the sole supported execution-checkpoint artifact.""" from __future__ import annotations import copy -import re from collections.abc import Mapping -from dataclasses import dataclass from functools import cache from typing import Any -from .codes import ( - CheckpointArtifactFailureCode as CheckpointCode, -) -from .codes import ( - PersistenceFailureCode as PersistenceCode, -) -from .errors import ArtifactError -from .wire import ( - ArtifactSource, - DefinitionResolver, - RestoredAggregate, - _schema_registry, - artifact_schema, - canonical_bytes, - hash_value, - load_json_artifact, - restore_aggregate, -) - -_DECIMAL = re.compile(r"(?:0|[1-9][0-9]*)\Z") -_MAX_DECIMAL_DIGITS = 4096 - - -@dataclass(frozen=True) -class RestoredExecutionCheckpoint: - """One verified checkpoint and its optional retained aggregate.""" - - document: dict[str, Any] - aggregate: RestoredAggregate | None - canonical_bytes: bytes - source_bytes: bytes +from .wire import _schema_registry, artifact_schema, canonical_bytes, hash_value def execution_checkpoint_digest(document: Mapping[str, Any]) -> str: - """Compute the exact versioned checkpoint digest.""" + """Compute the canonical schema-v2 checkpoint digest.""" body = copy.deepcopy(dict(document)) body.pop("execution_checkpoint_digest", None) - domain = ( - "determa-execution-checkpoint-digest-2" - if body.get("execution_checkpoint_schema_version") == 2 - else "determa-execution-checkpoint-digest-1" - ) - return hash_value([domain, body]) + return hash_value(["determa-execution-checkpoint-digest-2", body]) def seal_execution_checkpoint(document: Mapping[str, Any]) -> dict[str, Any]: @@ -67,482 +30,16 @@ def serialize_execution_checkpoint(document: Mapping[str, Any]) -> bytes: return canonical_bytes(seal_execution_checkpoint(document)) -def _invalid() -> ArtifactError: - return ArtifactError(CheckpointCode.INVALID_EXECUTION_CHECKPOINT) - - @cache def _member_validator(name: str) -> Any: import jsonschema - schema = artifact_schema("execution_checkpoint") + schema = artifact_schema("execution_checkpoint_v2") return jsonschema.Draft202012Validator( - {"$ref": f"{schema['$id']}#/$defs/{name}"}, - registry=_schema_registry(), + {"$ref": f"{schema['$id']}#/$defs/{name}"}, registry=_schema_registry() ) def validate_execution_checkpoint_member(name: str, value: Any) -> bool: """Return whether a value matches one closed checkpoint schema member.""" return next(_member_validator(name).iter_errors(value), None) is None - - -def _decimal(value: Any) -> int: - if ( - not isinstance(value, str) - or len(value) > _MAX_DECIMAL_DIGITS - or _DECIMAL.fullmatch(value) is None - ): - raise _invalid() - try: - return int(value) - except ValueError as exc: - raise _invalid() from exc - - -def _ordered_unique(values: list[int]) -> bool: - return values == sorted(values) and len(values) == len(set(values)) - - -def _target_root_instance_id(target: Any) -> str: - if not isinstance(target, dict) or len(target) != 1: - raise _invalid() - member = next(iter(target.values())) - if not isinstance(member, dict): - raise _invalid() - root_instance_id = member.get("root_instance_id") - if not isinstance(root_instance_id, str): - raise _invalid() - return root_instance_id - - -def _validate_receipts( - document: dict[str, Any], -) -> tuple[dict[str, dict[str, Any]], list[dict[str, Any]], set[str], set[str]]: - revision = _decimal(document["revision"]) - receipts = document["operation_receipts"] - creation = receipts[0] - if creation["operation_kind"] != "creation" or creation["receipt_sequence"] != "0": - raise _invalid() - - next_receipt = _decimal(document["next_operation_receipt_sequence"]) - sequences = [_decimal(receipt["receipt_sequence"]) for receipt in receipts] - if not _ordered_unique(sequences) or any(sequence >= next_receipt for sequence in sequences): - raise _invalid() - - retention = document["replay_retention"] - cutoff_value = retention["pruned_through_receipt_sequence"] - cutoff = _decimal(cutoff_value) if cutoff_value is not None else None - if retention["mode"] == "permanent" or cutoff is None: - expected = list(range(next_receipt)) - else: - expected = [0, *range(cutoff + 1, next_receipt)] - if sequences != expected: - raise _invalid() - - by_sequence = {receipt["receipt_sequence"]: receipt for receipt in receipts} - delivery_receipts: list[dict[str, Any]] = [] - referenced_effects: set[str] = set() - referenced_migrations: set[str] = set() - operation_ids: set[str] = set() - prior_revision = -1 - for receipt in receipts: - committed_revision = _decimal(receipt["committed_revision"]) - if committed_revision > revision or committed_revision <= prior_revision: - raise _invalid() - prior_revision = committed_revision - operation_kind = receipt["operation_kind"] - if operation_kind == "creation": - if committed_revision != 0 or receipt is not creation: - raise _invalid() - elif operation_kind == "delivery": - delivery_receipts.append(receipt) - accepted_revision = _decimal(receipt["accepted_revision"]) - accepted_sequence = _decimal(receipt["accepted_delivery_sequence"]) - if ( - accepted_revision > committed_revision - or accepted_sequence >= _decimal(document["next_delivery_sequence"]) - or ( - receipt["delivery_mode"] == "input" - and accepted_revision == 0 - ) - or ( - receipt["delivery_mode"] == "internal" - and accepted_revision == committed_revision - ) - ): - raise _invalid() - else: - operation_id = receipt["operation_id"] - if operation_id in operation_ids: - raise _invalid() - operation_ids.add(operation_id) - migration_sequences = [ - _decimal(sequence) for sequence in receipt["migration_sequences"] - ] - if migration_sequences != sorted(migration_sequences): - raise _invalid() - if ( - receipt["result_code"] == "migration_no_operation" - and receipt["source_aggregate_state_digest"] - != receipt["resulting_aggregate_state_digest"] - ): - raise _invalid() - referenced_migrations.update(receipt["migration_sequences"]) - - for index, emission in enumerate(receipt.get("emission_references", [])): - if _decimal(emission["emission_index"]) != index: - raise _invalid() - if emission["kind"] == "external_outbox": - referenced_effects.add(emission["effect_id"]) - return by_sequence, delivery_receipts, referenced_effects, referenced_migrations - - -def _validate_deliveries( - document: dict[str, Any], - receipts_by_sequence: dict[str, dict[str, Any]], - delivery_receipts: list[dict[str, Any]], -) -> None: - revision = _decimal(document["revision"]) - root_instance_id = document["root_instance_id"] - next_delivery = _decimal(document["next_delivery_sequence"]) - pending = document["pending_deliveries"] - pending_sequences = [_decimal(item["delivery_sequence"]) for item in pending] - if ( - not _ordered_unique(pending_sequences) - or any(sequence >= next_delivery for sequence in pending_sequences) - ): - raise _invalid() - - pending_event_ids = [item["envelope"]["event_id"] for item in pending] - receipt_event_ids = [receipt["event_id"] for receipt in delivery_receipts] - if ( - len(pending_event_ids) != len(set(pending_event_ids)) - or len(receipt_event_ids) != len(set(receipt_event_ids)) - or set(pending_event_ids) & set(receipt_event_ids) - ): - raise _invalid() - - allocated_sequences = [ - *pending_sequences, - *[_decimal(receipt["accepted_delivery_sequence"]) for receipt in delivery_receipts], - ] - if len(allocated_sequences) != len(set(allocated_sequences)): - raise _invalid() - if document["replay_retention"]["mode"] == "permanent" and sorted( - allocated_sequences - ) != list(range(next_delivery)): - raise _invalid() - - deliveries: dict[str, tuple[str, str, str, dict[str, Any]]] = {} - for item in pending: - parsed_accepted_revision = _decimal(item["accepted_revision"]) - if parsed_accepted_revision > revision or ( - item["delivery_mode"] == "input" - and parsed_accepted_revision == 0 - ): - raise _invalid() - expected_digest = hash_value( - [ - "determa-inbox-envelope-digest-1", - "1", - root_instance_id, - item["delivery_mode"], - item["envelope"], - ] - ) - if ( - item["envelope_digest"] != expected_digest - or _target_root_instance_id(item["envelope"]["target"]) != root_instance_id - ): - raise _invalid() - deliveries[item["delivery_sequence"]] = ( - item["envelope"]["event_id"], - item["accepted_revision"], - item["delivery_mode"], - item["origin"], - ) - for receipt in delivery_receipts: - deliveries[receipt["accepted_delivery_sequence"]] = ( - receipt["event_id"], - receipt["accepted_revision"], - receipt["delivery_mode"], - receipt["origin"], - ) - - for sequence, (event_id, accepted_revision, mode, origin) in deliveries.items(): - if mode != "internal": - continue - producer = receipts_by_sequence.get(origin["producing_receipt_sequence"]) - if producer is None or accepted_revision != producer["committed_revision"]: - raise _invalid() - emission_index = _decimal(origin["emission_index"]) - emissions = producer.get("emission_references", []) - if emission_index >= len(emissions): - raise _invalid() - emission = emissions[emission_index] - if ( - emission.get("kind") != "internal_delivery" - or emission.get("event_id") != event_id - or emission.get("delivery_sequence") != sequence - ): - raise _invalid() - - permanent = document["replay_retention"]["mode"] == "permanent" - for receipt in document["operation_receipts"]: - for emission in receipt.get("emission_references", []): - if emission["kind"] != "internal_delivery": - continue - linked = deliveries.get(emission["delivery_sequence"]) - if permanent and (linked is None or linked[0] != emission["event_id"]): - raise _invalid() - - -def _validate_outbox(document: dict[str, Any], referenced_effects: set[str]) -> None: - revision = _decimal(document["revision"]) - pending = document["pending_outbox_intents"] - terminal = document["terminal_outbox_records"] - tombstones = document["outbox_effect_tombstones"] - pending_sequences = [_decimal(item["intent"]["sequence"]) for item in pending] - terminal_sequences = [_decimal(item["terminal_sequence"]) for item in terminal] - tombstone_sequences = [_decimal(item["terminal_sequence"]) for item in tombstones] - if ( - pending_sequences != sorted(pending_sequences) - or terminal_sequences != sorted(terminal_sequences) - or tombstone_sequences != sorted(tombstone_sequences) - ): - raise _invalid() - - full_sequences = [ - *pending_sequences, - *[_decimal(item["intent"]["sequence"]) for item in terminal], - ] - if len(full_sequences) != len(set(full_sequences)): - raise _invalid() - all_terminal_sequences = [*terminal_sequences, *tombstone_sequences] - if ( - len(all_terminal_sequences) != len(set(all_terminal_sequences)) - or any( - sequence >= _decimal(document["next_outbox_terminal_sequence"]) - for sequence in all_terminal_sequences - ) - ): - raise _invalid() - - pending_effects = [item["intent"]["effect_id"] for item in pending] - terminal_effects = [item["intent"]["effect_id"] for item in terminal] - tombstone_effects = [item["effect_id"] for item in tombstones] - all_effects = [*pending_effects, *terminal_effects, *tombstone_effects] - if len(all_effects) != len(set(all_effects)): - raise _invalid() - effect_set = set(all_effects) - if not referenced_effects.issubset(effect_set): - raise _invalid() - if document["replay_retention"]["mode"] == "permanent" and effect_set != referenced_effects: - raise _invalid() - - for item in pending: - state_revision = _decimal(item["state_revision"]) - if state_revision > revision: - raise _invalid() - effect_id = item["intent"]["effect_id"] - producers = [ - receipt - for receipt in document["operation_receipts"] - if any( - emission.get("kind") == "external_outbox" - and emission.get("effect_id") == effect_id - for emission in receipt.get("emission_references", []) - ) - ] - if len(producers) != 1: - if document["replay_retention"]["mode"] == "permanent" or producers: - raise _invalid() - continue - producer_revision = _decimal(producers[0]["committed_revision"]) - if item["delivery_state"]["status"] == "not_attempted": - if state_revision != producer_revision: - raise _invalid() - elif state_revision <= producer_revision: - raise _invalid() - receipt_by_effect = { - emission["effect_id"]: receipt - for receipt in document["operation_receipts"] - for emission in receipt.get("emission_references", []) - if emission["kind"] == "external_outbox" - } - for item in [*terminal, *tombstones]: - committed_revision = _decimal(item["committed_revision"]) - if committed_revision > revision: - raise _invalid() - effect_id = ( - item["intent"]["effect_id"] - if "intent" in item - else item["effect_id"] - ) - producer = receipt_by_effect.get(effect_id) - if producer is not None and committed_revision <= _decimal( - producer["committed_revision"] - ): - raise _invalid() - - -def _validate_audit_and_root( - document: dict[str, Any], referenced_migrations: set[str] -) -> None: - root_instance_id = document["root_instance_id"] - root_record = document["root_record"] - creation = document["operation_receipts"][0] - if root_record["status"] == "retained": - aggregate = root_record["aggregate_state"] - if aggregate["root_instance_id"] != root_instance_id: - raise _invalid() - creation_id = aggregate["creation_id"] - root_runtime_id = aggregate["root_runtime_id"] - else: - creation_id = root_record["creation_id"] - root_runtime_id = root_record["root_runtime_id"] - if document["pending_deliveries"] or document["pending_outbox_intents"]: - raise _invalid() - if creation["creation_id"] != creation_id: - raise _invalid() - - audits = document["migration_audit_records"] - audit_sequences = [_decimal(item["migration_sequence"]) for item in audits] - if not _ordered_unique(audit_sequences): - raise _invalid() - available = {item["migration_sequence"] for item in audits} - if not referenced_migrations.issubset(available): - raise _invalid() - if document["replay_retention"]["mode"] == "permanent" and referenced_migrations != available: - raise _invalid() - if any( - audit["root_instance_id"] != root_instance_id - or audit["root_runtime_id"] != root_runtime_id - for audit in audits - ): - raise _invalid() - audit_by_sequence = {item["migration_sequence"]: item for item in audits} - for receipt in document["operation_receipts"]: - if receipt["operation_kind"] != "maintenance_migration": - continue - linked = [ - audit_by_sequence[sequence] - for sequence in receipt["migration_sequences"] - if sequence in audit_by_sequence - ] - if len(linked) != len(receipt["migration_sequences"]): - raise _invalid() - if linked and ( - linked[0]["source_aggregate_state_digest"] - != receipt["source_aggregate_state_digest"] - or linked[-1]["target_aggregate_state_digest"] - != receipt["resulting_aggregate_state_digest"] - or any( - left["target_aggregate_state_digest"] - != right["source_aggregate_state_digest"] - for left, right in zip(linked, linked[1:], strict=False) - ) - ): - raise _invalid() - - final_digest = ( - root_record["final_aggregate_state_digest"] - if root_record["status"] == "tombstone" - else root_record["aggregate_state"]["aggregate_state_digest"] - ) - retention = document["replay_retention"] - cutoff = retention["pruned_through_receipt_sequence"] - last_receipt = document["operation_receipts"][-1] - has_final_receipt_evidence = ( - cutoff is None or last_receipt["receipt_sequence"] != "0" - ) - if ( - has_final_receipt_evidence - and last_receipt["resulting_aggregate_state_digest"] != final_digest - ): - raise _invalid() - - status_evidence = next( - ( - receipt - for receipt in reversed(document["operation_receipts"]) - if receipt["operation_kind"] in {"creation", "delivery"} - ), - None, - ) - if ( - status_evidence is None - or ( - cutoff is not None - and status_evidence["receipt_sequence"] == "0" - ) - ): - return - if status_evidence["operation_kind"] == "creation": - status = status_evidence["status"] - fault = status_evidence["fault"] - else: - status = status_evidence["outcome"]["status"] - fault = status_evidence["outcome"]["fault"] - - if root_record["status"] == "tombstone": - if status != root_record["terminal_status"]: - raise _invalid() - return - - aggregate = root_record["aggregate_state"] - root_runtime = next( - ( - runtime - for runtime in aggregate["runtimes"] - if runtime["runtime_id"] == aggregate["root_runtime_id"] - ), - None, - ) - if ( - root_runtime is None - or root_runtime["status"] != status - or root_runtime["fault"] != fault - ): - raise _invalid() - - -def validate_execution_checkpoint_semantics(document: dict[str, Any]) -> None: - """Validate all schema-version-1 portable cross-field invariants.""" - receipts, deliveries, referenced_effects, referenced_migrations = _validate_receipts( - document - ) - _validate_deliveries(document, receipts, deliveries) - _validate_outbox(document, referenced_effects) - _validate_audit_and_root(document, referenced_migrations) - - -def restore_execution_checkpoint( - source: ArtifactSource, definition_resolver: DefinitionResolver -) -> RestoredExecutionCheckpoint: - """Parse, verify, and restore one strict execution checkpoint.""" - document, raw = load_json_artifact(source, "execution_checkpoint") - aggregate: RestoredAggregate | None = None - if document["root_record"]["status"] == "retained": - try: - aggregate = restore_aggregate( - document["root_record"]["aggregate_state"], definition_resolver - ) - except ArtifactError as exc: - if exc.code in { - PersistenceCode.SOURCE_DEFINITION_UNAVAILABLE, - PersistenceCode.DEFINITION_UNTRUSTED, - PersistenceCode.DEFINITION_FINGERPRINT_MISMATCH, - }: - raise - raise _invalid() from exc - if execution_checkpoint_digest(document) != document["execution_checkpoint_digest"]: - raise ArtifactError(CheckpointCode.EXECUTION_CHECKPOINT_DIGEST_MISMATCH) - validate_execution_checkpoint_semantics(document) - return RestoredExecutionCheckpoint( - document=copy.deepcopy(document), - aggregate=aggregate, - canonical_bytes=canonical_bytes(document), - source_bytes=raw, - ) diff --git a/src/determa/state/checkpoint_v2.py b/src/determa/state/checkpoint_v2.py index 0e4d04b..df50526 100644 --- a/src/determa/state/checkpoint_v2.py +++ b/src/determa/state/checkpoint_v2.py @@ -9,37 +9,35 @@ from .checkpoint import ( execution_checkpoint_digest, - restore_execution_checkpoint, seal_execution_checkpoint, - validate_execution_checkpoint_member, ) from .codes import CheckpointArtifactFailureCode, CheckpointHostFailureCode from .errors import ArtifactError -from .host import creation_request_digest, delivery_request_digest +from .host import ( + creation_request_digest, + maintenance_migration_request_digest, +) from .queueing import ( _entry_digest, - _runtime_id_for_target, _valid_envelope_shape, + _validate_new_deliveries, admit_aggregate_v2, create_aggregate_v2, restore_aggregate_v2, - seal_aggregate_v2, step_aggregate_v2, - upgrade_aggregate_v1_to_v2, ) from .wire import ( ArtifactSource, DefinitionResolver, canonical_bytes, decimal, - hash_value, load_json_artifact, ) @dataclass(frozen=True) -class RestoredExecutionCheckpointV2: - """One verified queue-bearing checkpoint.""" +class RestoredExecutionCheckpoint: + """One verified execution checkpoint.""" document: dict[str, Any] canonical_bytes: bytes @@ -108,6 +106,8 @@ def _validate_checkpoint_semantics(document: dict[str, Any]) -> None: raise _invalid() cutoff_value = document["replay_retention"]["pruned_through_receipt_sequence"] cutoff = decimal(cutoff_value) if cutoff_value is not None else None + if cutoff is not None and cutoff >= next_receipt: + raise _invalid() first_retained = 1 if cutoff is None else cutoff + 1 if len(sequences) != 1 + max(0, next_receipt - first_retained): raise _invalid() @@ -122,16 +122,15 @@ def _validate_checkpoint_semantics(document: dict[str, Any]) -> None: if aggregate is not None and (aggregate["root_instance_id"] != document["root_instance_id"]): raise _invalid() creation = receipts[0] - legacy_creation = creation.get("legacy_receipt", creation) root_creation_id = ( aggregate["creation_id"] if aggregate is not None else root.get("creation_id") ) - if legacy_creation.get("creation_id") != root_creation_id: + if creation.get("creation_id") != root_creation_id: raise _invalid() if ( aggregate is not None and revision == 0 - and legacy_creation.get("resulting_aggregate_state_digest") + and creation.get("resulting_aggregate_state_digest") != aggregate["aggregate_state_digest"] ): raise _invalid() @@ -146,13 +145,7 @@ def _validate_checkpoint_semantics(document: dict[str, Any]) -> None: referenced_terminal: dict[str, dict[str, Any]] = {} pending_producers: dict[str, dict[str, Any]] = {} terminal_producers: dict[str, dict[str, Any]] = {} - legacy_wrappers: list[dict[str, Any]] = [] - legacy_producer_references: dict[ - str, list[tuple[dict[str, Any], dict[str, Any]]] - ] = {} - legacy_terminal_events: dict[str, dict[str, Any]] = {} referenced_effects: set[str] = set() - committed_order: list[tuple[int, int]] = [] for receipt in receipts: kind = receipt["operation_kind"] if kind == "acceptance": @@ -169,31 +162,9 @@ def _validate_checkpoint_semantics(document: dict[str, Any]) -> None: if event_id in terminals: raise _invalid() terminals[event_id] = receipt - committed_order.append( - ( - decimal(receipt["committed_revision"]), - decimal(receipt["receipt_sequence"]), - ) - ) elif kind == "creation" and decimal(receipt["committed_revision"]) > revision: raise _invalid() - elif kind == "legacy_v1_operation": - legacy_receipt = receipt["legacy_receipt"] - if decimal(legacy_receipt["committed_revision"]) > revision: - raise _invalid() - if kind in {"legacy_v1_creation", "legacy_v1_operation"}: - legacy_wrappers.append(receipt) - legacy_receipt = receipt["legacy_receipt"] - if legacy_receipt["receipt_sequence"] != receipt["receipt_sequence"]: - raise _invalid() - for reference in legacy_receipt.get("emission_references", []): - if reference["kind"] == "internal_delivery": - legacy_producer_references.setdefault(reference["event_id"], []).append( - (receipt, reference) - ) references = receipt.get("emission_references") - if references is None and kind.startswith("legacy_v1_"): - references = receipt["legacy_receipt"].get("emission_references", []) for reference in references or []: reference_kind = reference["kind"] if reference_kind == "internal_mailbox": @@ -213,116 +184,6 @@ def _validate_checkpoint_semantics(document: dict[str, Any]) -> None: if effect_id in referenced_effects: raise _invalid() referenced_effects.add(effect_id) - if legacy_wrappers and receipts[: len(legacy_wrappers)] != legacy_wrappers: - raise _invalid() - if committed_order != sorted(committed_order) or len(committed_order) != len( - set(committed_order) - ): - raise _invalid() - - def validate_internal_legacy_producer( - event_id: str, - acceptance_sequence: str, - accepted_revision: str, - origin: Mapping[str, Any], - ) -> None: - producers = legacy_producer_references.get(event_id, []) - if ( - len(producers) != 1 - or producers[0][0]["receipt_sequence"] - != origin["producing_receipt_sequence"] - or producers[0][1]["emission_index"] != origin["emission_index"] - or producers[0][1]["delivery_sequence"] != acceptance_sequence - or producers[0][0]["legacy_receipt"]["committed_revision"] - != accepted_revision - ): - raise _invalid() - - legacy_terminal_acceptance_sequences: set[str] = set() - for wrapper in legacy_wrappers: - legacy_receipt = wrapper["legacy_receipt"] - if legacy_receipt["operation_kind"] != "delivery": - continue - event_id = legacy_receipt["event_id"] - acceptance_sequence = legacy_receipt["accepted_delivery_sequence"] - if ( - event_id in legacy_terminal_events - or acceptance_sequence in legacy_terminal_acceptance_sequences - ): - raise _invalid() - legacy_terminal_events[event_id] = legacy_receipt - legacy_terminal_acceptance_sequences.add(acceptance_sequence) - if legacy_receipt["delivery_mode"] == "internal": - origin = legacy_receipt["origin"] - if not ( - origin.get("kind") == "internal_emission" - and set(origin) - == {"kind", "producing_receipt_sequence", "emission_index"} - ): - raise _invalid() - validate_internal_legacy_producer( - event_id, - acceptance_sequence, - legacy_receipt["accepted_revision"], - origin, - ) - - def validate_legacy_evidence( - acceptance: Mapping[str, Any], entry: Mapping[str, Any] | None - ) -> None: - evidence = acceptance.get("legacy_v1_delivery") - if evidence is None: - return - origin = evidence["origin"] - mode = acceptance["delivery_mode"] - if ( - evidence["delivery_sequence"] != acceptance["acceptance_sequence"] - or evidence["envelope_digest"] == "sha256:" + ("0" * 64) - or evidence["envelope_digest"] == acceptance["request_digest"] - ): - raise _invalid() - if mode == "input": - if origin != {"kind": "host_input"}: - raise _invalid() - elif not ( - origin.get("kind") == "internal_emission" - and set(origin) == {"kind", "producing_receipt_sequence", "emission_index"} - ): - raise _invalid() - if entry is not None: - source = entry["envelope"]["source"] - legacy_envelope = copy.deepcopy(entry["envelope"]) - legacy_envelope.pop("cause_id") - legacy_envelope.pop("source") - expected_legacy_digest = hash_value( - [ - "determa-inbox-envelope-digest-1", - "1", - document["root_instance_id"], - entry["delivery_mode"], - legacy_envelope, - ] - ) - source_matches = (source == {"host": True} and mode == "input") or ( - mode == "internal" - and source - == { - "legacy_v1_internal": { - "producing_receipt_sequence": origin["producing_receipt_sequence"], - "emission_index": origin["emission_index"], - } - } - ) - if evidence["envelope_digest"] != expected_legacy_digest or not source_matches: - raise _invalid() - if mode == "internal": - validate_internal_legacy_producer( - acceptance["event_id"], - acceptance["acceptance_sequence"], - acceptance["accepted_revision"], - origin, - ) - for event_id, entry in pending_by_event.items(): acceptance = acceptances.get(event_id) producer = referenced_pending.get(event_id) @@ -341,14 +202,9 @@ def validate_legacy_evidence( ): raise _invalid() source = entry["envelope"]["source"] - if "legacy_v1_internal" in source: - if acceptance is None or acceptance.get("legacy_v1_delivery") is None: - raise _invalid() - validate_legacy_evidence(acceptance, entry) - elif source == {"host": True}: + if source == {"host": True}: if acceptance is None: raise _invalid() - validate_legacy_evidence(acceptance, entry) elif acceptance is not None: raise _invalid() if set(pending_by_event) & set(terminals): @@ -377,9 +233,6 @@ def validate_legacy_evidence( > decimal(terminal["committed_revision"]) ): raise _invalid() - if acceptance is not None: - validate_legacy_evidence(acceptance, None) - tombstones = document["event_identity_tombstones"] tombstone_events = [item["event_id"] for item in tombstones] tombstone_sequences = [decimal(item["terminal_receipt_sequence"]) for item in tombstones] @@ -395,13 +248,6 @@ def validate_legacy_evidence( ): raise _invalid() tombstones_by_event = {item["event_id"]: item for item in tombstones} - if ( - set(legacy_terminal_events) & set(pending_by_event) - or set(legacy_terminal_events) & set(terminals) - or set(legacy_terminal_events) & set(tombstones_by_event) - or set(legacy_terminal_events) & set(acceptances) - ): - raise _invalid() if any( event_id not in pending_by_event and event_id not in terminals for event_id in acceptances ): @@ -411,30 +257,6 @@ def validate_legacy_evidence( for producer in [*pending_producers.values(), *terminal_producers.values()] ): raise _invalid() - for event_id, references in legacy_producer_references.items(): - if len(references) != 1: - raise _invalid() - located_entry = pending_by_event.get(event_id) - located_terminal = terminals.get(event_id) - located_tombstone = tombstones_by_event.get(event_id) - located_legacy_terminal = legacy_terminal_events.get(event_id) - locations = [ - item - for item in ( - located_entry, - located_terminal, - located_tombstone, - located_legacy_terminal, - ) - if item is not None - ] - if len(locations) != 1: - raise _invalid() - allocation = locations[0].get( - "acceptance_sequence", locations[0].get("accepted_delivery_sequence") - ) - if references[0][1]["delivery_sequence"] != allocation: - raise _invalid() for event_id, reference in referenced_terminal.items(): terminal_evidence = terminals.get(event_id) tombstone = tombstones_by_event.get(event_id) @@ -449,15 +271,29 @@ def validate_legacy_evidence( if evidence_sequence is None or reference["terminal_receipt_sequence"] != evidence_sequence: raise _invalid() + receipt_chronology: list[tuple[int, int]] = [] + greatest_prior_revision = -1 + for receipt, receipt_sequence in zip(receipts, sequences, strict=True): + if "committed_revision" in receipt: + effective_revision = receipt["committed_revision"] + else: + effective_revision = receipt["accepted_revision"] + effective_revision_value = decimal(effective_revision) + if ( + receipt["operation_kind"] == "maintenance_migration" + and effective_revision_value <= greatest_prior_revision + ): + raise _invalid() + receipt_chronology.append((effective_revision_value, receipt_sequence)) + greatest_prior_revision = max(greatest_prior_revision, effective_revision_value) + if receipt_chronology != sorted(receipt_chronology): + raise _invalid() + allocation_acceptances = ( [decimal(entry["acceptance_sequence"]) for entry in pending_entries] + [decimal(item["acceptance_sequence"]) for item in acceptances.values()] + [decimal(item["acceptance_sequence"]) for item in terminals.values()] + [decimal(item["acceptance_sequence"]) for item in tombstones] - + [ - decimal(item["accepted_delivery_sequence"]) - for item in legacy_terminal_events.values() - ] ) allocation_queues = [decimal(entry["queue_sequence"]) for entry in pending_entries] + [ decimal(item["final_queue_sequence"]) for item in terminals.values() @@ -477,11 +313,6 @@ def validate_legacy_evidence( prior_owner = acceptance_owners.setdefault(sequence, event_id) if prior_owner != event_id: raise _invalid() - for event_id, item in legacy_terminal_events.items(): - sequence = decimal(item["accepted_delivery_sequence"]) - prior_owner = acceptance_owners.setdefault(sequence, event_id) - if prior_owner != event_id: - raise _invalid() if aggregate is not None and ( any( value >= decimal(aggregate["next_acceptance_sequence"]) @@ -553,10 +384,98 @@ def validate_legacy_evidence( ): raise _invalid() + audit_by_sequence = {item["migration_sequence"]: item for item in audits} + root_runtime_id = ( + aggregate["root_runtime_id"] if aggregate is not None else root["root_runtime_id"] + ) + if any(item["root_runtime_id"] != root_runtime_id for item in audits): + raise _invalid() + if aggregate is not None and audit_sequences and max(audit_sequences) > decimal( + aggregate["migration_sequence"] + ): + raise _invalid() + + operation_ids: set[str] = set() + referenced_audit_sequences: list[str] = [] + for receipt in receipts: + if receipt["operation_kind"] != "maintenance_migration": + continue + operation_id = receipt["operation_id"] + committed_revision = decimal(receipt["committed_revision"]) + if ( + operation_id in operation_ids + or committed_revision == 0 + or committed_revision > revision + ): + raise _invalid() + operation_ids.add(operation_id) + sequences_for_receipt = receipt["migration_sequences"] + selected = [audit_by_sequence.get(sequence) for sequence in sequences_for_receipt] + if any(item is None for item in selected): + raise _invalid() + linked = [item for item in selected if item is not None] + referenced_audit_sequences.extend(sequences_for_receipt) + if receipt["result_code"] == "migration_no_operation": + if ( + sequences_for_receipt + or receipt["source_aggregate_state_digest"] + != receipt["resulting_aggregate_state_digest"] + ): + raise _invalid() + descriptor_route: list[str] = [] + else: + if ( + not linked + or [item["migration_sequence"] for item in linked] + != sequences_for_receipt + or any( + decimal(right["migration_sequence"]) + != decimal(left["migration_sequence"]) + 1 + for left, right in zip(linked, linked[1:], strict=False) + ) + or linked[0]["source_aggregate_state_digest"] + != receipt["source_aggregate_state_digest"] + or linked[-1]["target_aggregate_state_digest"] + != receipt["resulting_aggregate_state_digest"] + or any( + left["target_aggregate_state_digest"] + != right["source_aggregate_state_digest"] + or left["target_validated_bundle_fingerprint"] + != right["source_validated_bundle_fingerprint"] + for left, right in zip(linked, linked[1:], strict=False) + ) + ): + raise _invalid() + descriptor_route = [item["migration_descriptor_digest"] for item in linked] + if ( + linked[-1]["target_validated_bundle_fingerprint"] + != receipt["target_validated_bundle_fingerprint"] + ): + raise _invalid() + possible_request_digests = { + maintenance_migration_request_digest( + document["root_instance_id"], + operation_id, + receipt["source_aggregate_state_digest"], + receipt["target_validated_bundle_fingerprint"], + descriptor_route, + maintenance_mode, + ) + for maintenance_mode in (False, True) + } + if receipt["request_digest"] not in possible_request_digests: + raise _invalid() + if len(referenced_audit_sequences) != len(set(referenced_audit_sequences)): + raise _invalid() + referenced_audits = set(referenced_audit_sequences) + available_audits = set(audit_by_sequence) + if not referenced_audits.issubset(available_audits): + raise _invalid() + def restore_execution_checkpoint_v2( source: ArtifactSource, definition_resolver: DefinitionResolver -) -> RestoredExecutionCheckpointV2: +) -> RestoredExecutionCheckpoint: """Restore one structurally and relationally valid version-2 checkpoint.""" document, raw = load_json_artifact(source, "execution_checkpoint_v2") if execution_checkpoint_digest(document) != document["execution_checkpoint_digest"]: @@ -580,118 +499,29 @@ def restore_execution_checkpoint_v2( ): raise raise _invalid() from error - return RestoredExecutionCheckpointV2( + return RestoredExecutionCheckpoint( document=copy.deepcopy(document), canonical_bytes=canonical_bytes(document), source_bytes=raw, ) -def upgrade_checkpoint_v1_to_v2( - source: ArtifactSource, definition_resolver: DefinitionResolver -) -> dict[str, Any]: - """Explicitly convert a complete version-1 checkpoint and pending history.""" - restored = restore_execution_checkpoint(source, definition_resolver) - document = restored.document - if document["root_record"]["status"] != "retained": - raise _invalid() - aggregate = upgrade_aggregate_v1_to_v2( - document["root_record"]["aggregate_state"], definition_resolver - ) - receipts: list[dict[str, Any]] = [] - for receipt in document["operation_receipts"]: - receipts.append( - { - "operation_kind": ( - "legacy_v1_creation" - if receipt["operation_kind"] == "creation" - else "legacy_v1_operation" - ), - "receipt_sequence": receipt["receipt_sequence"], - "legacy_receipt": copy.deepcopy(receipt), - } - ) - next_receipt = int(document["next_operation_receipt_sequence"]) - next_queue = 0 - aggregate["next_acceptance_sequence"] = document["next_delivery_sequence"] - runtime_by_id = {runtime["runtime_id"]: runtime for runtime in aggregate["runtimes"]} - for pending in sorted( - document["pending_deliveries"], key=lambda item: int(item["delivery_sequence"]) - ): - origin = pending["origin"] - envelope = copy.deepcopy(pending["envelope"]) - envelope["cause_id"] = envelope["event_id"] - envelope["source"] = ( - {"host": True} - if origin["kind"] == "host_input" - else { - "legacy_v1_internal": { - "producing_receipt_sequence": origin["producing_receipt_sequence"], - "emission_index": origin["emission_index"], - } - } - ) - digest = _entry_digest(document["root_instance_id"], pending["delivery_mode"], envelope) - entry = { - "acceptance_sequence": pending["delivery_sequence"], - "queue_sequence": str(next_queue), - "delivery_mode": pending["delivery_mode"], - "envelope": envelope, - "envelope_digest": digest, - "deferral_count": "0", - } - runtime_id = _runtime_id_for_target(envelope["target"]) - if runtime_id not in runtime_by_id: - raise _invalid() - runtime_by_id[runtime_id]["ready_mailbox"].append(entry) - acceptance = { - "operation_kind": "acceptance", - "receipt_sequence": str(next_receipt), - "event_id": envelope["event_id"], - "request_digest": digest, - "acceptance_sequence": pending["delivery_sequence"], - "accepted_revision": pending["accepted_revision"], - "delivery_mode": pending["delivery_mode"], - } - if pending["delivery_mode"] == "internal": - acceptance["legacy_v1_delivery"] = { - "delivery_sequence": pending["delivery_sequence"], - "envelope_digest": pending["envelope_digest"], - "origin": copy.deepcopy(origin), - } - receipts.append(acceptance) - next_receipt += 1 - next_queue += 1 - aggregate["next_queue_sequence"] = str(next_queue) - aggregate = seal_aggregate_v2(aggregate) - result = { - "execution_checkpoint_format": "determa.execution_checkpoint", - "execution_checkpoint_schema_version": 2, - "root_instance_id": document["root_instance_id"], - "revision": str(int(document["revision"]) + 1), - "root_record": {"status": "retained", "aggregate_state": aggregate}, - "replay_retention": copy.deepcopy(document["replay_retention"]), - "next_operation_receipt_sequence": str(next_receipt), - "operation_receipts": receipts, - "event_identity_tombstones": [], - "pending_outbox_intents": copy.deepcopy(document["pending_outbox_intents"]), - "next_outbox_terminal_sequence": document["next_outbox_terminal_sequence"], - "terminal_outbox_records": copy.deepcopy(document["terminal_outbox_records"]), - "outbox_effect_tombstones": copy.deepcopy(document["outbox_effect_tombstones"]), - "migration_audit_records": copy.deepcopy(document["migration_audit_records"]), - } - return seal_execution_checkpoint(result) - - def _append_external_intent( checkpoint: dict[str, Any], references: list[dict[str, Any]], emission: Mapping[str, Any], emission_index: int, ) -> None: + intent = { + "effect_id": emission["effect_id"], + "sequence": emission["sequence"], + "event": emission["event"], + "payload": copy.deepcopy(emission["payload"]), + "correlation_id": emission["correlation_id"], + } checkpoint["pending_outbox_intents"].append( { - "intent": copy.deepcopy(dict(emission)), + "intent": intent, "state_revision": checkpoint["revision"], "delivery_state": {"status": "not_attempted"}, } @@ -713,7 +543,14 @@ def create_checkpoint_v2( bindings: dict[str, dict[str, Any]] | None = None, ) -> dict[str, Any]: """Create a fresh queue-bearing checkpoint from the public v2 core result.""" - result = create_aggregate_v2(bundle, machine_id, root_instance_id, creation_id, bindings) + result = create_aggregate_v2( + bundle, + machine_id, + root_instance_id, + creation_id, + bindings, + _include_host_evidence=True, + ) aggregate = result["state"] if aggregate is None: raise ArtifactError(CheckpointHostFailureCode.CREATION_REJECTED) @@ -755,7 +592,10 @@ def create_checkpoint_v2( checkpoint["next_operation_receipt_sequence"] = str(1 + len(lifecycle_sequences)) for index, emission in enumerate(result["emissions"]): if "kind" not in emission: - _append_external_intent(checkpoint, receipt["emission_references"], emission, index) + emission_index = int(emission.get("_determa_v2_emission_index", index)) + _append_external_intent( + checkpoint, receipt["emission_references"], emission, emission_index + ) elif emission["kind"] == "internal_mailbox": receipt["emission_references"].append(copy.deepcopy(emission)) else: @@ -825,7 +665,7 @@ def _pending_replay( def _retained_replay( - document: Mapping[str, Any], event_id: str, digests: Mapping[str, str] + document: Mapping[str, Any], event_id: str, request_digest: str ) -> dict[str, Any] | None: terminal = next( ( @@ -844,7 +684,7 @@ def _retained_replay( None, ) if terminal is not None: - if terminal["request_digest"] != digests["determa-inbox-envelope-digest-2"]: + if terminal["request_digest"] != request_digest: raise ArtifactError(CheckpointHostFailureCode.EVENT_ID_CONFLICT) return { "result": "replay", @@ -858,47 +698,17 @@ def _retained_replay( None, ) if tombstone is not None: - if tombstone["request_digest"] != digests[tombstone["request_digest_domain"]]: + if tombstone["request_digest"] != request_digest: raise ArtifactError(CheckpointHostFailureCode.EVENT_ID_CONFLICT) result = { "result": "replay", "terminal_receipt_sequence": tombstone["terminal_receipt_sequence"], "terminal_disposition": tombstone["terminal_disposition"], } - if tombstone["request_digest_domain"] == "determa-inbox-envelope-digest-1": - result.update( - { - "event_id": event_id, - "acceptance_sequence": tombstone["acceptance_sequence"], - "request_digest_domain": tombstone["request_digest_domain"], - } - ) return result return None -def _legacy_replay( - document: Mapping[str, Any], event_id: str, request_digest: str -) -> dict[str, Any] | None: - for receipt in document["operation_receipts"]: - if receipt["operation_kind"] != "legacy_v1_operation": - continue - legacy = receipt["legacy_receipt"] - if legacy.get("operation_kind") != "delivery" or legacy.get("event_id") != event_id: - continue - if legacy["request_digest"] != request_digest: - raise ArtifactError(CheckpointHostFailureCode.EVENT_ID_CONFLICT) - return { - "result": "replay", - "event_id": event_id, - "acceptance_sequence": legacy["accepted_delivery_sequence"], - "terminal_receipt_sequence": receipt["receipt_sequence"], - "terminal_disposition": legacy["outcome"]["disposition"], - "request_digest_domain": "determa-inbox-envelope-digest-1", - } - return None - - def admit_checkpoint_v2( source: ArtifactSource, deliveries: Sequence[Mapping[str, Any]], @@ -937,19 +747,10 @@ def admit_checkpoint_v2( canonical_digests: list[str] = [] event_ids: list[str] = [] target_roots: list[Any] = [] - legacy_domains: list[bool] = [] for delivery in deliveries: - legacy_domain = ( - isinstance(delivery, Mapping) - and delivery.get("request_digest_domain") == "determa-inbox-envelope-digest-1" - ) expected_members = {"delivery_mode", "envelope", "envelope_digest"} - if legacy_domain: - expected_members.add("request_digest_domain") envelope_is_valid = ( - validate_execution_checkpoint_member("envelope", delivery.get("envelope")) - if legacy_domain and isinstance(delivery, Mapping) - else _valid_envelope_shape(delivery.get("envelope")) + _valid_envelope_shape(delivery.get("envelope")) if isinstance(delivery, Mapping) else False ) @@ -974,12 +775,7 @@ def admit_checkpoint_v2( raise ArtifactError("malformed_delivery") event_ids.append(event_id) target_roots.append(identity.get("root_instance_id")) - legacy_domains.append(legacy_domain) - digest = ( - delivery_request_digest(document["root_instance_id"], str(mode), envelope) - if legacy_domain - else _entry_digest(document["root_instance_id"], str(mode), envelope) - ) + digest = _entry_digest(document["root_instance_id"], str(mode), envelope) canonical_digests.append(digest) if any(root != document["root_instance_id"] for root in target_roots): @@ -988,30 +784,12 @@ def admit_checkpoint_v2( raise ArtifactError("duplicate_event_id_in_batch") replay_evidence: list[dict[str, Any] | None] = [] - for delivery, digest, legacy_domain in zip( - deliveries, canonical_digests, legacy_domains, strict=True - ): + for delivery, digest in zip(deliveries, canonical_digests, strict=True): envelope = delivery["envelope"] - mode = delivery["delivery_mode"] event_id = envelope["event_id"] - v1_envelope = { - key: copy.deepcopy(value) - for key, value in envelope.items() - if key not in {"cause_id", "source"} - } - replay_digests = { - "determa-inbox-envelope-digest-2": "" if legacy_domain else digest, - "determa-inbox-envelope-digest-1": delivery_request_digest( - document["root_instance_id"], str(mode), v1_envelope - ), - } replay = _pending_replay(document, event_id, digest) if replay is None: - replay = _retained_replay(document, event_id, replay_digests) - if replay is None: - replay = _legacy_replay( - document, event_id, replay_digests["determa-inbox-envelope-digest-1"] - ) + replay = _retained_replay(document, event_id, digest) replay_evidence.append(replay) if all(evidence is not None for evidence in replay_evidence): @@ -1026,8 +804,6 @@ def admit_checkpoint_v2( if len(replay_members) == 1: return dict(replay_members[0]["evidence"]) return {"result": "batch", "checkpoint": document, "members": replay_members} - if any(delivery.get("request_digest_domain") is not None for delivery in deliveries): - raise ArtifactError("malformed_delivery") if terminal_code is not None: raise ArtifactError(terminal_code) _check_cas(document, expected_revision, expected_checkpoint_digest) @@ -1038,6 +814,10 @@ def admit_checkpoint_v2( for delivery, evidence in zip(deliveries, replay_evidence, strict=True) if evidence is None ] + restored_aggregate = restore_aggregate_v2(aggregate, definition_resolver) + validation_code = _validate_new_deliveries(restored_aggregate, new_deliveries) + if validation_code is not None: + raise ArtifactError(validation_code) admission = admit_aggregate_v2(aggregate, new_deliveries, definition_resolver) if admission["result"] == "rejected": raise ArtifactError(admission["rejection"]["code"]) @@ -1100,7 +880,33 @@ def step_checkpoint_v2( selected = ( copy.deepcopy(runtime["ready_mailbox"][0]) if runtime and runtime["ready_mailbox"] else None ) - result = step_aggregate_v2(aggregate, target_runtime_id, definition_resolver) + if runtime is not None and selected is None: + root_status = next( + item["status"] + for item in aggregate["runtimes"] + if item["runtime_id"] == aggregate["root_runtime_id"] + ) + return { + "result": "not_committed", + "step_result": { + "core_step_result_format": "determa.core_step_result", + "core_step_result_schema_version": 2, + "status": root_status, + "disposition": "not_runnable", + "state": copy.deepcopy(aggregate), + "emissions": [], + "lifecycle_dispositions": [], + "fault": None, + "rejection": None, + }, + "checkpoint": document, + } + result = step_aggregate_v2( + aggregate, + target_runtime_id, + definition_resolver, + _include_host_evidence=True, + ) if selected is None or result["disposition"] in {"not_runnable", "rejected"}: return { "result": "not_committed", @@ -1133,7 +939,12 @@ def step_checkpoint_v2( references: list[dict[str, Any]] = [] for emission_index, emission in enumerate(result["emissions"]): if "kind" not in emission: - _append_external_intent(candidate, references, emission, emission_index) + action_emission_index = int( + emission.get("_determa_v2_emission_index", emission_index) + ) + _append_external_intent( + candidate, references, emission, action_emission_index + ) continue if emission["kind"] != "internal_disposed": references.append(copy.deepcopy(emission)) @@ -1194,18 +1005,97 @@ def step_checkpoint_v2( return seal_execution_checkpoint(candidate) +def process_delivery_checkpoint_v2( + source: ArtifactSource, + delivery: Mapping[str, Any], + definition_resolver: DefinitionResolver, + *, + target_validated_bundle_fingerprint: str, + migration_descriptor_digest_route: Sequence[str], + expected_revision: str, + expected_checkpoint_digest: str, +) -> dict[str, Any]: + """Migrate, admit, and process one delivery under one checkpoint revision.""" + from .queueing import _runtime_id_for_target, migrate_aggregate_v2 + + restored = restore_execution_checkpoint_v2(source, definition_resolver) + prior = restored.document + _check_cas(prior, expected_revision, expected_checkpoint_digest) + aggregate = prior["root_record"].get("aggregate_state") + if aggregate is None: + raise ArtifactError("tombstoned_root") + migration = migrate_aggregate_v2( + aggregate, + target_validated_bundle_fingerprint, + migration_descriptor_digest_route, + definition_resolver, + maintenance_mode=False, + ) + candidate = copy.deepcopy(prior) + committed_revision = str(int(prior["revision"]) + 1) + candidate["revision"] = committed_revision + candidate["root_record"]["aggregate_state"] = migration["aggregate_state"] + candidate["migration_audit_records"].extend(migration["audit_records"]) + candidate = seal_execution_checkpoint(candidate) + original_receipt_count = len(candidate["operation_receipts"]) + admitted = admit_checkpoint_v2( + candidate, + [delivery], + definition_resolver, + expected_revision=candidate["revision"], + expected_checkpoint_digest=candidate["execution_checkpoint_digest"], + ) + if admitted.get("execution_checkpoint_schema_version") != 2: + raise ArtifactError(CheckpointArtifactFailureCode.INVALID_EXECUTION_CHECKPOINT) + target_runtime_id = _runtime_id_for_target(delivery["envelope"]["target"]) + processed = step_checkpoint_v2( + admitted, + target_runtime_id, + definition_resolver, + expected_revision=admitted["revision"], + expected_checkpoint_digest=admitted["execution_checkpoint_digest"], + ) + if processed.get("execution_checkpoint_schema_version") != 2: + raise ArtifactError(CheckpointArtifactFailureCode.INVALID_EXECUTION_CHECKPOINT) + processed["revision"] = committed_revision + for receipt in processed["operation_receipts"][original_receipt_count:]: + if "accepted_revision" in receipt: + receipt["accepted_revision"] = committed_revision + if "committed_revision" in receipt: + receipt["committed_revision"] = committed_revision + for intent in processed["pending_outbox_intents"]: + if int(intent["state_revision"]) > int(prior["revision"]): + intent["state_revision"] = committed_revision + return seal_execution_checkpoint(processed) + + def prune_checkpoint_v2( source: ArtifactSource, cutoff_receipt_sequence: str, definition_resolver: DefinitionResolver, *, + target_mode: str | None = None, + policy_identifier: str | None = None, expected_revision: str, expected_checkpoint_digest: str, ) -> dict[str, Any]: """Advance bounded replay retention through one dependency-closed cutoff.""" restored = restore_execution_checkpoint_v2(source, definition_resolver) document = restored.document - if document["replay_retention"]["mode"] == "permanent": + current_mode = document["replay_retention"]["mode"] + selected_mode = current_mode if target_mode is None else target_mode + if selected_mode not in {"permanent", "bounded"}: + raise _invalid() + if current_mode == "bounded" and selected_mode != "bounded": + raise _invalid() + if selected_mode == "permanent": + raise _invalid() + selected_policy = ( + document["replay_retention"]["policy_identifier"] + if policy_identifier is None + else policy_identifier + ) + if not isinstance(selected_policy, str) or not selected_policy: raise _invalid() prior_value = document["replay_retention"]["pruned_through_receipt_sequence"] prior = decimal(prior_value) if prior_value is not None else -1 @@ -1273,21 +1163,16 @@ def prune_checkpoint_v2( acceptance = acceptance_by_event.get(receipt["event_id"]) if acceptance and acceptance["receipt_sequence"] in removed_sequences: raise _invalid() - legacy = receipt.get("legacy_receipt", {}) - origin = legacy.get("origin", {}) - if ( - origin.get("kind") == "internal_emission" - and origin.get("producing_receipt_sequence") in removed_sequences - ): - raise _invalid() - converted_origin = receipt.get("legacy_v1_delivery", {}).get("origin", {}) - if ( - converted_origin.get("kind") == "internal_emission" - and converted_origin.get("producing_receipt_sequence") in removed_sequences - ): - raise _invalid() candidate = copy.deepcopy(document) + candidate["replay_retention"] = { + "mode": "bounded", + "permanent_replay_eligible": False, + "pruned_through_receipt_sequence": candidate["replay_retention"][ + "pruned_through_receipt_sequence" + ], + "policy_identifier": selected_policy, + } tombstones = list(candidate["event_identity_tombstones"]) for receipt in removed: if receipt["operation_kind"] == "event_terminal": @@ -1301,22 +1186,18 @@ def prune_checkpoint_v2( "terminal_disposition": receipt["outcome"]["disposition"], } ) - elif ( - receipt["operation_kind"] == "legacy_v1_operation" - and receipt["legacy_receipt"].get("operation_kind") == "delivery" - ): - legacy = receipt["legacy_receipt"] - tombstones.append( - { - "event_id": legacy["event_id"], - "request_digest": legacy["request_digest"], - "request_digest_domain": "determa-inbox-envelope-digest-1", - "acceptance_sequence": legacy["accepted_delivery_sequence"], - "terminal_receipt_sequence": receipt["receipt_sequence"], - "terminal_disposition": legacy["outcome"]["disposition"], - } - ) candidate["operation_receipts"] = retained + referenced_migrations = { + sequence + for receipt in retained + if receipt["operation_kind"] == "maintenance_migration" + for sequence in receipt["migration_sequences"] + } + candidate["migration_audit_records"] = [ + audit + for audit in candidate["migration_audit_records"] + if audit["migration_sequence"] in referenced_migrations + ] candidate["event_identity_tombstones"] = sorted( tombstones, key=lambda item: int(item["terminal_receipt_sequence"]) ) diff --git a/src/determa/state/codes.py b/src/determa/state/codes.py index bc87130..0761e23 100644 --- a/src/determa/state/codes.py +++ b/src/determa/state/codes.py @@ -25,19 +25,16 @@ class CheckpointHostFailureCode(StrEnum): INJECTED_PRE_COMMIT_FAILURE = "injected_pre_commit_failure" INVALID_EXECUTION_CHECKPOINT = "invalid_execution_checkpoint" OPERATION_ID_CONFLICT = "operation_id_conflict" - PHYSICAL_DELETION_UNSUPPORTED = "physical_deletion_unsupported" RESPONSE_LOST_AFTER_COMMIT = "response_lost_after_commit" class CheckpointPreAcceptanceFailureCode(StrEnum): - CHECKPOINT_UPGRADE_REQUIRED = "checkpoint_upgrade_required" DELIVERY_DIGEST_MISMATCH = "delivery_digest_mismatch" DUPLICATE_EVENT_ID_IN_BATCH = "duplicate_event_id_in_batch" EVENT_ID_CONFLICT = "event_id_conflict" INACTIVE_COMPONENT_TARGET = "inactive_component_target" INVALID_CORRELATION = "invalid_correlation" INVALID_DELIVERY_MODE = "invalid_delivery_mode" - INVALID_DELIVERY_ORIGIN = "invalid_delivery_origin" INVALID_DELIVERY_SOURCE = "invalid_delivery_source" INVALID_EVENT = "invalid_event" INVALID_INSTANCE_TARGET = "invalid_instance_target" diff --git a/src/determa/state/data/aggregate-state-package-v2.schema.json b/src/determa/state/data/aggregate-state-package-v2.schema.json index 65ad286..6c40836 100644 --- a/src/determa/state/data/aggregate-state-package-v2.schema.json +++ b/src/determa/state/data/aggregate-state-package-v2.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://determa.dev/state/schema/aggregate-state-package-v2.schema.json", - "title": "Determa State self-contained aggregate-state package version 2", - "description": "Closed transport package for one queue-bearing aggregate and its verified attachments.", + "title": "Determa State self-contained aggregate-state package", + "description": "Closed transport package for the sole supported queue-bearing aggregate artifact and its verified attachments.", "type": "object", "required": [ "aggregate_state_package_format", diff --git a/src/determa/state/data/aggregate-state-package.schema.json b/src/determa/state/data/aggregate-state-package.schema.json deleted file mode 100644 index 75d8b6b..0000000 --- a/src/determa/state/data/aggregate-state-package.schema.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://determa.dev/state/schema/aggregate-state-package.schema.json", - "title": "Determa State self-contained aggregate-state package", - "description": "Closed transport package for one aggregate and its verified attachments.", - "type": "object", - "required": [ - "aggregate_state_package_format", - "aggregate_state_package_schema_version", - "aggregate_state", - "normalized_definitions", - "migration_descriptors", - "migration_route" - ], - "additionalProperties": false, - "properties": { - "aggregate_state_package_format": { - "const": "determa.aggregate_state_package" - }, - "aggregate_state_package_schema_version": { - "const": 1 - }, - "aggregate_state": { - "$ref": "aggregate-state.schema.json" - }, - "normalized_definitions": { - "type": "array", - "items": { - "type": "object", - "required": [ - "validated_bundle_fingerprint", - "normalized_bundle" - ], - "additionalProperties": false, - "properties": { - "validated_bundle_fingerprint": { - "type": "string", - "pattern": "^sha256:[0-9a-f]{64}$" - }, - "normalized_bundle": { - "$ref": "aggregate-state.schema.json#/$defs/typedValue" - } - } - } - }, - "migration_descriptors": { - "type": "array", - "items": { - "$ref": "migration-descriptor.schema.json" - } - }, - "migration_route": { - "type": "array", - "uniqueItems": true, - "items": { - "type": "string", - "pattern": "^sha256:[0-9a-f]{64}$" - } - } - } -} diff --git a/src/determa/state/data/aggregate-state-v2.schema.json b/src/determa/state/data/aggregate-state-v2.schema.json index 3aff5be..a82feaf 100644 --- a/src/determa/state/data/aggregate-state-v2.schema.json +++ b/src/determa/state/data/aggregate-state-v2.schema.json @@ -1,260 +1,604 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://determa.dev/state/schema/aggregate-state-v2.schema.json", - "title": "Determa State portable aggregate-state envelope version 2", - "description": "Closed wire schema for aggregate_state_schema_version 2 with runtime-local mailboxes.", + "title": "Determa State portable aggregate state", + "description": "Closed wire schema for the sole supported queue-bearing aggregate state artifact.", "$ref": "#/$defs/aggregateState", "$defs": { "nonEmptyString": { - "$ref": "aggregate-state.schema.json#/$defs/nonEmptyString" + "type": "string", + "minLength": 1 }, "identifier": { - "$ref": "aggregate-state.schema.json#/$defs/identifier" + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" }, "namespace": { - "$ref": "aggregate-state.schema.json#/$defs/namespace" + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(\\.[A-Za-z_][A-Za-z0-9_]*)*$" }, "canonicalDecimal": { - "$ref": "aggregate-state.schema.json#/$defs/canonicalDecimal" + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" }, "positiveCanonicalDecimal": { - "$ref": "aggregate-state.schema.json#/$defs/positiveCanonicalDecimal" + "type": "string", + "pattern": "^[1-9][0-9]*$" }, "signedCanonicalDecimal": { - "$ref": "aggregate-state.schema.json#/$defs/signedCanonicalDecimal" + "type": "string", + "pattern": "^(0|-?[1-9][0-9]*)$" }, "sha256": { - "$ref": "aggregate-state.schema.json#/$defs/sha256" + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" }, "jsonPointer": { - "$ref": "aggregate-state.schema.json#/$defs/jsonPointer" + "type": "string", + "pattern": "^(?:/(?:[^~/]|~[01])*)*$" }, "sourceLocator": { - "$ref": "aggregate-state.schema.json#/$defs/sourceLocator" + "type": "string", + "pattern": "^(?:(?:/(?:[^~/]|~[01])*)+|system:[A-Za-z_][A-Za-z0-9_]*)$" }, "typedValue": { - "$ref": "aggregate-state.schema.json#/$defs/typedValue" - }, - "eventName": { "oneOf": [ { - "$ref": "#/$defs/identifier" + "type": "array", + "prefixItems": [ + { + "const": "null" + } + ], + "minItems": 1, + "maxItems": 1 }, { - "enum": [ - "determa.component_completed", - "determa.component_failed", - "determa.spawned_instance_failed" - ] - } - ] - }, - "typedMap": { - "type": "array", - "prefixItems": [ + "type": "array", + "prefixItems": [ + { + "const": "boolean" + }, + { + "type": "boolean" + } + ], + "minItems": 2, + "maxItems": 2 + }, { - "const": "map" + "type": "array", + "prefixItems": [ + { + "const": "string" + }, + { + "type": "string" + } + ], + "minItems": 2, + "maxItems": 2 }, { "type": "array", - "items": { - "type": "array", - "prefixItems": [ - { - "type": "string" - }, - { + "prefixItems": [ + { + "const": "integer" + }, + { + "$ref": "#/$defs/signedCanonicalDecimal" + } + ], + "minItems": 2, + "maxItems": 2 + }, + { + "type": "array", + "prefixItems": [ + { + "const": "float" + }, + { + "type": "string", + "pattern": "^[0-9a-f]{16}$" + } + ], + "minItems": 2, + "maxItems": 2 + }, + { + "type": "array", + "prefixItems": [ + { + "const": "list" + }, + { + "type": "array", + "items": { "$ref": "#/$defs/typedValue" } - ], - "minItems": 2, - "maxItems": 2 - } + } + ], + "minItems": 2, + "maxItems": 2 + }, + { + "type": "array", + "prefixItems": [ + { + "const": "map" + }, + { + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/typedValue" + } + ], + "minItems": 2, + "maxItems": 2 + } + } + ], + "minItems": 2, + "maxItems": 2 } - ], - "minItems": 2, - "maxItems": 2 + ] }, "machineIdentity": { - "$ref": "aggregate-state.schema.json#/$defs/machineIdentity" + "type": "object", + "required": [ + "namespace", + "machine_id", + "machine_version", + "root_definition_pointer" + ], + "additionalProperties": false, + "properties": { + "namespace": { + "$ref": "#/$defs/namespace" + }, + "machine_id": { + "$ref": "#/$defs/identifier" + }, + "machine_version": { + "$ref": "#/$defs/positiveCanonicalDecimal" + }, + "root_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + } + } }, "definitionBinding": { - "$ref": "aggregate-state.schema.json#/$defs/definitionBinding" + "type": "object", + "required": [ + "validated_bundle_fingerprint", + "machine" + ], + "additionalProperties": false, + "properties": { + "validated_bundle_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "machine": { + "$ref": "#/$defs/machineIdentity" + } + } }, "rootIdentityOrigin": { - "$ref": "aggregate-state.schema.json#/$defs/rootIdentityOrigin" + "type": "object", + "required": [ + "kind", + "definition", + "root_instance_id" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "root" + }, + "definition": { + "$ref": "#/$defs/definitionBinding" + }, + "root_instance_id": { + "$ref": "#/$defs/nonEmptyString" + } + } }, "componentIdentityOrigin": { - "$ref": "aggregate-state.schema.json#/$defs/componentIdentityOrigin" + "type": "object", + "required": [ + "kind", + "definition", + "owner_runtime_id", + "component_definition_pointer", + "activation_sequence", + "declaration_index" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "component" + }, + "definition": { + "$ref": "#/$defs/definitionBinding" + }, + "owner_runtime_id": { + "$ref": "#/$defs/sha256" + }, + "component_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "activation_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "declaration_index": { + "$ref": "#/$defs/canonicalDecimal" + } + } }, "spawnedIdentityOrigin": { - "$ref": "aggregate-state.schema.json#/$defs/spawnedIdentityOrigin" + "type": "object", + "required": [ + "kind", + "definition", + "owner_runtime_id", + "spawn_action_pointer", + "spawn_sequence" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "owned_spawned_instance" + }, + "definition": { + "$ref": "#/$defs/definitionBinding" + }, + "owner_runtime_id": { + "$ref": "#/$defs/sha256" + }, + "spawn_action_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "spawn_sequence": { + "$ref": "#/$defs/canonicalDecimal" + } + } }, "identityOrigin": { - "$ref": "aggregate-state.schema.json#/$defs/identityOrigin" + "oneOf": [ + { + "$ref": "#/$defs/rootIdentityOrigin" + }, + { + "$ref": "#/$defs/componentIdentityOrigin" + }, + { + "$ref": "#/$defs/spawnedIdentityOrigin" + } + ] }, "instanceReference": { - "$ref": "aggregate-state.schema.json#/$defs/instanceReference" + "type": "object", + "required": [ + "root_instance_id", + "instance_id", + "machine_id", + "machine_version" + ], + "additionalProperties": false, + "properties": { + "root_instance_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "instance_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "machine_id": { + "$ref": "#/$defs/identifier" + }, + "machine_version": { + "$ref": "#/$defs/positiveCanonicalDecimal" + } + } }, "targetIdentity": { - "$ref": "aggregate-state.schema.json#/$defs/targetIdentity" - }, - "envelopeSource": { "oneOf": [ { "type": "object", "required": [ - "host" + "root" ], "additionalProperties": false, "properties": { - "host": { - "const": true + "root": { + "type": "object", + "required": [ + "root_instance_id", + "root_runtime_id" + ], + "additionalProperties": false, + "properties": { + "root_instance_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "root_runtime_id": { + "$ref": "#/$defs/nonEmptyString" + } + } } } }, { "type": "object", "required": [ - "runtime" + "component" ], "additionalProperties": false, "properties": { - "runtime": { - "$ref": "#/$defs/targetIdentity" + "component": { + "type": "object", + "required": [ + "root_instance_id", + "owner_runtime_id", + "component_id", + "component_runtime_id", + "activation_sequence" + ], + "additionalProperties": false, + "properties": { + "root_instance_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "owner_runtime_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "component_id": { + "$ref": "#/$defs/identifier" + }, + "component_runtime_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "activation_sequence": { + "$ref": "#/$defs/canonicalDecimal" + } + } } } }, { "type": "object", "required": [ - "system" + "spawned_instance" ], "additionalProperties": false, "properties": { - "system": { - "$ref": "#/$defs/sourceLocator" + "spawned_instance": { + "$ref": "#/$defs/instanceReference" } } + } + ] + }, + "lifetimeHolder": { + "type": "object", + "required": [ + "holder_runtime_id", + "variable_declaration_pointer", + "holder_state_activation_sequence" + ], + "additionalProperties": false, + "properties": { + "holder_runtime_id": { + "$ref": "#/$defs/sha256" }, + "variable_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "holder_state_activation_sequence": { + "$ref": "#/$defs/canonicalDecimal" + } + } + }, + "relation": { + "oneOf": [ { "type": "object", "required": [ - "legacy_v1_internal" + "kind" ], "additionalProperties": false, "properties": { - "legacy_v1_internal": { - "type": "object", - "required": [ - "producing_receipt_sequence", - "emission_index" - ], - "additionalProperties": false, - "properties": { - "producing_receipt_sequence": { - "$ref": "#/$defs/canonicalDecimal" + "kind": { + "const": "root" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "owner_runtime_id", + "component_id", + "current_component_definition_pointer", + "activation_sequence", + "declaration_index" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "component" + }, + "owner_runtime_id": { + "$ref": "#/$defs/sha256" + }, + "component_id": { + "$ref": "#/$defs/identifier" + }, + "current_component_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "activation_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "declaration_index": { + "$ref": "#/$defs/canonicalDecimal" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "owner_runtime_id", + "current_spawn_action_pointer", + "spawn_sequence", + "lifetime_holder" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "owned_spawned_instance" + }, + "owner_runtime_id": { + "$ref": "#/$defs/sha256" + }, + "current_spawn_action_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "spawn_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "lifetime_holder": { + "oneOf": [ + { + "type": "null" }, - "emission_index": { - "$ref": "#/$defs/canonicalDecimal" + { + "$ref": "#/$defs/lifetimeHolder" } - } + ] } } } ] }, - "envelope": { + "stateActivation": { "type": "object", "required": [ - "event", - "event_id", - "cause_id", - "source", - "target", - "payload" + "state_definition_pointer", + "activation_sequence" ], "additionalProperties": false, "properties": { - "event": { - "$ref": "#/$defs/eventName" - }, - "event_id": { - "$ref": "#/$defs/nonEmptyString" + "state_definition_pointer": { + "$ref": "#/$defs/jsonPointer" }, - "cause_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "source": { - "$ref": "#/$defs/envelopeSource" + "activation_sequence": { + "$ref": "#/$defs/canonicalDecimal" + } + } + }, + "variable": { + "type": "object", + "required": [ + "variable_declaration_pointer", + "declaring_state_activation_sequence", + "value" + ], + "additionalProperties": false, + "properties": { + "variable_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" }, - "target": { - "$ref": "#/$defs/targetIdentity" + "declaring_state_activation_sequence": { + "$ref": "#/$defs/canonicalDecimal" }, - "payload": { - "$ref": "#/$defs/typedMap" + "value": { + "$ref": "#/$defs/typedValue" + } + } + }, + "history": { + "type": "object", + "required": [ + "history_declaration_pointer", + "recorded_state_definition_pointers" + ], + "additionalProperties": false, + "properties": { + "history_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" }, - "correlation_id": { - "$ref": "#/$defs/nonEmptyString" + "recorded_state_definition_pointers": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/jsonPointer" + }, + "uniqueItems": true + } + ] } } }, - "mailboxEntry": { + "nextCounter": { "type": "object", "required": [ - "acceptance_sequence", - "queue_sequence", - "delivery_mode", - "envelope", - "envelope_digest", - "deferral_count" + "definition_pointer", + "next_sequence" ], "additionalProperties": false, "properties": { - "acceptance_sequence": { - "$ref": "#/$defs/canonicalDecimal" + "definition_pointer": { + "$ref": "#/$defs/jsonPointer" }, - "queue_sequence": { + "next_sequence": { "$ref": "#/$defs/canonicalDecimal" + } + } + }, + "fault": { + "type": "object", + "required": [ + "definition_fingerprint", + "runtime_id", + "cause_id", + "code", + "step_sequence", + "source_locator" + ], + "additionalProperties": false, + "properties": { + "definition_fingerprint": { + "$ref": "#/$defs/sha256" }, - "delivery_mode": { - "enum": [ - "input", - "internal" - ] + "runtime_id": { + "$ref": "#/$defs/sha256" }, - "envelope": { - "$ref": "#/$defs/envelope" + "cause_id": { + "$ref": "#/$defs/nonEmptyString" }, - "envelope_digest": { - "$ref": "#/$defs/sha256" + "code": { + "$ref": "#/$defs/identifier" }, - "deferral_count": { + "step_sequence": { "$ref": "#/$defs/canonicalDecimal" + }, + "source_locator": { + "$ref": "#/$defs/sourceLocator" } } }, - "lifetimeHolder": { - "$ref": "aggregate-state.schema.json#/$defs/lifetimeHolder" - }, - "relation": { - "$ref": "aggregate-state.schema.json#/$defs/relation" - }, - "stateActivation": { - "$ref": "aggregate-state.schema.json#/$defs/stateActivation" - }, - "variable": { - "$ref": "aggregate-state.schema.json#/$defs/variable" - }, - "history": { - "$ref": "aggregate-state.schema.json#/$defs/history" - }, - "nextCounter": { - "$ref": "aggregate-state.schema.json#/$defs/nextCounter" - }, - "fault": { - "$ref": "aggregate-state.schema.json#/$defs/fault" - }, "runtime": { "type": "object", "required": [ @@ -442,6 +786,156 @@ "$ref": "#/$defs/sha256" } } + }, + "eventName": { + "oneOf": [ + { + "$ref": "#/$defs/identifier" + }, + { + "enum": [ + "determa.component_completed", + "determa.component_failed", + "determa.spawned_instance_failed" + ] + } + ] + }, + "typedMap": { + "type": "array", + "prefixItems": [ + { + "const": "map" + }, + { + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/typedValue" + } + ], + "minItems": 2, + "maxItems": 2 + } + } + ], + "minItems": 2, + "maxItems": 2 + }, + "envelopeSource": { + "oneOf": [ + { + "type": "object", + "required": [ + "host" + ], + "additionalProperties": false, + "properties": { + "host": { + "const": true + } + } + }, + { + "type": "object", + "required": [ + "runtime" + ], + "additionalProperties": false, + "properties": { + "runtime": { + "$ref": "#/$defs/targetIdentity" + } + } + }, + { + "type": "object", + "required": [ + "system" + ], + "additionalProperties": false, + "properties": { + "system": { + "$ref": "#/$defs/sourceLocator" + } + } + } + ] + }, + "envelope": { + "type": "object", + "required": [ + "event", + "event_id", + "cause_id", + "source", + "target", + "payload" + ], + "additionalProperties": false, + "properties": { + "event": { + "$ref": "#/$defs/eventName" + }, + "event_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "cause_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "source": { + "$ref": "#/$defs/envelopeSource" + }, + "target": { + "$ref": "#/$defs/targetIdentity" + }, + "payload": { + "$ref": "#/$defs/typedMap" + }, + "correlation_id": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "mailboxEntry": { + "type": "object", + "required": [ + "acceptance_sequence", + "queue_sequence", + "delivery_mode", + "envelope", + "envelope_digest", + "deferral_count" + ], + "additionalProperties": false, + "properties": { + "acceptance_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "queue_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "delivery_mode": { + "enum": [ + "input", + "internal" + ] + }, + "envelope": { + "$ref": "#/$defs/envelope" + }, + "envelope_digest": { + "$ref": "#/$defs/sha256" + }, + "deferral_count": { + "$ref": "#/$defs/canonicalDecimal" + } + } } } } diff --git a/src/determa/state/data/aggregate-state.schema.json b/src/determa/state/data/aggregate-state.schema.json deleted file mode 100644 index db57842..0000000 --- a/src/determa/state/data/aggregate-state.schema.json +++ /dev/null @@ -1,769 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://determa.dev/state/schema/aggregate-state.schema.json", - "title": "Determa State portable aggregate-state envelope", - "description": "Closed wire schema for aggregate_state_schema_version 1.", - "$ref": "#/$defs/aggregateState", - "$defs": { - "nonEmptyString": { - "type": "string", - "minLength": 1 - }, - "identifier": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" - }, - "namespace": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*(\\.[A-Za-z_][A-Za-z0-9_]*)*$" - }, - "canonicalDecimal": { - "type": "string", - "pattern": "^(0|[1-9][0-9]*)$" - }, - "positiveCanonicalDecimal": { - "type": "string", - "pattern": "^[1-9][0-9]*$" - }, - "signedCanonicalDecimal": { - "type": "string", - "pattern": "^(0|-?[1-9][0-9]*)$" - }, - "sha256": { - "type": "string", - "pattern": "^sha256:[0-9a-f]{64}$" - }, - "jsonPointer": { - "type": "string", - "pattern": "^(?:/(?:[^~/]|~[01])*)*$" - }, - "sourceLocator": { - "type": "string", - "pattern": "^(?:(?:/(?:[^~/]|~[01])*)+|system:[A-Za-z_][A-Za-z0-9_]*)$" - }, - "typedValue": { - "oneOf": [ - { - "type": "array", - "prefixItems": [ - { - "const": "null" - } - ], - "minItems": 1, - "maxItems": 1 - }, - { - "type": "array", - "prefixItems": [ - { - "const": "boolean" - }, - { - "type": "boolean" - } - ], - "minItems": 2, - "maxItems": 2 - }, - { - "type": "array", - "prefixItems": [ - { - "const": "string" - }, - { - "type": "string" - } - ], - "minItems": 2, - "maxItems": 2 - }, - { - "type": "array", - "prefixItems": [ - { - "const": "integer" - }, - { - "$ref": "#/$defs/signedCanonicalDecimal" - } - ], - "minItems": 2, - "maxItems": 2 - }, - { - "type": "array", - "prefixItems": [ - { - "const": "float" - }, - { - "type": "string", - "pattern": "^[0-9a-f]{16}$" - } - ], - "minItems": 2, - "maxItems": 2 - }, - { - "type": "array", - "prefixItems": [ - { - "const": "list" - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/typedValue" - } - } - ], - "minItems": 2, - "maxItems": 2 - }, - { - "type": "array", - "prefixItems": [ - { - "const": "map" - }, - { - "type": "array", - "items": { - "type": "array", - "prefixItems": [ - { - "type": "string" - }, - { - "$ref": "#/$defs/typedValue" - } - ], - "minItems": 2, - "maxItems": 2 - } - } - ], - "minItems": 2, - "maxItems": 2 - } - ] - }, - "machineIdentity": { - "type": "object", - "required": [ - "namespace", - "machine_id", - "machine_version", - "root_definition_pointer" - ], - "additionalProperties": false, - "properties": { - "namespace": { - "$ref": "#/$defs/namespace" - }, - "machine_id": { - "$ref": "#/$defs/identifier" - }, - "machine_version": { - "$ref": "#/$defs/positiveCanonicalDecimal" - }, - "root_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - } - } - }, - "definitionBinding": { - "type": "object", - "required": [ - "validated_bundle_fingerprint", - "machine" - ], - "additionalProperties": false, - "properties": { - "validated_bundle_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "machine": { - "$ref": "#/$defs/machineIdentity" - } - } - }, - "rootIdentityOrigin": { - "type": "object", - "required": [ - "kind", - "definition", - "root_instance_id" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "root" - }, - "definition": { - "$ref": "#/$defs/definitionBinding" - }, - "root_instance_id": { - "$ref": "#/$defs/nonEmptyString" - } - } - }, - "componentIdentityOrigin": { - "type": "object", - "required": [ - "kind", - "definition", - "owner_runtime_id", - "component_definition_pointer", - "activation_sequence", - "declaration_index" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "component" - }, - "definition": { - "$ref": "#/$defs/definitionBinding" - }, - "owner_runtime_id": { - "$ref": "#/$defs/sha256" - }, - "component_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "activation_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "declaration_index": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "spawnedIdentityOrigin": { - "type": "object", - "required": [ - "kind", - "definition", - "owner_runtime_id", - "spawn_action_pointer", - "spawn_sequence" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "owned_spawned_instance" - }, - "definition": { - "$ref": "#/$defs/definitionBinding" - }, - "owner_runtime_id": { - "$ref": "#/$defs/sha256" - }, - "spawn_action_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "spawn_sequence": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "identityOrigin": { - "oneOf": [ - { - "$ref": "#/$defs/rootIdentityOrigin" - }, - { - "$ref": "#/$defs/componentIdentityOrigin" - }, - { - "$ref": "#/$defs/spawnedIdentityOrigin" - } - ] - }, - "instanceReference": { - "type": "object", - "required": [ - "root_instance_id", - "instance_id", - "machine_id", - "machine_version" - ], - "additionalProperties": false, - "properties": { - "root_instance_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "instance_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "machine_id": { - "$ref": "#/$defs/identifier" - }, - "machine_version": { - "$ref": "#/$defs/positiveCanonicalDecimal" - } - } - }, - "targetIdentity": { - "oneOf": [ - { - "type": "object", - "required": [ - "root" - ], - "additionalProperties": false, - "properties": { - "root": { - "type": "object", - "required": [ - "root_instance_id", - "root_runtime_id" - ], - "additionalProperties": false, - "properties": { - "root_instance_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "root_runtime_id": { - "$ref": "#/$defs/nonEmptyString" - } - } - } - } - }, - { - "type": "object", - "required": [ - "component" - ], - "additionalProperties": false, - "properties": { - "component": { - "type": "object", - "required": [ - "root_instance_id", - "owner_runtime_id", - "component_id", - "component_runtime_id", - "activation_sequence" - ], - "additionalProperties": false, - "properties": { - "root_instance_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "owner_runtime_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "component_id": { - "$ref": "#/$defs/identifier" - }, - "component_runtime_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "activation_sequence": { - "$ref": "#/$defs/canonicalDecimal" - } - } - } - } - }, - { - "type": "object", - "required": [ - "spawned_instance" - ], - "additionalProperties": false, - "properties": { - "spawned_instance": { - "$ref": "#/$defs/instanceReference" - } - } - } - ] - }, - "lifetimeHolder": { - "type": "object", - "required": [ - "holder_runtime_id", - "variable_declaration_pointer", - "holder_state_activation_sequence" - ], - "additionalProperties": false, - "properties": { - "holder_runtime_id": { - "$ref": "#/$defs/sha256" - }, - "variable_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "holder_state_activation_sequence": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "relation": { - "oneOf": [ - { - "type": "object", - "required": [ - "kind" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "root" - } - } - }, - { - "type": "object", - "required": [ - "kind", - "owner_runtime_id", - "component_id", - "current_component_definition_pointer", - "activation_sequence", - "declaration_index" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "component" - }, - "owner_runtime_id": { - "$ref": "#/$defs/sha256" - }, - "component_id": { - "$ref": "#/$defs/identifier" - }, - "current_component_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "activation_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "declaration_index": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - { - "type": "object", - "required": [ - "kind", - "owner_runtime_id", - "current_spawn_action_pointer", - "spawn_sequence", - "lifetime_holder" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "owned_spawned_instance" - }, - "owner_runtime_id": { - "$ref": "#/$defs/sha256" - }, - "current_spawn_action_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "spawn_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "lifetime_holder": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/$defs/lifetimeHolder" - } - ] - } - } - } - ] - }, - "stateActivation": { - "type": "object", - "required": [ - "state_definition_pointer", - "activation_sequence" - ], - "additionalProperties": false, - "properties": { - "state_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "activation_sequence": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "variable": { - "type": "object", - "required": [ - "variable_declaration_pointer", - "declaring_state_activation_sequence", - "value" - ], - "additionalProperties": false, - "properties": { - "variable_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "declaring_state_activation_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "value": { - "$ref": "#/$defs/typedValue" - } - } - }, - "history": { - "type": "object", - "required": [ - "history_declaration_pointer", - "recorded_state_definition_pointers" - ], - "additionalProperties": false, - "properties": { - "history_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "recorded_state_definition_pointers": { - "oneOf": [ - { - "type": "null" - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/jsonPointer" - }, - "uniqueItems": true - } - ] - } - } - }, - "nextCounter": { - "type": "object", - "required": [ - "definition_pointer", - "next_sequence" - ], - "additionalProperties": false, - "properties": { - "definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "next_sequence": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "fault": { - "type": "object", - "required": [ - "definition_fingerprint", - "runtime_id", - "cause_id", - "code", - "step_sequence", - "source_locator" - ], - "additionalProperties": false, - "properties": { - "definition_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "runtime_id": { - "$ref": "#/$defs/sha256" - }, - "cause_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "code": { - "$ref": "#/$defs/identifier" - }, - "step_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "source_locator": { - "$ref": "#/$defs/sourceLocator" - } - } - }, - "runtime": { - "type": "object", - "required": [ - "runtime_id", - "identity_origin", - "target_identity", - "current_definition", - "relation", - "status", - "active_leaf_state_definition_pointers", - "active_state_activations", - "variables", - "history", - "next_spawn_sequence", - "next_state_activation_sequences", - "next_component_activation_sequences", - "fault" - ], - "additionalProperties": false, - "properties": { - "runtime_id": { - "$ref": "#/$defs/sha256" - }, - "identity_origin": { - "$ref": "#/$defs/identityOrigin" - }, - "target_identity": { - "$ref": "#/$defs/targetIdentity" - }, - "current_definition": { - "$ref": "#/$defs/definitionBinding" - }, - "relation": { - "$ref": "#/$defs/relation" - }, - "status": { - "enum": [ - "running", - "completed", - "faulted" - ] - }, - "active_leaf_state_definition_pointers": { - "type": "array", - "items": { - "$ref": "#/$defs/jsonPointer" - }, - "uniqueItems": true - }, - "active_state_activations": { - "type": "array", - "items": { - "$ref": "#/$defs/stateActivation" - } - }, - "variables": { - "type": "array", - "items": { - "$ref": "#/$defs/variable" - } - }, - "history": { - "type": "array", - "items": { - "$ref": "#/$defs/history" - } - }, - "next_spawn_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "next_state_activation_sequences": { - "type": "array", - "items": { - "$ref": "#/$defs/nextCounter" - } - }, - "next_component_activation_sequences": { - "type": "array", - "items": { - "$ref": "#/$defs/nextCounter" - } - }, - "fault": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/$defs/fault" - } - ] - } - } - }, - "aggregateState": { - "type": "object", - "required": [ - "aggregate_state_format", - "aggregate_state_schema_version", - "machine_format", - "validated_bundle_fingerprint", - "namespace", - "root_machine_id", - "root_machine_version", - "root_instance_id", - "creation_id", - "root_runtime_id", - "migration_sequence", - "next_logical_step_sequence", - "next_output_sequence", - "runtimes", - "aggregate_state_digest" - ], - "additionalProperties": false, - "properties": { - "aggregate_state_format": { - "const": "determa.aggregate_state" - }, - "aggregate_state_schema_version": { - "const": 1 - }, - "machine_format": { - "const": 1 - }, - "validated_bundle_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "namespace": { - "$ref": "#/$defs/namespace" - }, - "root_machine_id": { - "$ref": "#/$defs/identifier" - }, - "root_machine_version": { - "$ref": "#/$defs/positiveCanonicalDecimal" - }, - "root_instance_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "creation_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "root_runtime_id": { - "$ref": "#/$defs/sha256" - }, - "migration_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "next_logical_step_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "next_output_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "runtimes": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/$defs/runtime" - } - }, - "aggregate_state_digest": { - "$ref": "#/$defs/sha256" - } - } - } - } -} diff --git a/src/determa/state/data/core-step-result-v2.schema.json b/src/determa/state/data/core-step-result-v2.schema.json index 24309ae..10f7b6b 100644 --- a/src/determa/state/data/core-step-result-v2.schema.json +++ b/src/determa/state/data/core-step-result-v2.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://determa.dev/state/schema/core-step-result-v2.schema.json", - "title": "Determa State queue-bearing core step result version 2", - "description": "Closed result for one explicitly targeted aggregate-state version-2 mailbox step.", + "title": "Determa State queue-bearing core step result", + "description": "Closed result for one explicitly targeted portable aggregate-state mailbox step.", "$ref": "#/$defs/coreStepResult", "$defs": { "canonicalDecimal": { @@ -28,11 +28,21 @@ ], "additionalProperties": false, "properties": { - "kind": { "const": "internal_mailbox" }, - "emission_index": { "$ref": "#/$defs/canonicalDecimal" }, - "event_id": { "$ref": "#/$defs/nonEmptyString" }, - "acceptance_sequence": { "$ref": "#/$defs/canonicalDecimal" }, - "queue_sequence": { "$ref": "#/$defs/canonicalDecimal" } + "kind": { + "const": "internal_mailbox" + }, + "emission_index": { + "$ref": "#/$defs/canonicalDecimal" + }, + "event_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "acceptance_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "queue_sequence": { + "$ref": "#/$defs/canonicalDecimal" + } } }, "internalDisposedEmission": { @@ -46,18 +56,34 @@ ], "additionalProperties": false, "properties": { - "kind": { "const": "internal_disposed" }, - "emission_index": { "$ref": "#/$defs/canonicalDecimal" }, - "event_id": { "$ref": "#/$defs/nonEmptyString" }, - "acceptance_sequence": { "$ref": "#/$defs/canonicalDecimal" }, - "lifecycle_disposition_index": { "$ref": "#/$defs/canonicalDecimal" } + "kind": { + "const": "internal_disposed" + }, + "emission_index": { + "$ref": "#/$defs/canonicalDecimal" + }, + "event_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "acceptance_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "lifecycle_disposition_index": { + "$ref": "#/$defs/canonicalDecimal" + } } }, "emission": { "oneOf": [ - { "$ref": "#/$defs/internalMailboxEmission" }, - { "$ref": "#/$defs/internalDisposedEmission" }, - { "$ref": "execution-checkpoint.schema.json#/$defs/outboxIntent" } + { + "$ref": "#/$defs/internalMailboxEmission" + }, + { + "$ref": "#/$defs/internalDisposedEmission" + }, + { + "$ref": "execution-checkpoint-v2.schema.json#/$defs/outboxIntent" + } ] }, "lifecycleDisposition": { @@ -72,11 +98,21 @@ ], "additionalProperties": false, "properties": { - "event_id": { "$ref": "#/$defs/nonEmptyString" }, - "request_digest": { "$ref": "#/$defs/sha256" }, - "acceptance_sequence": { "$ref": "#/$defs/canonicalDecimal" }, - "final_queue_sequence": { "$ref": "#/$defs/canonicalDecimal" }, - "target_runtime_id": { "$ref": "#/$defs/sha256" }, + "event_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "request_digest": { + "$ref": "#/$defs/sha256" + }, + "acceptance_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "final_queue_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "target_runtime_id": { + "$ref": "#/$defs/sha256" + }, "reason": { "enum": [ "runtime_cancelled", @@ -88,7 +124,9 @@ }, "rejection": { "type": "object", - "required": ["code"], + "required": [ + "code" + ], "additionalProperties": false, "properties": { "code": { @@ -123,7 +161,11 @@ "const": 2 }, "status": { - "enum": ["running", "completed", "faulted"] + "enum": [ + "running", + "completed", + "faulted" + ] }, "disposition": { "enum": [ @@ -140,22 +182,34 @@ }, "emissions": { "type": "array", - "items": { "$ref": "#/$defs/emission" } + "items": { + "$ref": "#/$defs/emission" + } }, "lifecycle_dispositions": { "type": "array", - "items": { "$ref": "#/$defs/lifecycleDisposition" } + "items": { + "$ref": "#/$defs/lifecycleDisposition" + } }, "fault": { "oneOf": [ - { "type": "null" }, - { "$ref": "aggregate-state-v2.schema.json#/$defs/fault" } + { + "type": "null" + }, + { + "$ref": "aggregate-state-v2.schema.json#/$defs/fault" + } ] }, "rejection": { "oneOf": [ - { "type": "null" }, - { "$ref": "#/$defs/rejection" } + { + "type": "null" + }, + { + "$ref": "#/$defs/rejection" + } ] } }, @@ -163,21 +217,35 @@ { "if": { "properties": { - "disposition": { "const": "rejected" } + "disposition": { + "const": "rejected" + } }, - "required": ["disposition"] + "required": [ + "disposition" + ] }, "then": { "properties": { - "emissions": { "maxItems": 0 }, - "lifecycle_dispositions": { "maxItems": 0 }, - "fault": { "type": "null" }, - "rejection": { "$ref": "#/$defs/rejection" } + "emissions": { + "maxItems": 0 + }, + "lifecycle_dispositions": { + "maxItems": 0 + }, + "fault": { + "type": "null" + }, + "rejection": { + "$ref": "#/$defs/rejection" + } } }, "else": { "properties": { - "rejection": { "type": "null" } + "rejection": { + "type": "null" + } } } } diff --git a/src/determa/state/data/execution-checkpoint-v2.schema.json b/src/determa/state/data/execution-checkpoint-v2.schema.json index d6726a0..d7e881b 100644 --- a/src/determa/state/data/execution-checkpoint-v2.schema.json +++ b/src/determa/state/data/execution-checkpoint-v2.schema.json @@ -1,10 +1,290 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://determa.dev/state/schema/execution-checkpoint-v2.schema.json", - "title": "Determa State portable execution checkpoint version 2", - "description": "Closed durable-host checkpoint whose aggregate owns accepted ready and deferred envelopes.", + "title": "Determa State portable execution checkpoint", + "description": "Closed durable-host checkpoint for the sole supported queue-bearing aggregate artifact.", "$ref": "#/$defs/executionCheckpoint", "$defs": { + "nonEmptyString": { + "type": "string", + "minLength": 1 + }, + "canonicalDecimal": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "positiveCanonicalDecimal": { + "type": "string", + "pattern": "^[1-9][0-9]*$" + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "externalOutboxEmissionReference": { + "type": "object", + "required": [ + "kind", + "emission_index", + "effect_id" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "external_outbox" + }, + "emission_index": { + "$ref": "#/$defs/canonicalDecimal", + "description": "The referenced emission's action-local, zero-based emission_index from SPEC section 9; not a transaction-wide or receipt-array index. Array position preserves cross-action order and effect_id disambiguates the referenced emission." + }, + "effect_id": { + "$ref": "#/$defs/sha256" + } + } + }, + "handledOutcome": { + "type": "object", + "required": [ + "status", + "disposition", + "fault", + "rejection" + ], + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "running", + "completed" + ] + }, + "disposition": { + "const": "handled" + }, + "fault": { + "type": "null" + }, + "rejection": { + "type": "null" + } + } + }, + "unhandledOutcome": { + "type": "object", + "required": [ + "status", + "disposition", + "fault", + "rejection" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "running" + }, + "disposition": { + "const": "unhandled" + }, + "fault": { + "type": "null" + }, + "rejection": { + "type": "null" + } + } + }, + "faultedOutcome": { + "type": "object", + "required": [ + "status", + "disposition", + "fault", + "rejection" + ], + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "running", + "faulted" + ] + }, + "disposition": { + "const": "faulted" + }, + "fault": { + "$ref": "aggregate-state-v2.schema.json#/$defs/fault" + }, + "rejection": { + "type": "null" + } + } + }, + "pendingOutboxIntent": { + "type": "object", + "required": [ + "intent", + "state_revision", + "delivery_state" + ], + "additionalProperties": false, + "properties": { + "intent": { + "$ref": "#/$defs/outboxIntent" + }, + "state_revision": { + "$ref": "#/$defs/canonicalDecimal" + }, + "delivery_state": { + "$ref": "#/$defs/pendingOutboxState" + } + } + }, + "terminalOutboxRecord": { + "type": "object", + "required": [ + "terminal_sequence", + "intent", + "committed_revision", + "outcome" + ], + "additionalProperties": false, + "properties": { + "terminal_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "intent": { + "$ref": "#/$defs/outboxIntent" + }, + "committed_revision": { + "$ref": "#/$defs/canonicalDecimal" + }, + "outcome": { + "$ref": "#/$defs/terminalOutboxOutcome" + } + } + }, + "outboxEffectTombstone": { + "type": "object", + "required": [ + "terminal_sequence", + "effect_id", + "intent_digest", + "committed_revision", + "outcome" + ], + "additionalProperties": false, + "properties": { + "terminal_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "effect_id": { + "$ref": "#/$defs/sha256" + }, + "intent_digest": { + "$ref": "#/$defs/sha256" + }, + "committed_revision": { + "$ref": "#/$defs/canonicalDecimal" + }, + "outcome": { + "$ref": "#/$defs/terminalOutboxOutcome" + } + } + }, + "migrationAuditRecord": { + "type": "object", + "required": [ + "migration_audit_record_schema_version", + "root_instance_id", + "root_runtime_id", + "migration_sequence", + "source_validated_bundle_fingerprint", + "target_validated_bundle_fingerprint", + "migration_descriptor_digest", + "source_aggregate_state_digest", + "target_aggregate_state_digest", + "result_code" + ], + "additionalProperties": false, + "properties": { + "migration_audit_record_schema_version": { + "const": 2 + }, + "root_instance_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "root_runtime_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "migration_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "source_validated_bundle_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "target_validated_bundle_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "migration_descriptor_digest": { + "$ref": "#/$defs/sha256" + }, + "source_aggregate_state_digest": { + "$ref": "#/$defs/sha256" + }, + "target_aggregate_state_digest": { + "$ref": "#/$defs/sha256" + }, + "result_code": { + "const": "migration_applied" + } + } + }, + "rootTombstone": { + "type": "object", + "required": [ + "status", + "root_runtime_id", + "creation_id", + "terminal_status", + "final_aggregate_state_digest", + "tombstone_operation_id" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "tombstone" + }, + "root_runtime_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "creation_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "terminal_status": { + "enum": [ + "completed", + "faulted" + ] + }, + "final_aggregate_state_digest": { + "$ref": "#/$defs/sha256" + }, + "tombstone_operation_id": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "replayRetention": { + "oneOf": [ + { + "$ref": "#/$defs/permanentReplayRetention" + }, + { + "$ref": "#/$defs/boundedReplayRetention" + } + ] + }, "internalMailboxEmissionReference": { "type": "object", "required": [ @@ -20,16 +300,17 @@ "const": "internal_mailbox" }, "emission_index": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal", + "description": "Copies the referenced internal emission's zero-based emission_ordinal from SPEC section 9: action-local for an author emission or lifecycle-operation-local for an engine lifecycle emission. It is not a transaction-wide or receipt-array index. Array position preserves cross-action and cross-lifecycle-operation order, and event_id disambiguates the referenced emission." }, "event_id": { - "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + "$ref": "#/$defs/nonEmptyString" }, "acceptance_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "queue_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" } } }, @@ -48,16 +329,17 @@ "const": "internal_terminal" }, "emission_index": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal", + "description": "Copies the referenced internal emission's zero-based emission_ordinal from SPEC section 9: action-local for an author emission or lifecycle-operation-local for an engine lifecycle emission. It is not a transaction-wide or receipt-array index. Array position preserves cross-action and cross-lifecycle-operation order, and event_id disambiguates the referenced emission." }, "event_id": { - "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + "$ref": "#/$defs/nonEmptyString" }, "acceptance_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "terminal_receipt_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" } } }, @@ -70,7 +352,7 @@ "$ref": "#/$defs/internalTerminalEmissionReference" }, { - "$ref": "execution-checkpoint.schema.json#/$defs/externalOutboxEmissionReference" + "$ref": "#/$defs/externalOutboxEmissionReference" } ] }, @@ -96,16 +378,16 @@ "const": "0" }, "creation_id": { - "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + "$ref": "#/$defs/nonEmptyString" }, "request_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + "$ref": "#/$defs/sha256" }, "committed_revision": { "const": "0" }, "resulting_aggregate_state_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + "$ref": "#/$defs/sha256" }, "status": { "enum": [ @@ -177,67 +459,22 @@ "const": "acceptance" }, "receipt_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "event_id": { - "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + "$ref": "#/$defs/nonEmptyString" }, "request_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + "$ref": "#/$defs/sha256" }, "acceptance_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "accepted_revision": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "delivery_mode": { - "enum": [ - "input", - "internal" - ] - }, - "legacy_v1_delivery": { - "$ref": "#/$defs/legacyV1DeliveryEvidence" - } - }, - "allOf": [ - { - "if": { - "properties": { - "delivery_mode": { - "const": "internal" - } - }, - "required": [ - "delivery_mode" - ] - }, - "then": { - "required": [ - "legacy_v1_delivery" - ] - } - } - ] - }, - "legacyV1DeliveryEvidence": { - "type": "object", - "required": [ - "delivery_sequence", - "envelope_digest", - "origin" - ], - "additionalProperties": false, - "properties": { - "delivery_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" - }, - "envelope_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" - }, - "origin": { - "$ref": "execution-checkpoint.schema.json#/$defs/deliveryOrigin" + "const": "input" } } }, @@ -291,16 +528,20 @@ "additionalProperties": false, "properties": { "status": { - "enum": ["running", "completed", "faulted"] + "enum": [ + "running", + "completed", + "faulted" + ] }, "disposition": { "const": "migration_disposed" }, "reason": { - "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + "$ref": "#/$defs/nonEmptyString" }, "migration_descriptor_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + "$ref": "#/$defs/sha256" }, "fault": { "type": "null" @@ -313,13 +554,13 @@ "terminalEventOutcome": { "oneOf": [ { - "$ref": "execution-checkpoint.schema.json#/$defs/handledOutcome" + "$ref": "#/$defs/handledOutcome" }, { - "$ref": "execution-checkpoint.schema.json#/$defs/unhandledOutcome" + "$ref": "#/$defs/unhandledOutcome" }, { - "$ref": "execution-checkpoint.schema.json#/$defs/faultedOutcome" + "$ref": "#/$defs/faultedOutcome" }, { "$ref": "#/$defs/disposedOutcome" @@ -329,45 +570,6 @@ } ] }, - "legacyV1CreationReceipt": { - "type": "object", - "required": ["operation_kind", "receipt_sequence", "legacy_receipt"], - "additionalProperties": false, - "properties": { - "operation_kind": { - "const": "legacy_v1_creation" - }, - "receipt_sequence": { - "const": "0" - }, - "legacy_receipt": { - "$ref": "execution-checkpoint.schema.json#/$defs/creationReceipt" - } - } - }, - "legacyV1OperationReceipt": { - "type": "object", - "required": ["operation_kind", "receipt_sequence", "legacy_receipt"], - "additionalProperties": false, - "properties": { - "operation_kind": { - "const": "legacy_v1_operation" - }, - "receipt_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" - }, - "legacy_receipt": { - "oneOf": [ - { - "$ref": "execution-checkpoint.schema.json#/$defs/deliveryReceipt" - }, - { - "$ref": "execution-checkpoint.schema.json#/$defs/maintenanceMigrationReceipt" - } - ] - } - } - }, "terminalEventReceipt": { "type": "object", "required": [ @@ -388,25 +590,25 @@ "const": "event_terminal" }, "receipt_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "event_id": { - "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + "$ref": "#/$defs/nonEmptyString" }, "request_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + "$ref": "#/$defs/sha256" }, "acceptance_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "final_queue_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "committed_revision": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "resulting_aggregate_state_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + "$ref": "#/$defs/sha256" }, "outcome": { "$ref": "#/$defs/terminalEventOutcome" @@ -419,6 +621,89 @@ } } }, + "maintenanceMigrationReceipt": { + "type": "object", + "required": [ + "operation_kind", + "receipt_sequence", + "operation_id", + "request_digest", + "committed_revision", + "source_aggregate_state_digest", + "target_validated_bundle_fingerprint", + "resulting_aggregate_state_digest", + "migration_sequences", + "result_code" + ], + "additionalProperties": false, + "properties": { + "operation_kind": { + "const": "maintenance_migration" + }, + "receipt_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "operation_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "request_digest": { + "$ref": "#/$defs/sha256" + }, + "committed_revision": { + "$ref": "#/$defs/canonicalDecimal" + }, + "source_aggregate_state_digest": { + "$ref": "#/$defs/sha256" + }, + "target_validated_bundle_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "resulting_aggregate_state_digest": { + "$ref": "#/$defs/sha256" + }, + "migration_sequences": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/$defs/positiveCanonicalDecimal" + } + }, + "result_code": { + "enum": [ + "migration_applied", + "migration_no_operation" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "result_code": { + "const": "migration_applied" + } + }, + "required": [ + "result_code" + ] + }, + "then": { + "properties": { + "migration_sequences": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "migration_sequences": { + "maxItems": 0 + } + } + } + } + ] + }, "operationReceipt": { "oneOf": [ { @@ -431,10 +716,7 @@ "$ref": "#/$defs/terminalEventReceipt" }, { - "$ref": "#/$defs/legacyV1CreationReceipt" - }, - { - "$ref": "#/$defs/legacyV1OperationReceipt" + "$ref": "#/$defs/maintenanceMigrationReceipt" } ] }, @@ -451,55 +733,30 @@ "additionalProperties": false, "properties": { "event_id": { - "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + "$ref": "#/$defs/nonEmptyString" }, "request_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + "$ref": "#/$defs/sha256" }, "request_digest_domain": { - "enum": [ - "determa-inbox-envelope-digest-1", - "determa-inbox-envelope-digest-2" - ] + "const": "determa-inbox-envelope-digest-2" }, "acceptance_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "terminal_receipt_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "terminal_disposition": { "enum": [ "handled", "unhandled", - "rejected", "faulted", "disposed", "migration_disposed" ] } - }, - "allOf": [ - { - "if": { - "properties": { - "terminal_disposition": { - "const": "rejected" - } - }, - "required": [ - "terminal_disposition" - ] - }, - "then": { - "properties": { - "request_digest_domain": { - "const": "determa-inbox-envelope-digest-1" - } - } - } - } - ] + } }, "retainedRootRecord": { "type": "object", @@ -523,7 +780,7 @@ "$ref": "#/$defs/retainedRootRecord" }, { - "$ref": "execution-checkpoint.schema.json#/$defs/rootTombstone" + "$ref": "#/$defs/rootTombstone" } ] }, @@ -555,33 +812,26 @@ "const": 2 }, "root_instance_id": { - "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + "$ref": "#/$defs/nonEmptyString" }, "revision": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "root_record": { "$ref": "#/$defs/rootRecord" }, "replay_retention": { - "$ref": "execution-checkpoint.schema.json#/$defs/replayRetention" + "$ref": "#/$defs/replayRetention" }, "next_operation_receipt_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "operation_receipts": { "type": "array", "minItems": 1, "prefixItems": [ { - "oneOf": [ - { - "$ref": "#/$defs/creationReceipt" - }, - { - "$ref": "#/$defs/legacyV1CreationReceipt" - } - ] + "$ref": "#/$defs/creationReceipt" } ], "items": { @@ -597,32 +847,32 @@ "pending_outbox_intents": { "type": "array", "items": { - "$ref": "execution-checkpoint.schema.json#/$defs/pendingOutboxIntent" + "$ref": "#/$defs/pendingOutboxIntent" } }, "next_outbox_terminal_sequence": { - "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + "$ref": "#/$defs/canonicalDecimal" }, "terminal_outbox_records": { "type": "array", "items": { - "$ref": "execution-checkpoint.schema.json#/$defs/terminalOutboxRecord" + "$ref": "#/$defs/terminalOutboxRecord" } }, "outbox_effect_tombstones": { "type": "array", "items": { - "$ref": "execution-checkpoint.schema.json#/$defs/outboxEffectTombstone" + "$ref": "#/$defs/outboxEffectTombstone" } }, "migration_audit_records": { "type": "array", "items": { - "$ref": "execution-checkpoint.schema.json#/$defs/migrationAuditRecord" + "$ref": "#/$defs/migrationAuditRecord" } }, "execution_checkpoint_digest": { - "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + "$ref": "#/$defs/sha256" } }, "allOf": [ @@ -653,6 +903,219 @@ } } ] + }, + "pendingOutboxState": { + "oneOf": [ + { + "$ref": "#/$defs/notAttemptedOutboxState" + }, + { + "$ref": "#/$defs/retryableFailureOutboxState" + }, + { + "$ref": "#/$defs/ambiguousOutboxState" + } + ] + }, + "outboxIntent": { + "type": "object", + "required": [ + "effect_id", + "sequence", + "event", + "payload", + "correlation_id" + ], + "additionalProperties": false, + "properties": { + "effect_id": { + "$ref": "#/$defs/sha256" + }, + "sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "event": { + "$ref": "#/$defs/identifier" + }, + "payload": { + "$ref": "#/$defs/typedMap" + }, + "correlation_id": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "terminalOutboxOutcome": { + "oneOf": [ + { + "$ref": "#/$defs/confirmedOutboxOutcome" + }, + { + "$ref": "#/$defs/reasonedTerminalOutboxOutcome" + } + ] + }, + "boundedReplayRetention": { + "type": "object", + "required": [ + "mode", + "permanent_replay_eligible", + "pruned_through_receipt_sequence", + "policy_identifier" + ], + "additionalProperties": false, + "properties": { + "mode": { + "const": "bounded" + }, + "permanent_replay_eligible": { + "const": false + }, + "pruned_through_receipt_sequence": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/positiveCanonicalDecimal" + } + ] + }, + "policy_identifier": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "permanentReplayRetention": { + "type": "object", + "required": [ + "mode", + "permanent_replay_eligible", + "pruned_through_receipt_sequence", + "policy_identifier" + ], + "additionalProperties": false, + "properties": { + "mode": { + "const": "permanent" + }, + "permanent_replay_eligible": { + "const": true + }, + "pruned_through_receipt_sequence": { + "type": "null" + }, + "policy_identifier": { + "type": "null" + } + } + }, + "ambiguousOutboxState": { + "type": "object", + "required": [ + "status", + "reason_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "ambiguous" + }, + "reason_code": { + "$ref": "#/$defs/identifier" + } + } + }, + "confirmedOutboxOutcome": { + "type": "object", + "required": [ + "status" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "confirmed" + } + } + }, + "notAttemptedOutboxState": { + "type": "object", + "required": [ + "status" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "not_attempted" + } + } + }, + "reasonedTerminalOutboxOutcome": { + "type": "object", + "required": [ + "status", + "reason_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "permanently_rejected", + "operator_cancelled", + "discarded", + "dead_lettered" + ] + }, + "reason_code": { + "$ref": "#/$defs/identifier" + } + } + }, + "identifier": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "typedMap": { + "type": "array", + "prefixItems": [ + { + "const": "map" + }, + { + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "aggregate-state-v2.schema.json#/$defs/typedValue" + } + ], + "minItems": 2, + "maxItems": 2 + } + } + ], + "minItems": 2, + "maxItems": 2 + }, + "retryableFailureOutboxState": { + "type": "object", + "required": [ + "status", + "reason_code" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "retryable_failure" + }, + "reason_code": { + "$ref": "#/$defs/identifier" + } + } } } } diff --git a/src/determa/state/data/execution-checkpoint.schema.json b/src/determa/state/data/execution-checkpoint.schema.json deleted file mode 100644 index e30ad6d..0000000 --- a/src/determa/state/data/execution-checkpoint.schema.json +++ /dev/null @@ -1,1330 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://determa.dev/state/schema/execution-checkpoint.schema.json", - "title": "Determa State portable execution checkpoint", - "description": "Closed durable-host checkpoint for one root ownership aggregate.", - "$ref": "#/$defs/executionCheckpoint", - "$defs": { - "nonEmptyString": { - "type": "string", - "minLength": 1 - }, - "identifier": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" - }, - "eventName": { - "oneOf": [ - { - "$ref": "#/$defs/identifier" - }, - { - "enum": [ - "determa.component_completed", - "determa.component_failed", - "determa.spawned_instance_failed" - ] - } - ] - }, - "canonicalDecimal": { - "type": "string", - "pattern": "^(0|[1-9][0-9]*)$" - }, - "positiveCanonicalDecimal": { - "type": "string", - "pattern": "^[1-9][0-9]*$" - }, - "sha256": { - "type": "string", - "pattern": "^sha256:[0-9a-f]{64}$" - }, - "typedMap": { - "type": "array", - "prefixItems": [ - { - "const": "map" - }, - { - "type": "array", - "items": { - "type": "array", - "prefixItems": [ - { - "type": "string" - }, - { - "$ref": "aggregate-state.schema.json#/$defs/typedValue" - } - ], - "minItems": 2, - "maxItems": 2 - } - } - ], - "minItems": 2, - "maxItems": 2 - }, - "envelope": { - "type": "object", - "required": [ - "event", - "event_id", - "target", - "payload" - ], - "additionalProperties": false, - "properties": { - "event": { - "$ref": "#/$defs/eventName" - }, - "event_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "target": { - "$ref": "aggregate-state.schema.json#/$defs/targetIdentity" - }, - "payload": { - "$ref": "#/$defs/typedMap" - }, - "correlation_id": { - "$ref": "#/$defs/nonEmptyString" - } - } - }, - "hostInputOrigin": { - "type": "object", - "required": [ - "kind" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "host_input" - } - } - }, - "internalEmissionOrigin": { - "type": "object", - "required": [ - "kind", - "producing_receipt_sequence", - "emission_index" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "internal_emission" - }, - "producing_receipt_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "emission_index": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "deliveryOrigin": { - "oneOf": [ - { - "$ref": "#/$defs/hostInputOrigin" - }, - { - "$ref": "#/$defs/internalEmissionOrigin" - } - ] - }, - "preAcceptanceFailure": { - "type": "object", - "required": [ - "code" - ], - "additionalProperties": false, - "properties": { - "code": { - "enum": [ - "malformed_delivery", - "wrong_root", - "invalid_delivery_mode", - "invalid_delivery_origin", - "delivery_digest_mismatch", - "event_id_conflict", - "tombstoned_root" - ] - } - } - }, - "notAcceptedResult": { - "type": "object", - "required": [ - "result", - "failure" - ], - "additionalProperties": false, - "properties": { - "result": { - "const": "not_accepted" - }, - "failure": { - "$ref": "#/$defs/preAcceptanceFailure" - } - } - }, - "pendingAcceptanceResult": { - "type": "object", - "required": [ - "result", - "event_id", - "delivery_sequence", - "accepted_revision" - ], - "additionalProperties": false, - "properties": { - "result": { - "const": "pending" - }, - "event_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "delivery_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "accepted_revision": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "committedDeliveryResult": { - "type": "object", - "required": [ - "result", - "receipt" - ], - "additionalProperties": false, - "properties": { - "result": { - "const": "committed" - }, - "receipt": { - "$ref": "#/$defs/deliveryReceipt" - } - } - }, - "acceptanceResult": { - "oneOf": [ - { - "$ref": "#/$defs/pendingAcceptanceResult" - }, - { - "$ref": "#/$defs/committedDeliveryResult" - }, - { - "$ref": "#/$defs/notAcceptedResult" - } - ] - }, - "pendingDelivery": { - "type": "object", - "required": [ - "delivery_sequence", - "accepted_revision", - "delivery_mode", - "origin", - "envelope", - "envelope_digest" - ], - "additionalProperties": false, - "properties": { - "delivery_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "accepted_revision": { - "$ref": "#/$defs/canonicalDecimal" - }, - "delivery_mode": { - "enum": [ - "input", - "internal" - ] - }, - "origin": { - "$ref": "#/$defs/deliveryOrigin" - }, - "envelope": { - "$ref": "#/$defs/envelope" - }, - "envelope_digest": { - "$ref": "#/$defs/sha256" - } - }, - "allOf": [ - { - "if": { - "properties": { - "delivery_mode": { - "const": "input" - } - }, - "required": [ - "delivery_mode" - ] - }, - "then": { - "properties": { - "origin": { - "$ref": "#/$defs/hostInputOrigin" - } - } - } - }, - { - "if": { - "properties": { - "delivery_mode": { - "const": "internal" - } - }, - "required": [ - "delivery_mode" - ] - }, - "then": { - "properties": { - "origin": { - "$ref": "#/$defs/internalEmissionOrigin" - } - } - } - } - ] - }, - "internalDeliveryEmissionReference": { - "type": "object", - "required": [ - "kind", - "emission_index", - "event_id", - "delivery_sequence" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "internal_delivery" - }, - "emission_index": { - "$ref": "#/$defs/canonicalDecimal" - }, - "event_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "delivery_sequence": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "externalOutboxEmissionReference": { - "type": "object", - "required": [ - "kind", - "emission_index", - "effect_id" - ], - "additionalProperties": false, - "properties": { - "kind": { - "const": "external_outbox" - }, - "emission_index": { - "$ref": "#/$defs/canonicalDecimal" - }, - "effect_id": { - "$ref": "#/$defs/sha256" - } - } - }, - "emissionReference": { - "oneOf": [ - { - "$ref": "#/$defs/internalDeliveryEmissionReference" - }, - { - "$ref": "#/$defs/externalOutboxEmissionReference" - } - ] - }, - "rejection": { - "type": "object", - "required": [ - "code" - ], - "additionalProperties": false, - "properties": { - "code": { - "enum": [ - "invalid_event", - "invalid_payload", - "invalid_correlation", - "invalid_instance_target", - "inactive_component_target", - "invalid_prior_state", - "incompatible_bundle" - ] - } - } - }, - "handledOutcome": { - "type": "object", - "required": [ - "status", - "disposition", - "fault", - "rejection" - ], - "additionalProperties": false, - "properties": { - "status": { - "enum": [ - "running", - "completed" - ] - }, - "disposition": { - "const": "handled" - }, - "fault": { - "type": "null" - }, - "rejection": { - "type": "null" - } - } - }, - "unhandledOutcome": { - "type": "object", - "required": [ - "status", - "disposition", - "fault", - "rejection" - ], - "additionalProperties": false, - "properties": { - "status": { - "const": "running" - }, - "disposition": { - "const": "unhandled" - }, - "fault": { - "type": "null" - }, - "rejection": { - "type": "null" - } - } - }, - "rejectedOutcome": { - "type": "object", - "required": [ - "status", - "disposition", - "fault", - "rejection" - ], - "additionalProperties": false, - "properties": { - "status": { - "enum": [ - "running", - "completed", - "faulted" - ] - }, - "disposition": { - "const": "rejected" - }, - "fault": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "aggregate-state.schema.json#/$defs/fault" - } - ] - }, - "rejection": { - "$ref": "#/$defs/rejection" - } - }, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "faulted" - } - }, - "required": [ - "status" - ] - }, - "then": { - "properties": { - "fault": { - "$ref": "aggregate-state.schema.json#/$defs/fault" - } - } - }, - "else": { - "properties": { - "fault": { - "type": "null" - } - } - } - } - ] - }, - "faultedOutcome": { - "type": "object", - "required": [ - "status", - "disposition", - "fault", - "rejection" - ], - "additionalProperties": false, - "properties": { - "status": { - "enum": [ - "running", - "faulted" - ] - }, - "disposition": { - "const": "faulted" - }, - "fault": { - "$ref": "aggregate-state.schema.json#/$defs/fault" - }, - "rejection": { - "type": "null" - } - } - }, - "deliveryOutcome": { - "oneOf": [ - { - "$ref": "#/$defs/handledOutcome" - }, - { - "$ref": "#/$defs/unhandledOutcome" - }, - { - "$ref": "#/$defs/rejectedOutcome" - }, - { - "$ref": "#/$defs/faultedOutcome" - } - ] - }, - "creationReceipt": { - "type": "object", - "required": [ - "operation_kind", - "receipt_sequence", - "creation_id", - "request_digest", - "committed_revision", - "resulting_aggregate_state_digest", - "status", - "fault", - "emission_references" - ], - "additionalProperties": false, - "properties": { - "operation_kind": { - "const": "creation" - }, - "receipt_sequence": { - "const": "0" - }, - "creation_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "request_digest": { - "$ref": "#/$defs/sha256" - }, - "committed_revision": { - "const": "0" - }, - "resulting_aggregate_state_digest": { - "$ref": "#/$defs/sha256" - }, - "status": { - "enum": [ - "running", - "completed", - "faulted" - ] - }, - "fault": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "aggregate-state.schema.json#/$defs/fault" - } - ] - }, - "emission_references": { - "type": "array", - "items": { - "$ref": "#/$defs/emissionReference" - } - } - }, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "faulted" - } - }, - "required": [ - "status" - ] - }, - "then": { - "properties": { - "fault": { - "$ref": "aggregate-state.schema.json#/$defs/fault" - } - } - }, - "else": { - "properties": { - "fault": { - "type": "null" - } - } - } - } - ] - }, - "deliveryReceipt": { - "type": "object", - "required": [ - "operation_kind", - "receipt_sequence", - "event_id", - "request_digest", - "accepted_delivery_sequence", - "accepted_revision", - "delivery_mode", - "origin", - "committed_revision", - "resulting_aggregate_state_digest", - "outcome", - "emission_references" - ], - "additionalProperties": false, - "properties": { - "operation_kind": { - "const": "delivery" - }, - "receipt_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "event_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "request_digest": { - "$ref": "#/$defs/sha256" - }, - "accepted_delivery_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "accepted_revision": { - "$ref": "#/$defs/canonicalDecimal" - }, - "delivery_mode": { - "enum": [ - "input", - "internal" - ] - }, - "origin": { - "$ref": "#/$defs/deliveryOrigin" - }, - "committed_revision": { - "$ref": "#/$defs/canonicalDecimal" - }, - "resulting_aggregate_state_digest": { - "$ref": "#/$defs/sha256" - }, - "outcome": { - "$ref": "#/$defs/deliveryOutcome" - }, - "emission_references": { - "type": "array", - "items": { - "$ref": "#/$defs/emissionReference" - } - } - }, - "allOf": [ - { - "if": { - "properties": { - "delivery_mode": { - "const": "input" - } - }, - "required": [ - "delivery_mode" - ] - }, - "then": { - "properties": { - "origin": { - "$ref": "#/$defs/hostInputOrigin" - } - } - } - }, - { - "if": { - "properties": { - "delivery_mode": { - "const": "internal" - } - }, - "required": [ - "delivery_mode" - ] - }, - "then": { - "properties": { - "origin": { - "$ref": "#/$defs/internalEmissionOrigin" - } - } - } - } - ] - }, - "maintenanceMigrationReceipt": { - "type": "object", - "required": [ - "operation_kind", - "receipt_sequence", - "operation_id", - "request_digest", - "committed_revision", - "source_aggregate_state_digest", - "resulting_aggregate_state_digest", - "migration_sequences", - "result_code" - ], - "additionalProperties": false, - "properties": { - "operation_kind": { - "const": "maintenance_migration" - }, - "receipt_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "operation_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "request_digest": { - "$ref": "#/$defs/sha256" - }, - "committed_revision": { - "$ref": "#/$defs/canonicalDecimal" - }, - "source_aggregate_state_digest": { - "$ref": "#/$defs/sha256" - }, - "resulting_aggregate_state_digest": { - "$ref": "#/$defs/sha256" - }, - "migration_sequences": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/$defs/positiveCanonicalDecimal" - } - }, - "result_code": { - "enum": [ - "migration_applied", - "migration_no_operation" - ] - } - }, - "allOf": [ - { - "if": { - "properties": { - "result_code": { - "const": "migration_applied" - } - }, - "required": [ - "result_code" - ] - }, - "then": { - "properties": { - "migration_sequences": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "migration_sequences": { - "maxItems": 0 - } - } - } - } - ] - }, - "operationReceipt": { - "oneOf": [ - { - "$ref": "#/$defs/creationReceipt" - }, - { - "$ref": "#/$defs/deliveryReceipt" - }, - { - "$ref": "#/$defs/maintenanceMigrationReceipt" - } - ] - }, - "outboxIntent": { - "type": "object", - "required": [ - "effect_id", - "sequence", - "event", - "payload", - "correlation_id" - ], - "additionalProperties": false, - "properties": { - "effect_id": { - "$ref": "#/$defs/sha256" - }, - "sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "event": { - "$ref": "#/$defs/identifier" - }, - "payload": { - "$ref": "#/$defs/typedMap" - }, - "correlation_id": { - "$ref": "#/$defs/nonEmptyString" - } - } - }, - "notAttemptedOutboxState": { - "type": "object", - "required": [ - "status" - ], - "additionalProperties": false, - "properties": { - "status": { - "const": "not_attempted" - } - } - }, - "retryableFailureOutboxState": { - "type": "object", - "required": [ - "status", - "reason_code" - ], - "additionalProperties": false, - "properties": { - "status": { - "const": "retryable_failure" - }, - "reason_code": { - "$ref": "#/$defs/identifier" - } - } - }, - "ambiguousOutboxState": { - "type": "object", - "required": [ - "status", - "reason_code" - ], - "additionalProperties": false, - "properties": { - "status": { - "const": "ambiguous" - }, - "reason_code": { - "$ref": "#/$defs/identifier" - } - } - }, - "pendingOutboxState": { - "oneOf": [ - { - "$ref": "#/$defs/notAttemptedOutboxState" - }, - { - "$ref": "#/$defs/retryableFailureOutboxState" - }, - { - "$ref": "#/$defs/ambiguousOutboxState" - } - ] - }, - "pendingOutboxIntent": { - "type": "object", - "required": [ - "intent", - "state_revision", - "delivery_state" - ], - "additionalProperties": false, - "properties": { - "intent": { - "$ref": "#/$defs/outboxIntent" - }, - "state_revision": { - "$ref": "#/$defs/canonicalDecimal" - }, - "delivery_state": { - "$ref": "#/$defs/pendingOutboxState" - } - } - }, - "pendingOutboxUpdateResult": { - "type": "object", - "required": [ - "result", - "record" - ], - "additionalProperties": false, - "properties": { - "result": { - "const": "committed" - }, - "record": { - "$ref": "#/$defs/pendingOutboxIntent" - } - } - }, - "confirmedOutboxOutcome": { - "type": "object", - "required": [ - "status" - ], - "additionalProperties": false, - "properties": { - "status": { - "const": "confirmed" - } - } - }, - "reasonedTerminalOutboxOutcome": { - "type": "object", - "required": [ - "status", - "reason_code" - ], - "additionalProperties": false, - "properties": { - "status": { - "enum": [ - "permanently_rejected", - "operator_cancelled", - "discarded", - "dead_lettered" - ] - }, - "reason_code": { - "$ref": "#/$defs/identifier" - } - } - }, - "terminalOutboxOutcome": { - "oneOf": [ - { - "$ref": "#/$defs/confirmedOutboxOutcome" - }, - { - "$ref": "#/$defs/reasonedTerminalOutboxOutcome" - } - ] - }, - "terminalOutboxRecord": { - "type": "object", - "required": [ - "terminal_sequence", - "intent", - "committed_revision", - "outcome" - ], - "additionalProperties": false, - "properties": { - "terminal_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "intent": { - "$ref": "#/$defs/outboxIntent" - }, - "committed_revision": { - "$ref": "#/$defs/canonicalDecimal" - }, - "outcome": { - "$ref": "#/$defs/terminalOutboxOutcome" - } - } - }, - "outboxEffectTombstone": { - "type": "object", - "required": [ - "terminal_sequence", - "effect_id", - "intent_digest", - "committed_revision", - "outcome" - ], - "additionalProperties": false, - "properties": { - "terminal_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "effect_id": { - "$ref": "#/$defs/sha256" - }, - "intent_digest": { - "$ref": "#/$defs/sha256" - }, - "committed_revision": { - "$ref": "#/$defs/canonicalDecimal" - }, - "outcome": { - "$ref": "#/$defs/terminalOutboxOutcome" - } - } - }, - "migrationAuditRecord": { - "type": "object", - "required": [ - "migration_audit_record_schema_version", - "root_instance_id", - "root_runtime_id", - "migration_sequence", - "source_validated_bundle_fingerprint", - "target_validated_bundle_fingerprint", - "migration_descriptor_digest", - "source_aggregate_state_digest", - "target_aggregate_state_digest", - "result_code" - ], - "additionalProperties": false, - "properties": { - "migration_audit_record_schema_version": { - "const": 1 - }, - "root_instance_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "root_runtime_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "migration_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "source_validated_bundle_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "target_validated_bundle_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "migration_descriptor_digest": { - "$ref": "#/$defs/sha256" - }, - "source_aggregate_state_digest": { - "$ref": "#/$defs/sha256" - }, - "target_aggregate_state_digest": { - "$ref": "#/$defs/sha256" - }, - "result_code": { - "const": "migration_applied" - } - } - }, - "retainedRootRecord": { - "type": "object", - "required": [ - "status", - "aggregate_state" - ], - "additionalProperties": false, - "properties": { - "status": { - "const": "retained" - }, - "aggregate_state": { - "$ref": "aggregate-state.schema.json" - } - } - }, - "rootTombstone": { - "type": "object", - "required": [ - "status", - "root_runtime_id", - "creation_id", - "terminal_status", - "final_aggregate_state_digest", - "tombstone_operation_id" - ], - "additionalProperties": false, - "properties": { - "status": { - "const": "tombstone" - }, - "root_runtime_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "creation_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "terminal_status": { - "enum": [ - "completed", - "faulted" - ] - }, - "final_aggregate_state_digest": { - "$ref": "#/$defs/sha256" - }, - "tombstone_operation_id": { - "$ref": "#/$defs/nonEmptyString" - } - } - }, - "rootRecord": { - "oneOf": [ - { - "$ref": "#/$defs/retainedRootRecord" - }, - { - "$ref": "#/$defs/rootTombstone" - } - ] - }, - "permanentReplayRetention": { - "type": "object", - "required": [ - "mode", - "permanent_replay_eligible", - "pruned_through_receipt_sequence", - "policy_identifier" - ], - "additionalProperties": false, - "properties": { - "mode": { - "const": "permanent" - }, - "permanent_replay_eligible": { - "const": true - }, - "pruned_through_receipt_sequence": { - "type": "null" - }, - "policy_identifier": { - "type": "null" - } - } - }, - "boundedReplayRetention": { - "type": "object", - "required": [ - "mode", - "permanent_replay_eligible", - "pruned_through_receipt_sequence", - "policy_identifier" - ], - "additionalProperties": false, - "properties": { - "mode": { - "const": "bounded" - }, - "permanent_replay_eligible": { - "const": false - }, - "pruned_through_receipt_sequence": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/$defs/positiveCanonicalDecimal" - } - ] - }, - "policy_identifier": { - "$ref": "#/$defs/nonEmptyString" - } - } - }, - "replayRetention": { - "oneOf": [ - { - "$ref": "#/$defs/permanentReplayRetention" - }, - { - "$ref": "#/$defs/boundedReplayRetention" - } - ] - }, - "executionCheckpoint": { - "type": "object", - "required": [ - "execution_checkpoint_format", - "execution_checkpoint_schema_version", - "root_instance_id", - "revision", - "root_record", - "replay_retention", - "next_delivery_sequence", - "pending_deliveries", - "next_operation_receipt_sequence", - "operation_receipts", - "pending_outbox_intents", - "next_outbox_terminal_sequence", - "terminal_outbox_records", - "outbox_effect_tombstones", - "migration_audit_records", - "execution_checkpoint_digest" - ], - "additionalProperties": false, - "properties": { - "execution_checkpoint_format": { - "const": "determa.execution_checkpoint" - }, - "execution_checkpoint_schema_version": { - "const": 1 - }, - "root_instance_id": { - "$ref": "#/$defs/nonEmptyString" - }, - "revision": { - "$ref": "#/$defs/canonicalDecimal" - }, - "root_record": { - "$ref": "#/$defs/rootRecord" - }, - "replay_retention": { - "$ref": "#/$defs/replayRetention" - }, - "next_delivery_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "pending_deliveries": { - "type": "array", - "items": { - "$ref": "#/$defs/pendingDelivery" - } - }, - "next_operation_receipt_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "operation_receipts": { - "type": "array", - "minItems": 1, - "prefixItems": [ - { - "$ref": "#/$defs/creationReceipt" - } - ], - "items": { - "$ref": "#/$defs/operationReceipt" - } - }, - "pending_outbox_intents": { - "type": "array", - "items": { - "$ref": "#/$defs/pendingOutboxIntent" - } - }, - "next_outbox_terminal_sequence": { - "$ref": "#/$defs/canonicalDecimal" - }, - "terminal_outbox_records": { - "type": "array", - "items": { - "$ref": "#/$defs/terminalOutboxRecord" - } - }, - "outbox_effect_tombstones": { - "type": "array", - "items": { - "$ref": "#/$defs/outboxEffectTombstone" - } - }, - "migration_audit_records": { - "type": "array", - "items": { - "$ref": "#/$defs/migrationAuditRecord" - } - }, - "execution_checkpoint_digest": { - "$ref": "#/$defs/sha256" - } - }, - "allOf": [ - { - "if": { - "properties": { - "root_record": { - "properties": { - "status": { - "const": "tombstone" - } - }, - "required": [ - "status" - ] - } - }, - "required": [ - "root_record" - ] - }, - "then": { - "properties": { - "pending_deliveries": { - "maxItems": 0 - }, - "pending_outbox_intents": { - "maxItems": 0 - } - } - } - } - ] - } - } -} diff --git a/src/determa/state/data/migration-descriptor-v2.schema.json b/src/determa/state/data/migration-descriptor-v2.schema.json index 4a328f2..01b596c 100644 --- a/src/determa/state/data/migration-descriptor-v2.schema.json +++ b/src/determa/state/data/migration-descriptor-v2.schema.json @@ -1,46 +1,600 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://determa.dev/state/schema/migration-descriptor-v2.schema.json", - "title": "Determa State aggregate migration descriptor version 2", - "description": "Closed migration descriptor for queue-bearing aggregate-state version 2.", - "type": "object", - "required": [ - "migration_descriptor_format", - "migration_descriptor_schema_version", - "base_descriptor", - "queued_event_default", - "queued_event_rules", - "migration_descriptor_digest" - ], - "additionalProperties": false, - "properties": { - "migration_descriptor_format": { - "const": "determa.aggregate_migration" - }, - "migration_descriptor_schema_version": { - "const": 2 - }, - "base_descriptor": { - "$ref": "migration-descriptor.schema.json" - }, - "queued_event_default": { - "const": "preserve_if_compatible" - }, - "queued_event_rules": { - "type": "array", - "items": { - "$ref": "#/$defs/queuedEventRule" - } - }, - "migration_descriptor_digest": { - "$ref": "#/$defs/sha256" - } - }, + "title": "Determa State aggregate migration descriptor", + "description": "Closed declarative migration schema for the sole supported queue-bearing aggregate artifact.", + "$ref": "#/$defs/migrationDescriptor", "$defs": { "identifier": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" }, + "nonEmptyString": { + "type": "string", + "minLength": 1 + }, + "canonicalDecimal": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "jsonPointer": { + "type": "string", + "pattern": "^(?:/(?:[^~/]|~[01])*)*$" + }, + "pointerMapping": { + "type": "object", + "required": [ + "source_definition_pointer", + "target_definition_pointer" + ], + "additionalProperties": false, + "properties": { + "source_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + } + } + }, + "activeStateMapping": { + "type": "object", + "required": [ + "source_leaf_state_definition_pointer", + "target_leaf_state_definition_pointers" + ], + "additionalProperties": false, + "properties": { + "source_leaf_state_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_leaf_state_definition_pointers": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/jsonPointer" + } + } + } + }, + "variableRule": { + "oneOf": [ + { + "type": "object", + "required": [ + "operation", + "source_declaration_pointer", + "target_declaration_pointer" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "copy" + }, + "source_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + } + } + }, + { + "type": "object", + "required": [ + "operation", + "source_declaration_pointers", + "target_declaration_pointer", + "expression" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "transform" + }, + "source_declaration_pointers": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/jsonPointer" + } + }, + "target_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "expression": { + "type": "string", + "minLength": 1 + } + } + }, + { + "type": "object", + "required": [ + "operation", + "source_declaration_pointer", + "reason" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "drop" + }, + "source_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "reason": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + { + "type": "object", + "required": [ + "operation", + "target_declaration_pointer", + "expression" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "initialize" + }, + "target_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "expression": { + "type": "string", + "minLength": 1 + } + } + } + ] + }, + "historyRule": { + "oneOf": [ + { + "type": "object", + "required": [ + "operation", + "source_history_declaration_pointer", + "target_history_declaration_pointer", + "recorded_state_mappings" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "map" + }, + "source_history_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_history_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "recorded_state_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/pointerMapping" + } + } + } + }, + { + "type": "object", + "required": [ + "operation", + "source_history_declaration_pointer", + "reason" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "drop" + }, + "source_history_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "reason": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + { + "type": "object", + "required": [ + "operation", + "target_history_declaration_pointer" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "initialize_null" + }, + "target_history_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + } + } + } + ] + }, + "componentRule": { + "type": "object", + "required": [ + "source_component_definition_pointer", + "target_component_definition_pointer", + "target_component_id" + ], + "additionalProperties": false, + "properties": { + "source_component_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_component_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_component_id": { + "$ref": "#/$defs/identifier" + } + } + }, + "ownedRuntimeRule": { + "type": "object", + "required": [ + "source_spawn_action_pointer", + "target_spawn_action_pointer", + "source_machine_id", + "target_machine_id" + ], + "additionalProperties": false, + "properties": { + "source_spawn_action_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_spawn_action_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "source_machine_id": { + "$ref": "#/$defs/identifier" + }, + "target_machine_id": { + "$ref": "#/$defs/identifier" + } + } + }, + "lifetimeHolderRule": { + "type": "object", + "required": [ + "source_variable_declaration_pointer", + "target_variable_declaration_pointer" + ], + "additionalProperties": false, + "properties": { + "source_variable_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_variable_declaration_pointer": { + "$ref": "#/$defs/jsonPointer" + } + } + }, + "counterRule": { + "oneOf": [ + { + "type": "object", + "required": [ + "operation", + "source_definition_pointer", + "target_definition_pointer" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "map" + }, + "source_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + }, + "target_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + } + } + }, + { + "type": "object", + "required": [ + "operation", + "target_definition_pointer" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "initialize_zero" + }, + "target_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + } + } + }, + { + "type": "object", + "required": [ + "operation", + "source_definition_pointers", + "target_definition_pointer" + ], + "additionalProperties": false, + "properties": { + "operation": { + "const": "merge_maximum" + }, + "source_definition_pointers": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/jsonPointer" + } + }, + "target_definition_pointer": { + "$ref": "#/$defs/jsonPointer" + } + } + } + ] + }, + "terminalPolicy": { + "type": "object", + "required": [ + "completed", + "faulted" + ], + "additionalProperties": false, + "properties": { + "completed": { + "enum": [ + "preserve", + "reject" + ] + }, + "faulted": { + "enum": [ + "preserve", + "reject" + ] + } + } + }, + "resourceRequirements": { + "type": "object", + "required": [ + "maximum_transformed_output_bytes", + "maximum_cel_expression_length", + "maximum_cel_ast_nodes", + "maximum_cel_evaluation_steps" + ], + "additionalProperties": false, + "properties": { + "maximum_transformed_output_bytes": { + "$ref": "#/$defs/canonicalDecimal" + }, + "maximum_cel_expression_length": { + "$ref": "#/$defs/canonicalDecimal" + }, + "maximum_cel_ast_nodes": { + "$ref": "#/$defs/canonicalDecimal" + }, + "maximum_cel_evaluation_steps": { + "$ref": "#/$defs/canonicalDecimal" + } + } + }, + "mappings": { + "type": "object", + "required": [ + "machines", + "active_states", + "variables", + "history", + "components", + "owned_runtimes", + "lifetime_holders", + "counters" + ], + "additionalProperties": false, + "properties": { + "machines": { + "type": "array", + "items": { + "$ref": "#/$defs/pointerMapping" + } + }, + "active_states": { + "type": "array", + "items": { + "$ref": "#/$defs/activeStateMapping" + } + }, + "variables": { + "type": "array", + "items": { + "$ref": "#/$defs/variableRule" + } + }, + "history": { + "type": "array", + "items": { + "$ref": "#/$defs/historyRule" + } + }, + "components": { + "type": "array", + "items": { + "$ref": "#/$defs/componentRule" + } + }, + "owned_runtimes": { + "type": "array", + "items": { + "$ref": "#/$defs/ownedRuntimeRule" + } + }, + "lifetime_holders": { + "type": "array", + "items": { + "$ref": "#/$defs/lifetimeHolderRule" + } + }, + "counters": { + "type": "array", + "items": { + "$ref": "#/$defs/counterRule" + } + } + } + }, + "migrationDescriptor": { + "type": "object", + "required": [ + "migration_descriptor_format", + "migration_descriptor_schema_version", + "source_machine_format", + "target_machine_format", + "source_validated_bundle_fingerprint", + "target_validated_bundle_fingerprint", + "source_aggregate_shape_fingerprint", + "target_aggregate_shape_fingerprint", + "mode", + "mappings", + "terminal_policy", + "resource_requirements", + "queued_event_default", + "queued_event_rules", + "migration_descriptor_digest" + ], + "additionalProperties": false, + "properties": { + "migration_descriptor_format": { + "const": "determa.aggregate_migration" + }, + "migration_descriptor_schema_version": { + "const": 2 + }, + "source_machine_format": { + "const": 1 + }, + "target_machine_format": { + "const": 1 + }, + "source_validated_bundle_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "target_validated_bundle_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "source_aggregate_shape_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "target_aggregate_shape_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "mode": { + "enum": [ + "compatible", + "transform" + ] + }, + "mappings": { + "$ref": "#/$defs/mappings" + }, + "terminal_policy": { + "$ref": "#/$defs/terminalPolicy" + }, + "resource_requirements": { + "$ref": "#/$defs/resourceRequirements" + }, + "migration_descriptor_digest": { + "$ref": "#/$defs/sha256" + }, + "queued_event_default": { + "const": "preserve_if_compatible" + }, + "queued_event_rules": { + "type": "array", + "items": { + "$ref": "#/$defs/queuedEventRule" + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "mode": { + "const": "compatible" + } + }, + "required": [ + "mode" + ] + }, + "then": { + "properties": { + "mappings": { + "properties": { + "machines": { + "maxItems": 0 + }, + "active_states": { + "maxItems": 0 + }, + "variables": { + "maxItems": 0 + }, + "history": { + "maxItems": 0 + }, + "components": { + "maxItems": 0 + }, + "owned_runtimes": { + "maxItems": 0 + }, + "lifetime_holders": { + "maxItems": 0 + }, + "counters": { + "maxItems": 0 + } + } + }, + "resource_requirements": { + "properties": { + "maximum_transformed_output_bytes": { + "const": "0" + }, + "maximum_cel_expression_length": { + "const": "0" + }, + "maximum_cel_ast_nodes": { + "const": "0" + }, + "maximum_cel_evaluation_steps": { + "const": "0" + } + } + } + } + } + } + ] + }, "eventName": { "oneOf": [ { @@ -55,10 +609,6 @@ } ] }, - "sha256": { - "type": "string", - "pattern": "^sha256:[0-9a-f]{64}$" - }, "disposeRule": { "type": "object", "required": [ @@ -77,7 +627,10 @@ "$ref": "#/$defs/eventName" }, "delivery_mode": { - "enum": ["input", "internal"] + "enum": [ + "input", + "internal" + ] }, "action": { "const": "dispose" diff --git a/src/determa/state/data/migration-descriptor.schema.json b/src/determa/state/data/migration-descriptor.schema.json deleted file mode 100644 index e186bab..0000000 --- a/src/determa/state/data/migration-descriptor.schema.json +++ /dev/null @@ -1,588 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://determa.dev/state/schema/migration-descriptor.schema.json", - "title": "Determa State aggregate migration descriptor", - "description": "Closed declarative migration schema for migration_descriptor_schema_version 1.", - "$ref": "#/$defs/migrationDescriptor", - "$defs": { - "identifier": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" - }, - "nonEmptyString": { - "type": "string", - "minLength": 1 - }, - "canonicalDecimal": { - "type": "string", - "pattern": "^(0|[1-9][0-9]*)$" - }, - "sha256": { - "type": "string", - "pattern": "^sha256:[0-9a-f]{64}$" - }, - "jsonPointer": { - "type": "string", - "pattern": "^(?:/(?:[^~/]|~[01])*)*$" - }, - "pointerMapping": { - "type": "object", - "required": [ - "source_definition_pointer", - "target_definition_pointer" - ], - "additionalProperties": false, - "properties": { - "source_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - } - } - }, - "activeStateMapping": { - "type": "object", - "required": [ - "source_leaf_state_definition_pointer", - "target_leaf_state_definition_pointers" - ], - "additionalProperties": false, - "properties": { - "source_leaf_state_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_leaf_state_definition_pointers": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "$ref": "#/$defs/jsonPointer" - } - } - } - }, - "variableRule": { - "oneOf": [ - { - "type": "object", - "required": [ - "operation", - "source_declaration_pointer", - "target_declaration_pointer" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "copy" - }, - "source_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - } - } - }, - { - "type": "object", - "required": [ - "operation", - "source_declaration_pointers", - "target_declaration_pointer", - "expression" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "transform" - }, - "source_declaration_pointers": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "$ref": "#/$defs/jsonPointer" - } - }, - "target_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "expression": { - "type": "string", - "minLength": 1 - } - } - }, - { - "type": "object", - "required": [ - "operation", - "source_declaration_pointer", - "reason" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "drop" - }, - "source_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "reason": { - "$ref": "#/$defs/nonEmptyString" - } - } - }, - { - "type": "object", - "required": [ - "operation", - "target_declaration_pointer", - "expression" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "initialize" - }, - "target_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "expression": { - "type": "string", - "minLength": 1 - } - } - } - ] - }, - "historyRule": { - "oneOf": [ - { - "type": "object", - "required": [ - "operation", - "source_history_declaration_pointer", - "target_history_declaration_pointer", - "recorded_state_mappings" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "map" - }, - "source_history_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_history_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "recorded_state_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/pointerMapping" - } - } - } - }, - { - "type": "object", - "required": [ - "operation", - "source_history_declaration_pointer", - "reason" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "drop" - }, - "source_history_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "reason": { - "$ref": "#/$defs/nonEmptyString" - } - } - }, - { - "type": "object", - "required": [ - "operation", - "target_history_declaration_pointer" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "initialize_null" - }, - "target_history_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - } - } - } - ] - }, - "componentRule": { - "type": "object", - "required": [ - "source_component_definition_pointer", - "target_component_definition_pointer", - "target_component_id" - ], - "additionalProperties": false, - "properties": { - "source_component_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_component_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_component_id": { - "$ref": "#/$defs/identifier" - } - } - }, - "ownedRuntimeRule": { - "type": "object", - "required": [ - "source_spawn_action_pointer", - "target_spawn_action_pointer", - "source_machine_id", - "target_machine_id" - ], - "additionalProperties": false, - "properties": { - "source_spawn_action_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_spawn_action_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "source_machine_id": { - "$ref": "#/$defs/identifier" - }, - "target_machine_id": { - "$ref": "#/$defs/identifier" - } - } - }, - "lifetimeHolderRule": { - "type": "object", - "required": [ - "source_variable_declaration_pointer", - "target_variable_declaration_pointer" - ], - "additionalProperties": false, - "properties": { - "source_variable_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_variable_declaration_pointer": { - "$ref": "#/$defs/jsonPointer" - } - } - }, - "counterRule": { - "oneOf": [ - { - "type": "object", - "required": [ - "operation", - "source_definition_pointer", - "target_definition_pointer" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "map" - }, - "source_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - }, - "target_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - } - } - }, - { - "type": "object", - "required": [ - "operation", - "target_definition_pointer" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "initialize_zero" - }, - "target_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - } - } - }, - { - "type": "object", - "required": [ - "operation", - "source_definition_pointers", - "target_definition_pointer" - ], - "additionalProperties": false, - "properties": { - "operation": { - "const": "merge_maximum" - }, - "source_definition_pointers": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "$ref": "#/$defs/jsonPointer" - } - }, - "target_definition_pointer": { - "$ref": "#/$defs/jsonPointer" - } - } - } - ] - }, - "terminalPolicy": { - "type": "object", - "required": [ - "completed", - "faulted" - ], - "additionalProperties": false, - "properties": { - "completed": { - "enum": [ - "preserve", - "reject" - ] - }, - "faulted": { - "enum": [ - "preserve", - "reject" - ] - } - } - }, - "resourceRequirements": { - "type": "object", - "required": [ - "maximum_transformed_output_bytes", - "maximum_cel_expression_length", - "maximum_cel_ast_nodes", - "maximum_cel_evaluation_steps" - ], - "additionalProperties": false, - "properties": { - "maximum_transformed_output_bytes": { - "$ref": "#/$defs/canonicalDecimal" - }, - "maximum_cel_expression_length": { - "$ref": "#/$defs/canonicalDecimal" - }, - "maximum_cel_ast_nodes": { - "$ref": "#/$defs/canonicalDecimal" - }, - "maximum_cel_evaluation_steps": { - "$ref": "#/$defs/canonicalDecimal" - } - } - }, - "mappings": { - "type": "object", - "required": [ - "machines", - "active_states", - "variables", - "history", - "components", - "owned_runtimes", - "lifetime_holders", - "counters" - ], - "additionalProperties": false, - "properties": { - "machines": { - "type": "array", - "items": { - "$ref": "#/$defs/pointerMapping" - } - }, - "active_states": { - "type": "array", - "items": { - "$ref": "#/$defs/activeStateMapping" - } - }, - "variables": { - "type": "array", - "items": { - "$ref": "#/$defs/variableRule" - } - }, - "history": { - "type": "array", - "items": { - "$ref": "#/$defs/historyRule" - } - }, - "components": { - "type": "array", - "items": { - "$ref": "#/$defs/componentRule" - } - }, - "owned_runtimes": { - "type": "array", - "items": { - "$ref": "#/$defs/ownedRuntimeRule" - } - }, - "lifetime_holders": { - "type": "array", - "items": { - "$ref": "#/$defs/lifetimeHolderRule" - } - }, - "counters": { - "type": "array", - "items": { - "$ref": "#/$defs/counterRule" - } - } - } - }, - "migrationDescriptor": { - "type": "object", - "required": [ - "migration_descriptor_format", - "migration_descriptor_schema_version", - "source_machine_format", - "target_machine_format", - "source_validated_bundle_fingerprint", - "target_validated_bundle_fingerprint", - "source_aggregate_shape_fingerprint", - "target_aggregate_shape_fingerprint", - "mode", - "mappings", - "terminal_policy", - "resource_requirements", - "migration_descriptor_digest" - ], - "additionalProperties": false, - "properties": { - "migration_descriptor_format": { - "const": "determa.aggregate_migration" - }, - "migration_descriptor_schema_version": { - "const": 1 - }, - "source_machine_format": { - "const": 1 - }, - "target_machine_format": { - "const": 1 - }, - "source_validated_bundle_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "target_validated_bundle_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "source_aggregate_shape_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "target_aggregate_shape_fingerprint": { - "$ref": "#/$defs/sha256" - }, - "mode": { - "enum": [ - "compatible", - "transform" - ] - }, - "mappings": { - "$ref": "#/$defs/mappings" - }, - "terminal_policy": { - "$ref": "#/$defs/terminalPolicy" - }, - "resource_requirements": { - "$ref": "#/$defs/resourceRequirements" - }, - "migration_descriptor_digest": { - "$ref": "#/$defs/sha256" - } - }, - "allOf": [ - { - "if": { - "properties": { - "mode": { - "const": "compatible" - } - }, - "required": [ - "mode" - ] - }, - "then": { - "properties": { - "mappings": { - "properties": { - "machines": { - "maxItems": 0 - }, - "active_states": { - "maxItems": 0 - }, - "variables": { - "maxItems": 0 - }, - "history": { - "maxItems": 0 - }, - "components": { - "maxItems": 0 - }, - "owned_runtimes": { - "maxItems": 0 - }, - "lifetime_holders": { - "maxItems": 0 - }, - "counters": { - "maxItems": 0 - } - } - }, - "resource_requirements": { - "properties": { - "maximum_transformed_output_bytes": { - "const": "0" - }, - "maximum_cel_expression_length": { - "const": "0" - }, - "maximum_cel_ast_nodes": { - "const": "0" - }, - "maximum_cel_evaluation_steps": { - "const": "0" - } - } - } - } - } - } - ] - } - } -} diff --git a/src/determa/state/engine.py b/src/determa/state/engine.py index 0ce8f1d..1db4b76 100644 --- a/src/determa/state/engine.py +++ b/src/determa/state/engine.py @@ -39,8 +39,8 @@ validate_unicode, ) -Result = dict[str, Any] -Delivery = dict[str, dict[str, Any]] | None +_Result = dict[str, Any] +_Delivery = dict[str, dict[str, Any]] | None _INT_MIN = -(2**63) _INT_MAX = 2**63 - 1 @@ -275,7 +275,7 @@ def _normalize_payload(declaration: dict[str, Any], payload: Any) -> dict[str, A return normalized -def _empty_result(*, status: str, state: dict[str, Any] | None, disposition: str | None) -> Result: +def _empty_result(*, status: str, state: dict[str, Any] | None, disposition: str | None) -> _Result: return { "status": status, "disposition": disposition, @@ -286,7 +286,7 @@ def _empty_result(*, status: str, state: dict[str, Any] | None, disposition: str } -def create( +def _create( bundle: Bundle | BundleSource, machine_id: str, root_instance_id: str, @@ -294,7 +294,7 @@ def create( bindings: dict[str, dict[str, Any]] | None = None, *, _capture_emission_provenance: bool = False, -) -> Result: +) -> _Result: """Create and synchronously initialize one root ownership aggregate.""" validated = _coerce_bundle(bundle) if ( @@ -386,13 +386,13 @@ def create( return result -def dispatch( +def _dispatch( bundle: Bundle | BundleSource, prior_state: dict[str, Any], - delivery: Delivery = None, + delivery: _Delivery = None, *, _capture_emission_provenance: bool = False, -) -> Result: +) -> _Result: """Validate and process at most one envelope against an aggregate copy.""" validated = _coerce_bundle(bundle) if not _valid_prior_state(prior_state, validated): @@ -506,7 +506,7 @@ def dispatch( return result -def _rejected(prior_state: dict[str, Any], code: DispatchCode) -> Result: +def _rejected(prior_state: dict[str, Any], code: DispatchCode) -> _Result: result = _empty_result( status=prior_state["status"], state=prior_state, @@ -1315,6 +1315,10 @@ def _validate_envelope( expected_direction = "input" if mode == "input" else "internal" if declaration["direction"] != expected_direction: return DispatchCode.INVALID_EVENT + if "payload" not in envelope: + return DispatchCode.INVALID_PAYLOAD + if _normalize_payload(declaration, envelope.get("payload")) is None: + return DispatchCode.INVALID_PAYLOAD correlation = envelope.get("correlation_id") if correlation is not None and ( not isinstance(correlation, str) or not correlation or not validate_unicode(correlation) @@ -1322,10 +1326,6 @@ def _validate_envelope( return DispatchCode.INVALID_CORRELATION if declaration.get("correlates_to") and correlation is None: return DispatchCode.INVALID_CORRELATION - if "payload" not in envelope: - return DispatchCode.INVALID_PAYLOAD - if _normalize_payload(declaration, envelope.get("payload")) is None: - return DispatchCode.INVALID_PAYLOAD return None @@ -2132,6 +2132,8 @@ def send( ), "sequence": sequence, } + if self.capture_emission_provenance: + emission["_determa_v2_provenance"] = {"emission_index": index} else: assert isinstance(target, dict) target_runtime_id = _target_runtime_id(target) @@ -2151,6 +2153,8 @@ def send( } if correlation is not None: emission["correlation_id"] = correlation + if self.capture_emission_provenance: + emission["_determa_v2_emission_index"] = index self.append_emission(emission, runtime) def resolve_send_target( diff --git a/src/determa/state/host.py b/src/determa/state/host.py index 511a5f1..26bfaef 100644 --- a/src/determa/state/host.py +++ b/src/determa/state/host.py @@ -6,11 +6,9 @@ from collections.abc import Callable, Mapping, Sequence from contextlib import nullcontext from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from .checkpoint import ( - RestoredExecutionCheckpoint, - restore_execution_checkpoint, seal_execution_checkpoint, serialize_execution_checkpoint, validate_execution_checkpoint_member, @@ -28,10 +26,8 @@ PersistenceFailureCode as PersistenceCode, ) from .definition import Bundle, BundleSource, load_bundle -from .engine import create as core_create -from .engine import dispatch as core_dispatch from .errors import DetermaError -from .migration import MigrationLimits, migrate_aggregate +from .migration import MigrationLimits from .stores import ( COMPACT_EFFECT_IDENTITY_RETENTION, DURABLE_CONCURRENT, @@ -44,35 +40,23 @@ ExecutionStoreRegistry, ExecutionStoreTransaction, ) +from .stores.base import checkpoint_metadata from .wire import ( ArtifactResolver, - aggregate_envelope, decoded_typed_value, hash_value, - strict_json, + load_json_artifact, + migration_descriptor_digest, typed_value, ) if TYPE_CHECKING: - from .checkpoint_v2 import RestoredExecutionCheckpointV2 + from .checkpoint_v2 import RestoredExecutionCheckpoint FaultInjector = Callable[[str], None] _MAX_DECIMAL_DIGITS = 4096 -def _bundle_requires_v2(bundle: Bundle) -> bool: - def has_deferral(value: Any) -> bool: - if isinstance(value, Mapping): - return "deferred_events" in value or any( - has_deferral(child) for child in value.values() - ) - if isinstance(value, list): - return any(has_deferral(child) for child in value) - return False - - return has_deferral(bundle.raw["machines"]) - - class ExecutionHostError(DetermaError): """A closed host-layer failure.""" @@ -130,8 +114,8 @@ def creation_request_digest( machine_version = "0" if machine is None else str(machine["version"]) return hash_value( [ - "determa-creation-request-digest-1", - "1", + "determa-creation-request-digest-2", + "2", validated.fingerprint, validated.namespace, machine_id, @@ -149,8 +133,8 @@ def delivery_request_digest( """Compute one canonical pending/receipt delivery identity.""" return hash_value( [ - "determa-inbox-envelope-digest-1", - "1", + "determa-inbox-envelope-digest-2", + "2", root_instance_id, delivery_mode, dict(envelope), @@ -166,16 +150,20 @@ def portable_envelope( *, correlation_id: str | None = None, ) -> dict[str, Any]: - """Project one native host envelope into the checkpoint wire shape.""" + """Project one host-input envelope into the aggregate wire shape.""" result = { "event": event, "event_id": event_id, + "cause_id": event_id, + "source": {"host": True}, "target": copy.deepcopy(dict(target)), "payload": typed_value(dict(payload)), } if correlation_id is not None: result["correlation_id"] = correlation_id - if not validate_execution_checkpoint_member("envelope", result): + from .queueing import _valid_envelope_shape + + if not _valid_envelope_shape(result): raise ExecutionHostError(PreAcceptanceCode.MALFORMED_DELIVERY) return result @@ -191,8 +179,8 @@ def maintenance_migration_request_digest( """Compute one canonical keyed maintenance-migration identity.""" return hash_value( [ - "determa-maintenance-migration-request-digest-1", - "1", + "determa-maintenance-migration-request-digest-2", + "2", root_instance_id, operation_id, source_aggregate_state_digest, @@ -207,14 +195,132 @@ def outbox_intent_digest(root_instance_id: str, intent: Mapping[str, Any]) -> st """Compute the compact evidence digest for one complete outbox intent.""" return hash_value( [ - "determa-outbox-intent-digest-1", - "1", + "determa-outbox-intent-digest-2", + "2", root_instance_id, dict(intent), ] ) +_BACKUP_DEFINITION_DIGEST_FIELDS = frozenset( + { + "validated_bundle_fingerprint", + "definition_fingerprint", + "source_validated_bundle_fingerprint", + "target_validated_bundle_fingerprint", + } +) +_BACKUP_DESCRIPTOR_DIGEST_FIELDS = frozenset( + {"migration_descriptor_digest", "migration_descriptor_digest_route"} +) + + +def _required_backup_artifacts( + value: Any, +) -> tuple[frozenset[str], frozenset[str]]: + """Collect definitions and migration descriptors needed for recovery.""" + definitions: set[str] = set() + descriptors: set[str] = set() + + def visit(item: Any) -> None: + if isinstance(item, Mapping): + for key, member in item.items(): + if key in _BACKUP_DEFINITION_DIGEST_FIELDS and isinstance(member, str): + definitions.add(member) + elif key in _BACKUP_DESCRIPTOR_DIGEST_FIELDS and isinstance( + member, str + ): + descriptors.add(member) + elif key == "migration_descriptor_digest_route" and isinstance( + member, Sequence + ) and not isinstance(member, str | bytes): + descriptors.update( + digest for digest in member if isinstance(digest, str) + ) + visit(member) + elif isinstance(item, Sequence) and not isinstance(item, str | bytes): + for member in item: + visit(member) + + if ( + isinstance(value, Mapping) + and value.get("execution_checkpoint_schema_version") == 2 + ): + aggregate = value.get("root_record", {}).get("aggregate_state") + if aggregate is not None: + visit(aggregate) + visit(value.get("migration_audit_records", [])) + for receipt in value.get("operation_receipts", []): + if receipt.get("operation_kind") == "maintenance_migration": + visit( + { + "target_validated_bundle_fingerprint": receipt.get( + "target_validated_bundle_fingerprint" + ) + } + ) + outcome = receipt.get("outcome") + if isinstance(outcome, Mapping) and "migration_descriptor_digest" in outcome: + visit( + { + "migration_descriptor_digest": outcome[ + "migration_descriptor_digest" + ] + } + ) + else: + visit(value) + return frozenset(definitions), frozenset(descriptors) + + +def _required_backup_artifact_digests(value: Any) -> frozenset[str]: + """Collect every immutable artifact digest needed for recovery.""" + definitions, descriptors = _required_backup_artifacts(value) + return definitions | descriptors + + +def _validate_required_backup_artifacts( + resolver: ArtifactResolver, + trusted_artifact_digests: set[str], + value: Any, +) -> None: + """Require every recovery artifact to resolve to trusted addressed content.""" + definitions, descriptors = _required_backup_artifacts(value) + try: + for fingerprint in definitions: + if ( + fingerprint not in trusted_artifact_digests + or not resolver.definition_is_trusted(fingerprint) + ): + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + source = resolver.resolve_definition(fingerprint) + if source is None: + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + bundle = source if isinstance(source, Bundle) else load_bundle(source) + if bundle.fingerprint != fingerprint: + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + + for digest in descriptors: + if ( + digest not in trusted_artifact_digests + or not resolver.migration_descriptor_is_trusted(digest) + ): + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + descriptor_source = resolver.resolve_migration_descriptor(digest) + if descriptor_source is None: + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + descriptor, _ = load_json_artifact( + descriptor_source, "migration_descriptor_v2" + ) + if migration_descriptor_digest(descriptor) != digest: + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + except DetermaError as exc: + if isinstance(exc, ExecutionHostError): + raise + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) from exc + + def validate_host_profile( store: ExecutionStore, profile: str, @@ -228,7 +334,7 @@ def validate_host_profile( {DURABLE_SINGLE_WRITER, DURABLE_CONCURRENT}.intersection(capabilities) ) common = durable and ROOT_IDENTITY_RETENTION in capabilities - atomic = "atomic_checkpoint_processing" in host_features + atomic = "atomic_accept_process" in host_features valid = False if profile == "durable_embedded_processing": valid = common and atomic @@ -244,7 +350,7 @@ def validate_host_profile( common and atomic and { - "acknowledge_after_checkpoint_commit", + "ingress_ack_after_commit", "durable_redelivery", "outbox_worker", }.issubset(host_features) @@ -252,7 +358,6 @@ def validate_host_profile( elif profile == "strict_durable_outbox": valid = ( common - and atomic and PERMANENT_OUTBOX_TERMINAL_RETENTION in capabilities and { "outbox_worker", @@ -263,20 +368,18 @@ def validate_host_profile( elif profile == "compact_durable_outbox": valid = ( common - and atomic and COMPACT_EFFECT_IDENTITY_RETENTION in capabilities and { "outbox_worker", "total_outbox_lifecycle", - "retain_referenced_effect_tombstones", + "retain_receipt_references", }.issubset(host_features) ) elif profile == "shared_application_transaction": valid = ( common - and atomic and SHARED_APPLICATION_TRANSACTION in capabilities - and "native_shared_application_transaction" in host_features + and "native_shared_transaction_used" in host_features ) if not valid: raise ExecutionHostError(AdapterCode.ADAPTER_CAPABILITY_MISMATCH) @@ -290,167 +393,6 @@ class StagedExecutionResult: state: str = "staged" -def _project_fault( - result: Mapping[str, Any], aggregate: Mapping[str, Any] | None -) -> dict[str, Any] | None: - fault = result["fault"] - if fault is None or aggregate is None: - return None - for runtime in aggregate["runtimes"]: - candidate = runtime["fault"] - if candidate is not None and candidate["runtime_id"] == fault["runtime_id"]: - return cast(dict[str, Any], copy.deepcopy(candidate)) - raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) - - -def _project_emission(emission: Mapping[str, Any]) -> dict[str, Any]: - if emission["target"] == "external": - return { - "kind": "external", - "effect_id": emission["effect_id"], - "sequence": str(emission["sequence"]), - "event": emission["event"], - "payload": typed_value(emission["payload"]), - "correlation_id": emission["correlation_id"], - } - projected = { - "kind": "internal", - "event": emission["event"], - "event_id": emission["event_id"], - "target": copy.deepcopy(emission["target"]), - "payload": typed_value(emission["payload"]), - } - if "correlation_id" in emission: - projected["correlation_id"] = emission["correlation_id"] - return projected - - -def _project_core_result(bundle: Bundle, result: Mapping[str, Any]) -> dict[str, Any]: - aggregate = ( - aggregate_envelope(bundle, result["state"]) - if result["state"] is not None - else None - ) - return { - "status": result["status"], - "disposition": result["disposition"], - "aggregate_state": aggregate, - "emissions": [_project_emission(item) for item in result["emissions"]], - "fault": _project_fault(result, aggregate), - "rejection": copy.deepcopy(result["rejection"]), - } - - -def _append_emissions( - checkpoint: dict[str, Any], - receipt: dict[str, Any], - projected_result: Mapping[str, Any], -) -> None: - for index, emission in enumerate(projected_result["emissions"]): - if emission["kind"] == "internal": - sequence = checkpoint["next_delivery_sequence"] - checkpoint["next_delivery_sequence"] = _increment_checkpoint_number( - sequence - ) - origin = { - "kind": "internal_emission", - "producing_receipt_sequence": receipt["receipt_sequence"], - "emission_index": str(index), - } - envelope = { - name: copy.deepcopy(emission[name]) - for name in ("event", "event_id", "target", "payload") - } - if "correlation_id" in emission: - envelope["correlation_id"] = emission["correlation_id"] - checkpoint["pending_deliveries"].append( - { - "delivery_sequence": sequence, - "accepted_revision": checkpoint["revision"], - "delivery_mode": "internal", - "origin": origin, - "envelope": envelope, - "envelope_digest": delivery_request_digest( - checkpoint["root_instance_id"], "internal", envelope - ), - } - ) - receipt["emission_references"].append( - { - "kind": "internal_delivery", - "emission_index": str(index), - "event_id": emission["event_id"], - "delivery_sequence": sequence, - } - ) - else: - checkpoint["pending_outbox_intents"].append( - { - "intent": { - "effect_id": emission["effect_id"], - "sequence": emission["sequence"], - "event": emission["event"], - "payload": copy.deepcopy(emission["payload"]), - "correlation_id": emission["correlation_id"], - }, - "state_revision": checkpoint["revision"], - "delivery_state": {"status": "not_attempted"}, - } - ) - receipt["emission_references"].append( - { - "kind": "external_outbox", - "emission_index": str(index), - "effect_id": emission["effect_id"], - } - ) - - -def _new_checkpoint( - aggregate: dict[str, Any], - request_digest: str, - projected_result: Mapping[str, Any], -) -> dict[str, Any]: - receipt = { - "operation_kind": "creation", - "receipt_sequence": "0", - "creation_id": aggregate["creation_id"], - "request_digest": request_digest, - "committed_revision": "0", - "resulting_aggregate_state_digest": aggregate["aggregate_state_digest"], - "status": projected_result["status"], - "fault": copy.deepcopy(projected_result["fault"]), - "emission_references": [], - } - checkpoint = { - "execution_checkpoint_format": "determa.execution_checkpoint", - "execution_checkpoint_schema_version": 1, - "root_instance_id": aggregate["root_instance_id"], - "revision": "0", - "root_record": { - "status": "retained", - "aggregate_state": copy.deepcopy(aggregate), - }, - "replay_retention": { - "mode": "permanent", - "permanent_replay_eligible": True, - "pruned_through_receipt_sequence": None, - "policy_identifier": None, - }, - "next_delivery_sequence": "0", - "pending_deliveries": [], - "next_operation_receipt_sequence": "1", - "operation_receipts": [receipt], - "pending_outbox_intents": [], - "next_outbox_terminal_sequence": "0", - "terminal_outbox_records": [], - "outbox_effect_tombstones": [], - "migration_audit_records": [], - } - _append_emissions(checkpoint, receipt, projected_result) - return seal_execution_checkpoint(checkpoint) - - def _mutate(checkpoint: Mapping[str, Any]) -> dict[str, Any]: result = copy.deepcopy(dict(checkpoint)) result.pop("execution_checkpoint_digest", None) @@ -490,7 +432,7 @@ def __init__( required_capabilities: set[str] | frozenset[str] = frozenset(), profile: str | None = None, host_features: set[str] | frozenset[str] = frozenset( - {"atomic_checkpoint_processing"} + {"atomic_accept_process"} ), fault_injector: FaultInjector | None = None, ) -> None: @@ -516,7 +458,7 @@ def from_uri( required_capabilities: set[str] | frozenset[str] = frozenset(), profile: str | None = None, host_features: set[str] | frozenset[str] = frozenset( - {"atomic_checkpoint_processing"} + {"atomic_accept_process"} ), fault_injector: FaultInjector | None = None, ) -> ExecutionHost: @@ -547,18 +489,9 @@ def _restore( source: bytes, root_instance_id: str, ) -> Any: - document, _ = strict_json(source) - if ( - isinstance(document, Mapping) - and document.get("execution_checkpoint_schema_version") == 2 - ): - from .checkpoint_v2 import restore_execution_checkpoint_v2 + from .checkpoint_v2 import restore_execution_checkpoint_v2 - restored: Any = restore_execution_checkpoint_v2( - source, self.artifact_resolver - ) - else: - restored = restore_execution_checkpoint(source, self.artifact_resolver) + restored = restore_execution_checkpoint_v2(source, self.artifact_resolver) if restored.document["root_instance_id"] != root_instance_id: raise ExecutionHostError("transaction_root_mismatch") if ( @@ -655,57 +588,89 @@ def _stage_replace( def read_checkpoint( self, root_instance_id: str, - ) -> RestoredExecutionCheckpoint | RestoredExecutionCheckpointV2 | None: + ) -> RestoredExecutionCheckpoint | None: with self._transaction(root_instance_id) as transaction: source = transaction.load() return None if source is None else self._restore(source, root_instance_id) - def create( + def validate_backup_restore_v2( self, - bundle: Bundle | BundleSource, - machine_id: str, - root_instance_id: str, - creation_id: str, - bindings: Mapping[str, Mapping[str, Any]] | None = None, + *, + action: str, + checkpoint_members: Sequence[bytes], + checkpoint_digests: Sequence[str], + trusted_artifact_digests: Sequence[str], + adapter_metadata_digest: str, + retention_mode: str, + consistency_point: Mapping[str, Any], ) -> dict[str, Any]: - validated = bundle if isinstance(bundle, Bundle) else load_bundle(bundle) - normalized_bindings = { - name: copy.deepcopy(dict(value)) for name, value in (bindings or {}).items() - } - request_digest = creation_request_digest( - validated, - machine_id, - root_instance_id, - creation_id, - normalized_bindings, + """Validate one complete, byte-exact checkpoint backup/restore manifest.""" + if ( + action not in {"backup", "restore"} + or retention_mode not in {"permanent", "bounded"} + or not checkpoint_members + or len(checkpoint_members) != len(checkpoint_digests) + or not all(isinstance(item, bytes) for item in checkpoint_members) + or not all(isinstance(item, str) and item for item in checkpoint_digests) + or not all( + isinstance(item, str) and item for item in trusted_artifact_digests + ) + or set(consistency_point) != {"scope_id", "root_instance_ids"} + or not isinstance(consistency_point["scope_id"], str) + or not consistency_point["scope_id"] + or not isinstance(consistency_point["root_instance_ids"], Sequence) + or isinstance(consistency_point["root_instance_ids"], str | bytes) + ): + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + expected_metadata_digest = hash_value( + [ + "determa-backup-adapter-metadata-2", + consistency_point["scope_id"], + ] ) - with self._transaction(root_instance_id) as transaction: - source = transaction.load() - if source is not None: - checkpoint = self._restore(source, root_instance_id).document - receipt = checkpoint["operation_receipts"][0] - if ( - receipt["creation_id"] == creation_id - and receipt["request_digest"] == request_digest - ): - return {"result": "committed", "receipt": copy.deepcopy(receipt)} - raise ExecutionHostError(HostCode.CREATION_ID_CONFLICT) - result = core_create( - validated, - machine_id, - root_instance_id, - creation_id, - normalized_bindings, + if adapter_metadata_digest != expected_metadata_digest: + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + + restored_members = [] + for source, digest in zip( + checkpoint_members, checkpoint_digests, strict=True + ): + restored = self._restore(source, checkpoint_metadata(source)[0]) + if ( + restored.document["execution_checkpoint_digest"] != digest + or restored.document["replay_retention"]["mode"] != retention_mode + ): + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + restored_members.append(restored) + root_instance_ids = [ + restored.document["root_instance_id"] for restored in restored_members + ] + if ( + list(consistency_point["root_instance_ids"]) != root_instance_ids + or len(root_instance_ids) != len(set(root_instance_ids)) + ): + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + + trusted = set(trusted_artifact_digests) + for restored in restored_members: + _validate_required_backup_artifacts( + self.artifact_resolver, + trusted, + restored.document, ) - projected = _project_core_result(validated, result) - aggregate = projected["aggregate_state"] - if aggregate is None: - raise ExecutionHostError(HostCode.CREATION_REJECTED) - candidate = _new_checkpoint(aggregate, request_digest, projected) - self._stage_insert(transaction, candidate) - receipt = copy.deepcopy(candidate["operation_receipts"][0]) - self._after_commit() - return {"result": "committed", "receipt": receipt} + if action == "backup": + for source, root_instance_id in zip( + checkpoint_members, root_instance_ids, strict=True + ): + with self._transaction(root_instance_id) as transaction: + if transaction.load() != source: + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + return { + "result": "validated", + "mutation": "none", + "core_calls": 0, + "broker_acknowledged": False, + } def create_v2( self, @@ -727,10 +692,9 @@ def create_v2( request_digest = creation_request_digest( bundle, machine_id, root_instance_id, creation_id, normalized ) - receipt_value = receipt.get("legacy_receipt", receipt) if ( - receipt_value["creation_id"] == creation_id - and receipt_value["request_digest"] == request_digest + receipt["creation_id"] == creation_id + and receipt["request_digest"] == request_digest ): return {"result": "committed", "receipt": copy.deepcopy(receipt)} raise ExecutionHostError(HostCode.CREATION_ID_CONFLICT) @@ -830,32 +794,46 @@ def process_ready_v2( self._after_commit() return result - def upgrade_checkpoint_v2( + def process_delivery_v2( self, root_instance_id: str, + delivery: Mapping[str, Any], *, + target_validated_bundle_fingerprint: str, + migration_descriptor_digest_route: Sequence[str], expected_revision: str, expected_checkpoint_digest: str, ) -> dict[str, Any]: - """Atomically upgrade one retained v1 checkpoint to v2.""" - from .checkpoint_v2 import upgrade_checkpoint_v1_to_v2 + """Commit migration, admission, and one RTC step as one host transaction.""" + from .checkpoint_v2 import process_delivery_checkpoint_v2 with self._transaction(root_instance_id) as transaction: source = transaction.load() if source is None: raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) prior = self._restore(source, root_instance_id).document - self._check_expected(prior, expected_revision, expected_checkpoint_digest) - candidate = upgrade_checkpoint_v1_to_v2(prior, self.artifact_resolver) + candidate = process_delivery_checkpoint_v2( + prior, + delivery, + self.artifact_resolver, + target_validated_bundle_fingerprint=( + target_validated_bundle_fingerprint + ), + migration_descriptor_digest_route=migration_descriptor_digest_route, + expected_revision=expected_revision, + expected_checkpoint_digest=expected_checkpoint_digest, + ) self._stage_replace(transaction, prior, candidate) self._after_commit() - return candidate + return {"result": "committed", "checkpoint": candidate} def prune_v2( self, root_instance_id: str, cutoff_receipt_sequence: str, *, + target_mode: str | None = None, + policy_identifier: str | None = None, expected_revision: str, expected_checkpoint_digest: str, ) -> dict[str, Any]: @@ -871,6 +849,8 @@ def prune_v2( prior, cutoff_receipt_sequence, self.artifact_resolver, + target_mode=target_mode, + policy_identifier=policy_identifier, expected_revision=expected_revision, expected_checkpoint_digest=expected_checkpoint_digest, ) @@ -933,6 +913,7 @@ def _terminalize_v2_entries( def maintenance_migration_v2( self, root_instance_id: str, + operation_id: str, target_validated_bundle_fingerprint: str, migration_descriptor_digest_route: Sequence[str], *, @@ -941,19 +922,49 @@ def maintenance_migration_v2( maintenance_mode: bool = True, limits: MigrationLimits | None = None, ) -> dict[str, Any]: - """Migrate a v2 aggregate and all queue ownership in one transaction.""" + """Commit one keyed v2 aggregate migration and its durable receipt.""" from .checkpoint_v2 import _synchronize_mailbox_references from .queueing import migrate_aggregate_v2 + if not operation_id: + raise ExecutionHostError(PersistenceCode.INVALID_MIGRATION_REQUEST) with self._transaction(root_instance_id) as transaction: source = transaction.load() if source is None: raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) prior = self._restore(source, root_instance_id).document + for receipt in prior["operation_receipts"]: + if ( + receipt["operation_kind"] == "maintenance_migration" + and receipt["operation_id"] == operation_id + ): + replay_digest = maintenance_migration_request_digest( + root_instance_id, + operation_id, + receipt["source_aggregate_state_digest"], + target_validated_bundle_fingerprint, + migration_descriptor_digest_route, + maintenance_mode, + ) + if receipt["request_digest"] == replay_digest: + return { + "result": "committed", + "receipt": copy.deepcopy(receipt), + } + raise ExecutionHostError(HostCode.OPERATION_ID_CONFLICT) self._check_expected(prior, expected_revision, expected_checkpoint_digest) aggregate = prior["root_record"].get("aggregate_state") if aggregate is None: raise ExecutionHostError(PreAcceptanceCode.TOMBSTONED_ROOT) + source_aggregate_state_digest = aggregate["aggregate_state_digest"] + request_digest = maintenance_migration_request_digest( + root_instance_id, + operation_id, + source_aggregate_state_digest, + target_validated_bundle_fingerprint, + migration_descriptor_digest_route, + maintenance_mode, + ) migration = migrate_aggregate_v2( aggregate, target_validated_bundle_fingerprint, @@ -969,6 +980,32 @@ def maintenance_migration_v2( candidate["migration_audit_records"].extend( copy.deepcopy(migration["audit_records"]) ) + receipt_sequence = candidate["next_operation_receipt_sequence"] + candidate["next_operation_receipt_sequence"] = ( + _increment_checkpoint_number(receipt_sequence) + ) + migration_sequences = [ + item["migration_sequence"] for item in migration["audit_records"] + ] + receipt = { + "operation_kind": "maintenance_migration", + "receipt_sequence": receipt_sequence, + "operation_id": operation_id, + "request_digest": request_digest, + "committed_revision": candidate["revision"], + "source_aggregate_state_digest": source_aggregate_state_digest, + "target_validated_bundle_fingerprint": target_validated_bundle_fingerprint, + "resulting_aggregate_state_digest": migrated[ + "aggregate_state_digest" + ], + "migration_sequences": migration_sequences, + "result_code": ( + "migration_applied" + if migration_sequences + else "migration_no_operation" + ), + } + candidate["operation_receipts"].append(receipt) for entry, disposition in zip( migration["_disposed_entries"], migration["dispositions"], @@ -993,7 +1030,7 @@ def maintenance_migration_v2( candidate = seal_execution_checkpoint(candidate) self._stage_replace(transaction, prior, candidate) self._after_commit() - return candidate + return {"result": "committed", "receipt": copy.deepcopy(receipt)} def tombstone_root_v2( self, @@ -1014,7 +1051,10 @@ def tombstone_root_v2( root_record = prior["root_record"] if root_record["status"] == "tombstone": if root_record["tombstone_operation_id"] == operation_id: - return prior + return { + "result": "tombstoned", + "tombstone": copy.deepcopy(root_record), + } raise ExecutionHostError(HostCode.OPERATION_ID_CONFLICT) self._check_expected(prior, expected_revision, expected_checkpoint_digest) aggregate = root_record["aggregate_state"] @@ -1061,461 +1101,9 @@ def tombstone_root_v2( } candidate = seal_execution_checkpoint(candidate) self._stage_replace(transaction, prior, candidate) + tombstone = copy.deepcopy(candidate["root_record"]) self._after_commit() - return candidate - - def _delivery_candidate( - self, candidate: Any - ) -> tuple[str | None, str | None, Any, dict[str, Any] | None, str | None]: - if not isinstance(candidate, Mapping): - return None, None, None, None, None - allowed = { - "root_instance_id", - "delivery_mode", - "origin", - "envelope", - "envelope_digest", - } - required = {"root_instance_id", "delivery_mode", "origin", "envelope"} - if not required.issubset(candidate) or not set(candidate).issubset(allowed): - return None, None, None, None, None - root_instance_id = candidate["root_instance_id"] - mode = candidate["delivery_mode"] - origin = candidate["origin"] - envelope = candidate["envelope"] - supplied_digest = candidate.get("envelope_digest") - if ( - not isinstance(root_instance_id, str) - or not root_instance_id - or not isinstance(mode, str) - or not isinstance(envelope, dict) - or not validate_execution_checkpoint_member("envelope", envelope) - or (supplied_digest is not None and not isinstance(supplied_digest, str)) - ): - return None, None, None, None, None - return ( - root_instance_id, - mode, - copy.deepcopy(origin), - copy.deepcopy(envelope), - supplied_digest, - ) - - def _not_accepted(self, code: PreAcceptanceCode) -> dict[str, Any]: - return {"result": "not_accepted", "failure": {"code": code.value}} - - def _delivery_replay( - self, - checkpoint: Mapping[str, Any], - event_id: str, - digest: str, - ) -> dict[str, Any] | None: - for pending in checkpoint["pending_deliveries"]: - if pending["envelope"]["event_id"] == event_id: - if pending["envelope_digest"] != digest: - return self._not_accepted(PreAcceptanceCode.EVENT_ID_CONFLICT) - return { - "result": "pending", - "event_id": event_id, - "delivery_sequence": pending["delivery_sequence"], - "accepted_revision": pending["accepted_revision"], - } - for receipt in checkpoint["operation_receipts"]: - if ( - receipt["operation_kind"] == "delivery" - and receipt["event_id"] == event_id - ): - if receipt["request_digest"] != digest: - return self._not_accepted(PreAcceptanceCode.EVENT_ID_CONFLICT) - return {"result": "committed", "receipt": copy.deepcopy(receipt)} - return None - - def _prepare_acceptance( - self, - checkpoint: Mapping[str, Any], - candidate: Any, - bundle: Bundle | None = None, - ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: - parsed = self._delivery_candidate(candidate) - root_instance_id, mode, origin, envelope, supplied_digest = parsed - if root_instance_id is None or mode is None or envelope is None: - return None, self._not_accepted(PreAcceptanceCode.MALFORMED_DELIVERY) - if root_instance_id != checkpoint["root_instance_id"]: - return None, self._not_accepted(PreAcceptanceCode.WRONG_ROOT) - - digest = delivery_request_digest(root_instance_id, mode, envelope) - replay = self._delivery_replay(checkpoint, envelope["event_id"], digest) - if replay is not None: - return None, replay - if checkpoint["root_record"]["status"] == "tombstone": - return None, self._not_accepted(PreAcceptanceCode.TOMBSTONED_ROOT) - if bundle is not None and _bundle_requires_v2(bundle): - return None, self._not_accepted( - PreAcceptanceCode.CHECKPOINT_UPGRADE_REQUIRED - ) - - valid_mode = mode in {"input", "internal"} - valid_origin = validate_execution_checkpoint_member("deliveryOrigin", origin) - valid_pair = (mode == "input" and origin == {"kind": "host_input"}) or ( - mode == "internal" - and isinstance(origin, Mapping) - and origin.get("kind") == "internal_emission" - ) - if not valid_mode: - return None, self._not_accepted(PreAcceptanceCode.INVALID_DELIVERY_MODE) - if not valid_origin or not valid_pair: - return None, self._not_accepted(PreAcceptanceCode.INVALID_DELIVERY_ORIGIN) - if supplied_digest is not None and supplied_digest != digest: - return None, self._not_accepted(PreAcceptanceCode.DELIVERY_DIGEST_MISMATCH) - if ( - _target_root_instance_id(envelope["target"]) - != checkpoint["root_instance_id"] - ): - return None, self._not_accepted(PreAcceptanceCode.WRONG_ROOT) - return { - "root_instance_id": root_instance_id, - "delivery_mode": mode, - "origin": origin, - "envelope": envelope, - "envelope_digest": digest, - }, None - - def accept_delivery( - self, - root_instance_id: str, - candidate: Any, - *, - expected_revision: str, - expected_checkpoint_digest: str, - selected_bundle: Bundle | BundleSource | None = None, - ) -> dict[str, Any]: - with self._transaction(root_instance_id) as transaction: - source = transaction.load() - if source is None: - return self._not_accepted(PreAcceptanceCode.WRONG_ROOT) - restored = self._restore(source, root_instance_id) - checkpoint = restored.document - prepared, result = self._prepare_acceptance( - checkpoint, - candidate, - ( - selected_bundle - if isinstance(selected_bundle, Bundle) - else load_bundle(selected_bundle) - if selected_bundle is not None - else restored.aggregate.bundle - if restored.aggregate is not None - else None - ), - ) - if result is not None: - return result - assert prepared is not None - self._check_expected( - checkpoint, expected_revision, expected_checkpoint_digest - ) - next_checkpoint = _mutate(checkpoint) - sequence = next_checkpoint["next_delivery_sequence"] - next_checkpoint["next_delivery_sequence"] = _increment_checkpoint_number( - sequence - ) - pending = { - "delivery_sequence": sequence, - "accepted_revision": next_checkpoint["revision"], - "delivery_mode": prepared["delivery_mode"], - "origin": prepared["origin"], - "envelope": prepared["envelope"], - "envelope_digest": prepared["envelope_digest"], - } - next_checkpoint["pending_deliveries"].append(pending) - next_checkpoint = seal_execution_checkpoint(next_checkpoint) - self._stage_replace(transaction, checkpoint, next_checkpoint) - response = { - "result": "pending", - "event_id": pending["envelope"]["event_id"], - "delivery_sequence": sequence, - "accepted_revision": pending["accepted_revision"], - } - self._after_commit() - return response - - def _commit_delivery( - self, - checkpoint: Mapping[str, Any], - restored: RestoredExecutionCheckpoint, - request: dict[str, Any], - projected: Mapping[str, Any], - *, - foreground: bool, - pending: Mapping[str, Any] | None = None, - ) -> tuple[dict[str, Any], dict[str, Any]]: - candidate = _mutate(checkpoint) - if foreground: - delivery_sequence = candidate["next_delivery_sequence"] - candidate["next_delivery_sequence"] = _increment_checkpoint_number( - delivery_sequence - ) - accepted_revision = candidate["revision"] - else: - assert pending is not None - candidate["pending_deliveries"] = [ - item - for item in candidate["pending_deliveries"] - if item["delivery_sequence"] != pending["delivery_sequence"] - ] - delivery_sequence = pending["delivery_sequence"] - accepted_revision = pending["accepted_revision"] - request = { - "delivery_mode": pending["delivery_mode"], - "origin": copy.deepcopy(pending["origin"]), - "envelope": copy.deepcopy(pending["envelope"]), - "envelope_digest": pending["envelope_digest"], - } - receipt_sequence = candidate["next_operation_receipt_sequence"] - candidate["next_operation_receipt_sequence"] = _increment_checkpoint_number( - receipt_sequence - ) - aggregate = copy.deepcopy(projected["aggregate_state"]) - if aggregate is None or restored.aggregate is None: - raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) - candidate["root_record"]["aggregate_state"] = aggregate - receipt = { - "operation_kind": "delivery", - "receipt_sequence": receipt_sequence, - "event_id": request["envelope"]["event_id"], - "request_digest": request["envelope_digest"], - "accepted_delivery_sequence": delivery_sequence, - "accepted_revision": accepted_revision, - "delivery_mode": request["delivery_mode"], - "origin": copy.deepcopy(request["origin"]), - "committed_revision": candidate["revision"], - "resulting_aggregate_state_digest": aggregate["aggregate_state_digest"], - "outcome": { - "status": projected["status"], - "disposition": projected["disposition"], - "fault": copy.deepcopy(projected["fault"]), - "rejection": copy.deepcopy(projected["rejection"]), - }, - "emission_references": [], - } - candidate["operation_receipts"].append(receipt) - _append_emissions(candidate, receipt, projected) - return seal_execution_checkpoint(candidate), receipt - - def process_pending_delivery( - self, - root_instance_id: str, - candidate: Any, - *, - expected_revision: str, - expected_checkpoint_digest: str, - ) -> dict[str, Any]: - with self._transaction(root_instance_id) as transaction: - source = transaction.load() - if source is None: - raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) - restored = self._restore(source, root_instance_id) - checkpoint = restored.document - parsed = self._delivery_candidate(candidate) - candidate_root, mode, origin, envelope, supplied_digest = parsed - if ( - candidate_root != root_instance_id - or mode not in {"input", "internal"} - or origin is None - or envelope is None - ): - raise ExecutionHostError(PreAcceptanceCode.MALFORMED_DELIVERY) - digest = delivery_request_digest(root_instance_id, mode, envelope) - if supplied_digest is not None and supplied_digest != digest: - raise ExecutionHostError(PreAcceptanceCode.DELIVERY_DIGEST_MISMATCH) - replay = self._delivery_replay(checkpoint, envelope["event_id"], digest) - if replay is not None and replay["result"] == "committed": - return replay - if replay is not None and replay["result"] == "not_accepted": - raise ExecutionHostError(replay["failure"]["code"]) - pending = next( - ( - item - for item in checkpoint["pending_deliveries"] - if item["envelope"]["event_id"] == envelope["event_id"] - ), - None, - ) - if pending is None or pending["envelope_digest"] != digest: - raise ExecutionHostError(HostCode.EVENT_ID_CONFLICT) - self._check_expected( - checkpoint, expected_revision, expected_checkpoint_digest - ) - if restored.aggregate is None: - raise ExecutionHostError(PreAcceptanceCode.TOMBSTONED_ROOT) - result = core_dispatch( - restored.aggregate.bundle, - restored.aggregate.state, - _delivery_from_wire(pending["delivery_mode"], pending["envelope"]), - ) - projected = _project_core_result(restored.aggregate.bundle, result) - next_checkpoint, receipt = self._commit_delivery( - checkpoint, - restored, - {}, - projected, - foreground=False, - pending=pending, - ) - self._stage_replace(transaction, checkpoint, next_checkpoint) - response = {"result": "committed", "receipt": copy.deepcopy(receipt)} - self._after_commit() - return response - - def foreground_process_delivery( - self, - root_instance_id: str, - candidate: Any, - *, - expected_revision: str, - expected_checkpoint_digest: str, - ) -> dict[str, Any]: - with self._transaction(root_instance_id) as transaction: - source = transaction.load() - if source is None: - raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) - restored = self._restore(source, root_instance_id) - checkpoint = restored.document - prepared, replay = self._prepare_acceptance( - checkpoint, - candidate, - restored.aggregate.bundle if restored.aggregate is not None else None, - ) - if replay is not None: - return replay - assert prepared is not None - self._check_expected( - checkpoint, expected_revision, expected_checkpoint_digest - ) - if restored.aggregate is None: - raise ExecutionHostError(PreAcceptanceCode.TOMBSTONED_ROOT) - result = core_dispatch( - restored.aggregate.bundle, - restored.aggregate.state, - _delivery_from_wire(prepared["delivery_mode"], prepared["envelope"]), - ) - projected = _project_core_result(restored.aggregate.bundle, result) - next_checkpoint, receipt = self._commit_delivery( - checkpoint, - restored, - prepared, - projected, - foreground=True, - ) - self._stage_replace(transaction, checkpoint, next_checkpoint) - response = {"result": "committed", "receipt": copy.deepcopy(receipt)} - self._after_commit() - return response - - def maintenance_migration( - self, - root_instance_id: str, - operation_id: str, - target_validated_bundle_fingerprint: str, - migration_descriptor_digest_route: Sequence[str], - *, - source_aggregate_state_digest: str, - expected_revision: str, - expected_checkpoint_digest: str, - maintenance_mode: bool = True, - limits: MigrationLimits | None = None, - ) -> dict[str, Any]: - if not operation_id or not validate_execution_checkpoint_member( - "sha256", source_aggregate_state_digest - ): - raise ExecutionHostError(PersistenceCode.INVALID_MIGRATION_REQUEST) - request_digest = maintenance_migration_request_digest( - root_instance_id, - operation_id, - source_aggregate_state_digest, - target_validated_bundle_fingerprint, - migration_descriptor_digest_route, - maintenance_mode, - ) - with self._transaction(root_instance_id) as transaction: - source = transaction.load() - if source is None: - raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) - restored = self._restore(source, root_instance_id) - checkpoint = restored.document - for receipt in checkpoint["operation_receipts"]: - if ( - receipt["operation_kind"] == "maintenance_migration" - and receipt["operation_id"] == operation_id - ): - if receipt["request_digest"] == request_digest: - return { - "result": "committed", - "receipt": copy.deepcopy(receipt), - } - raise ExecutionHostError(HostCode.OPERATION_ID_CONFLICT) - if restored.aggregate is None: - raise ExecutionHostError(PreAcceptanceCode.TOMBSTONED_ROOT) - current_source_digest = restored.aggregate.aggregate_envelope[ - "aggregate_state_digest" - ] - if source_aggregate_state_digest != current_source_digest: - raise ExecutionHostError(PersistenceCode.INVALID_MIGRATION_REQUEST) - self._check_expected( - checkpoint, expected_revision, expected_checkpoint_digest - ) - result = migrate_aggregate( - restored.aggregate.aggregate_envelope, - target_validated_bundle_fingerprint, - migration_descriptor_digest_route, - self.artifact_resolver, - maintenance_mode=maintenance_mode, - resource_limits=limits, - ) - if result.failure is not None or result.aggregate_envelope is None: - code = ( - "migration_failed" - if result.failure is None - else result.failure.code - ) - raise ExecutionHostError(code) - candidate = _mutate(checkpoint) - receipt_sequence = candidate["next_operation_receipt_sequence"] - candidate["next_operation_receipt_sequence"] = _increment_checkpoint_number( - receipt_sequence - ) - migration_sequences = [ - item["migration_sequence"] for item in result.audit_records - ] - receipt = { - "operation_kind": "maintenance_migration", - "receipt_sequence": receipt_sequence, - "operation_id": operation_id, - "request_digest": request_digest, - "committed_revision": candidate["revision"], - "source_aggregate_state_digest": source_aggregate_state_digest, - "resulting_aggregate_state_digest": result.aggregate_envelope[ - "aggregate_state_digest" - ], - "migration_sequences": migration_sequences, - "result_code": ( - "migration_applied" - if migration_sequences - else "migration_no_operation" - ), - } - candidate["root_record"]["aggregate_state"] = copy.deepcopy( - result.aggregate_envelope - ) - candidate["operation_receipts"].append(receipt) - candidate["migration_audit_records"].extend( - copy.deepcopy(result.audit_records) - ) - candidate = seal_execution_checkpoint(candidate) - self._stage_replace(transaction, checkpoint, candidate) - response = {"result": "committed", "receipt": copy.deepcopy(receipt)} - self._after_commit() - return response + return {"result": "tombstoned", "tombstone": tombstone} def update_pending_outbox( self, @@ -1851,78 +1439,6 @@ def update_replay_retention( self._after_commit() return response - def tombstone_root( - self, - root_instance_id: str, - operation_id: str, - *, - expected_revision: str, - expected_checkpoint_digest: str, - ) -> dict[str, Any]: - if not operation_id: - raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) - with self._transaction(root_instance_id) as transaction: - source = transaction.load() - if source is None: - raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) - restored = self._restore(source, root_instance_id) - checkpoint = restored.document - root_record = checkpoint["root_record"] - if root_record["status"] == "tombstone": - if root_record["tombstone_operation_id"] == operation_id: - return { - "result": "tombstoned", - "tombstone": copy.deepcopy(root_record), - } - raise ExecutionHostError(HostCode.OPERATION_ID_CONFLICT) - self._check_expected( - checkpoint, expected_revision, expected_checkpoint_digest - ) - if restored.aggregate is None: - raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) - root_runtime = restored.aggregate.state["runtimes"][ - restored.aggregate.state["root_runtime_id"] - ] - if ( - root_runtime["status"] not in {"completed", "faulted"} - or checkpoint["pending_deliveries"] - or checkpoint["pending_outbox_intents"] - ): - raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) - aggregate = root_record["aggregate_state"] - candidate = _mutate(checkpoint) - tombstone = { - "status": "tombstone", - "root_runtime_id": aggregate["root_runtime_id"], - "creation_id": aggregate["creation_id"], - "terminal_status": root_runtime["status"], - "final_aggregate_state_digest": aggregate["aggregate_state_digest"], - "tombstone_operation_id": operation_id, - } - candidate["root_record"] = tombstone - candidate = seal_execution_checkpoint(candidate) - self._stage_replace(transaction, checkpoint, candidate) - response = { - "result": "tombstoned", - "tombstone": copy.deepcopy(tombstone), - } - self._after_commit() - return response - - def delete_checkpoint( - self, - root_instance_id: str, - *, - expected_revision: str, - expected_checkpoint_digest: str, - ) -> dict[str, Any]: - del root_instance_id, expected_revision, expected_checkpoint_digest - return { - "result": "unsupported", - "failure": {"code": HostCode.PHYSICAL_DELETION_UNSUPPORTED.value}, - } - - class SharedExecutionTransaction: """Root-bound staging surface for one host-owned shared transaction.""" @@ -1961,7 +1477,7 @@ def _finish(self) -> dict[str, Any]: def _deactivate(self) -> None: self._active = False - def create( + def create_v2( self, bundle: Bundle | BundleSource, machine_id: str, @@ -1969,8 +1485,8 @@ def create( bindings: Mapping[str, Mapping[str, Any]] | None = None, ) -> StagedExecutionResult: return self._stage( - "create", - lambda: self._host.create( + "create_v2", + lambda: self._host.create_v2( bundle, machine_id, self.root_instance_id, @@ -1979,77 +1495,102 @@ def create( ), ) - def accept_delivery( + def admit_v2( self, - candidate: Any, + deliveries: Sequence[Mapping[str, Any]], *, expected_revision: str, expected_checkpoint_digest: str, ) -> StagedExecutionResult: return self._stage( - "accept_delivery", - lambda: self._host.accept_delivery( + "admit_v2", + lambda: self._host.admit_v2( self.root_instance_id, - candidate, + deliveries, expected_revision=expected_revision, expected_checkpoint_digest=expected_checkpoint_digest, ), ) - def process_pending_delivery( + def process_ready_v2( self, - candidate: Any, + target_runtime_id: str, *, expected_revision: str, expected_checkpoint_digest: str, ) -> StagedExecutionResult: return self._stage( - "process_pending_delivery", - lambda: self._host.process_pending_delivery( + "process_ready_v2", + lambda: self._host.process_ready_v2( self.root_instance_id, - candidate, + target_runtime_id, expected_revision=expected_revision, expected_checkpoint_digest=expected_checkpoint_digest, ), ) - def foreground_process_delivery( + def process_delivery_v2( self, - candidate: Any, + delivery: Mapping[str, Any], *, + target_validated_bundle_fingerprint: str, + migration_descriptor_digest_route: Sequence[str], expected_revision: str, expected_checkpoint_digest: str, ) -> StagedExecutionResult: return self._stage( - "foreground_process_delivery", - lambda: self._host.foreground_process_delivery( + "process_delivery_v2", + lambda: self._host.process_delivery_v2( self.root_instance_id, - candidate, + delivery, + target_validated_bundle_fingerprint=( + target_validated_bundle_fingerprint + ), + migration_descriptor_digest_route=migration_descriptor_digest_route, + expected_revision=expected_revision, + expected_checkpoint_digest=expected_checkpoint_digest, + ), + ) + + def prune_v2( + self, + cutoff_receipt_sequence: str, + *, + target_mode: str | None = None, + policy_identifier: str | None = None, + expected_revision: str, + expected_checkpoint_digest: str, + ) -> StagedExecutionResult: + return self._stage( + "prune_v2", + lambda: self._host.prune_v2( + self.root_instance_id, + cutoff_receipt_sequence, + target_mode=target_mode, + policy_identifier=policy_identifier, expected_revision=expected_revision, expected_checkpoint_digest=expected_checkpoint_digest, ), ) - def maintenance_migration( + def maintenance_migration_v2( self, operation_id: str, target_validated_bundle_fingerprint: str, migration_descriptor_digest_route: Sequence[str], *, - source_aggregate_state_digest: str, expected_revision: str, expected_checkpoint_digest: str, maintenance_mode: bool = True, limits: MigrationLimits | None = None, ) -> StagedExecutionResult: return self._stage( - "maintenance_migration", - lambda: self._host.maintenance_migration( + "maintenance_migration_v2", + lambda: self._host.maintenance_migration_v2( self.root_instance_id, operation_id, target_validated_bundle_fingerprint, migration_descriptor_digest_route, - source_aggregate_state_digest=source_aggregate_state_digest, expected_revision=expected_revision, expected_checkpoint_digest=expected_checkpoint_digest, maintenance_mode=maintenance_mode, @@ -2057,6 +1598,23 @@ def maintenance_migration( ), ) + def tombstone_root_v2( + self, + operation_id: str, + *, + expected_revision: str, + expected_checkpoint_digest: str, + ) -> StagedExecutionResult: + return self._stage( + "tombstone_root_v2", + lambda: self._host.tombstone_root_v2( + self.root_instance_id, + operation_id, + expected_revision=expected_revision, + expected_checkpoint_digest=expected_checkpoint_digest, + ), + ) + def update_pending_outbox( self, effect_id: str, @@ -2146,24 +1704,6 @@ def update_replay_retention( ), ) - def tombstone_root( - self, - operation_id: str, - *, - expected_revision: str, - expected_checkpoint_digest: str, - ) -> StagedExecutionResult: - return self._stage( - "tombstone_root", - lambda: self._host.tombstone_root( - self.root_instance_id, - operation_id, - expected_revision=expected_revision, - expected_checkpoint_digest=expected_checkpoint_digest, - ), - ) - - def _target_root_instance_id(target: Mapping[str, Any]) -> str: member = next(iter(target.values())) return str(member["root_instance_id"]) diff --git a/src/determa/state/migration.py b/src/determa/state/migration.py index cad9f67..c842012 100644 --- a/src/determa/state/migration.py +++ b/src/determa/state/migration.py @@ -1,4 +1,4 @@ -"""Deterministic portable aggregate migration and atomic dispatch composition.""" +"""Deterministic portable aggregate migration.""" from __future__ import annotations @@ -10,7 +10,6 @@ from . import cel from .codes import PersistenceFailureCode as PersistenceCode from .definition import Bundle, _escape_pointer -from .engine import Delivery, dispatch from .errors import ArtifactError, CelError from .model import BundleModel, MachineModel, StateNode from .wire import ( @@ -82,35 +81,10 @@ def succeeded(self) -> bool: return self.failure is None -@dataclass(frozen=True) -class MigrationDispatchResult: - """One atomic migration plus optional ordinary dispatch result.""" - - aggregate_envelope: dict[str, Any] | None - aggregate_bytes: bytes | None - audit_records: tuple[dict[str, Any], ...] - status: str | None - disposition: str | None - emissions: tuple[dict[str, Any], ...] - fault: dict[str, Any] | None - rejection: dict[str, Any] | None - failure: MigrationFailure | None - - @property - def succeeded(self) -> bool: - return self.failure is None - - def _failure(code: str) -> MigrationResult: return MigrationResult(None, None, (), MigrationFailure(str(code))) -def _dispatch_failure(code: str) -> MigrationDispatchResult: - return MigrationDispatchResult( - None, None, (), None, None, (), None, None, MigrationFailure(code) - ) - - def _resource_metrics(value: Any, depth: int = 0) -> tuple[int, int, int, int]: maximum_depth = depth maximum_map_members = 0 @@ -439,13 +413,41 @@ def _resolve_descriptor( raise ArtifactError(PersistenceCode.MIGRATION_ROUTE_MISMATCH) if not resolver.migration_descriptor_is_trusted(digest): raise ArtifactError(PersistenceCode.MIGRATION_DESCRIPTOR_UNTRUSTED) - document, _raw = load_json_artifact(source, "migration_descriptor") + document, _raw = load_json_artifact(source, "migration_descriptor_v2") encoded = canonical_bytes(document) if migration_descriptor_digest(document) != digest: raise ArtifactError(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) return document, encoded +def restore_migration_descriptor_v2( + source: ArtifactSource, + artifact_resolver: ArtifactResolver, + *, + resource_limits: MigrationLimits | None = None, +) -> dict[str, Any]: + """Structurally and semantically restore one trusted v2 descriptor.""" + limits = resource_limits or MigrationLimits() + document, _raw = load_json_artifact(source, "migration_descriptor_v2") + digest = document["migration_descriptor_digest"] + if migration_descriptor_digest(document) != digest: + raise ArtifactError(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) + source_bundle = _bundle_from_resolver( + artifact_resolver, + document["source_validated_bundle_fingerprint"], + source=True, + require_trust=True, + ) + target_bundle = _bundle_from_resolver( + artifact_resolver, + document["target_validated_bundle_fingerprint"], + source=False, + require_trust=True, + ) + _validate_descriptor_semantics(document, source_bundle, target_bundle, limits) + return copy.deepcopy(document) + + def _compatible_candidate( source: dict[str, Any], target_bundle: Bundle, @@ -974,7 +976,7 @@ def migrate_aggregate( ) audits.append( { - "migration_audit_record_schema_version": 1, + "migration_audit_record_schema_version": 2, "root_instance_id": candidate["root_instance_id"], "root_runtime_id": candidate["root_runtime_id"], "migration_sequence": candidate["migration_sequence"], @@ -1001,50 +1003,3 @@ def migrate_aggregate( except (CelError, KeyError, TypeError, ValueError) as exc: del exc return _failure(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) - - -def migrate_and_dispatch( - aggregate: ArtifactSource, - target_validated_bundle_fingerprint: str, - migration_route: Sequence[str], - artifact_resolver: ArtifactResolver, - delivery: Delivery, - *, - maintenance_mode: bool, - resource_limits: MigrationLimits | None = None, -) -> MigrationDispatchResult: - """Migrate and dispatch as one commit-ready pure result boundary.""" - migrated = migrate_aggregate( - aggregate, - target_validated_bundle_fingerprint, - migration_route, - artifact_resolver, - maintenance_mode=maintenance_mode, - resource_limits=resource_limits, - ) - if migrated.failure is not None: - return _dispatch_failure(migrated.failure.code) - assert migrated.aggregate_bytes is not None - try: - restored = restore_aggregate(migrated.aggregate_bytes, artifact_resolver) - core = dispatch(restored.bundle, restored.state, delivery) - state = core["state"] - if state is None: - raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) - from .wire import aggregate_envelope - - envelope = aggregate_envelope(restored.bundle, state) - encoded = canonical_bytes(envelope) - return MigrationDispatchResult( - envelope, - encoded, - migrated.audit_records, - core["status"], - core["disposition"], - tuple(copy.deepcopy(core["emissions"])), - copy.deepcopy(core["fault"]), - copy.deepcopy(core["rejection"]), - None, - ) - except ArtifactError as exc: - return _dispatch_failure(exc.code) diff --git a/src/determa/state/persistence.py b/src/determa/state/persistence.py new file mode 100644 index 0000000..1b2113b --- /dev/null +++ b/src/determa/state/persistence.py @@ -0,0 +1,400 @@ +"""Production host transaction composition for checkpoint-backed applications.""" + +from __future__ import annotations + +import copy +import json +import sqlite3 +import threading +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +from .checkpoint import serialize_execution_checkpoint +from .codes import ExecutionStoreAdapterFailureCode as AdapterCode +from .errors import ArtifactError +from .host import ExecutionHost, ExecutionHostError, validate_host_profile +from .stores import ( + DURABLE_SINGLE_WRITER, + RESTART_PERSISTENT, + ROOT_IDENTITY_RETENTION, + SHARED_APPLICATION_TRANSACTION, + ExecutionStore, + ExecutionStoreError, + ExecutionStoreTransaction, +) +from .stores.base import checkpoint_metadata +from .wire import ArtifactResolver, canonical_bytes + +_CAPABILITIES = frozenset( + { + RESTART_PERSISTENT, + DURABLE_SINGLE_WRITER, + ROOT_IDENTITY_RETENTION, + SHARED_APPLICATION_TRANSACTION, + } +) + + +class _DocumentCheckpointTransaction(ExecutionStoreTransaction): + def __init__(self, document: dict[str, Any]) -> None: + self.document = document + self._root_instance_id = document["checkpoint"]["root_instance_id"] + + @property + def root_instance_id(self) -> str: + root_instance_id = self._root_instance_id + if not isinstance(root_instance_id, str): + raise ExecutionStoreError("transaction_root_mismatch") + return root_instance_id + + def load(self) -> bytes: + return serialize_execution_checkpoint(self.document["checkpoint"]) + + def insert(self, checkpoint: bytes) -> bool: + del checkpoint + return False + + def replace( + self, + expected_revision: str, + expected_checkpoint_digest: str, + checkpoint: bytes, + ) -> bool: + current = self.document["checkpoint"] + if ( + current["revision"] != expected_revision + or current["execution_checkpoint_digest"] + != expected_checkpoint_digest + ): + return False + root_instance_id, _revision, _digest = checkpoint_metadata(checkpoint) + if root_instance_id != self._root_instance_id: + raise ExecutionStoreError("transaction_root_mismatch") + self.document["checkpoint"] = json.loads(checkpoint) + return True + + +class DurableHostStore(ExecutionStore): + """Execution store extended with one atomic host-document transaction.""" + + @contextmanager + def host_transaction( + self, root_instance_id: str + ) -> Iterator[tuple[dict[str, Any], ExecutionStoreTransaction]]: + raise NotImplementedError + + def snapshot(self, root_instance_id: str) -> bytes: + raise NotImplementedError + + +class MemoryDurableHostStore(DurableHostStore): + """Atomic in-memory host-document store for embedded execution and tests.""" + + def __init__(self, documents: Mapping[str, bytes]) -> None: + self._documents = {key: bytes(value) for key, value in documents.items()} + self._lock = threading.RLock() + + @property + def capabilities(self) -> frozenset[str]: + return _CAPABILITIES + + @contextmanager + def host_transaction( + self, root_instance_id: str + ) -> Iterator[tuple[dict[str, Any], ExecutionStoreTransaction]]: + with self._lock: + source = self._documents.get(root_instance_id) + if source is None: + raise ExecutionStoreError("wrong_root") + document = json.loads(source) + transaction = _DocumentCheckpointTransaction(document) + yield document, transaction + self._documents[root_instance_id] = canonical_bytes(document) + + @contextmanager + def transaction( + self, root_instance_id: str + ) -> Iterator[ExecutionStoreTransaction]: + with self.host_transaction(root_instance_id) as (_document, transaction): + yield transaction + + def snapshot(self, root_instance_id: str) -> bytes: + with self._lock: + try: + return bytes(self._documents[root_instance_id]) + except KeyError as exc: + raise ExecutionStoreError("wrong_root") from exc + + def setup_schema(self) -> None: + return None + + def health(self) -> Mapping[str, Any]: + return {"healthy": True, "record_count": len(self._documents)} + + +class SQLiteDurableHostStore(DurableHostStore): + """SQLite store for one atomic checkpoint, inbox, outbox, and application row.""" + + def __init__(self, path: str | Path) -> None: + self.path = str(path) + if not self.path or self.path == ":memory:": + raise ExecutionStoreError(AdapterCode.INVALID_ADAPTER_CONFIGURATION) + + @property + def capabilities(self) -> frozenset[str]: + return _CAPABILITIES + + def setup_schema(self) -> None: + with sqlite3.connect(self.path) as connection: + connection.execute( + "CREATE TABLE IF NOT EXISTS determa_durable_host_documents (" + "root_instance_id TEXT PRIMARY KEY NOT NULL, document BLOB NOT NULL)" + ) + + def seed(self, root_instance_id: str, source: bytes) -> None: + with sqlite3.connect(self.path) as connection: + connection.execute( + "INSERT INTO determa_durable_host_documents " + "(root_instance_id, document) VALUES (?, ?)", + (root_instance_id, source), + ) + + @contextmanager + def host_transaction( + self, root_instance_id: str + ) -> Iterator[tuple[dict[str, Any], ExecutionStoreTransaction]]: + connection = sqlite3.connect(self.path, isolation_level=None) + try: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT document FROM determa_durable_host_documents " + "WHERE root_instance_id = ?", + (root_instance_id,), + ).fetchone() + if row is None: + raise ExecutionStoreError("wrong_root") + document = json.loads(bytes(row[0])) + transaction = _DocumentCheckpointTransaction(document) + yield document, transaction + connection.execute( + "UPDATE determa_durable_host_documents SET document = ? " + "WHERE root_instance_id = ?", + (canonical_bytes(document), root_instance_id), + ) + connection.commit() + except Exception: + connection.rollback() + raise + finally: + connection.close() + + @contextmanager + def transaction( + self, root_instance_id: str + ) -> Iterator[ExecutionStoreTransaction]: + with self.host_transaction(root_instance_id) as (_document, transaction): + yield transaction + + def snapshot(self, root_instance_id: str) -> bytes: + with sqlite3.connect(self.path) as connection: + row = connection.execute( + "SELECT document FROM determa_durable_host_documents " + "WHERE root_instance_id = ?", + (root_instance_id,), + ).fetchone() + if row is None: + raise ExecutionStoreError("wrong_root") + return bytes(row[0]) + + def validate_schema(self) -> None: + with sqlite3.connect(self.path) as connection: + columns = connection.execute( + "PRAGMA table_info(determa_durable_host_documents)" + ).fetchall() + if [row[1] for row in columns] != ["root_instance_id", "document"]: + raise ExecutionStoreError("execution_store_schema_mismatch") + + def health(self) -> Mapping[str, Any]: + try: + self.validate_schema() + except (sqlite3.Error, ExecutionStoreError): + return {"healthy": False, "schema_ready": False} + return {"healthy": True, "schema_ready": True} + + +class PersistenceHost: + """Coordinate durable delivery processing through a production ExecutionHost.""" + + def __init__( + self, + store: DurableHostStore, + artifact_resolver: ArtifactResolver, + ) -> None: + self.store = store + self.execution_host = ExecutionHost(store, artifact_resolver) + self.calls: list[str] = [] + self.core_calls = 0 + + @staticmethod + def _delivery(request: Mapping[str, Any]) -> dict[str, Any]: + return { + "delivery_mode": "input", + "envelope": copy.deepcopy(request["presented_envelope"]), + "envelope_digest": request["envelope_digest"], + } + + def _preflight(self, request: Mapping[str, Any]) -> None: + self.calls = ["select_scope"] + scope = request["scope"] + if scope["authorization"] != "authorized": + raise ExecutionHostError("invalid_store_scope") + self.calls.append("resolve_artifacts") + target = request.get("transaction_inputs", {}).get( + "target_validated_bundle_fingerprint" + ) + if target is not None: + source = self.execution_host.artifact_resolver.resolve_definition(target) + if source is None: + raise ArtifactError("definition_unavailable") + self.calls.append("validate_capabilities") + if not set(request.get("store_capabilities", [])).issubset( + self.store.capabilities + ): + raise ExecutionHostError(AdapterCode.ADAPTER_CAPABILITY_MISMATCH) + if "host_profile" in request: + validate_host_profile( + self.store, + request["host_profile"], + host_features=frozenset(request.get("host_guarantees", [])), + ) + + def process_v2(self, request: Mapping[str, Any]) -> dict[str, Any]: + self._preflight(request) + root_instance_id = request["expected_checkpoint"]["root_instance_id"] + event_id = request["presented_envelope"]["event_id"] + policy = request["transaction_inputs"]["failure_policy"] + if policy == "permanent_quarantine": + with self.store.host_transaction(root_instance_id) as (document, _tx): + document["quarantine"] = { + "event_id": event_id, + "reason_code": "permanent_processing_failure", + "released": False, + } + document["inbox"].append( + { + "event_id": event_id, + "request_digest": request["envelope_digest"], + "disposition": "quarantined", + } + ) + self.calls.append("quarantine") + return { + "result": "quarantined", + "mutation": "atomic", + "core_calls": 0, + "broker_acknowledged": False, + "code": "permanent_processing_failure", + } + + self.calls.append("begin_transaction") + with self.store.host_transaction(root_instance_id) as (document, transaction): + self.calls.extend(["read_checkpoint", "check_replay"]) + prior = next( + ( + item + for item in document["inbox"] + if item["event_id"] == event_id + ), + None, + ) + released = ( + prior is not None + and prior["disposition"] == "quarantined" + and (document.get("quarantine") or {}).get("event_id") == event_id + and document["quarantine"]["released"] is True + ) + if prior is not None and not released: + if prior["request_digest"] != request["envelope_digest"]: + raise ExecutionHostError("event_id_conflict") + self.calls.append("acknowledge") + return { + "result": "replayed", + "mutation": "none", + "core_calls": 0, + "broker_acknowledged": True, + } + if policy == "transient_retry": + self.calls.append("rollback") + raise ExecutionHostError("transient_processing_failure") + self.calls.append("call_core") + self.core_calls = 1 + expected = request["expected_checkpoint"] + bound = self.execution_host._bound(transaction) + bound.process_delivery_v2( + root_instance_id, + self._delivery(request), + target_validated_bundle_fingerprint=request["transaction_inputs"][ + "target_validated_bundle_fingerprint" + ], + migration_descriptor_digest_route=request["transaction_inputs"][ + "migration_descriptor_digest_route" + ], + expected_revision=expected["revision"], + expected_checkpoint_digest=expected["digest"], + ) + identity = { + "event_id": event_id, + "request_digest": request["envelope_digest"], + "disposition": "committed", + } + if released: + assert prior is not None + prior.clear() + prior.update(identity) + document["quarantine"] = None + else: + document["inbox"].append(identity) + document["application_rows"].update( + request["transaction_inputs"].get("application_writes", {}) + ) + self.calls.extend( + ["stage_checkpoint", "stage_inbox", "stage_outbox", "stage_audit"] + ) + if policy == "inject_pre_commit": + self.calls.append("rollback") + raise ExecutionHostError("injected_pre_commit_failure") + if request["transaction_inputs"].get("application_writes"): + self.calls.append("stage_application_rows") + self.calls.append("commit") + if policy == "inject_post_commit_response_loss": + raise ExecutionHostError("response_lost_after_commit") + self.calls.append("acknowledge") + return { + "result": "committed", + "mutation": "atomic", + "core_calls": 1, + "broker_acknowledged": True, + } + + def release_quarantine_v2(self, request: Mapping[str, Any]) -> dict[str, Any]: + self._preflight(request) + root_instance_id = request["expected_checkpoint"]["root_instance_id"] + with self.store.host_transaction(root_instance_id) as (document, _tx): + quarantine = document.get("quarantine") + if ( + quarantine is None + or quarantine["event_id"] != request["event_id"] + or quarantine["reason_code"] != request["quarantine_reason_code"] + or not request["release_authorization"] + ): + raise ExecutionHostError("invalid_execution_checkpoint") + quarantine["released"] = True + self.calls.append("release_quarantine") + return { + "result": "released", + "mutation": "atomic", + "core_calls": 0, + "broker_acknowledged": False, + } diff --git a/src/determa/state/queueing.py b/src/determa/state/queueing.py index f330a73..db7beb5 100644 --- a/src/determa/state/queueing.py +++ b/src/determa/state/queueing.py @@ -18,6 +18,8 @@ ) from .definition import Bundle, BundleSource, load_bundle from .engine import ( + _create, + _dispatch, _Execution, _normalize_payload, _normalize_value, @@ -25,8 +27,6 @@ _runtime_model, _validate_envelope, _validate_reserved_payload, - create, - dispatch, ) from .errors import ArtifactError, StepFault from .migration import MigrationLimits, migrate_aggregate @@ -86,71 +86,18 @@ def _utf8(value: str) -> bytes: return value.encode("utf-8", errors="strict") -def _project_v1(document: Mapping[str, Any]) -> dict[str, Any]: - result = copy.deepcopy(dict(document)) - result["aggregate_state_schema_version"] = 1 - result.pop("next_acceptance_sequence", None) - result.pop("next_queue_sequence", None) - for runtime in result["runtimes"]: - runtime.pop("ready_mailbox", None) - runtime.pop("deferred_mailbox", None) - result.pop("aggregate_state_digest", None) - result["aggregate_state_digest"] = aggregate_state_digest(result) - return result - - -def _upgrade_document(document: Mapping[str, Any]) -> dict[str, Any]: - result = copy.deepcopy(dict(document)) - result["aggregate_state_schema_version"] = 2 - result["next_acceptance_sequence"] = "0" - result["next_queue_sequence"] = "0" - for runtime in result["runtimes"]: - runtime["ready_mailbox"] = [] - runtime["deferred_mailbox"] = [] - return seal_aggregate_v2(result) - - -def upgrade_aggregate_v1_to_v2( - source: ArtifactSource, definition_resolver: DefinitionResolver -) -> dict[str, Any]: - """Explicitly convert a valid version-1 aggregate to empty version-2 mailboxes.""" - del definition_resolver - document, _ = load_json_artifact(source, "aggregate_state") - if aggregate_state_digest(document) != document["aggregate_state_digest"]: - raise ArtifactError(PersistenceFailureCode.AGGREGATE_STATE_DIGEST_MISMATCH) - return _upgrade_document(document) - - -def downgrade_aggregate_v2_to_v1( - source: ArtifactSource, definition_resolver: DefinitionResolver -) -> dict[str, Any]: - """Explicitly convert a version-2 aggregate only when no queued work exists.""" - del definition_resolver - document, _ = load_json_artifact(source, "aggregate_state_v2") - if aggregate_state_digest(document) != document["aggregate_state_digest"]: - raise ArtifactError(PersistenceFailureCode.AGGREGATE_STATE_DIGEST_MISMATCH) - _validate_mailboxes(document) - if document["next_acceptance_sequence"] != "0" or document["next_queue_sequence"] != "0": - raise ArtifactError(PersistenceFailureCode.MIGRATION_TOTALITY_FAILURE) - if any( - runtime[mailbox] - for runtime in document["runtimes"] - for mailbox in ("ready_mailbox", "deferred_mailbox") - ): - raise ArtifactError(PersistenceFailureCode.MIGRATION_TOTALITY_FAILURE) - return {"result": "success", "aggregate_state": _project_v1(document)} - - def create_aggregate_v2( bundle: Bundle | BundleSource, machine_id: str, root_instance_id: str, creation_id: str, bindings: dict[str, dict[str, Any]] | None = None, + *, + _include_host_evidence: bool = False, ) -> dict[str, Any]: """Create one queue-bearing aggregate and route initialization emissions.""" validated = bundle if isinstance(bundle, Bundle) else load_bundle(bundle) - result = create( + result = _create( validated, machine_id, root_instance_id, @@ -160,7 +107,7 @@ def create_aggregate_v2( ) if result["state"] is None: return {**result, "lifecycle_dispositions": []} - encoded = _upgrade_document(aggregate_envelope(validated, result["state"])) + encoded = aggregate_envelope(validated, result["state"]) root_runtime = next( runtime for runtime in encoded["runtimes"] @@ -173,6 +120,7 @@ def create_aggregate_v2( root_runtime, cast(list[Mapping[str, Any]], result["emissions"]), lifecycle, + include_host_evidence=_include_host_evidence, ) encoded = seal_aggregate_v2(encoded) restore_aggregate_v2(encoded, resolver_for_bundle(validated)) @@ -249,16 +197,12 @@ def _valid_envelope_source( entry: Mapping[str, Any], declaration: Mapping[str, Any] | None, payload: Mapping[str, Any], - *, - allow_legacy: bool, ) -> bool: envelope = entry["envelope"] source = envelope["source"] mode = entry["delivery_mode"] if mode == "input": return bool(source == {"host": True} and envelope["cause_id"] == envelope["event_id"]) - if "legacy_v1_internal" in source: - return allow_legacy event = envelope["event"] if declaration is not None or event == "env": if "runtime" not in source: @@ -369,7 +313,6 @@ def _validate_mailbox_semantics(document: Mapping[str, Any], restored: RestoredA entry, declaration, payload, - allow_legacy=True, ) and _mailbox_payload_is_normalized( restored, runtime, entry, declaration, payload @@ -393,7 +336,7 @@ def restore_aggregate_v2( ): raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) _validate_mailboxes(document) - restored = restore_aggregate(_project_v1(document), definition_resolver) + restored = restore_aggregate(document, definition_resolver) _validate_mailbox_semantics(document, restored) return RestoredAggregate( bundle=restored.bundle, @@ -472,6 +415,91 @@ def _dispatch_code_to_admission(code: DispatchRejectionCode) -> str: }[code] +def _validate_new_deliveries( + restored: RestoredAggregate, deliveries: Sequence[Mapping[str, Any]] +) -> str | None: + document = restored.aggregate_envelope + if any(delivery.get("delivery_mode") not in {"input", "internal"} for delivery in deliveries): + return AdmissionCode.INVALID_DELIVERY_MODE.value + runtime_by_id = {runtime["runtime_id"]: runtime for runtime in document["runtimes"]} + models = BundleModel(restored.bundle) + for delivery in deliveries: + mode = cast(Literal["input", "internal"], delivery["delivery_mode"]) + envelope = delivery["envelope"] + source_value = envelope.get("source") + if ( + (mode == "input" and source_value != {"host": True}) + or (mode == "internal" and not isinstance(source_value, Mapping)) + or (mode == "input" and envelope.get("cause_id") != envelope.get("event_id")) + ): + return AdmissionCode.INVALID_DELIVERY_SOURCE.value + if mode == "internal": + assert isinstance(source_value, Mapping) + source_runtime = source_value.get("runtime") + source_system = source_value.get("system") + valid_runtime_source = source_runtime is not None and any( + runtime["target_identity"] == source_runtime for runtime in document["runtimes"] + ) + valid_system_source = source_system in { + "system:component_completion", + "system:spawned_completion", + "system:component_failure", + "system:spawned_failure", + } + if not valid_runtime_source and not valid_system_source: + return AdmissionCode.INVALID_DELIVERY_SOURCE.value + try: + runtime_id = _runtime_id_for_target(envelope["target"]) + except (KeyError, TypeError): + return AdmissionCode.INVALID_INSTANCE_TARGET.value + target_runtime = runtime_by_id.get(runtime_id) + if target_runtime is None or target_runtime["target_identity"] != envelope["target"]: + return AdmissionCode.INVALID_INSTANCE_TARGET.value + relation = target_runtime["relation"]["kind"] + if target_runtime["status"] != "running": + return ( + AdmissionCode.INACTIVE_COMPONENT_TARGET.value + if relation == "component" + else AdmissionCode.INVALID_INSTANCE_TARGET.value + ) + if mode == "input" and relation == "component": + return AdmissionCode.INVALID_INSTANCE_TARGET.value + try: + native = _native_envelope(delivery) + except (ArtifactError, KeyError, TypeError): + return AdmissionCode.INVALID_PAYLOAD.value + rejection = _validate_envelope( + restored.bundle, + models, + restored.state, + native, + mode, + ) + if rejection is not None: + return _dispatch_code_to_admission(rejection) + machine_id = target_runtime["current_definition"]["machine"]["machine_id"] + machine = restored.bundle.machine(machine_id) + declarations = dict(restored.bundle.raw.get("events") or {}) + declarations.update((machine or {}).get("events") or {}) + declaration = declarations.get(envelope["event"]) + payload = decoded_typed_value(envelope["payload"]) + if not isinstance(payload, Mapping) or not _valid_envelope_source( + document, + delivery, + declaration, + payload, + ): + return AdmissionCode.INVALID_DELIVERY_SOURCE.value + if not _mailbox_payload_is_normalized( + restored, target_runtime, delivery, declaration, payload + ): + return AdmissionCode.INVALID_PAYLOAD.value + expected_digest = _entry_digest(document["root_instance_id"], mode, envelope) + if delivery.get("envelope_digest") != expected_digest: + return AdmissionCode.DELIVERY_DIGEST_MISMATCH.value + return None + + def admit_aggregate_v2( source: ArtifactSource, deliveries: Sequence[Mapping[str, Any]], @@ -546,79 +574,19 @@ def admit_aggregate_v2( "rejection": None, } - models = BundleModel(restored.bundle) + validation_code = _validate_new_deliveries(restored, new_deliveries) + if validation_code is not None: + return _admission_rejection(validation_code, document) accepted: list[dict[str, str]] = [] candidate = copy.deepcopy(document) runtime_by_id = {runtime["runtime_id"]: runtime for runtime in candidate["runtimes"]} for delivery in new_deliveries: mode = delivery.get("delivery_mode") envelope = delivery["envelope"] - if mode not in {"input", "internal"}: - return _admission_rejection(AdmissionCode.INVALID_DELIVERY_MODE.value, document) - source_value = envelope.get("source") - if ( - (mode == "input" and source_value != {"host": True}) - or (mode == "internal" and not isinstance(source_value, Mapping)) - or (mode == "input" and envelope.get("cause_id") != envelope.get("event_id")) - ): - return _admission_rejection(AdmissionCode.INVALID_DELIVERY_SOURCE.value, document) - if mode == "internal": - assert isinstance(source_value, Mapping) - source_runtime = source_value.get("runtime") - source_system = source_value.get("system") - valid_runtime_source = source_runtime is not None and any( - runtime["target_identity"] == source_runtime for runtime in document["runtimes"] - ) - valid_system_source = source_system in { - "system:component_completion", - "system:spawned_completion", - "system:component_failure", - "system:spawned_failure", - } - if not valid_runtime_source and not valid_system_source: - return _admission_rejection(AdmissionCode.INVALID_DELIVERY_SOURCE.value, document) supplied_digest = delivery.get("envelope_digest") expected_digest = _entry_digest(document["root_instance_id"], str(mode), envelope) - if supplied_digest != expected_digest: - return _admission_rejection(AdmissionCode.DELIVERY_DIGEST_MISMATCH.value, document) - try: - native = _native_envelope(delivery) - except (ArtifactError, KeyError, TypeError): - return _admission_rejection(AdmissionCode.INVALID_PAYLOAD.value, document) - try: - runtime_id = _runtime_id_for_target(envelope["target"]) - except (KeyError, TypeError): - return _admission_rejection(AdmissionCode.INVALID_INSTANCE_TARGET.value, document) - target_runtime = runtime_by_id.get(runtime_id) - if target_runtime is None or target_runtime["target_identity"] != envelope["target"]: - return _admission_rejection(AdmissionCode.INVALID_INSTANCE_TARGET.value, document) - rejection = _validate_envelope( - restored.bundle, - models, - restored.state, - native, - cast(Literal["input", "internal"], mode), - ) - if rejection is not None: - return _admission_rejection(_dispatch_code_to_admission(rejection), document) - machine_id = target_runtime["current_definition"]["machine"]["machine_id"] - machine = restored.bundle.machine(machine_id) - declarations = dict(restored.bundle.raw.get("events") or {}) - declarations.update((machine or {}).get("events") or {}) - declaration = declarations.get(envelope["event"]) - payload = decoded_typed_value(envelope["payload"]) - if not isinstance(payload, Mapping) or not _valid_envelope_source( - document, - delivery, - declaration, - payload, - allow_legacy=False, - ): - return _admission_rejection(AdmissionCode.INVALID_DELIVERY_SOURCE.value, document) - if not _mailbox_payload_is_normalized( - restored, target_runtime, delivery, declaration, payload - ): - return _admission_rejection(AdmissionCode.INVALID_PAYLOAD.value, document) + assert supplied_digest == expected_digest + runtime_id = _runtime_id_for_target(envelope["target"]) acceptance = candidate["next_acceptance_sequence"] queue = candidate["next_queue_sequence"] entry = { @@ -699,21 +667,31 @@ def _enqueue_emissions( selected_runtime: Mapping[str, Any], native_emissions: Sequence[Mapping[str, Any]], lifecycle: list[dict[str, Any]], + *, + include_host_evidence: bool = False, ) -> list[dict[str, Any]]: projected: list[dict[str, Any]] = [] runtimes = {runtime["runtime_id"]: runtime for runtime in encoded["runtimes"]} root_completed = _root_status(encoded) == "completed" - for index, emission in enumerate(native_emissions): + for emission in native_emissions: if emission.get("target") == "external": - projected.append( - { - "effect_id": emission["effect_id"], - "sequence": str(emission["sequence"]), - "event": emission["event"], - "payload": typed_value(emission["payload"]), - "correlation_id": emission["correlation_id"], - } - ) + projected_emission = { + "effect_id": emission["effect_id"], + "sequence": str(emission["sequence"]), + "event": emission["event"], + "payload": typed_value(emission["payload"]), + "correlation_id": emission["correlation_id"], + } + if include_host_evidence: + provenance = emission.get("_determa_v2_provenance") + if not isinstance(provenance, Mapping) or not isinstance( + provenance.get("emission_index"), int + ): + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) + projected_emission["_determa_v2_emission_index"] = str( + provenance["emission_index"] + ) + projected.append(projected_emission) continue target = cast(Mapping[str, Any], emission["target"]) target_wire = _wire_target(target) @@ -757,7 +735,9 @@ def _enqueue_emissions( projected.append( { "kind": "internal_mailbox", - "emission_index": str(index), + "emission_index": str( + emission.get("_determa_v2_emission_index", 0) + ), "event_id": emission["event_id"], "acceptance_sequence": acceptance, "queue_sequence": queue, @@ -769,7 +749,9 @@ def _enqueue_emissions( projected.append( { "kind": "internal_disposed", - "emission_index": str(index), + "emission_index": str( + emission.get("_determa_v2_emission_index", 0) + ), "event_id": emission["event_id"], "acceptance_sequence": acceptance, "lifecycle_disposition_index": disposition_index, @@ -784,7 +766,7 @@ def _encode_after_dispatch( before: Mapping[str, Any], mailboxes: Mapping[str, tuple[list[Any], list[Any]]], ) -> dict[str, Any]: - result = _upgrade_document(aggregate_envelope(bundle, state)) + result = aggregate_envelope(bundle, state) result["next_acceptance_sequence"] = before["next_acceptance_sequence"] result["next_queue_sequence"] = before["next_queue_sequence"] for runtime in result["runtimes"]: @@ -874,6 +856,8 @@ def step_aggregate_v2( source: ArtifactSource, target_runtime_id: str, definition_resolver: DefinitionResolver, + *, + _include_host_evidence: bool = False, ) -> dict[str, Any]: """Process at most the selected runtime's ready-mailbox head.""" restored = restore_aggregate_v2(source, definition_resolver) @@ -906,7 +890,7 @@ def step_aggregate_v2( mailboxes = _mailbox_maps(before) ready, deferred = mailboxes[target_runtime_id] native = _native_envelope(selected) - result = dispatch( + result = _dispatch( restored.bundle, restored.state, {selected["delivery_mode"]: native}, @@ -952,7 +936,12 @@ def step_aggregate_v2( encoded = _encode_after_dispatch(restored.bundle, state, before, mailboxes) overflow_lifecycle: list[dict[str, Any]] = [] emissions = _enqueue_emissions( - encoded, before, wire_runtime, execution.emissions, overflow_lifecycle + encoded, + before, + wire_runtime, + execution.emissions, + overflow_lifecycle, + include_host_evidence=_include_host_evidence, ) encoded_fault = copy.deepcopy(runtime["fault"]) encoded_fault["step_sequence"] = str(encoded_fault["step_sequence"]) @@ -1012,6 +1001,7 @@ def step_aggregate_v2( wire_runtime, cast(list[Mapping[str, Any]], result["emissions"]), lifecycle, + include_host_evidence=_include_host_evidence, ) if ( disposition == DispositionCode.HANDLED.value @@ -1056,28 +1046,6 @@ def resolver_for_bundle(bundle: Bundle | BundleSource) -> MemoryArtifactResolver return MemoryArtifactResolver(definitions={validated.fingerprint: validated}) -class _V1MigrationResolver: - def __init__( - self, - parent: DefinitionResolver, - descriptors: Mapping[str, Mapping[str, Any]], - ) -> None: - self.parent = parent - self.descriptors = descriptors - - def resolve_definition(self, fingerprint: str) -> Bundle | BundleSource | None: - return self.parent.resolve_definition(fingerprint) - - def definition_is_trusted(self, fingerprint: str) -> bool: - return self.parent.definition_is_trusted(fingerprint) - - def resolve_migration_descriptor(self, digest: str) -> ArtifactSource | None: - return self.descriptors.get(digest) - - def migration_descriptor_is_trusted(self, digest: str) -> bool: - return digest in self.descriptors - - def _queue_rule( descriptor: Mapping[str, Any], runtime: Mapping[str, Any], entry: Mapping[str, Any] ) -> Mapping[str, Any] | None: @@ -1153,7 +1121,6 @@ def migrate_aggregate_v2( if len(migration_route) > limits.maximum_chain_length: raise ArtifactError(PersistenceFailureCode.MIGRATION_RESOURCE_LIMIT_EXCEEDED) descriptors: list[dict[str, Any]] = [] - base_descriptors: dict[str, Mapping[str, Any]] = {} for digest in migration_route: source = artifact_resolver.resolve_migration_descriptor(digest) if source is None or not artifact_resolver.migration_descriptor_is_trusted(digest): @@ -1161,9 +1128,6 @@ def migrate_aggregate_v2( descriptor, _ = load_json_artifact(source, "migration_descriptor_v2") if migration_descriptor_digest(descriptor) != digest: raise ArtifactError(PersistenceFailureCode.INVALID_MIGRATION_DESCRIPTOR) - base = descriptor["base_descriptor"] - base_digest = base["migration_descriptor_digest"] - base_descriptors[base_digest] = base descriptors.append(descriptor) if not descriptors: if ( @@ -1182,8 +1146,8 @@ def migrate_aggregate_v2( return empty_result if len({item["migration_descriptor_digest"] for item in descriptors}) != len(descriptors): raise ArtifactError(PersistenceFailureCode.MIGRATION_ROUTE_MISMATCH) - fingerprints = [descriptors[0]["base_descriptor"]["source_validated_bundle_fingerprint"]] + [ - descriptor["base_descriptor"]["target_validated_bundle_fingerprint"] + fingerprints = [descriptors[0]["source_validated_bundle_fingerprint"]] + [ + descriptor["target_validated_bundle_fingerprint"] for descriptor in descriptors ] if ( @@ -1191,14 +1155,13 @@ def migrate_aggregate_v2( or fingerprints[-1] != target_validated_bundle_fingerprint or len(set(fingerprints)) != len(fingerprints) or any( - left["base_descriptor"]["target_validated_bundle_fingerprint"] - != right["base_descriptor"]["source_validated_bundle_fingerprint"] + left["target_validated_bundle_fingerprint"] + != right["source_validated_bundle_fingerprint"] for left, right in zip(descriptors, descriptors[1:], strict=False) ) ): raise ArtifactError(PersistenceFailureCode.MIGRATION_ROUTE_MISMATCH) - resolver = cast(Any, _V1MigrationResolver(artifact_resolver, base_descriptors)) candidate = copy.deepcopy(restored.aggregate_envelope) dispositions: list[dict[str, Any]] = [] disposed_entries: list[dict[str, Any]] = [] @@ -1206,24 +1169,21 @@ def migrate_aggregate_v2( for descriptor in descriptors: before_digest = candidate["aggregate_state_digest"] source_runtimes = candidate["runtimes"] - base = descriptor["base_descriptor"] migration = migrate_aggregate( - _project_v1(candidate), - base["target_validated_bundle_fingerprint"], - [base["migration_descriptor_digest"]], - resolver, + candidate, + descriptor["target_validated_bundle_fingerprint"], + [descriptor["migration_descriptor_digest"]], + artifact_resolver, maintenance_mode=maintenance_mode, resource_limits=limits, ) if not migration.succeeded or migration.aggregate_envelope is None: assert migration.failure is not None raise ArtifactError(migration.failure.code) - hop = _upgrade_document(migration.aggregate_envelope) - hop["next_acceptance_sequence"] = candidate["next_acceptance_sequence"] - hop["next_queue_sequence"] = candidate["next_queue_sequence"] + hop = copy.deepcopy(migration.aggregate_envelope) target_runtimes = {runtime["runtime_id"]: runtime for runtime in hop["runtimes"]} target_bundle_source = artifact_resolver.resolve_definition( - base["target_validated_bundle_fingerprint"] + descriptor["target_validated_bundle_fingerprint"] ) if target_bundle_source is None: raise ArtifactError(PersistenceFailureCode.TARGET_DEFINITION_UNAVAILABLE) @@ -1232,7 +1192,10 @@ def migrate_aggregate_v2( if isinstance(target_bundle_source, Bundle) else load_bundle(target_bundle_source) ) - target_state = restore_aggregate(_project_v1(hop), artifact_resolver).state + target_state = restore_aggregate(hop, artifact_resolver).state + for runtime in hop["runtimes"]: + runtime["ready_mailbox"] = [] + runtime["deferred_mailbox"] = [] for source_runtime in source_runtimes: runtime = target_runtimes.get(source_runtime["runtime_id"]) for mailbox_name in ("ready_mailbox", "deferred_mailbox"): @@ -1277,12 +1240,16 @@ def migrate_aggregate_v2( restore_aggregate_v2(candidate, artifact_resolver) audits.append( { - "migration_audit_record_schema_version": 1, + "migration_audit_record_schema_version": 2, "root_instance_id": candidate["root_instance_id"], "root_runtime_id": candidate["root_runtime_id"], "migration_sequence": candidate["migration_sequence"], - "source_validated_bundle_fingerprint": base["source_validated_bundle_fingerprint"], - "target_validated_bundle_fingerprint": base["target_validated_bundle_fingerprint"], + "source_validated_bundle_fingerprint": descriptor[ + "source_validated_bundle_fingerprint" + ], + "target_validated_bundle_fingerprint": descriptor[ + "target_validated_bundle_fingerprint" + ], "migration_descriptor_digest": descriptor["migration_descriptor_digest"], "source_aggregate_state_digest": before_digest, "target_aggregate_state_digest": candidate["aggregate_state_digest"], diff --git a/src/determa/state/wire.py b/src/determa/state/wire.py index 410009e..9ac7b94 100644 --- a/src/determa/state/wire.py +++ b/src/determa/state/wire.py @@ -137,18 +137,12 @@ def put_migration_descriptor( self, digest: str, descriptor: ArtifactSource, *, trusted: bool = True ) -> None: candidate, _ = strict_json(descriptor) - kind = ( - "migration_descriptor_v2" - if isinstance(candidate, dict) - and candidate.get("migration_descriptor_schema_version") == 2 - else "migration_descriptor" - ) - document, _ = load_json_artifact(candidate, kind) + document, _ = load_json_artifact(candidate, "migration_descriptor_v2") if migration_descriptor_digest(document) != digest: raise ArtifactError(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) existing = self._migration_descriptors.get(digest) if existing is not None: - current, _ = load_json_artifact(existing, kind) + current, _ = load_json_artifact(existing, "migration_descriptor_v2") if canonical_bytes(current) != canonical_bytes(document): raise ArtifactError(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) else: @@ -171,7 +165,7 @@ def _object_without_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: result: dict[str, Any] = {} for key, value in pairs: if key in result: - raise ArtifactError("duplicate_json_name") + raise ArtifactError(LoadCode.DUPLICATE_KEY) result[key] = value return result @@ -338,13 +332,9 @@ def decimal(value: Any, *, positive: bool = False) -> int: @cache def artifact_schema(kind: str) -> dict[str, Any]: filename = { - "aggregate_state": "aggregate-state.schema.json", "aggregate_state_v2": "aggregate-state-v2.schema.json", - "migration_descriptor": "migration-descriptor.schema.json", "migration_descriptor_v2": "migration-descriptor-v2.schema.json", - "aggregate_state_package": "aggregate-state-package.schema.json", "aggregate_state_package_v2": "aggregate-state-package-v2.schema.json", - "execution_checkpoint": "execution-checkpoint.schema.json", "execution_checkpoint_v2": "execution-checkpoint-v2.schema.json", "core_step_result_v2": "core-step-result-v2.schema.json", }[kind] @@ -357,13 +347,9 @@ def _schema_registry() -> Any: registry = Registry() for kind in ( - "aggregate_state", "aggregate_state_v2", - "migration_descriptor", "migration_descriptor_v2", - "aggregate_state_package", "aggregate_state_package_v2", - "execution_checkpoint", "execution_checkpoint_v2", "core_step_result_v2", ): @@ -376,14 +362,6 @@ def _format_code(document: Any, kind: str) -> str | None: if not isinstance(document, dict): return None definitions = { - "aggregate_state": ( - "aggregate_state_format", - "determa.aggregate_state", - "aggregate_state_schema_version", - 1, - PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_FORMAT, - PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_SCHEMA_VERSION, - ), "aggregate_state_v2": ( "aggregate_state_format", "determa.aggregate_state", @@ -392,14 +370,6 @@ def _format_code(document: Any, kind: str) -> str | None: PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_FORMAT, PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_SCHEMA_VERSION, ), - "migration_descriptor": ( - "migration_descriptor_format", - "determa.aggregate_migration", - "migration_descriptor_schema_version", - 1, - PersistenceCode.UNSUPPORTED_MIGRATION_DESCRIPTOR_FORMAT, - PersistenceCode.UNSUPPORTED_MIGRATION_DESCRIPTOR_SCHEMA_VERSION, - ), "migration_descriptor_v2": ( "migration_descriptor_format", "determa.aggregate_migration", @@ -408,14 +378,6 @@ def _format_code(document: Any, kind: str) -> str | None: PersistenceCode.UNSUPPORTED_MIGRATION_DESCRIPTOR_FORMAT, PersistenceCode.UNSUPPORTED_MIGRATION_DESCRIPTOR_SCHEMA_VERSION, ), - "aggregate_state_package": ( - "aggregate_state_package_format", - "determa.aggregate_state_package", - "aggregate_state_package_schema_version", - 1, - PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_PACKAGE_FORMAT, - PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_PACKAGE_SCHEMA_VERSION, - ), "aggregate_state_package_v2": ( "aggregate_state_package_format", "determa.aggregate_state_package", @@ -424,14 +386,6 @@ def _format_code(document: Any, kind: str) -> str | None: PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_PACKAGE_FORMAT, PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_PACKAGE_SCHEMA_VERSION, ), - "execution_checkpoint": ( - "execution_checkpoint_format", - "determa.execution_checkpoint", - "execution_checkpoint_schema_version", - 1, - CheckpointCode.UNSUPPORTED_EXECUTION_CHECKPOINT_FORMAT, - CheckpointCode.UNSUPPORTED_EXECUTION_CHECKPOINT_SCHEMA_VERSION, - ), "execution_checkpoint_v2": ( "execution_checkpoint_format", "determa.execution_checkpoint", @@ -469,14 +423,12 @@ def load_json_artifact(source: ArtifactSource, kind: str) -> tuple[dict[str, Any try: document, raw = strict_json(source) except ArtifactError as exc: + if exc.code in {LoadCode.DUPLICATE_KEY, LoadCode.INVALID_UNICODE}: + raise code = { - "aggregate_state": PersistenceCode.INVALID_AGGREGATE_STATE, "aggregate_state_v2": PersistenceCode.INVALID_AGGREGATE_STATE, - "migration_descriptor": PersistenceCode.INVALID_MIGRATION_DESCRIPTOR, "migration_descriptor_v2": PersistenceCode.INVALID_MIGRATION_DESCRIPTOR, - "aggregate_state_package": PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE, "aggregate_state_package_v2": PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE, - "execution_checkpoint": CheckpointCode.INVALID_EXECUTION_CHECKPOINT, "execution_checkpoint_v2": CheckpointCode.INVALID_EXECUTION_CHECKPOINT, "core_step_result_v2": "invalid_core_step_result", }[kind] @@ -489,13 +441,9 @@ def load_json_artifact(source: ArtifactSource, kind: str) -> tuple[dict[str, Any validator = jsonschema.Draft202012Validator(artifact_schema(kind), registry=_schema_registry()) if not isinstance(document, dict) or next(validator.iter_errors(document), None) is not None: code = { - "aggregate_state": PersistenceCode.INVALID_AGGREGATE_STATE, "aggregate_state_v2": PersistenceCode.INVALID_AGGREGATE_STATE, - "migration_descriptor": PersistenceCode.INVALID_MIGRATION_DESCRIPTOR, "migration_descriptor_v2": PersistenceCode.INVALID_MIGRATION_DESCRIPTOR, - "aggregate_state_package": PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE, "aggregate_state_package_v2": PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE, - "execution_checkpoint": CheckpointCode.INVALID_EXECUTION_CHECKPOINT, "execution_checkpoint_v2": CheckpointCode.INVALID_EXECUTION_CHECKPOINT, "core_step_result_v2": "invalid_core_step_result", }[kind] @@ -506,19 +454,13 @@ def load_json_artifact(source: ArtifactSource, kind: str) -> tuple[dict[str, Any def aggregate_state_digest(document: Mapping[str, Any]) -> str: body = copy.deepcopy(dict(document)) body.pop("aggregate_state_digest", None) - version = body.get("aggregate_state_schema_version") - domain = ( - "determa-aggregate-state-digest-2" if version == 2 else "determa-aggregate-state-digest-1" - ) - return hash_value([domain, body]) + return hash_value(["determa-aggregate-state-digest-2", body]) def migration_descriptor_digest(document: Mapping[str, Any]) -> str: body = copy.deepcopy(dict(document)) body.pop("migration_descriptor_digest", None) - version = body.get("migration_descriptor_schema_version") - domain = "determa-migration-descriptor-2" if version == 2 else "determa-migration-descriptor-1" - return hash_value([domain, body]) + return hash_value(["determa-migration-descriptor-2", body]) def normalized_definition_attachment(bundle: Bundle) -> dict[str, Any]: @@ -801,7 +743,7 @@ def aggregate_envelope(bundle: Bundle | BundleSource, state: dict[str, Any]) -> ) document: dict[str, Any] = { "aggregate_state_format": "determa.aggregate_state", - "aggregate_state_schema_version": 1, + "aggregate_state_schema_version": 2, "machine_format": 1, "validated_bundle_fingerprint": validated.fingerprint, "namespace": validated.namespace, @@ -811,11 +753,17 @@ def aggregate_envelope(bundle: Bundle | BundleSource, state: dict[str, Any]) -> "creation_id": state["creation_id"], "root_runtime_id": state["root_runtime_id"], "migration_sequence": str(state.get("migration_sequence", 0)), + "next_acceptance_sequence": "0", + "next_queue_sequence": "0", "next_logical_step_sequence": str(state["next_logical_step_sequence"]), "next_output_sequence": str(state["next_output_sequence"]), "runtimes": sorted( ( - _runtime_wire(validated, models, state, runtime) + { + **_runtime_wire(validated, models, state, runtime), + "ready_mailbox": [], + "deferred_mailbox": [], + } for runtime in state["runtimes"].values() ), key=( @@ -1161,7 +1109,7 @@ def restore_aggregate( source: ArtifactSource, definition_resolver: DefinitionResolver ) -> RestoredAggregate: """Verify and restore one portable aggregate without changing the source.""" - document, raw = load_json_artifact(source, "aggregate_state") + document, raw = load_json_artifact(source, "aggregate_state_v2") if aggregate_state_digest(document) != document["aggregate_state_digest"]: raise ArtifactError(PersistenceCode.AGGREGATE_STATE_DIGEST_MISMATCH) fingerprint = document["validated_bundle_fingerprint"] @@ -1225,16 +1173,7 @@ def restore_aggregate_package( source: ArtifactSource, artifact_resolver: ArtifactResolver ) -> RestoredAggregatePackage: """Verify a transport package and seed one mutable resolver atomically.""" - candidate, _raw = strict_json(source) - aggregate_version = ( - candidate.get("aggregate_state", {}).get("aggregate_state_schema_version") - if isinstance(candidate, dict) and isinstance(candidate.get("aggregate_state"), dict) - else None - ) - version = 2 if aggregate_version == 2 else 1 - package_kind = "aggregate_state_package_v2" if version == 2 else "aggregate_state_package" - descriptor_kind = "migration_descriptor_v2" if version == 2 else "migration_descriptor" - document, _raw = load_json_artifact(candidate, package_kind) + document, _raw = load_json_artifact(source, "aggregate_state_package_v2") definitions: dict[str, Bundle] = {} descriptors: dict[str, dict[str, Any]] = {} try: @@ -1266,7 +1205,9 @@ def restore_aggregate_package( for digest, descriptor in descriptors.items(): existing_descriptor = artifact_resolver.resolve_migration_descriptor(digest) if existing_descriptor is not None: - current_descriptor, _ = load_json_artifact(existing_descriptor, descriptor_kind) + current_descriptor, _ = load_json_artifact( + existing_descriptor, "migration_descriptor_v2" + ) if canonical_bytes(current_descriptor) != canonical_bytes(descriptor): raise ArtifactError(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) except (ArtifactError, KeyError, TypeError, ValidationError) as exc: @@ -1288,12 +1229,9 @@ def restore_aggregate_package( raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE) overlay = _PackageResolver(artifact_resolver, definitions, descriptors) try: - if version == 2: - from .queueing import restore_aggregate_v2 + from .queueing import restore_aggregate_v2 - aggregate = restore_aggregate_v2(document["aggregate_state"], overlay) - else: - aggregate = restore_aggregate(document["aggregate_state"], overlay) + aggregate = restore_aggregate_v2(document["aggregate_state"], overlay) store_definition = cast(Callable[[str, Bundle], None], put_definition) store_descriptor = cast(Callable[[str, Mapping[str, Any]], None], put_descriptor) for fingerprint, bundle in definitions.items(): diff --git a/tests/test_checkpoint_host.py b/tests/test_checkpoint_host.py deleted file mode 100644 index 3a1aea1..0000000 --- a/tests/test_checkpoint_host.py +++ /dev/null @@ -1,691 +0,0 @@ -from __future__ import annotations - -import copy -from contextlib import contextmanager - -import pytest - -from determa.state import ( - EPHEMERAL, - SHARED_APPLICATION_TRANSACTION, - ArtifactError, - ExecutionHost, - ExecutionHostError, - MemoryArtifactResolver, - MemoryExecutionStore, - StagedExecutionResult, - aggregate_shape_fingerprint, - delivery_request_digest, - load_bundle, - portable_envelope, - restore_execution_checkpoint, - seal_execution_checkpoint, -) -from determa.state.wire import migration_descriptor_digest - -MACHINE = """ -format: 1 -namespace: test.execution_checkpoint -events: - increment: - direction: input - payload: - amount: { type: int, required: true } - deliberate_fault: { direction: input } -machines: - - machine_id: counter - version: 1 - root: - type: simple - variables: - count: { type: int, init: 0 } - on_events: - increment: - action: - - assign: { count: "count + event.payload.amount" } - deliberate_fault: - action: - - assign: { count: "count / 0" } -""" - -TERMINAL_MACHINE = """ -format: 1 -namespace: test.execution_checkpoint_terminal -machines: - - machine_id: terminal - version: 1 - root: - type: composite - initial: { transition_to: done } - states: - done: { type: final } -""" - - -def _host( - *, - store: MemoryExecutionStore | None = None, - fault_injector=None, -) -> tuple[ExecutionHost, MemoryExecutionStore]: - bundle = load_bundle(MACHINE) - resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) - selected = store or MemoryExecutionStore() - return ( - ExecutionHost( - selected, resolver, fault_injector=fault_injector - ), - selected, - ) - - -def _created(host: ExecutionHost, root: str = "root") -> dict: - result = host.create( - load_bundle(MACHINE), "counter", root, f"{root}-create", {} - ) - assert result["result"] == "committed" - restored = host.read_checkpoint(root) - assert restored is not None - return restored.document - - -def _candidate(checkpoint: dict, event_id: str = "increment-1") -> dict: - aggregate = checkpoint["root_record"]["aggregate_state"] - envelope = portable_envelope( - "increment", - event_id, - { - "root": { - "root_instance_id": checkpoint["root_instance_id"], - "root_runtime_id": aggregate["root_runtime_id"], - } - }, - {"amount": 1}, - ) - return { - "root_instance_id": checkpoint["root_instance_id"], - "delivery_mode": "input", - "origin": {"kind": "host_input"}, - "envelope": envelope, - "envelope_digest": delivery_request_digest( - checkpoint["root_instance_id"], "input", envelope - ), - } - - -def test_creation_response_loss_replays_the_committed_receipt() -> None: - def response_loss(boundary: str) -> None: - if boundary == "after_commit_before_response": - raise ExecutionHostError("response_lost_after_commit") - - host, store = _host(fault_injector=response_loss) - with pytest.raises(ExecutionHostError, match="response_lost_after_commit"): - host.create(load_bundle(MACHINE), "counter", "root", "create", {}) - - replay, _ = _host(store=store) - result = replay.create( - load_bundle(MACHINE), "counter", "root", "create", {} - ) - assert result["receipt"]["receipt_sequence"] == "0" - assert replay.read_checkpoint("root") is not None - - -def test_pre_commit_failure_leaves_no_checkpoint() -> None: - def rollback(boundary: str) -> None: - if boundary == "before_commit": - raise ExecutionHostError("injected_pre_commit_failure") - - host, _ = _host(fault_injector=rollback) - with pytest.raises(ExecutionHostError, match="injected_pre_commit_failure"): - host.create(load_bundle(MACHINE), "counter", "root", "create", {}) - assert host.read_checkpoint("root") is None - - -class _SharedMemoryStore(MemoryExecutionStore): - def __init__(self, *, cross_root: bool = False) -> None: - super().__init__() - self.business_rows: list[str] = [] - self.cross_root = cross_root - - @property - def capabilities(self) -> frozenset[str]: - return frozenset({EPHEMERAL, SHARED_APPLICATION_TRANSACTION}) - - @contextmanager - def shared_transaction(self, root_instance_id: str): - transaction_root = "other-root" if self.cross_root else root_instance_id - pending_business_rows: list[str] = [] - with self.transaction(transaction_root) as transaction: - yield pending_business_rows, transaction - self.business_rows.extend(pending_business_rows) - - -def test_host_owned_shared_transaction_returns_only_after_commit() -> None: - store = _SharedMemoryStore() - host, _ = _host(store=store) - staged_results: list[StagedExecutionResult] = [] - - def callback(connection, execution) -> None: - connection.append("business-row") - staged = execution.create( - load_bundle(MACHINE), "counter", "create", {} - ) - assert not isinstance(staged, dict) - staged_results.append(staged) - - result = host.run_shared_transaction("root", callback) - - assert result["result"] == "committed" - assert staged_results == [StagedExecutionResult("create")] - assert store.business_rows == ["business-row"] - assert host.read_checkpoint("root") is not None - - -def test_host_owned_shared_transaction_rollback_returns_no_committed_result() -> None: - store = _SharedMemoryStore() - host, _ = _host(store=store) - - def callback(connection, execution) -> None: - connection.append("business-row") - execution.create(load_bundle(MACHINE), "counter", "create", {}) - raise RuntimeError("application rollback") - - with pytest.raises(RuntimeError, match="application rollback"): - host.run_shared_transaction("root", callback) - - assert store.business_rows == [] - assert host.read_checkpoint("root") is None - - -def test_shared_transaction_response_loss_occurs_only_after_outer_commit() -> None: - def response_loss(boundary: str) -> None: - if boundary == "after_commit_before_response": - raise ExecutionHostError("response_lost_after_commit") - - store = _SharedMemoryStore() - host, _ = _host(store=store, fault_injector=response_loss) - - def callback(connection, execution) -> None: - connection.append("business-row") - execution.create(load_bundle(MACHINE), "counter", "create", {}) - - with pytest.raises(ExecutionHostError, match="response_lost_after_commit"): - host.run_shared_transaction("root", callback) - - replay, _ = _host(store=store) - result = replay.create( - load_bundle(MACHINE), "counter", "root", "create", {} - ) - assert result["result"] == "committed" - assert store.business_rows == ["business-row"] - - -def test_shared_transaction_rejects_a_store_transaction_bound_to_another_root() -> None: - host, _ = _host(store=_SharedMemoryStore(cross_root=True)) - with pytest.raises(ExecutionHostError) as error: - host.run_shared_transaction("root", lambda _connection, _execution: None) - assert error.value.code == "transaction_root_mismatch" - - -def test_shared_transaction_accepts_exactly_one_host_operation() -> None: - store = _SharedMemoryStore() - host, _ = _host(store=store) - - def callback(_connection, execution) -> None: - execution.create(load_bundle(MACHINE), "counter", "create", {}) - execution.create(load_bundle(MACHINE), "counter", "create", {}) - - with pytest.raises(ExecutionHostError) as error: - host.run_shared_transaction("root", callback) - assert error.value.code == "shared_transaction_operation_conflict" - assert host.read_checkpoint("root") is None - - -def test_accept_process_and_replay_use_durable_host_receipts() -> None: - host, _ = _host() - created = _created(host) - candidate = _candidate(created) - pending = host.accept_delivery( - "root", - candidate, - expected_revision=created["revision"], - expected_checkpoint_digest=created["execution_checkpoint_digest"], - ) - assert pending["result"] == "pending" - - accepted = host.read_checkpoint("root") - assert accepted is not None - committed = host.process_pending_delivery( - "root", - candidate, - expected_revision=accepted.document["revision"], - expected_checkpoint_digest=accepted.document[ - "execution_checkpoint_digest" - ], - ) - replay = host.accept_delivery( - "root", - candidate, - expected_revision=created["revision"], - expected_checkpoint_digest=created["execution_checkpoint_digest"], - ) - assert replay == committed - - -def test_delivery_replay_precedes_origin_and_tombstone_validation() -> None: - host, _ = _host() - created = _created(host) - aggregate = created["root_record"]["aggregate_state"] - envelope = portable_envelope( - "deliberate_fault", - "fault-replay", - { - "root": { - "root_instance_id": "root", - "root_runtime_id": aggregate["root_runtime_id"], - } - }, - {}, - ) - candidate = { - "root_instance_id": "root", - "delivery_mode": "input", - "origin": {"kind": "host_input"}, - "envelope": envelope, - } - committed = host.foreground_process_delivery( - "root", - candidate, - expected_revision=created["revision"], - expected_checkpoint_digest=created["execution_checkpoint_digest"], - ) - faulted = host.read_checkpoint("root") - assert faulted is not None - host.tombstone_root( - "root", - "tombstone", - expected_revision=faulted.document["revision"], - expected_checkpoint_digest=faulted.document[ - "execution_checkpoint_digest" - ], - ) - invalid_origin = copy.deepcopy(candidate) - invalid_origin["origin"] = {"kind": "invalid"} - replay = host.accept_delivery( - "root", - invalid_origin, - expected_revision=created["revision"], - expected_checkpoint_digest=created["execution_checkpoint_digest"], - ) - assert replay == committed - - invalid_mode = copy.deepcopy(candidate) - invalid_mode["delivery_mode"] = "invalid" - mode_conflict = host.accept_delivery( - "root", - invalid_mode, - expected_revision=created["revision"], - expected_checkpoint_digest=created["execution_checkpoint_digest"], - ) - assert mode_conflict == { - "result": "not_accepted", - "failure": {"code": "event_id_conflict"}, - } - - conflicting = copy.deepcopy(invalid_origin) - conflicting["envelope"]["event"] = "increment" - conflict = host.accept_delivery( - "root", - conflicting, - expected_revision=created["revision"], - expected_checkpoint_digest=created["execution_checkpoint_digest"], - ) - assert conflict == { - "result": "not_accepted", - "failure": {"code": "event_id_conflict"}, - } - - -def test_checkpoint_digest_mismatch_is_classified_after_structure() -> None: - host, _ = _host() - checkpoint = _created(host) - checkpoint["execution_checkpoint_digest"] = "sha256:" + ("0" * 64) - resolver = host.artifact_resolver - with pytest.raises(ArtifactError) as error: - restore_execution_checkpoint(checkpoint, resolver) - assert error.value.code == "execution_checkpoint_digest_mismatch" - - -def test_unknown_checkpoint_member_is_structurally_rejected() -> None: - host, _ = _host() - checkpoint = _created(host) - checkpoint["extra"] = True - with pytest.raises(ArtifactError) as error: - restore_execution_checkpoint(checkpoint, host.artifact_resolver) - assert error.value.code == "invalid_execution_checkpoint" - - -def test_root_deletion_is_unsupported_and_preserves_bytes() -> None: - host, _ = _host() - checkpoint = _created(host) - result = host.delete_checkpoint( - "root", - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - assert result == { - "result": "unsupported", - "failure": {"code": "physical_deletion_unsupported"}, - } - restored = host.read_checkpoint("root") - assert restored is not None - assert restored.document == checkpoint - - -def test_bounded_retention_cannot_attest_unallocated_receipts() -> None: - host, _ = _host() - checkpoint = _created(host) - with pytest.raises(ExecutionHostError) as error: - host.update_replay_retention( - "root", - { - "mode": "bounded", - "permanent_replay_eligible": False, - "pruned_through_receipt_sequence": "1", - "policy_identifier": "bounded-test", - }, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint[ - "execution_checkpoint_digest" - ], - ) - assert error.value.code == "invalid_execution_checkpoint" - - -def test_checkpoint_restore_does_not_mutate_caller_document() -> None: - host, _ = _host() - checkpoint = _created(host) - original = copy.deepcopy(checkpoint) - restore_execution_checkpoint(checkpoint, host.artifact_resolver) - assert checkpoint == original - - -def test_restore_rejects_creation_status_inconsistent_with_aggregate() -> None: - host, _ = _host() - checkpoint = _created(host) - checkpoint["operation_receipts"][0]["status"] = "completed" - mutated = seal_execution_checkpoint(checkpoint) - with pytest.raises(ArtifactError) as error: - restore_execution_checkpoint(mutated, host.artifact_resolver) - assert error.value.code == "invalid_execution_checkpoint" - - -def test_restore_rejects_delivery_fault_inconsistent_with_aggregate() -> None: - host, _ = _host() - created = _created(host) - aggregate = created["root_record"]["aggregate_state"] - envelope = portable_envelope( - "deliberate_fault", - "fault-1", - { - "root": { - "root_instance_id": "root", - "root_runtime_id": aggregate["root_runtime_id"], - } - }, - {}, - ) - host.foreground_process_delivery( - "root", - { - "root_instance_id": "root", - "delivery_mode": "input", - "origin": {"kind": "host_input"}, - "envelope": envelope, - }, - expected_revision=created["revision"], - expected_checkpoint_digest=created["execution_checkpoint_digest"], - ) - restored = host.read_checkpoint("root") - assert restored is not None - checkpoint = restored.document - checkpoint["operation_receipts"][-1]["outcome"]["fault"]["code"] = ( - "different_fault" - ) - mutated = seal_execution_checkpoint(checkpoint) - with pytest.raises(ArtifactError) as error: - restore_execution_checkpoint(mutated, host.artifact_resolver) - assert error.value.code == "invalid_execution_checkpoint" - - -def test_restore_rejects_completed_tombstone_relabeled_faulted() -> None: - bundle = load_bundle(TERMINAL_MACHINE) - resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) - host = ExecutionHost(MemoryExecutionStore(), resolver) - host.create(bundle, "terminal", "terminal-root", "create", {}) - completed = host.read_checkpoint("terminal-root") - assert completed is not None - host.tombstone_root( - "terminal-root", - "tombstone", - expected_revision=completed.document["revision"], - expected_checkpoint_digest=completed.document[ - "execution_checkpoint_digest" - ], - ) - restored = host.read_checkpoint("terminal-root") - assert restored is not None - checkpoint = restored.document - checkpoint["root_record"]["terminal_status"] = "faulted" - mutated = seal_execution_checkpoint(checkpoint) - with pytest.raises(ArtifactError) as error: - restore_execution_checkpoint(mutated, resolver) - assert error.value.code == "invalid_execution_checkpoint" - - -def test_bounded_pruning_through_latest_receipt_remains_valid() -> None: - host, _ = _host() - created = _created(host) - host.foreground_process_delivery( - "root", - _candidate(created), - expected_revision=created["revision"], - expected_checkpoint_digest=created["execution_checkpoint_digest"], - ) - processed = host.read_checkpoint("root") - assert processed is not None - result = host.update_replay_retention( - "root", - { - "mode": "bounded", - "permanent_replay_eligible": False, - "pruned_through_receipt_sequence": "1", - "policy_identifier": "bounded-test", - }, - expected_revision=processed.document["revision"], - expected_checkpoint_digest=processed.document[ - "execution_checkpoint_digest" - ], - ) - assert result["result"] == "committed" - bounded = host.read_checkpoint("root") - assert bounded is not None - assert [ - receipt["receipt_sequence"] - for receipt in bounded.document["operation_receipts"] - ] == ["0"] - - -def test_maintenance_replay_precedes_tombstone_eligibility() -> None: - bundle = load_bundle(TERMINAL_MACHINE) - resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) - host = ExecutionHost(MemoryExecutionStore(), resolver) - host.create(bundle, "terminal", "terminal-root", "create", {}) - created = host.read_checkpoint("terminal-root") - assert created is not None - source_digest = created.document["root_record"]["aggregate_state"][ - "aggregate_state_digest" - ] - committed = host.maintenance_migration( - "terminal-root", - "migration", - bundle.fingerprint, - [], - source_aggregate_state_digest=source_digest, - expected_revision=created.document["revision"], - expected_checkpoint_digest=created.document[ - "execution_checkpoint_digest" - ], - ) - migrated = host.read_checkpoint("terminal-root") - assert migrated is not None - invalid_receipt = copy.deepcopy(migrated.document) - invalid_receipt["operation_receipts"][-1][ - "resulting_aggregate_state_digest" - ] = "sha256:" + ("0" * 64) - with pytest.raises(ArtifactError) as invalid_error: - restore_execution_checkpoint( - seal_execution_checkpoint(invalid_receipt), resolver - ) - assert invalid_error.value.code == "invalid_execution_checkpoint" - host.tombstone_root( - "terminal-root", - "tombstone", - expected_revision=migrated.document["revision"], - expected_checkpoint_digest=migrated.document[ - "execution_checkpoint_digest" - ], - ) - replay = host.maintenance_migration( - "terminal-root", - "migration", - bundle.fingerprint, - [], - source_aggregate_state_digest=source_digest, - expected_revision=created.document["revision"], - expected_checkpoint_digest=created.document[ - "execution_checkpoint_digest" - ], - ) - assert replay == committed - with pytest.raises(ExecutionHostError) as conflict: - host.maintenance_migration( - "terminal-root", - "migration", - bundle.fingerprint, - [], - source_aggregate_state_digest=source_digest, - expected_revision=created.document["revision"], - expected_checkpoint_digest=created.document[ - "execution_checkpoint_digest" - ], - maintenance_mode=False, - ) - assert conflict.value.code == "operation_id_conflict" - - -def test_restore_checks_status_evidence_across_maintenance_migration() -> None: - source = load_bundle(MACHINE) - target = load_bundle(MACHINE.replace( - "namespace: test.execution_checkpoint", - "namespace: test.execution_checkpoint\nmeta: {release: target}", - )) - shape = aggregate_shape_fingerprint(source) - assert aggregate_shape_fingerprint(target) == shape - descriptor = { - "migration_descriptor_format": "determa.aggregate_migration", - "migration_descriptor_schema_version": 1, - "source_machine_format": 1, - "target_machine_format": 1, - "source_validated_bundle_fingerprint": source.fingerprint, - "target_validated_bundle_fingerprint": target.fingerprint, - "source_aggregate_shape_fingerprint": shape, - "target_aggregate_shape_fingerprint": shape, - "mode": "compatible", - "mappings": { - "machines": [], - "active_states": [], - "variables": [], - "history": [], - "components": [], - "owned_runtimes": [], - "lifetime_holders": [], - "counters": [], - }, - "terminal_policy": {"completed": "preserve", "faulted": "preserve"}, - "resource_requirements": { - "maximum_transformed_output_bytes": "0", - "maximum_cel_expression_length": "0", - "maximum_cel_ast_nodes": "0", - "maximum_cel_evaluation_steps": "0", - }, - } - descriptor["migration_descriptor_digest"] = migration_descriptor_digest( - descriptor - ) - resolver = MemoryArtifactResolver( - definitions={ - source.fingerprint: source, - target.fingerprint: target, - }, - migration_descriptors={ - descriptor["migration_descriptor_digest"]: descriptor, - }, - ) - host = ExecutionHost(MemoryExecutionStore(), resolver) - host.create(source, "counter", "migration-root", "create", {}) - created = host.read_checkpoint("migration-root") - assert created is not None - source_digest = created.document["root_record"]["aggregate_state"][ - "aggregate_state_digest" - ] - host.maintenance_migration( - "migration-root", - "migration", - target.fingerprint, - [descriptor["migration_descriptor_digest"]], - source_aggregate_state_digest=source_digest, - expected_revision=created.document["revision"], - expected_checkpoint_digest=created.document[ - "execution_checkpoint_digest" - ], - ) - migrated = host.read_checkpoint("migration-root") - assert migrated is not None - checkpoint = migrated.document - assert ( - checkpoint["operation_receipts"][0]["resulting_aggregate_state_digest"] - != checkpoint["root_record"]["aggregate_state"]["aggregate_state_digest"] - ) - checkpoint["operation_receipts"][0]["status"] = "completed" - with pytest.raises(ArtifactError) as error: - restore_execution_checkpoint( - seal_execution_checkpoint(checkpoint), resolver - ) - assert error.value.code == "invalid_execution_checkpoint" - - -def test_maintenance_request_requires_exact_source_digest() -> None: - host, _ = _host() - created = _created(host) - bundle = load_bundle(MACHINE) - with pytest.raises(TypeError): - host.maintenance_migration( - "root", - "migration", - bundle.fingerprint, - [], - expected_revision=created["revision"], - expected_checkpoint_digest=created[ - "execution_checkpoint_digest" - ], - ) - - -def test_oversized_checkpoint_decimal_is_closed_invalidity() -> None: - host, _ = _host() - checkpoint = _created(host) - checkpoint["revision"] = "9" * 5000 - mutated = seal_execution_checkpoint(checkpoint) - with pytest.raises(ArtifactError) as error: - restore_execution_checkpoint(mutated, host.artifact_resolver) - assert error.value.code == "invalid_execution_checkpoint" diff --git a/tests/test_checkpoint_v2.py b/tests/test_checkpoint_v2.py new file mode 100644 index 0000000..0c433b4 --- /dev/null +++ b/tests/test_checkpoint_v2.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import copy + +import pytest + +import determa.state.checkpoint_v2 as checkpoint_v2 +from determa.state import ( + ArtifactError, + ExecutionHost, + ExecutionHostError, + MemoryArtifactResolver, + MemoryExecutionStore, + create_checkpoint_v2, + delivery_request_digest, + load_bundle, + portable_envelope, + restore_execution_checkpoint_v2, + seal_execution_checkpoint, +) +from determa.state.queueing import admit_aggregate_v2, step_aggregate_v2 + +MACHINE = """ +format: 1 +namespace: test.checkpoint_v2 +events: + increment: + direction: input + payload: + amount: { type: int, required: true } + work_requested: + direction: output + work_completed: + direction: input + correlates_to: work_requested + payload: + result: { type: string, required: true } +machines: + - machine_id: counter + version: 1 + root: + type: simple + variables: + count: { type: int, init: 0 } + on_events: + increment: + action: + - assign: { count: "count + event.payload.amount" } +""" + +TERMINAL_MACHINE = """ +format: 1 +namespace: test.checkpoint_v2_terminal +machines: + - machine_id: terminal + version: 1 + root: + type: final +""" + +OUTPUT_MACHINE = """ +format: 1 +namespace: test.checkpoint_v2_output +events: + trigger: + direction: input + output_record: + direction: output + payload: + index: { type: int, required: true } +machines: + - machine_id: outputter + version: 1 + root: + type: simple + on_events: + trigger: + action: + - send: + event: output_record + to: { external: true } + payload: { index: "0" } + correlation_id: "'batch'" + - send: + event: output_record + to: { external: true } + payload: { index: "1" } + correlation_id: "'batch'" +""" + + +def _bundle_and_resolver(): + bundle = load_bundle(MACHINE) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + return bundle, resolver + + +def _host() -> tuple[ExecutionHost, MemoryExecutionStore]: + _bundle, resolver = _bundle_and_resolver() + store = MemoryExecutionStore() + return ExecutionHost(store, resolver), store + + +def _created_checkpoint() -> tuple[dict, MemoryArtifactResolver]: + bundle, resolver = _bundle_and_resolver() + return create_checkpoint_v2(bundle, "counter", "root", "create", {}), resolver + + +def _delivery( + checkpoint: dict, + event: str, + event_id: str, + payload: dict, + *, + mode: str = "input", + correlation_id: str | None = None, +) -> dict: + aggregate = checkpoint["root_record"]["aggregate_state"] + root = next( + runtime + for runtime in aggregate["runtimes"] + if runtime["runtime_id"] == aggregate["root_runtime_id"] + ) + envelope = portable_envelope( + event, + event_id, + root["target_identity"], + payload, + correlation_id=correlation_id, + ) + return { + "delivery_mode": mode, + "envelope": envelope, + "envelope_digest": delivery_request_digest("root", mode, envelope), + } + + +def test_v2_is_the_only_supported_checkpoint_schema() -> None: + checkpoint, resolver = _created_checkpoint() + checkpoint["execution_checkpoint_schema_version"] = 1 + + with pytest.raises(ArtifactError) as error: + restore_execution_checkpoint_v2(checkpoint, resolver) + + assert error.value.code == "unsupported_execution_checkpoint_schema_version" + + +def test_tombstone_response_is_exact_and_replay_is_read_only() -> None: + bundle = load_bundle(TERMINAL_MACHINE) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + store = MemoryExecutionStore() + host = ExecutionHost(store, resolver) + host.create_v2(bundle, "terminal", "root", "create", {}) + before = host.read_checkpoint("root") + assert before is not None + + committed = host.tombstone_root_v2( + "root", + "tombstone", + expected_revision=before.document["revision"], + expected_checkpoint_digest=before.document["execution_checkpoint_digest"], + ) + assert set(committed) == {"result", "tombstone"} + assert committed["result"] == "tombstoned" + assert committed["tombstone"]["status"] == "tombstone" + after = host.read_checkpoint("root") + assert after is not None + + replay = host.tombstone_root_v2( + "root", + "tombstone", + expected_revision="stale", + expected_checkpoint_digest="sha256:" + "0" * 64, + ) + replay_state = host.read_checkpoint("root") + assert replay == committed + assert replay_state is not None + assert replay_state.source_bytes == after.source_bytes + + +def test_empty_maintenance_receipt_records_target_and_replays_before_cas() -> None: + bundle, _resolver = _bundle_and_resolver() + host, _store = _host() + host.create_v2(bundle, "counter", "root", "create", {}) + before = host.read_checkpoint("root") + assert before is not None + + committed = host.maintenance_migration_v2( + "root", + "migration-1", + bundle.fingerprint, + [], + expected_revision=before.document["revision"], + expected_checkpoint_digest=before.document["execution_checkpoint_digest"], + ) + receipt = committed["receipt"] + assert receipt["target_validated_bundle_fingerprint"] == bundle.fingerprint + assert receipt["result_code"] == "migration_no_operation" + assert receipt["migration_sequences"] == [] + + replay = host.maintenance_migration_v2( + "root", + "migration-1", + bundle.fingerprint, + [], + expected_revision="stale", + expected_checkpoint_digest="sha256:" + "0" * 64, + ) + assert replay == committed + + +def test_same_maintenance_operation_id_with_changed_target_conflicts() -> None: + bundle, _resolver = _bundle_and_resolver() + host, _store = _host() + host.create_v2(bundle, "counter", "root", "create", {}) + before = host.read_checkpoint("root") + assert before is not None + host.maintenance_migration_v2( + "root", + "migration-1", + bundle.fingerprint, + [], + expected_revision=before.document["revision"], + expected_checkpoint_digest=before.document["execution_checkpoint_digest"], + ) + + with pytest.raises(ExecutionHostError) as error: + host.maintenance_migration_v2( + "root", + "migration-1", + "sha256:" + "1" * 64, + [], + expected_revision="stale", + expected_checkpoint_digest="sha256:" + "0" * 64, + ) + + assert error.value.code == "operation_id_conflict" + + +def test_restore_rejects_maintenance_receipt_revision_regression() -> None: + bundle, resolver = _bundle_and_resolver() + host = ExecutionHost(MemoryExecutionStore(), resolver) + host.create_v2(bundle, "counter", "root", "create", {}) + before = host.read_checkpoint("root") + assert before is not None + host.maintenance_migration_v2( + "root", + "migration-1", + bundle.fingerprint, + [], + expected_revision=before.document["revision"], + expected_checkpoint_digest=before.document["execution_checkpoint_digest"], + ) + committed = host.read_checkpoint("root") + assert committed is not None + forged = copy.deepcopy(committed.document) + forged["operation_receipts"][1]["committed_revision"] = "0" + forged = seal_execution_checkpoint(forged) + + with pytest.raises(ArtifactError) as error: + restore_execution_checkpoint_v2(forged, resolver) + + assert error.value.code == "invalid_execution_checkpoint" + + +def test_checkpoint_admission_validates_before_calling_aggregate_core( + monkeypatch: pytest.MonkeyPatch, +) -> None: + checkpoint, resolver = _created_checkpoint() + before = copy.deepcopy(checkpoint) + delivery = _delivery(checkpoint, "increment", "invalid-payload", {}) + + def unexpected_core_call(*_args: object, **_kwargs: object) -> None: + raise AssertionError("aggregate admission core must not run") + + monkeypatch.setattr(checkpoint_v2, "admit_aggregate_v2", unexpected_core_call) + with pytest.raises(ArtifactError) as error: + checkpoint_v2.admit_checkpoint_v2( + checkpoint, + [delivery], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + + assert error.value.code == "invalid_payload" + assert checkpoint == before + + +def test_checkpoint_admission_payload_failure_precedes_missing_correlation() -> None: + checkpoint, resolver = _created_checkpoint() + delivery = _delivery(checkpoint, "work_completed", "invalid-contract", {}) + + with pytest.raises(ArtifactError) as error: + checkpoint_v2.admit_checkpoint_v2( + checkpoint, + [delivery], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + + assert error.value.code == "invalid_payload" + + +def test_checkpoint_admission_mode_failure_precedes_later_member_contract_failure() -> None: + checkpoint, resolver = _created_checkpoint() + invalid_event = _delivery(checkpoint, "not_declared", "invalid-event", {}) + invalid_mode = _delivery( + checkpoint, + "increment", + "invalid-mode", + {"amount": 1}, + mode="unsupported", + ) + + with pytest.raises(ArtifactError) as error: + checkpoint_v2.admit_checkpoint_v2( + checkpoint, + [invalid_event, invalid_mode], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + + assert error.value.code == "invalid_delivery_mode" + + +def test_checkpoint_empty_mailbox_does_not_call_aggregate_core( + monkeypatch: pytest.MonkeyPatch, +) -> None: + checkpoint, resolver = _created_checkpoint() + aggregate = checkpoint["root_record"]["aggregate_state"] + root_runtime_id = aggregate["root_runtime_id"] + before = copy.deepcopy(checkpoint) + + def unexpected_core_call(*_args: object, **_kwargs: object) -> None: + raise AssertionError("aggregate step core must not run") + + monkeypatch.setattr(checkpoint_v2, "step_aggregate_v2", unexpected_core_call) + result = checkpoint_v2.step_checkpoint_v2( + checkpoint, + root_runtime_id, + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + + assert result["result"] == "not_committed" + assert result["step_result"]["disposition"] == "not_runnable" + assert result["checkpoint"] == before + assert checkpoint == before + + +def test_external_action_ordinals_are_private_core_evidence_for_checkpoint_receipts() -> None: + bundle = load_bundle(OUTPUT_MACHINE) + resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + checkpoint = create_checkpoint_v2(bundle, "outputter", "root", "create", {}) + delivery = _delivery(checkpoint, "trigger", "trigger-1", {}) + + admitted_core = admit_aggregate_v2( + checkpoint["root_record"]["aggregate_state"], [delivery], resolver + ) + core_result = step_aggregate_v2( + admitted_core["state"], + admitted_core["state"]["root_runtime_id"], + resolver, + ) + assert len(core_result["emissions"]) == 2 + assert all( + "_determa_v2_emission_index" not in emission + for emission in core_result["emissions"] + ) + + admitted_checkpoint = checkpoint_v2.admit_checkpoint_v2( + checkpoint, + [delivery], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + processed = checkpoint_v2.step_checkpoint_v2( + admitted_checkpoint, + admitted_checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + resolver, + expected_revision=admitted_checkpoint["revision"], + expected_checkpoint_digest=admitted_checkpoint["execution_checkpoint_digest"], + ) + + references = processed["operation_receipts"][-1]["emission_references"] + assert [reference["emission_index"] for reference in references] == ["0", "0"] + assert all( + "_determa_v2_emission_index" not in pending["intent"] + for pending in processed["pending_outbox_intents"] + ) + restore_execution_checkpoint_v2(processed, resolver) diff --git a/tests/test_engine.py b/tests/test_engine.py index ec3c642..e27f21f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -5,7 +5,7 @@ import pytest -from determa.state import Bundle, create, dispatch, load_bundle +from determa.state import Bundle, load_bundle from determa.state.engine import ( _cause_id, _component_runtime_id, @@ -13,6 +13,8 @@ _root_runtime_id, _spawned_runtime_id, ) +from determa.state.engine import _create as create +from determa.state.engine import _dispatch as dispatch from determa.state.errors import StepFault from determa.state.model import BundleModel @@ -45,6 +47,22 @@ - assign: { count: "count / 0" } """ + +def test_public_execution_api_is_queue_bearing_only() -> None: + import determa.state as state + + assert {"create", "admit", "step"}.issubset(state.__all__) + for removed in ( + "dispatch", + "Delivery", + "Result", + "create_aggregate_v2", + "admit_aggregate_v2", + "step_aggregate_v2", + ): + assert removed not in state.__all__ + assert not hasattr(state, removed) + BINDING_BUNDLE = """ format: 1 namespace: example.binding diff --git a/tests/test_execution_stores.py b/tests/test_execution_stores.py index 0f7c9a6..34c0e71 100644 --- a/tests/test_execution_stores.py +++ b/tests/test_execution_stores.py @@ -27,10 +27,28 @@ SQLiteExecutionStore, bundled_execution_store_registry, load_bundle, - portable_envelope, ) -from .test_checkpoint_host import MACHINE +MACHINE = """ +format: 1 +namespace: test.execution_checkpoint +events: + increment: + direction: input + payload: + amount: { type: int, required: true } +machines: + - machine_id: counter + version: 1 + root: + type: simple + variables: + count: { type: int, init: 0 } + on_events: + increment: + action: + - assign: { count: "count + event.payload.amount" } +""" _STRONG_RETENTION_CAPABILITIES = { ROOT_IDENTITY_RETENTION, @@ -47,7 +65,7 @@ def _resolver() -> MemoryArtifactResolver: def _create(store: ExecutionStore, root: str = "root") -> ExecutionHost: host = ExecutionHost(store, _resolver()) - host.create(load_bundle(MACHINE), "counter", root, f"{root}-create", {}) + host.create_v2(load_bundle(MACHINE), "counter", root, f"{root}-create", {}) return host @@ -68,12 +86,50 @@ def test_shared_adapter_contract_round_trip(tmp_path: Path, index: int) -> None: host = _create(store) restored = host.read_checkpoint("root") assert restored is not None - replay = host.create( + replay = host.create_v2( load_bundle(MACHINE), "counter", "root", "root-create", {} ) assert replay["receipt"]["receipt_sequence"] == "0" +@pytest.mark.parametrize("index", range(3)) +def test_adapters_commit_and_replay_keyed_v2_maintenance_receipts( + tmp_path: Path, index: int +) -> None: + store = _factories(tmp_path)[index]() + store.setup_schema() + bundle = load_bundle(MACHINE) + host = ExecutionHost(store, _resolver()) + host.create_v2(bundle, "counter", "root", "root-create", {}) + checkpoint = host.read_checkpoint("root") + assert checkpoint is not None + aggregate = checkpoint.document["root_record"]["aggregate_state"] + + committed = host.maintenance_migration_v2( + "root", + "adapter-empty-migration", + aggregate["validated_bundle_fingerprint"], + [], + expected_revision=checkpoint.document["revision"], + expected_checkpoint_digest=checkpoint.document[ + "execution_checkpoint_digest" + ], + ) + replay = host.maintenance_migration_v2( + "root", + "adapter-empty-migration", + aggregate["validated_bundle_fingerprint"], + [], + expected_revision=checkpoint.document["revision"], + expected_checkpoint_digest=checkpoint.document[ + "execution_checkpoint_digest" + ], + ) + + assert replay == committed + assert host.read_checkpoint("root").document["revision"] == "1" + + @pytest.mark.parametrize("index", range(3)) def test_store_transactions_reject_checkpoint_bytes_for_another_root( tmp_path: Path, index: int @@ -163,27 +219,13 @@ def test_concurrent_stale_writer_cannot_overwrite( document = checkpoint.document aggregate = document["root_record"]["aggregate_state"] - def process(event_id: str) -> str: - candidate = { - "root_instance_id": "root", - "delivery_mode": "input", - "origin": {"kind": "host_input"}, - "envelope": portable_envelope( - "increment", - event_id, - { - "root": { - "root_instance_id": "root", - "root_runtime_id": aggregate["root_runtime_id"], - } - }, - {"amount": 1}, - ), - } + def process(operation_id: str) -> str: try: - ExecutionHost(store, _resolver()).foreground_process_delivery( + ExecutionHost(store, _resolver()).maintenance_migration_v2( "root", - candidate, + operation_id, + aggregate["validated_bundle_fingerprint"], + [], expected_revision=document["revision"], expected_checkpoint_digest=document[ "execution_checkpoint_digest" @@ -194,7 +236,7 @@ def process(event_id: str) -> str: return "committed" with ThreadPoolExecutor(max_workers=2) as executor: - outcomes = sorted(executor.map(process, ["event-a", "event-b"])) + outcomes = sorted(executor.map(process, ["operation-a", "operation-b"])) assert outcomes == ["checkpoint_revision_conflict", "committed"] @@ -368,7 +410,7 @@ def test_sqlite_persists_policy_and_forbids_native_root_or_policy_mutation( assert host.read_checkpoint("bank-root") is not None with pytest.raises(ExecutionHostError) as recreate: - host.create( + host.create_v2( load_bundle(MACHINE), "counter", "bank-root", "replacement", {} ) assert recreate.value.code == "creation_id_conflict" @@ -463,13 +505,13 @@ def test_configured_sqlite_satisfies_bank_and_outbox_profiles( _resolver(), profile="strict_durable_outbox", host_features={ - "atomic_checkpoint_processing", + "atomic_accept_process", "outbox_worker", "total_outbox_lifecycle", "retain_unresolved_outbox", }, ) - checkpoint = host.create( + checkpoint = host.create_v2( load_bundle(MACHINE), "counter", "bank-root", "create", {} ) assert checkpoint["result"] == "committed" @@ -515,10 +557,10 @@ def test_configured_sqlite_satisfies_compact_outbox_profile( _resolver(), profile="compact_durable_outbox", host_features={ - "atomic_checkpoint_processing", + "atomic_accept_process", "outbox_worker", "total_outbox_lifecycle", - "retain_referenced_effect_tombstones", + "retain_receipt_references", }, ) with pytest.raises(ExecutionHostError) as error: diff --git a/tests/test_persistence.py b/tests/test_persistence.py deleted file mode 100644 index 7856976..0000000 --- a/tests/test_persistence.py +++ /dev/null @@ -1,312 +0,0 @@ -from __future__ import annotations - -import copy - -import pytest - -from determa.state import ( - ArtifactError, - MemoryArtifactResolver, - aggregate_envelope, - aggregate_shape_fingerprint, - create, - load_bundle, - migrate_aggregate, - restore_aggregate, - serialize_aggregate, -) -from determa.state.wire import ( - aggregate_state_digest, - canonical_bytes, - load_json_artifact, - migration_descriptor_digest, - strict_json, -) - -SOURCE = """ -format: 1 -namespace: tests.persistence -machines: - - machine_id: job - version: 1 - root: - variables: - count: {type: int, init: 3} -""" - -COMPONENT_SOURCE = """ -format: 1 -namespace: tests.persistence_components -machines: - - machine_id: owner - root: - type: parallel - components: - - component_id: worker - machine_id: worker - - component_id: second_worker - machine_id: worker - - machine_id: worker - root: {} -""" - -SPAWN_SOURCE = """ -format: 1 -namespace: tests.persistence_spawn -machines: - - machine_id: owner - root: - variables: - worker_reference: - type: instance_reference - machine_id: worker - nullable: true - init: null - entry: - - spawn: - machine_id: worker - bind_to: worker_reference - - machine_id: worker - version: 1 - root: {} -""" - - -def _resolver_for(bundle): - return MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) - - -def _redigest(document): - document["aggregate_state_digest"] = aggregate_state_digest(document) - return canonical_bytes(document) - - -LEGACY_SNAPSHOT = b'''{ - "machine": { - "id": "legacy_job", - "version": "0.0.6" - }, - "configuration": ["idle"], - "extended_state_variables": { - "attempts": 1 - } -}\n''' - - -def test_selected_legacy_migration_descriptor_without_discriminator_is_unsupported() -> None: - source = bytes(LEGACY_SNAPSHOT) - resolver = MemoryArtifactResolver() - - with pytest.raises(ArtifactError) as raised: - resolver.put_migration_descriptor("sha256:" + "0" * 64, source) - - assert raised.value.code == "unsupported_migration_descriptor_format" - assert source == LEGACY_SNAPSHOT - - -@pytest.mark.parametrize( - ("kind", "code"), - [ - ("aggregate_state", "unsupported_aggregate_state_format"), - ("aggregate_state_package", "unsupported_aggregate_state_package_format"), - ("execution_checkpoint", "unsupported_execution_checkpoint_format"), - ], -) -def test_selected_legacy_artifact_decoder_uses_its_exact_format_code( - kind: str, code: str -) -> None: - with pytest.raises(ArtifactError) as raised: - load_json_artifact(LEGACY_SNAPSHOT, kind) - - assert raised.value.code == code - - -def test_aggregate_round_trip_is_canonical_and_does_not_mutate_state() -> None: - bundle = load_bundle(SOURCE) - created = create(bundle, "job", "job-1", "create-1", {}) - state = created["state"] - assert state is not None - snapshot = copy.deepcopy(state) - - encoded = serialize_aggregate(bundle, state) - resolver = MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) - restored = restore_aggregate(encoded, resolver) - - assert state == snapshot - assert restored.aggregate_envelope == aggregate_envelope(bundle, restored.state) - assert restored.canonical_bytes == encoded - assert canonical_bytes(restored.aggregate_envelope) == encoded - - -def test_restore_rejects_root_header_version_mismatch() -> None: - bundle = load_bundle(SOURCE) - created = create(bundle, "job", "job-1", "create-1", {}) - assert created["state"] is not None - document = aggregate_envelope(bundle, created["state"]) - document["root_machine_version"] = "2" - - with pytest.raises(ArtifactError) as raised: - restore_aggregate(_redigest(document), _resolver_for(bundle)) - - assert raised.value.code == "invalid_aggregate_state" - - -@pytest.mark.parametrize( - ("field", "forged"), - [ - ("root_instance_id", "forged-root"), - ("owner_runtime_id", "sha256:" + "0" * 64), - ("component_id", "forged_component"), - ("component_runtime_id", "sha256:" + "1" * 64), - ("activation_sequence", "7"), - ], -) -def test_restore_rejects_forged_component_target_identity( - field: str, forged: str -) -> None: - bundle = load_bundle(COMPONENT_SOURCE) - created = create(bundle, "owner", "owner-1", "create-1", {}) - assert created["state"] is not None - document = aggregate_envelope(bundle, created["state"]) - runtime = next( - item for item in document["runtimes"] if item["relation"]["kind"] == "component" - ) - runtime["target_identity"]["component"][field] = forged - - with pytest.raises(ArtifactError) as raised: - restore_aggregate(_redigest(document), _resolver_for(bundle)) - - assert raised.value.code == "invalid_aggregate_state" - - -@pytest.mark.parametrize( - ("field", "forged"), - [ - ("root_instance_id", "forged-root"), - ("instance_id", "sha256:" + "2" * 64), - ("machine_id", "owner"), - ("machine_version", "7"), - ], -) -def test_restore_rejects_forged_spawned_target_identity( - field: str, forged: str -) -> None: - bundle = load_bundle(SPAWN_SOURCE) - created = create(bundle, "owner", "owner-1", "create-1", {}) - assert created["state"] is not None - document = aggregate_envelope(bundle, created["state"]) - runtime = next( - item - for item in document["runtimes"] - if item["relation"]["kind"] == "owned_spawned_instance" - ) - runtime["target_identity"]["spawned_instance"][field] = forged - - with pytest.raises(ArtifactError) as raised: - restore_aggregate(_redigest(document), _resolver_for(bundle)) - - assert raised.value.code == "invalid_aggregate_state" - - -def test_migration_definition_byte_limit_uses_typed_normalized_bundle() -> None: - version = 9_007_199_254_740_992 - source = load_bundle( - f""" -format: 1 -namespace: tests.persistence_unsafe_version -meta: {{release: source}} -machines: - - machine_id: job - version: {version} - root: {{}} -""" - ) - target = load_bundle( - f""" -format: 1 -namespace: tests.persistence_unsafe_version -meta: {{release: target}} -events: - notice: {{direction: internal}} -machines: - - machine_id: job - version: {version} - root: {{}} -""" - ) - shape = aggregate_shape_fingerprint(source) - assert shape == aggregate_shape_fingerprint(target) - descriptor = { - "migration_descriptor_format": "determa.aggregate_migration", - "migration_descriptor_schema_version": 1, - "source_machine_format": 1, - "target_machine_format": 1, - "source_validated_bundle_fingerprint": source.fingerprint, - "target_validated_bundle_fingerprint": target.fingerprint, - "source_aggregate_shape_fingerprint": shape, - "target_aggregate_shape_fingerprint": shape, - "mode": "compatible", - "mappings": { - "machines": [], - "active_states": [], - "variables": [], - "history": [], - "components": [], - "owned_runtimes": [], - "lifetime_holders": [], - "counters": [], - }, - "terminal_policy": {"completed": "preserve", "faulted": "preserve"}, - "resource_requirements": { - "maximum_transformed_output_bytes": "0", - "maximum_cel_expression_length": "0", - "maximum_cel_ast_nodes": "0", - "maximum_cel_evaluation_steps": "0", - }, - } - descriptor["migration_descriptor_digest"] = migration_descriptor_digest(descriptor) - resolver = MemoryArtifactResolver( - definitions={source.fingerprint: source, target.fingerprint: target}, - migration_descriptors={ - descriptor["migration_descriptor_digest"]: descriptor, - }, - ) - created = create(source, "job", "job-1", "create-1", {}) - assert created["state"] is not None - - result = migrate_aggregate( - serialize_aggregate(source, created["state"]), - target.fingerprint, - [descriptor["migration_descriptor_digest"]], - resolver, - maintenance_mode=False, - ) - - assert result.failure is None - assert result.aggregate_envelope is not None - assert result.aggregate_envelope["root_machine_version"] == str(version) - - -def test_untrusted_definition_fails_closed() -> None: - bundle = load_bundle(SOURCE) - created = create(bundle, "job", "job-1", "create-1", {}) - assert created["state"] is not None - encoded = serialize_aggregate(bundle, created["state"]) - resolver = MemoryArtifactResolver( - definitions={bundle.fingerprint: bundle}, trusted_definitions=[] - ) - - with pytest.raises(ArtifactError, match="definition_untrusted") as raised: - restore_aggregate(encoded, resolver) - - assert raised.value.code == "definition_untrusted" - - -def test_strict_json_rejects_duplicate_members_and_nonfinite_numbers() -> None: - with pytest.raises(ArtifactError) as duplicate: - strict_json(b'{"value":1,"value":2}') - assert duplicate.value.code == "duplicate_json_name" - - with pytest.raises(ArtifactError) as nonfinite: - strict_json(b'{"value":NaN}') - assert nonfinite.value.code == "invalid_json_value" diff --git a/tests/test_postgresql_store.py b/tests/test_postgresql_store.py index 6e7382a..9cc0934 100644 --- a/tests/test_postgresql_store.py +++ b/tests/test_postgresql_store.py @@ -18,10 +18,9 @@ PostgreSQLExecutionStore, StagedExecutionResult, load_bundle, - portable_envelope, ) -from .test_checkpoint_host import MACHINE, _host +from .test_execution_stores import MACHINE, _resolver _STRONG_RETENTION_CAPABILITIES = { ROOT_IDENTITY_RETENTION, @@ -48,36 +47,21 @@ def test_postgresql_cas_and_shared_native_transaction() -> None: psycopg = pytest.importorskip("psycopg") store = _store() store.setup_schema() - local_host, _ = _host() - resolver = local_host.artifact_resolver + resolver = _resolver() host = ExecutionHost(store, resolver) - host.create(load_bundle(MACHINE), "counter", "root", "create", {}) + host.create_v2(load_bundle(MACHINE), "counter", "root", "create", {}) checkpoint = host.read_checkpoint("root") assert checkpoint is not None document = checkpoint.document aggregate = document["root_record"]["aggregate_state"] - def process(event_id: str) -> str: - candidate = { - "root_instance_id": "root", - "delivery_mode": "input", - "origin": {"kind": "host_input"}, - "envelope": portable_envelope( - "increment", - event_id, - { - "root": { - "root_instance_id": "root", - "root_runtime_id": aggregate["root_runtime_id"], - } - }, - {"amount": 1}, - ), - } + def process(operation_id: str) -> str: try: - ExecutionHost(store, resolver).foreground_process_delivery( + ExecutionHost(store, resolver).maintenance_migration_v2( "root", - candidate, + operation_id, + aggregate["validated_bundle_fingerprint"], + [], expected_revision=document["revision"], expected_checkpoint_digest=document[ "execution_checkpoint_digest" @@ -88,7 +72,7 @@ def process(event_id: str) -> str: return "committed" with ThreadPoolExecutor(max_workers=2) as executor: - outcomes = sorted(executor.map(process, ["event-a", "event-b"])) + outcomes = sorted(executor.map(process, ["operation-a", "operation-b"])) assert outcomes == ["checkpoint_revision_conflict", "committed"] application_table = f"determa_application_test_{uuid.uuid4().hex}" @@ -102,10 +86,10 @@ def commit_callback(connection, execution) -> None: f"INSERT INTO {application_table} (root_instance_id) VALUES (%s)", ("shared-root",), ) - staged = execution.create( + staged = execution.create_v2( load_bundle(MACHINE), "counter", "create", {} ) - assert staged == StagedExecutionResult("create") + assert staged == StagedExecutionResult("create_v2") committed = host.run_shared_transaction("shared-root", commit_callback) assert committed["result"] == "committed" @@ -118,12 +102,47 @@ def commit_callback(connection, execution) -> None: value = value.decode("ascii") assert value == "shared-root" + host.create_v2( + load_bundle(MACHINE), + "counter", + "migration-root", + "migration-root-create", + {}, + ) + migration_checkpoint = host.read_checkpoint("migration-root") + assert migration_checkpoint is not None + migration_aggregate = migration_checkpoint.document["root_record"][ + "aggregate_state" + ] + + def migrate_callback(connection, execution) -> None: + connection.execute( + f"INSERT INTO {application_table} (root_instance_id) VALUES (%s)", + ("migration-root",), + ) + staged = execution.maintenance_migration_v2( + "postgresql-empty-migration", + migration_aggregate["validated_bundle_fingerprint"], + [], + expected_revision=migration_checkpoint.document["revision"], + expected_checkpoint_digest=migration_checkpoint.document[ + "execution_checkpoint_digest" + ], + ) + assert staged == StagedExecutionResult("maintenance_migration_v2") + + migration_result = host.run_shared_transaction( + "migration-root", migrate_callback + ) + assert migration_result["receipt"]["result_code"] == "migration_no_operation" + assert host.read_checkpoint("migration-root").document["revision"] == "1" + def rollback_callback(connection, execution) -> None: connection.execute( f"INSERT INTO {application_table} (root_instance_id) VALUES (%s)", ("rolled-back-root",), ) - execution.create(load_bundle(MACHINE), "counter", "create", {}) + execution.create_v2(load_bundle(MACHINE), "counter", "create", {}) raise RuntimeError("application rollback") with pytest.raises(RuntimeError, match="application rollback"): @@ -196,18 +215,17 @@ def test_postgresql_configured_permanent_strict_profile() -> None: PERMANENT_RECEIPT_RETENTION, PERMANENT_OUTBOX_TERMINAL_RETENTION, }.issubset(store.capabilities) - local_host, _ = _host() ExecutionHost( store, - local_host.artifact_resolver, + _resolver(), profile="exactly_once_committed_processing", ) ExecutionHost( store, - local_host.artifact_resolver, + _resolver(), profile="strict_durable_outbox", host_features={ - "atomic_checkpoint_processing", + "atomic_accept_process", "outbox_worker", "total_outbox_lifecycle", "retain_unresolved_outbox", @@ -223,13 +241,13 @@ def test_postgresql_configured_permanent_strict_profile() -> None: assert COMPACT_EFFECT_IDENTITY_RETENTION in compact_store.capabilities ExecutionHost( compact_store, - local_host.artifact_resolver, + _resolver(), profile="compact_durable_outbox", host_features={ - "atomic_checkpoint_processing", + "atomic_accept_process", "outbox_worker", "total_outbox_lifecycle", - "retain_referenced_effect_tombstones", + "retain_receipt_references", }, ) @@ -244,13 +262,12 @@ def test_postgresql_persists_policy_and_forbids_native_deletion( outbox_retention="strict", ) store.setup_schema() - local_host, _ = _host() host = ExecutionHost( store, - local_host.artifact_resolver, + _resolver(), profile="exactly_once_committed_processing", ) - host.create(load_bundle(MACHINE), "counter", "bank-root", "create", {}) + host.create_v2(load_bundle(MACHINE), "counter", "bank-root", "create", {}) with psycopg.connect(store.conninfo) as connection: with pytest.raises(psycopg.Error, match="execution_store_immutable"): @@ -260,7 +277,7 @@ def test_postgresql_persists_policy_and_forbids_native_deletion( ) assert host.read_checkpoint("bank-root") is not None with pytest.raises(ExecutionHostError) as recreate: - host.create( + host.create_v2( load_bundle(MACHINE), "counter", "bank-root", "replacement", {} ) assert recreate.value.code == "creation_id_conflict" diff --git a/tests/test_version2_review_findings.py b/tests/test_version2_review_findings.py deleted file mode 100644 index e22d172..0000000 --- a/tests/test_version2_review_findings.py +++ /dev/null @@ -1,1766 +0,0 @@ -from __future__ import annotations - -import copy -from typing import Any - -import pytest - -from determa.state import ( - ArtifactError, - ExecutionHost, - MemoryArtifactResolver, - MemoryExecutionStore, - admit_aggregate_v2, - admit_checkpoint_v2, - aggregate_shape_fingerprint, - create_aggregate_v2, - create_checkpoint_v2, - downgrade_aggregate_v2_to_v1, - load_bundle, - migrate_aggregate_v2, - restore_aggregate_package, - restore_aggregate_v2, - restore_execution_checkpoint_v2, - seal_aggregate_v2, - seal_execution_checkpoint, - serialize_execution_checkpoint, - step_aggregate_v2, - step_checkpoint_v2, - upgrade_checkpoint_v1_to_v2, -) -from determa.state.queueing import _entry_digest -from determa.state.wire import ( - aggregate_state_digest, - load_json_artifact, - migration_descriptor_digest, - typed_value, -) - - -def _bundle(*, deferred: bool = False, external: bool = False) -> Any: - events: dict[str, Any] = { - "go": {"direction": "input"}, - "hold": {"direction": "input"}, - "tail": {"direction": "input"}, - "work": {"direction": "internal"}, - } - action: list[dict[str, Any]] = [] - if external: - events["outside"] = {"direction": "output"} - action = [ - { - "send": { - "event": "outside", - "to": {"external": True}, - "correlation_id": "'effect'", - } - } - ] - root: dict[str, Any] = { - "type": "simple", - "on_events": {"go": {"action": action}, "tail": {}, "work": {}}, - } - if deferred: - root["deferred_events"] = ["hold"] - return load_bundle( - { - "format": 1, - "namespace": f"tests.review82.{deferred}.{external}", - "events": events, - "machines": [{"machine_id": "machine", "root": root}], - } - ) - - -def _resolver(bundle: Any) -> MemoryArtifactResolver: - return MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) - - -def _created(bundle: Any, root: str = "root") -> dict[str, Any]: - result = create_aggregate_v2(bundle, "machine", root, "create", {}) - assert result["state"] is not None - return result["state"] - - -def _delivery(aggregate: dict[str, Any], event_id: str, event: str = "go") -> dict[str, Any]: - runtime = next( - item for item in aggregate["runtimes"] if item["runtime_id"] == aggregate["root_runtime_id"] - ) - envelope = { - "event": event, - "event_id": event_id, - "cause_id": event_id, - "source": {"host": True}, - "target": copy.deepcopy(runtime["target_identity"]), - "payload": ["map", []], - } - return { - "delivery_mode": "input", - "envelope": envelope, - "envelope_digest": _entry_digest(aggregate["root_instance_id"], "input", envelope), - } - - -def _admit(aggregate: dict[str, Any], bundle: Any, *events: tuple[str, str]) -> dict[str, Any]: - result = admit_aggregate_v2( - aggregate, - [_delivery(aggregate, event_id, event) for event_id, event in events], - _resolver(bundle), - ) - assert result["result"] == "accepted" - return result["state"] - - -def _compatible_v2_descriptor( - source: Any, - target: Any, - *, - queued_event_rules: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - base = { - "migration_descriptor_format": "determa.aggregate_migration", - "migration_descriptor_schema_version": 1, - "source_machine_format": 1, - "target_machine_format": 1, - "source_validated_bundle_fingerprint": source.fingerprint, - "target_validated_bundle_fingerprint": target.fingerprint, - "source_aggregate_shape_fingerprint": aggregate_shape_fingerprint(source), - "target_aggregate_shape_fingerprint": aggregate_shape_fingerprint(target), - "mode": "compatible", - "mappings": { - name: [] - for name in ( - "machines", - "active_states", - "variables", - "history", - "components", - "owned_runtimes", - "lifetime_holders", - "counters", - ) - }, - "terminal_policy": {"completed": "preserve", "faulted": "preserve"}, - "resource_requirements": { - "maximum_transformed_output_bytes": "0", - "maximum_cel_expression_length": "0", - "maximum_cel_ast_nodes": "0", - "maximum_cel_evaluation_steps": "0", - }, - } - base["migration_descriptor_digest"] = migration_descriptor_digest(base) - descriptor = { - "migration_descriptor_format": "determa.aggregate_migration", - "migration_descriptor_schema_version": 2, - "base_descriptor": base, - "queued_event_default": "preserve_if_compatible", - "queued_event_rules": queued_event_rules or [], - } - descriptor["migration_descriptor_digest"] = migration_descriptor_digest(descriptor) - return descriptor - - -def _upgraded_legacy_internal_checkpoint() -> tuple[Any, MemoryArtifactResolver, dict[str, Any]]: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.legacy_provenance", - "events": { - "go": {"direction": "input"}, - "work": {"direction": "internal"}, - }, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "simple", - "entry": [{"send": {"event": "work"}}], - "on_events": {"work": {}}, - }, - } - ], - } - ) - resolver = _resolver(bundle) - host = ExecutionHost(MemoryExecutionStore(), resolver) - host.create(bundle, "machine", "root", "create", {}) - upgraded = upgrade_checkpoint_v1_to_v2(host.read_checkpoint("root").document, resolver) - return bundle, resolver, upgraded - - -def _upgraded_legacy_terminal_with_native_allocations() -> tuple[ - MemoryArtifactResolver, dict[str, Any] -]: - bundle, resolver, _checkpoint = _upgraded_legacy_internal_checkpoint() - host = ExecutionHost(MemoryExecutionStore(), resolver) - host.create(bundle, "machine", "root", "create", {}) - created = host.read_checkpoint("root") - assert created is not None - pending = created.document["pending_deliveries"][0] - host.process_pending_delivery( - "root", - { - "root_instance_id": "root", - "delivery_mode": pending["delivery_mode"], - "origin": copy.deepcopy(pending["origin"]), - "envelope": copy.deepcopy(pending["envelope"]), - "envelope_digest": pending["envelope_digest"], - }, - expected_revision=created.document["revision"], - expected_checkpoint_digest=created.document["execution_checkpoint_digest"], - ) - processed = host.read_checkpoint("root") - assert processed is not None - checkpoint = upgrade_checkpoint_v1_to_v2(processed.document, resolver) - checkpoint = admit_checkpoint_v2( - checkpoint, - [ - _delivery(checkpoint["root_record"]["aggregate_state"], "native-1"), - _delivery(checkpoint["root_record"]["aggregate_state"], "native-2"), - ], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - assert checkpoint["root_record"]["aggregate_state"]["next_acceptance_sequence"] == "3" - return resolver, checkpoint - - -def test_create_v2_routes_initial_internal_and_external_work_with_provenance() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review82.creation", - "events": { - "inside": {"direction": "internal"}, - "outside": {"direction": "output"}, - }, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "simple", - "entry": [ - {"send": {"event": "inside"}}, - { - "send": { - "event": "outside", - "to": {"external": True}, - "correlation_id": "'creation'", - } - }, - ], - "on_events": {"inside": {}}, - }, - } - ], - } - ) - result = create_aggregate_v2(bundle, "machine", "root", "create", {}) - state = result["state"] - assert state["next_acceptance_sequence"] == "1" - entry = state["runtimes"][0]["ready_mailbox"][0] - assert entry["envelope"]["cause_id"] != entry["envelope"]["event_id"] - assert entry["envelope"]["source"] == {"runtime": state["runtimes"][0]["target_identity"]} - assert ["kind" not in item for item in result["emissions"]] == [False, True] - checkpoint = create_checkpoint_v2(bundle, "machine", "other", "create", {}) - assert len(checkpoint["pending_outbox_intents"]) == 1 - assert len(checkpoint["operation_receipts"][0]["emission_references"]) == 2 - - -def test_pre_step_rejection_preserves_ready_head( - monkeypatch: pytest.MonkeyPatch, -) -> None: - bundle = _bundle() - aggregate = _admit(_created(bundle), bundle, ("event", "go")) - before = copy.deepcopy(aggregate) - - def rejected(*args: Any, **kwargs: Any) -> dict[str, Any]: - return { - "status": "running", - "disposition": "rejected", - "state": args[1], - "emissions": [], - "fault": None, - "rejection": {"code": "invalid_instance_target"}, - } - - monkeypatch.setattr("determa.state.queueing.dispatch", rejected) - result = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) - assert result["state"] == before - - -def test_pending_replay_uses_canonical_digest_not_supplied_digest() -> None: - bundle = _bundle() - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "same") - admitted = admit_checkpoint_v2( - checkpoint, - [delivery], - _resolver(bundle), - expected_revision="0", - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - changed = copy.deepcopy(delivery) - changed["envelope"]["event"] = "tail" - with pytest.raises(ArtifactError, match="event_id_conflict"): - admit_checkpoint_v2( - admitted, - [changed], - _resolver(bundle), - expected_revision=admitted["revision"], - expected_checkpoint_digest=admitted["execution_checkpoint_digest"], - ) - - -def test_checkpoint_step_creates_external_outbox_reference() -> None: - bundle = _bundle(external=True) - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "go") - admitted = admit_checkpoint_v2( - checkpoint, - [delivery], - _resolver(bundle), - expected_revision="0", - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - stepped = step_checkpoint_v2( - admitted, - admitted["root_record"]["aggregate_state"]["root_runtime_id"], - _resolver(bundle), - expected_revision=admitted["revision"], - expected_checkpoint_digest=admitted["execution_checkpoint_digest"], - ) - assert len(stepped["pending_outbox_intents"]) == 1 - assert stepped["operation_receipts"][-1]["emission_references"][0]["kind"] == ( - "external_outbox" - ) - - -def test_v1_host_gate_is_real_and_store_is_unchanged() -> None: - bundle = _bundle(deferred=True) - host = ExecutionHost(MemoryExecutionStore(), _resolver(bundle)) - host.create(bundle, "machine", "root", "create", {}) - before = host.read_checkpoint("root") - assert before is not None - aggregate = before.document["root_record"]["aggregate_state"] - candidate = { - "root_instance_id": "root", - "delivery_mode": "input", - "origin": {"kind": "host_input"}, - "envelope": { - key: value - for key, value in _delivery(aggregate, "go")["envelope"].items() - if key not in {"cause_id", "source"} - }, - } - result = host.accept_delivery( - "root", - candidate, - expected_revision=before.document["revision"], - expected_checkpoint_digest=before.document["execution_checkpoint_digest"], - selected_bundle=bundle, - ) - assert result["failure"]["code"] == "checkpoint_upgrade_required" - assert host.read_checkpoint("root").source_bytes == before.source_bytes - - -def test_execution_host_round_trips_v2_transactions() -> None: - bundle = _bundle() - host = ExecutionHost(MemoryExecutionStore(), _resolver(bundle)) - host.create_v2(bundle, "machine", "root", "create", {}) - restored = host.read_checkpoint("root") - assert restored is not None - assert restored.document["execution_checkpoint_schema_version"] == 2 - delivery = _delivery(restored.document["root_record"]["aggregate_state"], "go") - host.admit_v2( - "root", - [delivery], - expected_revision=restored.document["revision"], - expected_checkpoint_digest=restored.document["execution_checkpoint_digest"], - ) - admitted = host.read_checkpoint("root") - host.process_ready_v2( - "root", - admitted.document["root_record"]["aggregate_state"]["root_runtime_id"], - expected_revision=admitted.document["revision"], - expected_checkpoint_digest=admitted.document["execution_checkpoint_digest"], - ) - assert ( - host.read_checkpoint("root").document["operation_receipts"][-1]["operation_kind"] - == "event_terminal" - ) - - -def test_execution_host_prunes_and_tombstones_v2_transactionally() -> None: - bundle = _bundle() - host = ExecutionHost(MemoryExecutionStore(), _resolver(bundle)) - host.create_v2(bundle, "machine", "root", "create", {}) - created = host.read_checkpoint("root") - delivery = _delivery(created.document["root_record"]["aggregate_state"], "go") - host.admit_v2( - "root", - [delivery], - expected_revision=created.document["revision"], - expected_checkpoint_digest=created.document["execution_checkpoint_digest"], - ) - admitted = host.read_checkpoint("root") - host.process_ready_v2( - "root", - admitted.document["root_record"]["aggregate_state"]["root_runtime_id"], - expected_revision=admitted.document["revision"], - expected_checkpoint_digest=admitted.document["execution_checkpoint_digest"], - ) - terminal = host.read_checkpoint("root") - host.update_replay_retention( - "root", - { - "mode": "bounded", - "permanent_replay_eligible": False, - "pruned_through_receipt_sequence": None, - "policy_identifier": "test-policy", - }, - expected_revision=terminal.document["revision"], - expected_checkpoint_digest=terminal.document["execution_checkpoint_digest"], - ) - bounded = host.read_checkpoint("root") - host.prune_v2( - "root", - "2", - expected_revision=bounded.document["revision"], - expected_checkpoint_digest=bounded.document["execution_checkpoint_digest"], - ) - assert host.read_checkpoint("root").document["event_identity_tombstones"][0]["event_id"] == "go" - - terminal_bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review82.tombstone", - "machines": [ - { - "machine_id": "machine", - "root": {"type": "simple", "entry": [{"stop": {}}]}, - } - ], - } - ) - terminal_host = ExecutionHost(MemoryExecutionStore(), _resolver(terminal_bundle)) - terminal_host.create_v2(terminal_bundle, "machine", "terminal", "create", {}) - before = terminal_host.read_checkpoint("terminal") - terminal_host.tombstone_root_v2( - "terminal", - "delete", - expected_revision=before.document["revision"], - expected_checkpoint_digest=before.document["execution_checkpoint_digest"], - ) - assert terminal_host.read_checkpoint("terminal").document["root_record"]["status"] == ( - "tombstone" - ) - - -@pytest.mark.parametrize( - "mutate", - [ - lambda delivery: delivery["envelope"].update({"extra": True}), - lambda delivery: delivery["envelope"].update({"source": {"host": False}}), - lambda delivery: delivery.update({"extra": True}), - lambda delivery: delivery["envelope"].update({"target": {}}), - ], -) -def test_admission_rejects_closed_envelope_shapes_without_leaking( - mutate: Any, -) -> None: - bundle = _bundle() - aggregate = _created(bundle) - delivery = _delivery(aggregate, "event") - mutate(delivery) - result = admit_aggregate_v2(aggregate, [delivery], _resolver(bundle)) - assert result["result"] == "rejected" - assert result["state"] == aggregate - - -def _terminal_checkpoint() -> tuple[Any, MemoryArtifactResolver, dict[str, Any]]: - bundle = _bundle() - resolver = _resolver(bundle) - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "go") - admitted = admit_checkpoint_v2( - checkpoint, - [delivery], - resolver, - expected_revision="0", - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - terminal = step_checkpoint_v2( - admitted, - admitted["root_record"]["aggregate_state"]["root_runtime_id"], - resolver, - expected_revision=admitted["revision"], - expected_checkpoint_digest=admitted["execution_checkpoint_digest"], - ) - return bundle, resolver, terminal - - -@pytest.mark.parametrize("case", ["counter", "digest", "sequence", "overlap", "evidence"]) -def test_checkpoint_restore_rejects_identity_allocation_corruption(case: str) -> None: - _bundle_value, resolver, checkpoint = _terminal_checkpoint() - candidate = copy.deepcopy(checkpoint) - aggregate = candidate["root_record"]["aggregate_state"] - terminal = candidate["operation_receipts"][-1] - if case == "counter": - aggregate["next_acceptance_sequence"] = "0" - elif case == "digest": - terminal["request_digest"] = "sha256:" + "0" * 64 - elif case == "sequence": - aggregate["next_acceptance_sequence"] = "2" - terminal["acceptance_sequence"] = "1" - elif case == "overlap": - candidate["event_identity_tombstones"].append( - { - "event_id": terminal["event_id"], - "request_digest": terminal["request_digest"], - "request_digest_domain": "determa-inbox-envelope-digest-2", - "acceptance_sequence": terminal["acceptance_sequence"], - "terminal_receipt_sequence": terminal["receipt_sequence"], - "terminal_disposition": terminal["outcome"]["disposition"], - } - ) - else: - terminal["event_id"] = "unowned-terminal" - candidate["root_record"]["aggregate_state"] = seal_aggregate_v2(aggregate) - candidate = seal_execution_checkpoint(candidate) - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(candidate, resolver) - - -def test_permanent_prune_rejects_without_mutation() -> None: - _bundle_value, resolver, checkpoint = _terminal_checkpoint() - before = copy.deepcopy(checkpoint) - from determa.state import prune_checkpoint_v2 - - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - prune_checkpoint_v2( - checkpoint, - "1", - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - assert checkpoint == before - - -def test_unhandled_delivery_allocates_no_logical_step() -> None: - bundle = _bundle() - aggregate = _admit(_created(bundle), bundle, ("unhandled", "hold")) - before = aggregate["next_logical_step_sequence"] - result = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) - assert result["disposition"] == "unhandled" - assert result["state"]["next_logical_step_sequence"] == before - - -def test_downgrade_requires_zero_mailbox_counters() -> None: - bundle = _bundle() - aggregate = _created(bundle) - aggregate["next_queue_sequence"] = "1" - aggregate = seal_aggregate_v2(aggregate) - with pytest.raises(ArtifactError, match="migration_totality_failure"): - downgrade_aggregate_v2_to_v1(aggregate, _resolver(bundle)) - - -def test_resource_bound_counter_validation_does_not_expand_counter() -> None: - bundle = _bundle() - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - checkpoint["next_operation_receipt_sequence"] = "9" * 4097 - checkpoint = seal_execution_checkpoint(checkpoint) - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(checkpoint, _resolver(bundle)) - - -def test_lifecycle_dispositions_are_ready_then_deferred() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review82.lifecycle", - "events": { - "finish": {"direction": "input"}, - "hold": {"direction": "input"}, - "tail": {"direction": "input"}, - }, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "simple", - "deferred_events": ["hold"], - "on_events": { - "finish": {"action": [{"stop": {}}]}, - "tail": {}, - }, - }, - } - ], - } - ) - aggregate = _admit( - _created(bundle), - bundle, - ("hold", "hold"), - ("finish", "finish"), - ("tail", "tail"), - ) - first = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) - result = step_aggregate_v2(first["state"], aggregate["root_runtime_id"], _resolver(bundle)) - assert [item["event_id"] for item in result["lifecycle_dispositions"]] == [ - "tail", - "hold", - ] - - -def test_current_step_emissions_precede_structural_recall() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review82.recall_order", - "events": { - "go": {"direction": "input"}, - "hold": {"direction": "input"}, - "work": {"direction": "internal"}, - }, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "composite", - "initial": {"transition_to": "waiting"}, - "states": { - "waiting": { - "deferred_events": ["hold"], - "on_events": { - "go": { - "action": [{"send": {"event": "work"}}], - "transition_to": "active", - } - }, - }, - "active": {"on_events": {"work": {}}}, - }, - }, - } - ], - } - ) - aggregate = _admit(_created(bundle), bundle, ("hold", "hold"), ("go", "go")) - aggregate = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle))[ - "state" - ] - result = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) - ready = result["state"]["runtimes"][0]["ready_mailbox"] - assert [entry["envelope"]["event"] for entry in ready] == ["work", "hold"], result - assert int(ready[0]["queue_sequence"]) < int(ready[1]["queue_sequence"]) - - -def test_contained_capacity_fault_notifies_owner_and_freezes_other_work() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review82.contained_capacity", - "events": { - "start": {"direction": "input"}, - "work": {"direction": "internal"}, - }, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "parallel", - "on_events": { - "start": { - "action": [ - { - "send": { - "event": "work", - "to": {"component": "worker"}, - } - }, - { - "send": { - "event": "work", - "to": {"component": "worker"}, - } - }, - ] - }, - "determa.component_failed": {}, - }, - "components": [ - { - "component_id": "worker", - "root": { - "type": "simple", - "deferred_event_capacity": 0, - "deferred_events": ["work"], - }, - }, - {"component_id": "peer", "root": {"type": "simple"}}, - ], - }, - } - ], - } - ) - aggregate = _admit(_created(bundle), bundle, ("start", "start")) - emitted = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle))["state"] - child = next( - runtime for runtime in emitted["runtimes"] if runtime["relation"]["kind"] == "component" - ) - result = step_aggregate_v2(emitted, child["runtime_id"], _resolver(bundle)) - child_after = next( - runtime - for runtime in result["state"]["runtimes"] - if runtime["runtime_id"] == child["runtime_id"] - ) - assert child_after["status"] == "faulted" - assert len(child_after["ready_mailbox"]) == 1 - owner = next( - runtime - for runtime in result["state"]["runtimes"] - if runtime["runtime_id"] == result["state"]["root_runtime_id"] - ) - notification = owner["ready_mailbox"][-1]["envelope"] - assert notification["event"] == "determa.component_failed" - assert notification["source"] == {"system": "system:component_failure"} - - -def test_migration_structurally_recalls_with_fresh_queue_sequence() -> None: - source = _bundle(deferred=True) - target_document = copy.deepcopy(source.raw) - target_document["machines"][0]["root"].pop("deferred_events") - target = load_bundle(target_document) - aggregate = _admit(_created(source), source, ("hold", "hold")) - aggregate = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(source))[ - "state" - ] - shape = aggregate_shape_fingerprint(source) - base = { - "migration_descriptor_format": "determa.aggregate_migration", - "migration_descriptor_schema_version": 1, - "source_machine_format": 1, - "target_machine_format": 1, - "source_validated_bundle_fingerprint": source.fingerprint, - "target_validated_bundle_fingerprint": target.fingerprint, - "source_aggregate_shape_fingerprint": shape, - "target_aggregate_shape_fingerprint": aggregate_shape_fingerprint(target), - "mode": "compatible", - "mappings": { - name: [] - for name in ( - "machines", - "active_states", - "variables", - "history", - "components", - "owned_runtimes", - "lifetime_holders", - "counters", - ) - }, - "terminal_policy": {"completed": "preserve", "faulted": "preserve"}, - "resource_requirements": { - "maximum_transformed_output_bytes": "0", - "maximum_cel_expression_length": "0", - "maximum_cel_ast_nodes": "0", - "maximum_cel_evaluation_steps": "0", - }, - } - base["migration_descriptor_digest"] = migration_descriptor_digest(base) - descriptor = { - "migration_descriptor_format": "determa.aggregate_migration", - "migration_descriptor_schema_version": 2, - "base_descriptor": base, - "queued_event_default": "preserve_if_compatible", - "queued_event_rules": [], - } - descriptor["migration_descriptor_digest"] = migration_descriptor_digest(descriptor) - resolver = MemoryArtifactResolver( - definitions={source.fingerprint: source, target.fingerprint: target}, - migration_descriptors={descriptor["migration_descriptor_digest"]: descriptor}, - ) - prior_next = int(aggregate["next_queue_sequence"]) - migrated = migrate_aggregate_v2( - aggregate, - target.fingerprint, - [descriptor["migration_descriptor_digest"]], - resolver, - maintenance_mode=True, - )["aggregate_state"] - runtime = migrated["runtimes"][0] - assert not runtime["deferred_mailbox"] - assert runtime["ready_mailbox"][0]["queue_sequence"] == str(prior_next) - - checkpoint = create_checkpoint_v2(source, "machine", "host-root", "create", {}) - host_delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "host-hold", "hold") - checkpoint = admit_checkpoint_v2( - checkpoint, - [host_delivery], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - checkpoint = step_checkpoint_v2( - checkpoint, - checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - host = ExecutionHost( - MemoryExecutionStore({"host-root": serialize_execution_checkpoint(checkpoint)}), - resolver, - ) - host.maintenance_migration_v2( - "host-root", - target.fingerprint, - [descriptor["migration_descriptor_digest"]], - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - restored = host.read_checkpoint("host-root") - assert ( - restored.document["root_record"]["aggregate_state"]["runtimes"][0]["ready_mailbox"][0][ - "envelope" - ]["event"] - == "hold" - ) - - -def test_restore_aggregate_package_accepts_strict_v2_package() -> None: - bundle = _bundle() - aggregate = _created(bundle) - package = { - "aggregate_state_package_format": "determa.aggregate_state_package", - "aggregate_state_package_schema_version": 2, - "aggregate_state": aggregate, - "normalized_definitions": [ - { - "validated_bundle_fingerprint": bundle.fingerprint, - "normalized_bundle": typed_value(bundle.raw), - } - ], - "migration_descriptors": [], - "migration_route": [], - } - restored = restore_aggregate_package(package, MemoryArtifactResolver()) - assert restored.aggregate.aggregate_envelope == aggregate - - -def test_created_component_emission_uses_canonical_wire_source_and_restores() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.component_source", - "events": {"notice": {"direction": "internal"}}, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "parallel", - "on_events": {"notice": {}}, - "components": [ - { - "component_id": "sender", - "root": { - "type": "simple", - "entry": [ - { - "send": { - "event": "notice", - "to": {"owner": True}, - } - }, - {"stop": {}}, - ], - }, - }, - {"component_id": "peer", "root": {"type": "simple"}}, - ], - }, - } - ], - } - ) - result = create_aggregate_v2(bundle, "machine", "root", "create", {}) - aggregate = result["state"] - entry = next( - entry - for runtime in aggregate["runtimes"] - for entry in runtime["ready_mailbox"] - if entry["envelope"]["event"] == "notice" - ) - source = entry["envelope"]["source"]["runtime"]["component"] - assert isinstance(source["activation_sequence"], str) - assert aggregate_state_digest(aggregate) == aggregate["aggregate_state_digest"] - assert restore_aggregate_v2(aggregate, _resolver(bundle)).aggregate_envelope == aggregate - - -def test_checkpoint_updates_internal_reference_on_deferral_and_recall() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.reference_recall", - "events": { - "go": {"direction": "input"}, - "work": {"direction": "internal"}, - }, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "composite", - "entry": [{"send": {"event": "work"}}], - "initial": {"transition_to": "waiting"}, - "states": { - "waiting": { - "deferred_events": ["work"], - "on_events": {"go": {"transition_to": "active"}}, - }, - "active": {"on_events": {"work": {}}}, - }, - }, - } - ], - } - ) - resolver = _resolver(bundle) - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - runtime_id = checkpoint["root_record"]["aggregate_state"]["root_runtime_id"] - checkpoint = step_checkpoint_v2( - checkpoint, - runtime_id, - resolver, - expected_revision="0", - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - deferred = checkpoint["root_record"]["aggregate_state"]["runtimes"][0]["deferred_mailbox"][0] - reference = checkpoint["operation_receipts"][0]["emission_references"][0] - assert reference["queue_sequence"] == deferred["queue_sequence"] - delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "go") - checkpoint = admit_checkpoint_v2( - checkpoint, - [delivery], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - checkpoint = step_checkpoint_v2( - checkpoint, - runtime_id, - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - recalled = checkpoint["root_record"]["aggregate_state"]["runtimes"][0]["ready_mailbox"][0] - reference = checkpoint["operation_receipts"][0]["emission_references"][0] - assert reference["queue_sequence"] == recalled["queue_sequence"] - restore_execution_checkpoint_v2(checkpoint, resolver) - - -def test_checkpoint_terminalizes_selected_and_lifecycle_disposed_references() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.reference_terminal", - "events": { - "start": {"direction": "input"}, - "finish": {"direction": "internal"}, - "work": {"direction": "internal"}, - }, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "parallel", - "on_events": { - "start": { - "action": [ - { - "send": { - "event": "finish", - "to": {"component": "worker"}, - } - }, - { - "send": { - "event": "work", - "to": {"component": "worker"}, - } - }, - ] - } - }, - "components": [ - { - "component_id": "worker", - "root": { - "type": "simple", - "on_events": { - "finish": {"action": [{"stop": {}}]}, - "work": {}, - }, - }, - }, - {"component_id": "peer", "root": {"type": "simple"}}, - ], - }, - } - ], - } - ) - resolver = _resolver(bundle) - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - checkpoint = admit_checkpoint_v2( - checkpoint, - [_delivery(checkpoint["root_record"]["aggregate_state"], "start", "start")], - resolver, - expected_revision="0", - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - checkpoint = step_checkpoint_v2( - checkpoint, - checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - worker = next( - runtime - for runtime in checkpoint["root_record"]["aggregate_state"]["runtimes"] - if runtime["relation"].get("component_id") == "worker" - ) - producer = checkpoint["operation_receipts"][-1] - checkpoint = step_checkpoint_v2( - checkpoint, - worker["runtime_id"], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - assert [reference["kind"] for reference in producer["emission_references"]] == [ - "internal_mailbox", - "internal_mailbox", - ] - committed_producer = next( - receipt - for receipt in checkpoint["operation_receipts"] - if receipt.get("event_id") == "start" and receipt["operation_kind"] == "event_terminal" - ) - assert [reference["kind"] for reference in committed_producer["emission_references"]] == [ - "internal_terminal", - "internal_terminal", - ] - restore_execution_checkpoint_v2(checkpoint, resolver) - - -@pytest.mark.parametrize("case", ["orphan", "receipt_order", "revision_order"]) -def test_checkpoint_rejects_reverse_orphan_and_terminal_chronology(case: str) -> None: - _bundle_value, resolver, checkpoint = _terminal_checkpoint() - candidate = copy.deepcopy(checkpoint) - acceptance = candidate["operation_receipts"][1] - terminal = candidate["operation_receipts"][2] - if case == "orphan": - candidate["operation_receipts"].pop() - candidate["next_operation_receipt_sequence"] = "2" - elif case == "receipt_order": - acceptance["receipt_sequence"] = "2" - terminal["receipt_sequence"] = "1" - candidate["operation_receipts"] = [ - candidate["operation_receipts"][0], - terminal, - acceptance, - ] - else: - acceptance["accepted_revision"] = "2" - terminal["committed_revision"] = "1" - candidate = seal_execution_checkpoint(candidate) - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(candidate, resolver) - - -@pytest.mark.parametrize("case", ["cause", "source", "event", "direction"]) -def test_aggregate_restore_validates_mailbox_envelope_semantics(case: str) -> None: - bundle = _bundle() - aggregate = _admit(_created(bundle), bundle, ("event", "go")) - entry = aggregate["runtimes"][0]["ready_mailbox"][0] - if case == "cause": - entry["envelope"]["cause_id"] = "other" - elif case == "source": - entry["envelope"]["source"] = { - "runtime": copy.deepcopy(aggregate["runtimes"][0]["target_identity"]) - } - elif case == "event": - entry["envelope"]["event"] = "missing" - else: - entry["envelope"]["event"] = "work" - entry["envelope_digest"] = _entry_digest( - aggregate["root_instance_id"], entry["delivery_mode"], entry["envelope"] - ) - aggregate = seal_aggregate_v2(aggregate) - with pytest.raises(ArtifactError, match="invalid_aggregate_state"): - restore_aggregate_v2(aggregate, _resolver(bundle)) - - -@pytest.mark.parametrize("forgery", ["wrong_type", "missing_default"]) -def test_restore_aggregate_v2_rejects_resealed_noncanonical_payload(forgery: str) -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.payload_restore", - "events": { - "go": { - "direction": "input", - "payload": { - "count": {"type": "int", "required": True}, - "label": {"type": "string", "default": "defaulted"}, - }, - } - }, - "machines": [ - {"machine_id": "machine", "root": {"type": "simple", "on_events": {"go": {}}}} - ], - } - ) - aggregate = _created(bundle) - delivery = _delivery(aggregate, "event") - delivery["envelope"]["payload"] = typed_value({"count": 1, "label": "defaulted"}) - delivery["envelope_digest"] = _entry_digest( - aggregate["root_instance_id"], "input", delivery["envelope"] - ) - admitted = admit_aggregate_v2(aggregate, [delivery], _resolver(bundle)) - assert admitted["result"] == "accepted" - candidate = admitted["state"] - entry = candidate["runtimes"][0]["ready_mailbox"][0] - entry["envelope"]["payload"] = typed_value( - {"count": "1", "label": "defaulted"} - if forgery == "wrong_type" - else {"count": 1} - ) - entry["envelope_digest"] = _entry_digest( - candidate["root_instance_id"], entry["delivery_mode"], entry["envelope"] - ) - candidate = seal_aggregate_v2(candidate) - - with pytest.raises(ArtifactError, match="invalid_aggregate_state"): - restore_aggregate_v2(candidate, _resolver(bundle)) - - -def test_admission_rejects_payload_that_would_materialize_a_default() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.payload_admission", - "events": { - "go": { - "direction": "input", - "payload": {"label": {"type": "string", "default": "defaulted"}}, - } - }, - "machines": [ - {"machine_id": "machine", "root": {"type": "simple", "on_events": {"go": {}}}} - ], - } - ) - aggregate = _created(bundle) - delivery = _delivery(aggregate, "event") - result = admit_aggregate_v2(aggregate, [delivery], _resolver(bundle)) - - assert result["result"] == "rejected" - assert result["rejection"]["code"] == "invalid_payload" - assert result["state"] == aggregate - - -def test_restore_rejects_resealed_author_event_with_system_source() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.author_source", - "events": {"work": {"direction": "internal"}}, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "simple", - "entry": [{"send": {"event": "work"}}], - "on_events": {"work": {}}, - }, - } - ], - } - ) - aggregate = _created(bundle) - entry = aggregate["runtimes"][0]["ready_mailbox"][0] - entry["envelope"]["source"] = {"system": "system:component_completion"} - entry["envelope_digest"] = _entry_digest( - aggregate["root_instance_id"], entry["delivery_mode"], entry["envelope"] - ) - aggregate = seal_aggregate_v2(aggregate) - - with pytest.raises(ArtifactError, match="invalid_aggregate_state"): - restore_aggregate_v2(aggregate, _resolver(bundle)) - - -def test_restore_rejects_resealed_reserved_event_with_wrong_system_source() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.reserved_source", - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "parallel", - "components": [ - { - "component_id": "worker", - "root": {"type": "simple", "entry": [{"stop": {}}]}, - }, - {"component_id": "peer", "root": {"type": "simple"}}, - ], - }, - } - ], - } - ) - aggregate = _created(bundle) - entry = next( - entry - for runtime in aggregate["runtimes"] - for entry in runtime["ready_mailbox"] - if entry["envelope"]["event"] == "determa.component_completed" - ) - entry["envelope"]["source"] = {"system": "system:component_failure"} - entry["envelope_digest"] = _entry_digest( - aggregate["root_instance_id"], entry["delivery_mode"], entry["envelope"] - ) - aggregate = seal_aggregate_v2(aggregate) - - with pytest.raises(ArtifactError, match="invalid_aggregate_state"): - restore_aggregate_v2(aggregate, _resolver(bundle)) - - -def test_checkpoint_restore_rejects_resealed_false_creation_result_digest() -> None: - bundle = _bundle() - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - checkpoint["operation_receipts"][0]["resulting_aggregate_state_digest"] = ( - "sha256:" + "0" * 64 - ) - checkpoint = seal_execution_checkpoint(checkpoint) - - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(checkpoint, _resolver(bundle)) - - -@pytest.mark.parametrize("field", ["producing_receipt_sequence", "emission_index"]) -def test_checkpoint_restore_rejects_resealed_legacy_origin_forgery(field: str) -> None: - _bundle_value, resolver, checkpoint = _upgraded_legacy_internal_checkpoint() - acceptance = next( - receipt - for receipt in checkpoint["operation_receipts"] - if receipt["operation_kind"] == "acceptance" - ) - acceptance["legacy_v1_delivery"]["origin"][field] = "999" - checkpoint = seal_execution_checkpoint(checkpoint) - - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(checkpoint, resolver) - - -def test_checkpoint_restore_rejects_resealed_legacy_producer_reference_forgery() -> None: - _bundle_value, resolver, checkpoint = _upgraded_legacy_internal_checkpoint() - producer = next( - receipt - for receipt in checkpoint["operation_receipts"] - if receipt["operation_kind"] == "legacy_v1_creation" - ) - producer["legacy_receipt"]["emission_references"][0]["emission_index"] = "999" - checkpoint = seal_execution_checkpoint(checkpoint) - - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(checkpoint, resolver) - - -def test_checkpoint_restore_rejects_resealed_legacy_mailbox_relabelled_native() -> None: - _bundle_value, resolver, checkpoint = _upgraded_legacy_internal_checkpoint() - aggregate = checkpoint["root_record"]["aggregate_state"] - entry = aggregate["runtimes"][0]["ready_mailbox"][0] - entry["envelope"]["source"] = { - "runtime": copy.deepcopy(aggregate["runtimes"][0]["target_identity"]) - } - entry["envelope_digest"] = _entry_digest( - aggregate["root_instance_id"], entry["delivery_mode"], entry["envelope"] - ) - acceptance = next( - receipt - for receipt in checkpoint["operation_receipts"] - if receipt["operation_kind"] == "acceptance" - ) - acceptance["request_digest"] = entry["envelope_digest"] - checkpoint["root_record"]["aggregate_state"] = seal_aggregate_v2(aggregate) - checkpoint = seal_execution_checkpoint(checkpoint) - - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(checkpoint, resolver) - - -def test_checkpoint_restore_rejects_resealed_processed_legacy_terminal_origin() -> None: - _bundle_value, resolver, checkpoint = _upgraded_legacy_internal_checkpoint() - checkpoint = step_checkpoint_v2( - checkpoint, - checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - acceptance = next( - receipt - for receipt in checkpoint["operation_receipts"] - if receipt["operation_kind"] == "acceptance" - ) - acceptance["legacy_v1_delivery"]["origin"]["emission_index"] = "999" - checkpoint = seal_execution_checkpoint(checkpoint) - - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(checkpoint, resolver) - - -def _set_wrapped_legacy_terminal_allocation( - checkpoint: dict[str, Any], sequence: str -) -> None: - wrapper = next( - receipt - for receipt in checkpoint["operation_receipts"] - if receipt["operation_kind"] == "legacy_v1_operation" - and receipt["legacy_receipt"]["operation_kind"] == "delivery" - ) - legacy = wrapper["legacy_receipt"] - legacy["accepted_delivery_sequence"] = sequence - producer_sequence = legacy["origin"]["producing_receipt_sequence"] - producer = next( - receipt - for receipt in checkpoint["operation_receipts"] - if receipt["receipt_sequence"] == producer_sequence - ) - reference = next( - item - for item in producer["legacy_receipt"]["emission_references"] - if item["event_id"] == legacy["event_id"] - ) - reference["delivery_sequence"] = sequence - - -def test_checkpoint_restore_rejects_legacy_allocation_collision_with_native() -> None: - resolver, checkpoint = _upgraded_legacy_terminal_with_native_allocations() - native_acceptance = next( - receipt - for receipt in checkpoint["operation_receipts"] - if receipt["operation_kind"] == "acceptance" - ) - _set_wrapped_legacy_terminal_allocation( - checkpoint, native_acceptance["acceptance_sequence"] - ) - checkpoint = seal_execution_checkpoint(checkpoint) - - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(checkpoint, resolver) - - -def test_checkpoint_restore_rejects_legacy_allocation_at_or_above_next_counter() -> None: - resolver, checkpoint = _upgraded_legacy_terminal_with_native_allocations() - assert checkpoint["root_record"]["aggregate_state"]["next_acceptance_sequence"] == "3" - _set_wrapped_legacy_terminal_allocation(checkpoint, "999") - checkpoint = seal_execution_checkpoint(checkpoint) - - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - restore_execution_checkpoint_v2(checkpoint, resolver) - - -def test_v1_upgrade_gate_finds_nested_inline_component_deferral() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.inline_gate", - "events": {"hold": {"direction": "internal"}}, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "parallel", - "components": [ - { - "component_id": "nested", - "root": { - "type": "composite", - "initial": {"transition_to": "waiting"}, - "states": {"waiting": {"deferred_events": ["hold"]}}, - }, - }, - {"component_id": "peer", "root": {"type": "simple"}}, - ], - }, - } - ], - } - ) - host = ExecutionHost(MemoryExecutionStore(), _resolver(bundle)) - host.create(bundle, "machine", "root", "create", {}) - before = host.read_checkpoint("root") - aggregate = before.document["root_record"]["aggregate_state"] - result = host.accept_delivery( - "root", - { - "root_instance_id": "root", - "delivery_mode": "input", - "origin": {"kind": "host_input"}, - "envelope": { - key: value - for key, value in _delivery(aggregate, "event")["envelope"].items() - if key not in {"cause_id", "source"} - }, - }, - expected_revision=before.document["revision"], - expected_checkpoint_digest=before.document["execution_checkpoint_digest"], - selected_bundle=bundle, - ) - assert result["failure"]["code"] == "checkpoint_upgrade_required" - assert host.read_checkpoint("root").source_bytes == before.source_bytes - - -def test_pruning_rejects_pending_outbox_producer_removal() -> None: - bundle = _bundle(external=True) - resolver = _resolver(bundle) - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - checkpoint = admit_checkpoint_v2( - checkpoint, - [_delivery(checkpoint["root_record"]["aggregate_state"], "go")], - resolver, - expected_revision="0", - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - checkpoint = step_checkpoint_v2( - checkpoint, - checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - checkpoint["replay_retention"] = { - "mode": "bounded", - "permanent_replay_eligible": False, - "pruned_through_receipt_sequence": None, - "policy_identifier": "test", - } - checkpoint = seal_execution_checkpoint(checkpoint) - with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): - from determa.state import prune_checkpoint_v2 - - prune_checkpoint_v2( - checkpoint, - "2", - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - - -def test_migration_rejects_payload_default_materialization() -> None: - source = _bundle(deferred=True) - target_document = copy.deepcopy(source.raw) - target_document["events"]["hold"]["payload"] = { - "added": {"type": "string", "default": "defaulted"} - } - target = load_bundle(target_document) - descriptor = _compatible_v2_descriptor(source, target) - aggregate = _admit(_created(source), source, ("hold", "hold")) - aggregate = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(source))[ - "state" - ] - resolver = MemoryArtifactResolver( - definitions={source.fingerprint: source, target.fingerprint: target}, - migration_descriptors={descriptor["migration_descriptor_digest"]: descriptor}, - ) - with pytest.raises(ArtifactError, match="migration_totality_failure"): - migrate_aggregate_v2( - aggregate, - target.fingerprint, - [descriptor["migration_descriptor_digest"]], - resolver, - maintenance_mode=True, - ) - - -def test_public_v2_migration_returns_exact_ordered_audit_records() -> None: - source = _bundle() - target_document = copy.deepcopy(source.raw) - target_document["events"]["extra"] = {"direction": "input"} - target = load_bundle(target_document) - descriptor = _compatible_v2_descriptor(source, target) - aggregate = _created(source) - resolver = MemoryArtifactResolver( - definitions={source.fingerprint: source, target.fingerprint: target}, - migration_descriptors={descriptor["migration_descriptor_digest"]: descriptor}, - ) - - result = migrate_aggregate_v2( - aggregate, - target.fingerprint, - [descriptor["migration_descriptor_digest"]], - resolver, - maintenance_mode=True, - ) - - assert set(result) == {"result", "aggregate_state", "dispositions", "audit_records"} - assert result["dispositions"] == [] - assert result["audit_records"] == [ - { - "migration_audit_record_schema_version": 1, - "root_instance_id": aggregate["root_instance_id"], - "root_runtime_id": aggregate["root_runtime_id"], - "migration_sequence": "1", - "source_validated_bundle_fingerprint": source.fingerprint, - "target_validated_bundle_fingerprint": target.fingerprint, - "migration_descriptor_digest": descriptor["migration_descriptor_digest"], - "source_aggregate_state_digest": aggregate["aggregate_state_digest"], - "target_aggregate_state_digest": result["aggregate_state"][ - "aggregate_state_digest" - ], - "result_code": "migration_applied", - } - ] - - no_operation = migrate_aggregate_v2( - aggregate, - source.fingerprint, - [], - resolver, - maintenance_mode=False, - ) - assert no_operation == { - "result": "success", - "aggregate_state": aggregate, - "dispositions": [], - "audit_records": [], - } - - -def test_host_v2_empty_route_maintenance_migration_commits_no_operation() -> None: - bundle = _bundle() - resolver = _resolver(bundle) - checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) - host = ExecutionHost( - MemoryExecutionStore({"root": serialize_execution_checkpoint(checkpoint)}), - resolver, - ) - - result = host.maintenance_migration_v2( - "root", - bundle.fingerprint, - [], - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - maintenance_mode=False, - ) - - assert int(result["revision"]) == int(checkpoint["revision"]) + 1 - assert result["root_record"]["aggregate_state"] == checkpoint["root_record"][ - "aggregate_state" - ] - assert result["migration_audit_records"] == checkpoint["migration_audit_records"] - assert host.read_checkpoint("root").document == result - - -def test_multihop_migration_recalls_each_hop_and_host_appends_exact_audit() -> None: - source = _bundle(deferred=True) - middle_document = copy.deepcopy(source.raw) - middle_document["machines"][0]["root"].pop("deferred_events") - middle = load_bundle(middle_document) - target_document = copy.deepcopy(source.raw) - target_document["events"]["extra"] = {"direction": "input"} - target = load_bundle(target_document) - first = _compatible_v2_descriptor(source, middle) - second = _compatible_v2_descriptor(middle, target) - resolver = MemoryArtifactResolver( - definitions={ - source.fingerprint: source, - middle.fingerprint: middle, - target.fingerprint: target, - }, - migration_descriptors={ - first["migration_descriptor_digest"]: first, - second["migration_descriptor_digest"]: second, - }, - ) - checkpoint = create_checkpoint_v2(source, "machine", "root", "create", {}) - checkpoint = admit_checkpoint_v2( - checkpoint, - [_delivery(checkpoint["root_record"]["aggregate_state"], "hold", "hold")], - resolver, - expected_revision="0", - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - checkpoint = step_checkpoint_v2( - checkpoint, - checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - public_result = migrate_aggregate_v2( - checkpoint["root_record"]["aggregate_state"], - target.fingerprint, - [first["migration_descriptor_digest"], second["migration_descriptor_digest"]], - resolver, - maintenance_mode=True, - ) - assert [ - record["migration_descriptor_digest"] for record in public_result["audit_records"] - ] == [first["migration_descriptor_digest"], second["migration_descriptor_digest"]] - old_queue = checkpoint["root_record"]["aggregate_state"]["runtimes"][0]["deferred_mailbox"][0][ - "queue_sequence" - ] - host = ExecutionHost( - MemoryExecutionStore({"root": serialize_execution_checkpoint(checkpoint)}), - resolver, - ) - migrated = host.maintenance_migration_v2( - "root", - target.fingerprint, - [first["migration_descriptor_digest"], second["migration_descriptor_digest"]], - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - runtime = migrated["root_record"]["aggregate_state"]["runtimes"][0] - assert not runtime["deferred_mailbox"] - assert int(runtime["ready_mailbox"][0]["queue_sequence"]) > int(old_queue) - audits = migrated["migration_audit_records"] - assert len(audits) == 2 - assert ( - audits[0]["source_aggregate_state_digest"] - == checkpoint["root_record"]["aggregate_state"]["aggregate_state_digest"] - ) - assert audits[0]["target_aggregate_state_digest"] == audits[1]["source_aggregate_state_digest"] - assert ( - audits[1]["target_aggregate_state_digest"] - == migrated["root_record"]["aggregate_state"]["aggregate_state_digest"] - ) - restore_execution_checkpoint_v2(migrated, resolver) - - -def test_lifecycle_cleanup_orders_component_mailboxes_reverse_declaration() -> None: - bundle = load_bundle( - { - "format": 1, - "namespace": "tests.review83.cleanup_order", - "events": { - "start": {"direction": "input"}, - "finish": {"direction": "input"}, - "work": {"direction": "internal"}, - }, - "machines": [ - { - "machine_id": "machine", - "root": { - "type": "parallel", - "on_events": { - "start": { - "action": [ - { - "send": { - "event": "work", - "to": {"component": "first"}, - } - }, - { - "send": { - "event": "work", - "to": {"component": "second"}, - } - }, - ] - }, - "finish": {"action": [{"stop": {}}]}, - }, - "components": [ - {"component_id": "first", "root": {"type": "simple"}}, - {"component_id": "second", "root": {"type": "simple"}}, - ], - }, - } - ], - } - ) - aggregate = _admit(_created(bundle), bundle, ("start", "start")) - aggregate = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle))[ - "state" - ] - aggregate = _admit(aggregate, bundle, ("finish", "finish")) - result = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) - component_by_runtime = { - runtime["runtime_id"]: runtime["relation"].get("component_id") - for runtime in aggregate["runtimes"] - } - assert [ - component_by_runtime[item["target_runtime_id"]] for item in result["lifecycle_dispositions"] - ] == ["second", "first"] - - -@pytest.mark.parametrize("first_failure", ["wrong_root", "conflict"]) -def test_batch_malformed_precedes_wrong_root_and_conflict(first_failure: str) -> None: - bundle = _bundle() - resolver = _resolver(bundle) - aggregate = _created(bundle) - if first_failure == "conflict": - aggregate = _admit(aggregate, bundle, ("same", "go")) - first = _delivery(aggregate, "same", "tail") - else: - first = _delivery(aggregate, "first") - first["envelope"]["target"]["root"]["root_instance_id"] = "wrong" - first["envelope_digest"] = _entry_digest("root", "input", first["envelope"]) - malformed = _delivery(aggregate, "malformed") - malformed["envelope"]["extra"] = True - result = admit_aggregate_v2(aggregate, [first, malformed], resolver) - assert result["rejection"]["code"] == "malformed_delivery" - - checkpoint = create_checkpoint_v2(bundle, "machine", "checkpoint", "create", {}) - if first_failure == "conflict": - original = _delivery(checkpoint["root_record"]["aggregate_state"], "same") - checkpoint = admit_checkpoint_v2( - checkpoint, - [original], - resolver, - expected_revision="0", - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - first = _delivery(checkpoint["root_record"]["aggregate_state"], "same", "tail") - else: - first = _delivery(checkpoint["root_record"]["aggregate_state"], "first") - first["envelope"]["target"]["root"]["root_instance_id"] = "wrong" - first["envelope_digest"] = _entry_digest("checkpoint", "input", first["envelope"]) - malformed = _delivery(checkpoint["root_record"]["aggregate_state"], "malformed") - malformed["envelope"]["extra"] = True - with pytest.raises(ArtifactError, match="malformed_delivery"): - admit_checkpoint_v2( - checkpoint, - [first, malformed], - resolver, - expected_revision=checkpoint["revision"], - expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], - ) - - -def test_invalid_core_step_result_reports_its_public_artifact_code() -> None: - with pytest.raises(ArtifactError, match="invalid_core_step_result"): - load_json_artifact( - { - "core_step_result_format": "determa.core_step_result", - "core_step_result_schema_version": 2, - }, - "core_step_result_v2", - )