diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 133ecf6..9d52a7b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,13 +58,13 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-conformance - ref: d6a45d31614ee25de20476ed93f10e14997d882c + ref: 531468c59c7a2dc32f5cbe92cfabf89805d27f6a path: .pinned/determa-state-conformance - name: Check out pinned specification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: fruwehq/determa-state-spec - ref: 7782671b56165a59caa61a65c29fefc63105ebf8 + ref: 2e33036563cb966b07124197db672159b4b7e1f4 path: .pinned/determa-state-spec - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/AGENTS.md b/AGENTS.md index 0240d24..55e8237 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,15 +9,14 @@ This is the Python implementation of Determa State. The distribution is package so it can coexist with the umbrella `determa` launcher. The implementation is conformant only when it passes the language-neutral suite. -The synchronized 0.2.0 release uses these immutable inputs: +The implementation uses these immutable inputs: -- specification: `7782671b56165a59caa61a65c29fefc63105ebf8`; -- conformance: `d6a45d31614ee25de20476ed93f10e14997d882c` (111 core cases, - 108 persistence vectors, 12 persistence-profile steps, 91 execution-checkpoint - vectors, and 89 closed-code registry entries). +- specification: `2e33036563cb966b07124197db672159b4b7e1f4`; +- conformance: `531468c59c7a2dc32f5cbe92cfabf89805d27f6a` (114 core cases, + 108 persistence vectors, 12 persistence-profile steps, 99 execution-checkpoint + vectors, 106 version-2 vectors, and 101 closed-code registry entries). -The package metadata is `0.2.0` for the next synchronized release; the specification, -conformance suite, Python engine, and Rust engine version together. +The package metadata remains `0.2.0`. ## Boundaries diff --git a/README.md b/README.md index 511c41e..dd55f17 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,14 @@ Python implementation of [Determa State](https://github.com/fruwehq/determa-stat a language-agnostic statechart engine with a shared normative conformance suite. This implementation supports Determa State `format: 1` at specification -commit `7782671b56165a59caa61a65c29fefc63105ebf8` (`v0.2.0`). Correctness is -determined by the 111-case core suite, 108 persistence vectors, 12 persistence-profile -steps, 91 execution-checkpoint vectors, and 89 closed-code registry entries at -conformance commit `d6a45d31614ee25de20476ed93f10e14997d882c` (`v0.2.0`). - -The package metadata is `0.2.0` for the next synchronized release of the specification, -conformance suite, Python engine, and Rust engine. This release adds the portable -execution-checkpoint host, selected legacy artifact decoding, and authoritative -portable code sets. +commit `2e33036563cb966b07124197db672159b4b7e1f4`. Correctness is determined by +the 114-case core suite, 108 persistence vectors, 12 persistence-profile steps, +99 execution-checkpoint vectors, 106 version-2 vectors, and 101 closed-code registry +entries at conformance commit `531468c59c7a2dc32f5cbe92cfabf89805d27f6a`. + +The package metadata remains `0.2.0`. The implementation includes the portable +execution-checkpoint host, selected legacy artifact decoding, and authoritative portable +code sets. ## Install diff --git a/conformance/execution_checkpoint.py b/conformance/execution_checkpoint.py index b3b4320..415c407 100644 --- a/conformance/execution_checkpoint.py +++ b/conformance/execution_checkpoint.py @@ -72,7 +72,9 @@ def execution_checkpoint_vectors() -> list[ExecutionCheckpointVector]: return [ ExecutionCheckpointVector(case, vector) for case in execution_checkpoint_cases() - for vector in case.test["execution_checkpoint_profile"]["vectors"] + for vector in case.test.get("execution_checkpoint_profile", {}).get( + "vectors", [] + ) ] @@ -90,7 +92,7 @@ def _pointer(document: Any, pointer: str) -> Any: def _resolver(case: ExecutionCheckpointCase) -> MemoryArtifactResolver: definitions = {} descriptors = {} - for path in case.path.glob("*.yaml"): + for path in case.path.parent.glob("**/*.yaml"): try: bundle = load_bundle(path.read_text(encoding="utf-8")) except Exception: diff --git a/conformance/harness.py b/conformance/harness.py index b66a3f4..9aa69ef 100644 --- a/conformance/harness.py +++ b/conformance/harness.py @@ -232,6 +232,7 @@ def _assert_result( "fault", "caller_still_owns_input", "caller_still_owns_state", + "state_bytes_unchanged", "state", "config", "variables", @@ -270,6 +271,9 @@ def _assert_result( assert prior_state_snapshot is not None assert prior_state == prior_state_snapshot assert result["state"] is prior_state + if expected.get("state_bytes_unchanged"): + assert prior_state_snapshot is not None + assert result["state"] == prior_state_snapshot if result["state"] is None: return state = result["state"] diff --git a/conformance/pins.py b/conformance/pins.py index 3948e11..046de39 100644 --- a/conformance/pins.py +++ b/conformance/pins.py @@ -4,8 +4,8 @@ from pathlib import Path -CONFORMANCE_COMMIT = "d6a45d31614ee25de20476ed93f10e14997d882c" -SPEC_COMMIT = "7782671b56165a59caa61a65c29fefc63105ebf8" +CONFORMANCE_COMMIT = "531468c59c7a2dc32f5cbe92cfabf89805d27f6a" +SPEC_COMMIT = "2e33036563cb966b07124197db672159b4b7e1f4" ROOT = Path(__file__).resolve().parent.parent CONFORMANCE_CACHE = ROOT / ".cache" / f"determa-state-conformance-{CONFORMANCE_COMMIT[:12]}" diff --git a/conformance/test_conformance.py b/conformance/test_conformance.py index e883955..a828df3 100644 --- a/conformance/test_conformance.py +++ b/conformance/test_conformance.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +import yaml from jsonschema import Draft202012Validator from determa.state import PORTABLE_CODE_SETS, load_bundle @@ -30,6 +31,15 @@ persistence_profile_cases, run_persistence_profile, ) +from .version2 import ( + run_version2_vector, + validate_version2_artifact, + version2_vectors, +) + + +def _load_case(case: CoreCase) -> dict: + return yaml.safe_load(case.test_file.read_text(encoding="utf-8")) or {} def _spec_schema() -> dict | None: @@ -50,8 +60,9 @@ def _spec_root() -> Path | None: def test_suite_present() -> None: assert CORE_DIR.exists(), "pinned conformance suite is unavailable" - assert len(core_cases()) == 111 - assert len(execution_checkpoint_vectors()) == 91 + assert len(core_cases()) == 114 + assert len(execution_checkpoint_vectors()) == 99 + assert len(version2_vectors()) == 106 def test_portable_code_sets_match_authoritative_registry() -> None: @@ -162,6 +173,11 @@ def test_bundled_schema_matches_pinned_spec() -> None: ("migration-descriptor.schema.json", "migration_descriptor"), ("aggregate-state-package.schema.json", "aggregate_state_package"), ("execution-checkpoint.schema.json", "execution_checkpoint"), + ("aggregate-state-v2.schema.json", "aggregate_state_v2"), + ("migration-descriptor-v2.schema.json", "migration_descriptor_v2"), + ("aggregate-state-package-v2.schema.json", "aggregate_state_package_v2"), + ("execution-checkpoint-v2.schema.json", "execution_checkpoint_v2"), + ("core-step-result-v2.schema.json", "core_step_result_v2"), ], ) def test_bundled_artifact_schemas_match_pinned_spec(name: str, kind: str) -> None: @@ -178,6 +194,11 @@ def test_bundled_artifact_schemas_match_pinned_spec(name: str, kind: str) -> Non "migration_descriptor", "aggregate_state_package", "execution_checkpoint", + "aggregate_state_v2", + "migration_descriptor_v2", + "aggregate_state_package_v2", + "execution_checkpoint_v2", + "core_step_result_v2", ], ) def test_bundled_artifact_schema_is_valid_draft_2020_12(kind: str) -> None: @@ -224,6 +245,11 @@ def test_execution_checkpoint_profile(item) -> None: run_execution_checkpoint_vector(item) +@pytest.mark.parametrize("item", version2_vectors(), ids=lambda item: item.name) +def test_version2_vector(item) -> None: + run_version2_vector(item) + + @pytest.mark.parametrize( ("case", "artifact"), [ @@ -242,3 +268,37 @@ def test_execution_checkpoint_profile(item) -> None: ) def test_execution_checkpoint_artifact(case, artifact) -> None: validate_execution_checkpoint_artifact(case, artifact) + + +@pytest.mark.parametrize( + ("case", "artifact"), + [ + (case, artifact) + for case in core_cases() + for artifact in (_load_case(case).get("artifacts", {}).get("documents", [])) + if artifact["kind"] + in { + "aggregate_state_v2", + "migration_descriptor_v2", + "aggregate_state_package_v2", + "execution_checkpoint_v2", + "core_step_result_v2", + } + ] + + [ + (case.path, artifact) + for case in execution_checkpoint_cases() + for artifact in case.test.get("artifacts", {}).get("documents", []) + if artifact["kind"] + in { + "aggregate_state_v2", + "migration_descriptor_v2", + "aggregate_state_package_v2", + "execution_checkpoint_v2", + "core_step_result_v2", + } + ], +) +def test_version2_artifact(case, artifact) -> None: + path = case.path if isinstance(case, CoreCase) else case + validate_version2_artifact(path, artifact) diff --git a/conformance/version2.py b/conformance/version2.py new file mode 100644 index 0000000..e3137fe --- /dev/null +++ b/conformance/version2.py @@ -0,0 +1,249 @@ +"""Driver for the pinned version-2 aggregate and checkpoint vectors.""" + +from __future__ import annotations + +import copy +import json +from dataclasses import dataclass +from functools import cache +from pathlib import Path +from typing import Any + +import yaml + +from determa.state import ( + ArtifactError, + ExecutionHost, + MemoryArtifactResolver, + MemoryExecutionStore, + load_bundle, + restore_aggregate_package, + serialize_execution_checkpoint, +) +from determa.state.checkpoint_v2 import ( + admit_checkpoint_v2, + prune_checkpoint_v2, + restore_execution_checkpoint_v2, + step_checkpoint_v2, + upgrade_checkpoint_v1_to_v2, +) +from determa.state.queueing import ( + admit_aggregate_v2, + create_aggregate_v2, + downgrade_aggregate_v2_to_v1, + migrate_aggregate_v2, + restore_aggregate_v2, + step_aggregate_v2, + upgrade_aggregate_v1_to_v2, +) +from determa.state.wire import load_json_artifact, migration_descriptor_digest + +from .harness import conformance_root + + +@dataclass(frozen=True) +class Version2Vector: + path: Path + vector: dict[str, Any] + + @property + def name(self) -> str: + return f"{self.path.name}/{self.vector['name']}" + + +def version2_vectors() -> list[Version2Vector]: + roots = [ + conformance_root() / "conformance" / "core", + conformance_root() / "conformance" / "profiles" / "execution-checkpoint", + ] + result = [] + for root in roots: + for path in sorted(root.iterdir()): + test_path = path / "test.yaml" + if not test_path.exists(): + continue + test = yaml.safe_load(test_path.read_text(encoding="utf-8")) or {} + result.extend(Version2Vector(path, item) for item in test.get("version2_vectors", [])) + return result + + +def _json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _pointer(document: Any, pointer: str) -> Any: + current = document + for part in pointer.removeprefix("/").split("/"): + current = current[part.replace("~1", "/").replace("~0", "~")] + return current + + +@cache +def _conformance_definitions() -> dict[str, Any]: + definitions = {} + for candidate in (conformance_root() / "conformance").glob("**/*.yaml"): + try: + bundle = load_bundle(candidate.read_text(encoding="utf-8")) + except Exception: + continue + definitions[bundle.fingerprint] = bundle + return definitions + + +def _resolver(path: Path) -> MemoryArtifactResolver: + descriptors = {} + for candidate in path.glob("*descriptor*.json"): + document = _json(candidate) + try: + descriptors[migration_descriptor_digest(document)] = document + except ArtifactError: + continue + return MemoryArtifactResolver( + definitions=_conformance_definitions(), migration_descriptors=descriptors + ) + + +def _invoke(item: Version2Vector, request: dict[str, Any]) -> Any: + path = item.path + vector = item.vector + operation = vector["operation"] + resolver = _resolver(path) + before_name = vector.get("state_before") or vector.get("checkpoint_before") + before = _json(path / before_name) if before_name else None + bundle = ( + load_bundle((path / vector["bundle"]).read_text(encoding="utf-8")) + if vector.get("bundle") + else None + ) + if operation == "create_v2": + assert bundle is not None + return create_aggregate_v2( + bundle, + request["machine_id"], + request["root_instance_id"], + request["creation_id"], + request["bindings"], + )["state"] + if operation == "admit_v2": + return admit_aggregate_v2(before, request["deliveries"], resolver) + if operation == "step_v2": + return step_aggregate_v2(before, request["target_runtime_id"], resolver) + if operation == "upgrade_aggregate_v1_to_v2": + return upgrade_aggregate_v1_to_v2(before, resolver) + if operation == "downgrade_aggregate_v2_to_v1": + return downgrade_aggregate_v2_to_v1(before, resolver) + if operation == "migrate_aggregate_v2": + return migrate_aggregate_v2( + before, + request["target_bundle"]["validated_bundle_fingerprint"], + request["migration_descriptor_digest_route"], + resolver, + maintenance_mode=request["maintenance_mode"], + ) + if operation == "upgrade_checkpoint_v1_to_v2": + return upgrade_checkpoint_v1_to_v2(before, resolver) + if operation == "checkpoint_admit_v2": + return admit_checkpoint_v2( + before, + request["deliveries"], + resolver, + expected_revision=request["expected_revision"], + expected_checkpoint_digest=request["expected_checkpoint_digest"], + ) + if operation == "checkpoint_step_v2": + return step_checkpoint_v2( + before, + request["target_runtime_id"], + resolver, + expected_revision=request["expected_revision"], + expected_checkpoint_digest=request["expected_checkpoint_digest"], + ) + if operation == "checkpoint_prune_v2": + return prune_checkpoint_v2( + before, + request["cutoff_receipt_sequence"], + resolver, + expected_revision=request["expected_revision"], + expected_checkpoint_digest=request["expected_checkpoint_digest"], + ) + if operation == "checkpoint_v1_accept": + assert bundle is not None + delivery = request["deliveries"][0] + envelope = { + key: copy.deepcopy(value) + for key, value in delivery["envelope"].items() + if key not in {"cause_id", "source"} + } + candidate = { + "root_instance_id": before["root_instance_id"], + "delivery_mode": delivery["delivery_mode"], + "origin": {"kind": "host_input"}, + "envelope": envelope, + } + host = ExecutionHost( + MemoryExecutionStore( + {before["root_instance_id"]: serialize_execution_checkpoint(before)} + ), + resolver, + ) + result = host.accept_delivery( + before["root_instance_id"], + candidate, + expected_revision=request["expected_revision"], + expected_checkpoint_digest=request["expected_checkpoint_digest"], + selected_bundle=bundle, + ) + if result["result"] == "not_accepted": + raise ArtifactError(result["failure"]["code"]) + return result + raise AssertionError(f"unsupported version-2 operation: {operation}") + + +def run_version2_vector(item: Version2Vector) -> None: + vector = item.vector + request = copy.deepcopy( + _pointer(_json(item.path / vector["request_file"]), vector["request_pointer"]) + ) + request_snapshot = copy.deepcopy(request) + try: + actual = _invoke(item, request) + code = None + except ArtifactError as error: + actual = None + code = error.code + expected = vector["expect"] + if isinstance(actual, dict) and actual.get("result") == "rejected": + code = actual["rejection"]["code"] + actual = None + if expected["result"] == "failure": + assert code == expected["code"] + else: + assert code is None + assert actual == _json(item.path / expected["exact_result_file"]) + if expected.get("caller_still_owns_input"): + assert request == request_snapshot + + +def validate_version2_artifact(path: Path, artifact: dict[str, Any]) -> None: + resolver = _resolver(path) + source = (path / artifact["file"]).read_bytes() + try: + if artifact["kind"] == "aggregate_state_v2": + restored = restore_aggregate_v2(source, resolver) + if artifact.get("canonical_of"): + assert restored.canonical_bytes == source + elif artifact["kind"] == "execution_checkpoint_v2": + restore_execution_checkpoint_v2(source, resolver) + elif artifact["kind"] == "aggregate_state_package_v2": + restore_aggregate_package(source, resolver) + elif artifact["kind"] in { + "migration_descriptor_v2", + "core_step_result_v2", + }: + load_json_artifact(source, artifact["kind"]) + else: + return + code = None + except ArtifactError as error: + code = error.code + assert code == (None if artifact["valid"] else artifact["error"]) diff --git a/pyproject.toml b/pyproject.toml index 44aa2f2..203a007 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,9 +64,14 @@ packages = ["src/determa"] [tool.hatch.build.targets.wheel.force-include] "src/determa/state/data/machine.schema.json" = "determa/state/data/machine.schema.json" "src/determa/state/data/aggregate-state.schema.json" = "determa/state/data/aggregate-state.schema.json" +"src/determa/state/data/aggregate-state-v2.schema.json" = "determa/state/data/aggregate-state-v2.schema.json" "src/determa/state/data/migration-descriptor.schema.json" = "determa/state/data/migration-descriptor.schema.json" +"src/determa/state/data/migration-descriptor-v2.schema.json" = "determa/state/data/migration-descriptor-v2.schema.json" "src/determa/state/data/aggregate-state-package.schema.json" = "determa/state/data/aggregate-state-package.schema.json" +"src/determa/state/data/aggregate-state-package-v2.schema.json" = "determa/state/data/aggregate-state-package-v2.schema.json" "src/determa/state/data/execution-checkpoint.schema.json" = "determa/state/data/execution-checkpoint.schema.json" +"src/determa/state/data/execution-checkpoint-v2.schema.json" = "determa/state/data/execution-checkpoint-v2.schema.json" +"src/determa/state/data/core-step-result-v2.schema.json" = "determa/state/data/core-step-result-v2.schema.json" [tool.ruff] line-length = 100 diff --git a/scripts/sync_schema.py b/scripts/sync_schema.py index 0c166f2..0209e12 100644 --- a/scripts/sync_schema.py +++ b/scripts/sync_schema.py @@ -22,11 +22,16 @@ SCHEMAS = ( "machine.schema.json", "aggregate-state.schema.json", + "aggregate-state-v2.schema.json", "migration-descriptor.schema.json", + "migration-descriptor-v2.schema.json", "aggregate-state-package.schema.json", + "aggregate-state-package-v2.schema.json", "execution-checkpoint.schema.json", + "execution-checkpoint-v2.schema.json", + "core-step-result-v2.schema.json", ) -SPEC_COMMIT = "7782671b56165a59caa61a65c29fefc63105ebf8" +SPEC_COMMIT = "2e33036563cb966b07124197db672159b4b7e1f4" def _fetch(name: str) -> str: @@ -49,7 +54,7 @@ def main() -> int: text = _fetch(name) json.loads(text) destination = DEST / name - if destination.read_text(encoding="utf-8") == text: + if destination.exists() and destination.read_text(encoding="utf-8") == text: print(f"{destination.relative_to(ROOT)} already up to date") continue destination.write_text(text, encoding="utf-8") diff --git a/src/determa/state/__init__.py b/src/determa/state/__init__.py index a708faa..096fdc8 100644 --- a/src/determa/state/__init__.py +++ b/src/determa/state/__init__.py @@ -14,6 +14,15 @@ validate_execution_checkpoint_member, validate_execution_checkpoint_semantics, ) +from .checkpoint_v2 import ( + RestoredExecutionCheckpointV2, + admit_checkpoint_v2, + create_checkpoint_v2, + prune_checkpoint_v2, + restore_execution_checkpoint_v2, + step_checkpoint_v2, + upgrade_checkpoint_v1_to_v2, +) from .codes import ( PORTABLE_CODE_SETS, CheckpointArtifactFailureCode, @@ -57,6 +66,16 @@ migrate_aggregate, migrate_and_dispatch, ) +from .queueing import ( + admit_aggregate_v2, + create_aggregate_v2, + downgrade_aggregate_v2_to_v1, + migrate_aggregate_v2, + restore_aggregate_v2, + seal_aggregate_v2, + step_aggregate_v2, + upgrade_aggregate_v1_to_v2, +) from .stores import ( COMPACT_EFFECT_IDENTITY_RETENTION, DURABLE_CONCURRENT, @@ -102,6 +121,8 @@ __all__ = [ "ArtifactError", "ArtifactResolver", + "admit_aggregate_v2", + "admit_checkpoint_v2", "Bundle", "BundleSource", "CelError", @@ -115,7 +136,10 @@ "DefinitionResolver", "Delivery", "CreationRejectionCode", + "create_aggregate_v2", + "create_checkpoint_v2", "DispatchRejectionCode", + "downgrade_aggregate_v2_to_v1", "DispositionCode", "ErrorRecord", "EPHEMERAL", @@ -137,6 +161,7 @@ "MigrationFailure", "MigrationLimits", "MigrationResult", + "migrate_aggregate_v2", "PERMANENT_OUTBOX_TERMINAL_RETENTION", "PERMANENT_RECEIPT_RETENTION", "PostgreSQLExecutionStore", @@ -148,6 +173,7 @@ "RestoredAggregate", "RestoredAggregatePackage", "RestoredExecutionCheckpoint", + "RestoredExecutionCheckpointV2", "SHARED_APPLICATION_TRANSACTION", "STANDARD_CAPABILITIES", "SchemaError", @@ -174,18 +200,26 @@ "outbox_intent_digest", "portable_envelope", "postgresql_execution_store_factory", + "prune_checkpoint_v2", "register_bundled_execution_stores", "restore_aggregate", + "restore_aggregate_v2", "restore_aggregate_package", "restore_execution_checkpoint", + "restore_execution_checkpoint_v2", + "seal_aggregate_v2", "seal_execution_checkpoint", "serialize_aggregate", "serialize_execution_checkpoint", + "step_aggregate_v2", + "step_checkpoint_v2", "sqlite_execution_store_factory", "validate", "validate_execution_checkpoint_member", "validate_execution_checkpoint_semantics", "validate_host_profile", + "upgrade_aggregate_v1_to_v2", + "upgrade_checkpoint_v1_to_v2", ] logging.getLogger("determa.state").addHandler(logging.NullHandler()) diff --git a/src/determa/state/checkpoint.py b/src/determa/state/checkpoint.py index b7fcab4..4b1c317 100644 --- a/src/determa/state/checkpoint.py +++ b/src/determa/state/checkpoint.py @@ -43,10 +43,15 @@ class RestoredExecutionCheckpoint: def execution_checkpoint_digest(document: Mapping[str, Any]) -> str: - """Compute the exact schema-version-1 checkpoint digest.""" + """Compute the exact versioned checkpoint digest.""" body = copy.deepcopy(dict(document)) body.pop("execution_checkpoint_digest", None) - return hash_value(["determa-execution-checkpoint-digest-1", body]) + domain = ( + "determa-execution-checkpoint-digest-2" + if body.get("execution_checkpoint_schema_version") == 2 + else "determa-execution-checkpoint-digest-1" + ) + return hash_value([domain, body]) def seal_execution_checkpoint(document: Mapping[str, Any]) -> dict[str, Any]: diff --git a/src/determa/state/checkpoint_v2.py b/src/determa/state/checkpoint_v2.py new file mode 100644 index 0000000..0e4d04b --- /dev/null +++ b/src/determa/state/checkpoint_v2.py @@ -0,0 +1,1325 @@ +"""Queue-bearing execution-checkpoint version 2 operations.""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from .checkpoint import ( + execution_checkpoint_digest, + restore_execution_checkpoint, + seal_execution_checkpoint, + validate_execution_checkpoint_member, +) +from .codes import CheckpointArtifactFailureCode, CheckpointHostFailureCode +from .errors import ArtifactError +from .host import creation_request_digest, delivery_request_digest +from .queueing import ( + _entry_digest, + _runtime_id_for_target, + _valid_envelope_shape, + admit_aggregate_v2, + create_aggregate_v2, + restore_aggregate_v2, + seal_aggregate_v2, + step_aggregate_v2, + upgrade_aggregate_v1_to_v2, +) +from .wire import ( + ArtifactSource, + DefinitionResolver, + canonical_bytes, + decimal, + hash_value, + load_json_artifact, +) + + +@dataclass(frozen=True) +class RestoredExecutionCheckpointV2: + """One verified queue-bearing checkpoint.""" + + document: dict[str, Any] + canonical_bytes: bytes + source_bytes: bytes + + +def _invalid() -> ArtifactError: + return ArtifactError(CheckpointArtifactFailureCode.INVALID_EXECUTION_CHECKPOINT) + + +def _mailbox_entries(aggregate: Mapping[str, Any]) -> list[dict[str, Any]]: + return [ + entry + for runtime in aggregate["runtimes"] + for mailbox in ("ready_mailbox", "deferred_mailbox") + for entry in runtime[mailbox] + ] + + +def _synchronize_mailbox_references(checkpoint: dict[str, Any]) -> None: + aggregate = checkpoint["root_record"].get("aggregate_state") + if aggregate is None: + return + locations = { + (entry["envelope"]["event_id"], entry["acceptance_sequence"]): entry["queue_sequence"] + for entry in _mailbox_entries(aggregate) + } + for receipt in checkpoint["operation_receipts"]: + for reference in receipt.get("emission_references", []): + if reference.get("kind") != "internal_mailbox": + continue + key = (reference["event_id"], reference["acceptance_sequence"]) + if key in locations: + reference["queue_sequence"] = locations[key] + + +def _terminalize_mailbox_reference( + checkpoint: dict[str, Any], + entry: Mapping[str, Any], + terminal_receipt_sequence: str, +) -> None: + for receipt in checkpoint["operation_receipts"]: + for reference in receipt.get("emission_references", []): + if ( + reference.get("kind") == "internal_mailbox" + and reference.get("event_id") == entry["envelope"]["event_id"] + and reference.get("acceptance_sequence") == entry["acceptance_sequence"] + ): + reference.pop("queue_sequence") + reference["kind"] = "internal_terminal" + reference["terminal_receipt_sequence"] = terminal_receipt_sequence + + +def _validate_checkpoint_semantics(document: dict[str, Any]) -> None: + revision = decimal(document["revision"]) + next_receipt = decimal(document["next_operation_receipt_sequence"]) + receipts = document["operation_receipts"] + sequences = [decimal(receipt["receipt_sequence"]) for receipt in receipts] + if ( + not receipts + or sequences != sorted(sequences) + or len(sequences) != len(set(sequences)) + or any(sequence >= next_receipt for sequence in sequences) + or receipts[0]["receipt_sequence"] != "0" + ): + raise _invalid() + cutoff_value = document["replay_retention"]["pruned_through_receipt_sequence"] + cutoff = decimal(cutoff_value) if cutoff_value is not None else None + first_retained = 1 if cutoff is None else cutoff + 1 + if len(sequences) != 1 + max(0, next_receipt - first_retained): + raise _invalid() + if any( + sequence != (0 if index == 0 else first_retained + index - 1) + for index, sequence in enumerate(sequences) + ): + raise _invalid() + + root = document["root_record"] + aggregate = root.get("aggregate_state") + if aggregate is not None and (aggregate["root_instance_id"] != document["root_instance_id"]): + raise _invalid() + creation = receipts[0] + legacy_creation = creation.get("legacy_receipt", creation) + root_creation_id = ( + aggregate["creation_id"] if aggregate is not None else root.get("creation_id") + ) + if legacy_creation.get("creation_id") != root_creation_id: + raise _invalid() + if ( + aggregate is not None + and revision == 0 + and legacy_creation.get("resulting_aggregate_state_digest") + != aggregate["aggregate_state_digest"] + ): + raise _invalid() + + pending_entries = _mailbox_entries(aggregate) if aggregate is not None else [] + pending_by_event = {entry["envelope"]["event_id"]: entry for entry in pending_entries} + if len(pending_by_event) != len(pending_entries): + raise _invalid() + acceptances: dict[str, dict[str, Any]] = {} + terminals: dict[str, dict[str, Any]] = {} + referenced_pending: dict[str, dict[str, Any]] = {} + referenced_terminal: dict[str, dict[str, Any]] = {} + pending_producers: dict[str, dict[str, Any]] = {} + terminal_producers: dict[str, dict[str, Any]] = {} + legacy_wrappers: list[dict[str, Any]] = [] + legacy_producer_references: dict[ + str, list[tuple[dict[str, Any], dict[str, Any]]] + ] = {} + legacy_terminal_events: dict[str, dict[str, Any]] = {} + referenced_effects: set[str] = set() + committed_order: list[tuple[int, int]] = [] + for receipt in receipts: + kind = receipt["operation_kind"] + if kind == "acceptance": + if decimal(receipt["accepted_revision"]) > revision: + raise _invalid() + event_id = receipt["event_id"] + if event_id in acceptances: + raise _invalid() + acceptances[event_id] = receipt + elif kind == "event_terminal": + if decimal(receipt["committed_revision"]) > revision: + raise _invalid() + event_id = receipt["event_id"] + if event_id in terminals: + raise _invalid() + terminals[event_id] = receipt + committed_order.append( + ( + decimal(receipt["committed_revision"]), + decimal(receipt["receipt_sequence"]), + ) + ) + elif kind == "creation" and decimal(receipt["committed_revision"]) > revision: + raise _invalid() + elif kind == "legacy_v1_operation": + legacy_receipt = receipt["legacy_receipt"] + if decimal(legacy_receipt["committed_revision"]) > revision: + raise _invalid() + if kind in {"legacy_v1_creation", "legacy_v1_operation"}: + legacy_wrappers.append(receipt) + legacy_receipt = receipt["legacy_receipt"] + if legacy_receipt["receipt_sequence"] != receipt["receipt_sequence"]: + raise _invalid() + for reference in legacy_receipt.get("emission_references", []): + if reference["kind"] == "internal_delivery": + legacy_producer_references.setdefault(reference["event_id"], []).append( + (receipt, reference) + ) + references = receipt.get("emission_references") + if references is None and kind.startswith("legacy_v1_"): + references = receipt["legacy_receipt"].get("emission_references", []) + for reference in references or []: + reference_kind = reference["kind"] + if reference_kind == "internal_mailbox": + event_id = reference["event_id"] + if event_id in referenced_pending: + raise _invalid() + referenced_pending[event_id] = reference + pending_producers[event_id] = receipt + elif reference_kind == "internal_terminal": + event_id = reference["event_id"] + if event_id in referenced_terminal: + raise _invalid() + referenced_terminal[event_id] = reference + terminal_producers[event_id] = receipt + elif reference_kind == "external_outbox": + effect_id = reference["effect_id"] + if effect_id in referenced_effects: + raise _invalid() + referenced_effects.add(effect_id) + if legacy_wrappers and receipts[: len(legacy_wrappers)] != legacy_wrappers: + raise _invalid() + if committed_order != sorted(committed_order) or len(committed_order) != len( + set(committed_order) + ): + raise _invalid() + + def validate_internal_legacy_producer( + event_id: str, + acceptance_sequence: str, + accepted_revision: str, + origin: Mapping[str, Any], + ) -> None: + producers = legacy_producer_references.get(event_id, []) + if ( + len(producers) != 1 + or producers[0][0]["receipt_sequence"] + != origin["producing_receipt_sequence"] + or producers[0][1]["emission_index"] != origin["emission_index"] + or producers[0][1]["delivery_sequence"] != acceptance_sequence + or producers[0][0]["legacy_receipt"]["committed_revision"] + != accepted_revision + ): + raise _invalid() + + legacy_terminal_acceptance_sequences: set[str] = set() + for wrapper in legacy_wrappers: + legacy_receipt = wrapper["legacy_receipt"] + if legacy_receipt["operation_kind"] != "delivery": + continue + event_id = legacy_receipt["event_id"] + acceptance_sequence = legacy_receipt["accepted_delivery_sequence"] + if ( + event_id in legacy_terminal_events + or acceptance_sequence in legacy_terminal_acceptance_sequences + ): + raise _invalid() + legacy_terminal_events[event_id] = legacy_receipt + legacy_terminal_acceptance_sequences.add(acceptance_sequence) + if legacy_receipt["delivery_mode"] == "internal": + origin = legacy_receipt["origin"] + if not ( + origin.get("kind") == "internal_emission" + and set(origin) + == {"kind", "producing_receipt_sequence", "emission_index"} + ): + raise _invalid() + validate_internal_legacy_producer( + event_id, + acceptance_sequence, + legacy_receipt["accepted_revision"], + origin, + ) + + def validate_legacy_evidence( + acceptance: Mapping[str, Any], entry: Mapping[str, Any] | None + ) -> None: + evidence = acceptance.get("legacy_v1_delivery") + if evidence is None: + return + origin = evidence["origin"] + mode = acceptance["delivery_mode"] + if ( + evidence["delivery_sequence"] != acceptance["acceptance_sequence"] + or evidence["envelope_digest"] == "sha256:" + ("0" * 64) + or evidence["envelope_digest"] == acceptance["request_digest"] + ): + raise _invalid() + if mode == "input": + if origin != {"kind": "host_input"}: + raise _invalid() + elif not ( + origin.get("kind") == "internal_emission" + and set(origin) == {"kind", "producing_receipt_sequence", "emission_index"} + ): + raise _invalid() + if entry is not None: + source = entry["envelope"]["source"] + legacy_envelope = copy.deepcopy(entry["envelope"]) + legacy_envelope.pop("cause_id") + legacy_envelope.pop("source") + expected_legacy_digest = hash_value( + [ + "determa-inbox-envelope-digest-1", + "1", + document["root_instance_id"], + entry["delivery_mode"], + legacy_envelope, + ] + ) + source_matches = (source == {"host": True} and mode == "input") or ( + mode == "internal" + and source + == { + "legacy_v1_internal": { + "producing_receipt_sequence": origin["producing_receipt_sequence"], + "emission_index": origin["emission_index"], + } + } + ) + if evidence["envelope_digest"] != expected_legacy_digest or not source_matches: + raise _invalid() + if mode == "internal": + validate_internal_legacy_producer( + acceptance["event_id"], + acceptance["acceptance_sequence"], + acceptance["accepted_revision"], + origin, + ) + + for event_id, entry in pending_by_event.items(): + acceptance = acceptances.get(event_id) + producer = referenced_pending.get(event_id) + if acceptance is None and not producer: + raise _invalid() + if acceptance is not None and producer is not None: + raise _invalid() + if acceptance is not None and ( + acceptance["request_digest"] != entry["envelope_digest"] + or acceptance["acceptance_sequence"] != entry["acceptance_sequence"] + ): + raise _invalid() + if producer is not None and ( + producer["acceptance_sequence"] != entry["acceptance_sequence"] + or producer["queue_sequence"] != entry["queue_sequence"] + ): + raise _invalid() + source = entry["envelope"]["source"] + if "legacy_v1_internal" in source: + if acceptance is None or acceptance.get("legacy_v1_delivery") is None: + raise _invalid() + validate_legacy_evidence(acceptance, entry) + elif source == {"host": True}: + if acceptance is None: + raise _invalid() + validate_legacy_evidence(acceptance, entry) + elif acceptance is not None: + raise _invalid() + if set(pending_by_event) & set(terminals): + raise _invalid() + + for event_id, terminal in terminals.items(): + acceptance = acceptances.get(event_id) + producer = referenced_terminal.get(event_id) + if acceptance is None and producer is None and cutoff is None: + raise _invalid() + if acceptance is not None and producer is not None: + raise _invalid() + if acceptance is not None and ( + terminal["request_digest"] != acceptance["request_digest"] + or terminal["acceptance_sequence"] != acceptance["acceptance_sequence"] + or decimal(acceptance["receipt_sequence"]) >= decimal(terminal["receipt_sequence"]) + or decimal(acceptance["accepted_revision"]) > decimal(terminal["committed_revision"]) + ): + raise _invalid() + if producer is not None and ( + producer["terminal_receipt_sequence"] != terminal["receipt_sequence"] + or producer["acceptance_sequence"] != terminal["acceptance_sequence"] + or decimal(terminal_producers[event_id]["receipt_sequence"]) + >= decimal(terminal["receipt_sequence"]) + or decimal(terminal_producers[event_id].get("committed_revision", "0")) + > decimal(terminal["committed_revision"]) + ): + raise _invalid() + if acceptance is not None: + validate_legacy_evidence(acceptance, None) + + tombstones = document["event_identity_tombstones"] + tombstone_events = [item["event_id"] for item in tombstones] + tombstone_sequences = [decimal(item["terminal_receipt_sequence"]) for item in tombstones] + if ( + len(tombstone_events) != len(set(tombstone_events)) + or set(tombstone_events) & set(pending_by_event) + or set(tombstone_events) & set(acceptances) + or set(tombstone_events) & set(terminals) + or set(tombstone_sequences) + & {decimal(item["receipt_sequence"]) for item in terminals.values()} + or any(sequence >= next_receipt for sequence in tombstone_sequences) + or tombstone_sequences != sorted(tombstone_sequences) + ): + raise _invalid() + tombstones_by_event = {item["event_id"]: item for item in tombstones} + if ( + set(legacy_terminal_events) & set(pending_by_event) + or set(legacy_terminal_events) & set(terminals) + or set(legacy_terminal_events) & set(tombstones_by_event) + or set(legacy_terminal_events) & set(acceptances) + ): + raise _invalid() + if any( + event_id not in pending_by_event and event_id not in terminals for event_id in acceptances + ): + raise _invalid() + if any( + decimal(producer["receipt_sequence"]) >= next_receipt + for producer in [*pending_producers.values(), *terminal_producers.values()] + ): + raise _invalid() + for event_id, references in legacy_producer_references.items(): + if len(references) != 1: + raise _invalid() + located_entry = pending_by_event.get(event_id) + located_terminal = terminals.get(event_id) + located_tombstone = tombstones_by_event.get(event_id) + located_legacy_terminal = legacy_terminal_events.get(event_id) + locations = [ + item + for item in ( + located_entry, + located_terminal, + located_tombstone, + located_legacy_terminal, + ) + if item is not None + ] + if len(locations) != 1: + raise _invalid() + allocation = locations[0].get( + "acceptance_sequence", locations[0].get("accepted_delivery_sequence") + ) + if references[0][1]["delivery_sequence"] != allocation: + raise _invalid() + for event_id, reference in referenced_terminal.items(): + terminal_evidence = terminals.get(event_id) + tombstone = tombstones_by_event.get(event_id) + evidence = terminal_evidence if terminal_evidence is not None else tombstone + evidence_sequence = ( + evidence["receipt_sequence"] + if terminal_evidence is not None and evidence is not None + else evidence["terminal_receipt_sequence"] + if evidence is not None + else None + ) + if evidence_sequence is None or reference["terminal_receipt_sequence"] != evidence_sequence: + raise _invalid() + + allocation_acceptances = ( + [decimal(entry["acceptance_sequence"]) for entry in pending_entries] + + [decimal(item["acceptance_sequence"]) for item in acceptances.values()] + + [decimal(item["acceptance_sequence"]) for item in terminals.values()] + + [decimal(item["acceptance_sequence"]) for item in tombstones] + + [ + decimal(item["accepted_delivery_sequence"]) + for item in legacy_terminal_events.values() + ] + ) + allocation_queues = [decimal(entry["queue_sequence"]) for entry in pending_entries] + [ + decimal(item["final_queue_sequence"]) for item in terminals.values() + ] + if len(allocation_queues) != len(set(allocation_queues)): + raise _invalid() + acceptance_owners: dict[int, str] = {} + for item in [ + *pending_entries, + *acceptances.values(), + *terminals.values(), + *tombstones, + ]: + sequence = decimal(item["acceptance_sequence"]) + envelope = item.get("envelope", {}) + event_id = item.get("event_id", envelope.get("event_id")) + prior_owner = acceptance_owners.setdefault(sequence, event_id) + if prior_owner != event_id: + raise _invalid() + for event_id, item in legacy_terminal_events.items(): + sequence = decimal(item["accepted_delivery_sequence"]) + prior_owner = acceptance_owners.setdefault(sequence, event_id) + if prior_owner != event_id: + raise _invalid() + if aggregate is not None and ( + any( + value >= decimal(aggregate["next_acceptance_sequence"]) + for value in allocation_acceptances + ) + or any(value >= decimal(aggregate["next_queue_sequence"]) for value in allocation_queues) + ): + raise _invalid() + current_aggregate_digest = ( + aggregate["aggregate_state_digest"] + if aggregate is not None + else root.get("final_aggregate_state_digest") + ) + digests_by_revision: dict[int, str] = {} + for terminal in terminals.values(): + committed_revision = decimal(terminal["committed_revision"]) + digest = terminal["resulting_aggregate_state_digest"] + prior_digest = digests_by_revision.setdefault(committed_revision, digest) + if prior_digest != digest or ( + committed_revision == revision and digest != current_aggregate_digest + ): + raise _invalid() + + pending_effects = {item["intent"]["effect_id"] for item in document["pending_outbox_intents"]} + terminal_effects = {item["intent"]["effect_id"] for item in document["terminal_outbox_records"]} + effect_tombstones = {item["effect_id"] for item in document["outbox_effect_tombstones"]} + if ( + len(pending_effects) != len(document["pending_outbox_intents"]) + or len(terminal_effects) != len(document["terminal_outbox_records"]) + or len(effect_tombstones) != len(document["outbox_effect_tombstones"]) + or pending_effects & terminal_effects + or pending_effects & effect_tombstones + or terminal_effects & effect_tombstones + or referenced_effects != pending_effects | terminal_effects | effect_tombstones + ): + raise _invalid() + terminal_sequences = [ + decimal(item["terminal_sequence"]) for item in document["terminal_outbox_records"] + ] + [decimal(item["terminal_sequence"]) for item in document["outbox_effect_tombstones"]] + terminal_record_sequences = [ + decimal(item["terminal_sequence"]) for item in document["terminal_outbox_records"] + ] + effect_tombstone_sequences = [ + decimal(item["terminal_sequence"]) for item in document["outbox_effect_tombstones"] + ] + if ( + len(terminal_sequences) != len(set(terminal_sequences)) + or terminal_record_sequences != sorted(terminal_record_sequences) + or effect_tombstone_sequences != sorted(effect_tombstone_sequences) + or any( + value >= decimal(document["next_outbox_terminal_sequence"]) + for value in terminal_sequences + ) + ): + raise _invalid() + + audits = document["migration_audit_records"] + audit_sequences = [decimal(item["migration_sequence"]) for item in audits] + if ( + audit_sequences != sorted(audit_sequences) + or len(audit_sequences) != len(set(audit_sequences)) + or any(item["root_instance_id"] != document["root_instance_id"] for item in audits) + or ( + aggregate is not None + and any( + sequence > decimal(aggregate["migration_sequence"]) for sequence in audit_sequences + ) + ) + ): + raise _invalid() + + +def restore_execution_checkpoint_v2( + source: ArtifactSource, definition_resolver: DefinitionResolver +) -> RestoredExecutionCheckpointV2: + """Restore one structurally and relationally valid version-2 checkpoint.""" + document, raw = load_json_artifact(source, "execution_checkpoint_v2") + if execution_checkpoint_digest(document) != document["execution_checkpoint_digest"]: + raise ArtifactError(CheckpointArtifactFailureCode.EXECUTION_CHECKPOINT_DIGEST_MISMATCH) + aggregate = document["root_record"].get("aggregate_state") + if aggregate is not None: + try: + restore_aggregate_v2(aggregate, definition_resolver) + except ArtifactError as error: + if error.code in { + "invalid_aggregate_state", + "aggregate_state_digest_mismatch", + }: + raise _invalid() from error + raise + try: + _validate_checkpoint_semantics(document) + except (ArtifactError, KeyError, TypeError, ValueError) as error: + if isinstance(error, ArtifactError) and error.code == str( + CheckpointArtifactFailureCode.INVALID_EXECUTION_CHECKPOINT.value + ): + raise + raise _invalid() from error + return RestoredExecutionCheckpointV2( + document=copy.deepcopy(document), + canonical_bytes=canonical_bytes(document), + source_bytes=raw, + ) + + +def upgrade_checkpoint_v1_to_v2( + source: ArtifactSource, definition_resolver: DefinitionResolver +) -> dict[str, Any]: + """Explicitly convert a complete version-1 checkpoint and pending history.""" + restored = restore_execution_checkpoint(source, definition_resolver) + document = restored.document + if document["root_record"]["status"] != "retained": + raise _invalid() + aggregate = upgrade_aggregate_v1_to_v2( + document["root_record"]["aggregate_state"], definition_resolver + ) + receipts: list[dict[str, Any]] = [] + for receipt in document["operation_receipts"]: + receipts.append( + { + "operation_kind": ( + "legacy_v1_creation" + if receipt["operation_kind"] == "creation" + else "legacy_v1_operation" + ), + "receipt_sequence": receipt["receipt_sequence"], + "legacy_receipt": copy.deepcopy(receipt), + } + ) + next_receipt = int(document["next_operation_receipt_sequence"]) + next_queue = 0 + aggregate["next_acceptance_sequence"] = document["next_delivery_sequence"] + runtime_by_id = {runtime["runtime_id"]: runtime for runtime in aggregate["runtimes"]} + for pending in sorted( + document["pending_deliveries"], key=lambda item: int(item["delivery_sequence"]) + ): + origin = pending["origin"] + envelope = copy.deepcopy(pending["envelope"]) + envelope["cause_id"] = envelope["event_id"] + envelope["source"] = ( + {"host": True} + if origin["kind"] == "host_input" + else { + "legacy_v1_internal": { + "producing_receipt_sequence": origin["producing_receipt_sequence"], + "emission_index": origin["emission_index"], + } + } + ) + digest = _entry_digest(document["root_instance_id"], pending["delivery_mode"], envelope) + entry = { + "acceptance_sequence": pending["delivery_sequence"], + "queue_sequence": str(next_queue), + "delivery_mode": pending["delivery_mode"], + "envelope": envelope, + "envelope_digest": digest, + "deferral_count": "0", + } + runtime_id = _runtime_id_for_target(envelope["target"]) + if runtime_id not in runtime_by_id: + raise _invalid() + runtime_by_id[runtime_id]["ready_mailbox"].append(entry) + acceptance = { + "operation_kind": "acceptance", + "receipt_sequence": str(next_receipt), + "event_id": envelope["event_id"], + "request_digest": digest, + "acceptance_sequence": pending["delivery_sequence"], + "accepted_revision": pending["accepted_revision"], + "delivery_mode": pending["delivery_mode"], + } + if pending["delivery_mode"] == "internal": + acceptance["legacy_v1_delivery"] = { + "delivery_sequence": pending["delivery_sequence"], + "envelope_digest": pending["envelope_digest"], + "origin": copy.deepcopy(origin), + } + receipts.append(acceptance) + next_receipt += 1 + next_queue += 1 + aggregate["next_queue_sequence"] = str(next_queue) + aggregate = seal_aggregate_v2(aggregate) + result = { + "execution_checkpoint_format": "determa.execution_checkpoint", + "execution_checkpoint_schema_version": 2, + "root_instance_id": document["root_instance_id"], + "revision": str(int(document["revision"]) + 1), + "root_record": {"status": "retained", "aggregate_state": aggregate}, + "replay_retention": copy.deepcopy(document["replay_retention"]), + "next_operation_receipt_sequence": str(next_receipt), + "operation_receipts": receipts, + "event_identity_tombstones": [], + "pending_outbox_intents": copy.deepcopy(document["pending_outbox_intents"]), + "next_outbox_terminal_sequence": document["next_outbox_terminal_sequence"], + "terminal_outbox_records": copy.deepcopy(document["terminal_outbox_records"]), + "outbox_effect_tombstones": copy.deepcopy(document["outbox_effect_tombstones"]), + "migration_audit_records": copy.deepcopy(document["migration_audit_records"]), + } + return seal_execution_checkpoint(result) + + +def _append_external_intent( + checkpoint: dict[str, Any], + references: list[dict[str, Any]], + emission: Mapping[str, Any], + emission_index: int, +) -> None: + checkpoint["pending_outbox_intents"].append( + { + "intent": copy.deepcopy(dict(emission)), + "state_revision": checkpoint["revision"], + "delivery_state": {"status": "not_attempted"}, + } + ) + references.append( + { + "kind": "external_outbox", + "emission_index": str(emission_index), + "effect_id": emission["effect_id"], + } + ) + + +def create_checkpoint_v2( + bundle: Any, + machine_id: str, + root_instance_id: str, + creation_id: str, + bindings: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Create a fresh queue-bearing checkpoint from the public v2 core result.""" + result = create_aggregate_v2(bundle, machine_id, root_instance_id, creation_id, bindings) + aggregate = result["state"] + if aggregate is None: + raise ArtifactError(CheckpointHostFailureCode.CREATION_REJECTED) + receipt = { + "operation_kind": "creation", + "receipt_sequence": "0", + "creation_id": creation_id, + "request_digest": creation_request_digest( + bundle, machine_id, root_instance_id, creation_id, bindings or {} + ), + "committed_revision": "0", + "resulting_aggregate_state_digest": aggregate["aggregate_state_digest"], + "status": result["status"], + "fault": copy.deepcopy(result["fault"]), + "emission_references": [], + } + checkpoint: dict[str, Any] = { + "execution_checkpoint_format": "determa.execution_checkpoint", + "execution_checkpoint_schema_version": 2, + "root_instance_id": root_instance_id, + "revision": "0", + "root_record": {"status": "retained", "aggregate_state": aggregate}, + "replay_retention": { + "mode": "permanent", + "permanent_replay_eligible": True, + "pruned_through_receipt_sequence": None, + "policy_identifier": None, + }, + "next_operation_receipt_sequence": "1", + "operation_receipts": [receipt], + "event_identity_tombstones": [], + "pending_outbox_intents": [], + "next_outbox_terminal_sequence": "0", + "terminal_outbox_records": [], + "outbox_effect_tombstones": [], + "migration_audit_records": [], + } + lifecycle_sequences = [str(index + 1) for index in range(len(result["lifecycle_dispositions"]))] + checkpoint["next_operation_receipt_sequence"] = str(1 + len(lifecycle_sequences)) + for index, emission in enumerate(result["emissions"]): + if "kind" not in emission: + _append_external_intent(checkpoint, receipt["emission_references"], emission, index) + elif emission["kind"] == "internal_mailbox": + receipt["emission_references"].append(copy.deepcopy(emission)) + else: + lifecycle_index = int(emission["lifecycle_disposition_index"]) + receipt["emission_references"].append( + { + "kind": "internal_terminal", + "emission_index": emission["emission_index"], + "event_id": emission["event_id"], + "acceptance_sequence": emission["acceptance_sequence"], + "terminal_receipt_sequence": lifecycle_sequences[lifecycle_index], + } + ) + for lifecycle, sequence in zip( + result["lifecycle_dispositions"], lifecycle_sequences, strict=True + ): + checkpoint["operation_receipts"].append( + { + "operation_kind": "event_terminal", + "receipt_sequence": sequence, + "event_id": lifecycle["event_id"], + "request_digest": lifecycle["request_digest"], + "acceptance_sequence": lifecycle["acceptance_sequence"], + "final_queue_sequence": lifecycle["final_queue_sequence"], + "committed_revision": "0", + "resulting_aggregate_state_digest": aggregate["aggregate_state_digest"], + "outcome": { + "status": result["status"], + "disposition": "disposed", + "reason": lifecycle["reason"], + "fault": None, + "rejection": None, + }, + "emission_references": [], + } + ) + return seal_execution_checkpoint(checkpoint) + + +def _check_cas(document: Mapping[str, Any], revision: str, digest: str) -> None: + if document["revision"] != revision or document["execution_checkpoint_digest"] != digest: + raise ArtifactError(CheckpointHostFailureCode.CHECKPOINT_REVISION_CONFLICT) + + +def _pending_replay( + document: Mapping[str, Any], event_id: str, request_digest: str +) -> dict[str, Any] | None: + aggregate = document["root_record"].get("aggregate_state") + if aggregate is None: + return None + for runtime in aggregate["runtimes"]: + for mailbox, location in ( + ("ready_mailbox", "ready"), + ("deferred_mailbox", "deferred"), + ): + for entry in runtime[mailbox]: + if entry["envelope"]["event_id"] == event_id: + if entry["envelope_digest"] != request_digest: + raise ArtifactError(CheckpointHostFailureCode.EVENT_ID_CONFLICT) + return { + "result": "replay", + "event_id": event_id, + "acceptance_sequence": entry["acceptance_sequence"], + "location": location, + } + return None + + +def _retained_replay( + document: Mapping[str, Any], event_id: str, digests: Mapping[str, str] +) -> dict[str, Any] | None: + terminal = next( + ( + receipt + for receipt in document["operation_receipts"] + if receipt["operation_kind"] == "event_terminal" and receipt["event_id"] == event_id + ), + None, + ) + acceptance = next( + ( + receipt + for receipt in document["operation_receipts"] + if receipt["operation_kind"] == "acceptance" and receipt["event_id"] == event_id + ), + None, + ) + if terminal is not None: + if terminal["request_digest"] != digests["determa-inbox-envelope-digest-2"]: + raise ArtifactError(CheckpointHostFailureCode.EVENT_ID_CONFLICT) + return { + "result": "replay", + "acceptance_receipt_sequence": ( + acceptance["receipt_sequence"] if acceptance is not None else "0" + ), + "terminal_receipt_sequence": terminal["receipt_sequence"], + } + tombstone = next( + (item for item in document["event_identity_tombstones"] if item["event_id"] == event_id), + None, + ) + if tombstone is not None: + if tombstone["request_digest"] != digests[tombstone["request_digest_domain"]]: + raise ArtifactError(CheckpointHostFailureCode.EVENT_ID_CONFLICT) + result = { + "result": "replay", + "terminal_receipt_sequence": tombstone["terminal_receipt_sequence"], + "terminal_disposition": tombstone["terminal_disposition"], + } + if tombstone["request_digest_domain"] == "determa-inbox-envelope-digest-1": + result.update( + { + "event_id": event_id, + "acceptance_sequence": tombstone["acceptance_sequence"], + "request_digest_domain": tombstone["request_digest_domain"], + } + ) + return result + return None + + +def _legacy_replay( + document: Mapping[str, Any], event_id: str, request_digest: str +) -> dict[str, Any] | None: + for receipt in document["operation_receipts"]: + if receipt["operation_kind"] != "legacy_v1_operation": + continue + legacy = receipt["legacy_receipt"] + if legacy.get("operation_kind") != "delivery" or legacy.get("event_id") != event_id: + continue + if legacy["request_digest"] != request_digest: + raise ArtifactError(CheckpointHostFailureCode.EVENT_ID_CONFLICT) + return { + "result": "replay", + "event_id": event_id, + "acceptance_sequence": legacy["accepted_delivery_sequence"], + "terminal_receipt_sequence": receipt["receipt_sequence"], + "terminal_disposition": legacy["outcome"]["disposition"], + "request_digest_domain": "determa-inbox-envelope-digest-1", + } + return None + + +def admit_checkpoint_v2( + source: ArtifactSource, + deliveries: Sequence[Mapping[str, Any]], + definition_resolver: DefinitionResolver, + *, + expected_revision: str, + expected_checkpoint_digest: str, +) -> dict[str, Any]: + """Atomically admit or replay a delivery batch against one v2 checkpoint.""" + restored = restore_execution_checkpoint_v2(source, definition_resolver) + document = restored.document + snapshot = copy.deepcopy(deliveries) + terminal_code: str | None + if document["root_record"]["status"] == "tombstone": + terminal_code = "tombstoned_root" + else: + aggregate = document["root_record"].get("aggregate_state") + terminal_code = ( + "terminal_root" + if aggregate is not None + and next( + runtime + for runtime in aggregate["runtimes"] + if runtime["runtime_id"] == aggregate["root_runtime_id"] + )["status"] + != "running" + else None + ) + if ( + not isinstance(deliveries, Sequence) + or isinstance(deliveries, str | bytes) + or not deliveries + ): + raise ArtifactError("malformed_delivery") + + canonical_digests: list[str] = [] + event_ids: list[str] = [] + target_roots: list[Any] = [] + legacy_domains: list[bool] = [] + for delivery in deliveries: + legacy_domain = ( + isinstance(delivery, Mapping) + and delivery.get("request_digest_domain") == "determa-inbox-envelope-digest-1" + ) + expected_members = {"delivery_mode", "envelope", "envelope_digest"} + if legacy_domain: + expected_members.add("request_digest_domain") + envelope_is_valid = ( + validate_execution_checkpoint_member("envelope", delivery.get("envelope")) + if legacy_domain and isinstance(delivery, Mapping) + else _valid_envelope_shape(delivery.get("envelope")) + if isinstance(delivery, Mapping) + else False + ) + if ( + not isinstance(delivery, Mapping) + or set(delivery) != expected_members + or not isinstance(delivery.get("delivery_mode"), str) + or not isinstance(delivery.get("envelope_digest"), str) + or not envelope_is_valid + ): + raise ArtifactError("malformed_delivery") + envelope = delivery["envelope"] + mode = delivery.get("delivery_mode") + event_id = envelope.get("event_id") + if not isinstance(event_id, str) or not event_id: + raise ArtifactError("malformed_delivery") + target = envelope.get("target") + if not isinstance(target, Mapping) or len(target) != 1: + raise ArtifactError("malformed_delivery") + identity = next(iter(target.values())) + if not isinstance(identity, Mapping): + raise ArtifactError("malformed_delivery") + event_ids.append(event_id) + target_roots.append(identity.get("root_instance_id")) + legacy_domains.append(legacy_domain) + digest = ( + delivery_request_digest(document["root_instance_id"], str(mode), envelope) + if legacy_domain + else _entry_digest(document["root_instance_id"], str(mode), envelope) + ) + canonical_digests.append(digest) + + if any(root != document["root_instance_id"] for root in target_roots): + raise ArtifactError("wrong_root") + if len(event_ids) != len(set(event_ids)): + raise ArtifactError("duplicate_event_id_in_batch") + + replay_evidence: list[dict[str, Any] | None] = [] + for delivery, digest, legacy_domain in zip( + deliveries, canonical_digests, legacy_domains, strict=True + ): + envelope = delivery["envelope"] + mode = delivery["delivery_mode"] + event_id = envelope["event_id"] + v1_envelope = { + key: copy.deepcopy(value) + for key, value in envelope.items() + if key not in {"cause_id", "source"} + } + replay_digests = { + "determa-inbox-envelope-digest-2": "" if legacy_domain else digest, + "determa-inbox-envelope-digest-1": delivery_request_digest( + document["root_instance_id"], str(mode), v1_envelope + ), + } + replay = _pending_replay(document, event_id, digest) + if replay is None: + replay = _retained_replay(document, event_id, replay_digests) + if replay is None: + replay = _legacy_replay( + document, event_id, replay_digests["determa-inbox-envelope-digest-1"] + ) + replay_evidence.append(replay) + + if all(evidence is not None for evidence in replay_evidence): + replay_members = [ + { + "event_id": delivery["envelope"]["event_id"], + "disposition": "replay", + "evidence": evidence, + } + for delivery, evidence in zip(deliveries, replay_evidence, strict=True) + ] + if len(replay_members) == 1: + return dict(replay_members[0]["evidence"]) + return {"result": "batch", "checkpoint": document, "members": replay_members} + if any(delivery.get("request_digest_domain") is not None for delivery in deliveries): + raise ArtifactError("malformed_delivery") + if terminal_code is not None: + raise ArtifactError(terminal_code) + _check_cas(document, expected_revision, expected_checkpoint_digest) + assert document["root_record"].get("aggregate_state") is not None + aggregate = document["root_record"]["aggregate_state"] + new_deliveries = [ + delivery + for delivery, evidence in zip(deliveries, replay_evidence, strict=True) + if evidence is None + ] + admission = admit_aggregate_v2(aggregate, new_deliveries, definition_resolver) + if admission["result"] == "rejected": + raise ArtifactError(admission["rejection"]["code"]) + + candidate = copy.deepcopy(document) + candidate["revision"] = str(int(candidate["revision"]) + 1) + candidate["root_record"]["aggregate_state"] = admission["state"] + members: list[dict[str, Any]] = [] + accepted_by_id = {item["event_id"]: item for item in admission.get("accepted", [])} + for delivery, evidence, digest in zip( + deliveries, replay_evidence, canonical_digests, strict=True + ): + event_id = delivery["envelope"]["event_id"] + if evidence is not None: + members.append({"event_id": event_id, "disposition": "replay", "evidence": evidence}) + continue + accepted = accepted_by_id[event_id] + receipt_sequence = candidate["next_operation_receipt_sequence"] + candidate["next_operation_receipt_sequence"] = str(int(receipt_sequence) + 1) + candidate["operation_receipts"].append( + { + "operation_kind": "acceptance", + "receipt_sequence": receipt_sequence, + "event_id": event_id, + "request_digest": digest, + "acceptance_sequence": accepted["acceptance_sequence"], + "accepted_revision": candidate["revision"], + "delivery_mode": delivery["delivery_mode"], + } + ) + members.append({"event_id": event_id, "disposition": "accepted", **accepted}) + members[-1].pop("event_id", None) + members[-1] = {"event_id": event_id, **members[-1]} + sealed = seal_execution_checkpoint(candidate) + assert deliveries == snapshot + if all(member["disposition"] == "accepted" for member in members): + return sealed + return {"result": "batch", "checkpoint": sealed, "members": members} + + +def step_checkpoint_v2( + source: ArtifactSource, + target_runtime_id: str, + definition_resolver: DefinitionResolver, + *, + expected_revision: str, + expected_checkpoint_digest: str, +) -> dict[str, Any]: + """Process one ready mailbox head and append its terminal receipt.""" + restored = restore_execution_checkpoint_v2(source, definition_resolver) + document = restored.document + _check_cas(document, expected_revision, expected_checkpoint_digest) + aggregate = document["root_record"].get("aggregate_state") + if aggregate is None: + raise ArtifactError("tombstoned_root") + runtime = next( + (item for item in aggregate["runtimes"] if item["runtime_id"] == target_runtime_id), + None, + ) + selected = ( + copy.deepcopy(runtime["ready_mailbox"][0]) if runtime and runtime["ready_mailbox"] else None + ) + result = step_aggregate_v2(aggregate, target_runtime_id, definition_resolver) + if selected is None or result["disposition"] in {"not_runnable", "rejected"}: + return { + "result": "not_committed", + "step_result": result, + "checkpoint": document, + } + candidate = copy.deepcopy(document) + candidate["revision"] = str(int(candidate["revision"]) + 1) + candidate["root_record"]["aggregate_state"] = result["state"] + if result["disposition"] == "deferred": + _synchronize_mailbox_references(candidate) + return seal_execution_checkpoint(candidate) + receipt_sequence = candidate["next_operation_receipt_sequence"] + lifecycle_sequences = [ + str(int(receipt_sequence) + index + 1) + for index in range(len(result["lifecycle_dispositions"])) + ] + candidate["next_operation_receipt_sequence"] = str( + int(receipt_sequence) + 1 + len(lifecycle_sequences) + ) + _terminalize_mailbox_reference(candidate, selected, receipt_sequence) + for lifecycle, terminal_sequence in zip( + result["lifecycle_dispositions"], lifecycle_sequences, strict=True + ): + lifecycle_entry = { + "acceptance_sequence": lifecycle["acceptance_sequence"], + "envelope": {"event_id": lifecycle["event_id"]}, + } + _terminalize_mailbox_reference(candidate, lifecycle_entry, terminal_sequence) + references: list[dict[str, Any]] = [] + for emission_index, emission in enumerate(result["emissions"]): + if "kind" not in emission: + _append_external_intent(candidate, references, emission, emission_index) + continue + if emission["kind"] != "internal_disposed": + references.append(copy.deepcopy(emission)) + continue + index = int(emission["lifecycle_disposition_index"]) + references.append( + { + "kind": "internal_terminal", + "emission_index": emission["emission_index"], + "event_id": emission["event_id"], + "acceptance_sequence": emission["acceptance_sequence"], + "terminal_receipt_sequence": lifecycle_sequences[index], + } + ) + candidate["operation_receipts"].append( + { + "operation_kind": "event_terminal", + "receipt_sequence": receipt_sequence, + "event_id": selected["envelope"]["event_id"], + "request_digest": selected["envelope_digest"], + "acceptance_sequence": selected["acceptance_sequence"], + "final_queue_sequence": selected["queue_sequence"], + "committed_revision": candidate["revision"], + "resulting_aggregate_state_digest": result["state"]["aggregate_state_digest"], + "outcome": { + "status": result["status"], + "disposition": result["disposition"], + "fault": copy.deepcopy(result["fault"]), + "rejection": copy.deepcopy(result["rejection"]), + }, + "emission_references": references, + } + ) + for lifecycle, terminal_sequence in zip( + result["lifecycle_dispositions"], lifecycle_sequences, strict=True + ): + candidate["operation_receipts"].append( + { + "operation_kind": "event_terminal", + "receipt_sequence": terminal_sequence, + "event_id": lifecycle["event_id"], + "request_digest": lifecycle["request_digest"], + "acceptance_sequence": lifecycle["acceptance_sequence"], + "final_queue_sequence": lifecycle["final_queue_sequence"], + "committed_revision": candidate["revision"], + "resulting_aggregate_state_digest": result["state"]["aggregate_state_digest"], + "outcome": { + "status": result["status"], + "disposition": "disposed", + "reason": lifecycle["reason"], + "fault": None, + "rejection": None, + }, + "emission_references": [], + } + ) + _synchronize_mailbox_references(candidate) + return seal_execution_checkpoint(candidate) + + +def prune_checkpoint_v2( + source: ArtifactSource, + cutoff_receipt_sequence: str, + definition_resolver: DefinitionResolver, + *, + expected_revision: str, + expected_checkpoint_digest: str, +) -> dict[str, Any]: + """Advance bounded replay retention through one dependency-closed cutoff.""" + restored = restore_execution_checkpoint_v2(source, definition_resolver) + document = restored.document + if document["replay_retention"]["mode"] == "permanent": + raise _invalid() + prior_value = document["replay_retention"]["pruned_through_receipt_sequence"] + prior = decimal(prior_value) if prior_value is not None else -1 + try: + cutoff = decimal(cutoff_receipt_sequence) + except ArtifactError as error: + raise _invalid() from error + if cutoff == prior: + return document + if cutoff < prior or cutoff >= int(document["next_operation_receipt_sequence"]): + raise _invalid() + _check_cas(document, expected_revision, expected_checkpoint_digest) + receipts = document["operation_receipts"] + removed = [ + receipt + for receipt in receipts + if receipt["receipt_sequence"] != "0" and int(receipt["receipt_sequence"]) <= cutoff + ] + retained = [receipt for receipt in receipts if receipt not in removed] + pending_ids = { + entry["envelope"]["event_id"] + for entry in _mailbox_entries( + document["root_record"].get("aggregate_state") or {"runtimes": []} + ) + } + removed_sequences = {receipt["receipt_sequence"] for receipt in removed} + pending_effects = {item["intent"]["effect_id"] for item in document["pending_outbox_intents"]} + acceptance_by_event = { + receipt["event_id"]: receipt + for receipt in receipts + if receipt["operation_kind"] == "acceptance" + } + terminals_by_event = { + receipt["event_id"]: receipt + for receipt in receipts + if receipt["operation_kind"] == "event_terminal" + } + if any( + receipt["operation_kind"] == "acceptance" + and ( + receipt["event_id"] in pending_ids + or terminals_by_event.get(receipt["event_id"], {}).get("receipt_sequence") + not in removed_sequences + ) + for receipt in removed + ): + raise _invalid() + if any( + reference.get("kind") == "internal_mailbox" and reference.get("event_id") in pending_ids + for receipt in removed + for reference in receipt.get("emission_references", []) + ): + raise _invalid() + if any( + reference.get("kind") == "external_outbox" and reference.get("effect_id") in pending_effects + for receipt in removed + for reference in receipt.get("emission_references", []) + ): + raise _invalid() + for receipt in retained: + for reference in receipt.get("emission_references", []): + if reference.get("kind") == "internal_mailbox" and reference["event_id"] in pending_ids: + continue + if receipt["operation_kind"] == "event_terminal": + acceptance = acceptance_by_event.get(receipt["event_id"]) + if acceptance and acceptance["receipt_sequence"] in removed_sequences: + raise _invalid() + legacy = receipt.get("legacy_receipt", {}) + origin = legacy.get("origin", {}) + if ( + origin.get("kind") == "internal_emission" + and origin.get("producing_receipt_sequence") in removed_sequences + ): + raise _invalid() + converted_origin = receipt.get("legacy_v1_delivery", {}).get("origin", {}) + if ( + converted_origin.get("kind") == "internal_emission" + and converted_origin.get("producing_receipt_sequence") in removed_sequences + ): + raise _invalid() + + candidate = copy.deepcopy(document) + tombstones = list(candidate["event_identity_tombstones"]) + for receipt in removed: + if receipt["operation_kind"] == "event_terminal": + tombstones.append( + { + "event_id": receipt["event_id"], + "request_digest": receipt["request_digest"], + "request_digest_domain": "determa-inbox-envelope-digest-2", + "acceptance_sequence": receipt["acceptance_sequence"], + "terminal_receipt_sequence": receipt["receipt_sequence"], + "terminal_disposition": receipt["outcome"]["disposition"], + } + ) + elif ( + receipt["operation_kind"] == "legacy_v1_operation" + and receipt["legacy_receipt"].get("operation_kind") == "delivery" + ): + legacy = receipt["legacy_receipt"] + tombstones.append( + { + "event_id": legacy["event_id"], + "request_digest": legacy["request_digest"], + "request_digest_domain": "determa-inbox-envelope-digest-1", + "acceptance_sequence": legacy["accepted_delivery_sequence"], + "terminal_receipt_sequence": receipt["receipt_sequence"], + "terminal_disposition": legacy["outcome"]["disposition"], + } + ) + candidate["operation_receipts"] = retained + candidate["event_identity_tombstones"] = sorted( + tombstones, key=lambda item: int(item["terminal_receipt_sequence"]) + ) + candidate["replay_retention"]["pruned_through_receipt_sequence"] = cutoff_receipt_sequence + candidate["revision"] = str(int(candidate["revision"]) + 1) + return seal_execution_checkpoint(candidate) diff --git a/src/determa/state/codes.py b/src/determa/state/codes.py index 290a6d9..bc87130 100644 --- a/src/determa/state/codes.py +++ b/src/determa/state/codes.py @@ -30,11 +30,20 @@ class CheckpointHostFailureCode(StrEnum): class CheckpointPreAcceptanceFailureCode(StrEnum): + CHECKPOINT_UPGRADE_REQUIRED = "checkpoint_upgrade_required" DELIVERY_DIGEST_MISMATCH = "delivery_digest_mismatch" + DUPLICATE_EVENT_ID_IN_BATCH = "duplicate_event_id_in_batch" EVENT_ID_CONFLICT = "event_id_conflict" + INACTIVE_COMPONENT_TARGET = "inactive_component_target" + INVALID_CORRELATION = "invalid_correlation" INVALID_DELIVERY_MODE = "invalid_delivery_mode" INVALID_DELIVERY_ORIGIN = "invalid_delivery_origin" + INVALID_DELIVERY_SOURCE = "invalid_delivery_source" + INVALID_EVENT = "invalid_event" + INVALID_INSTANCE_TARGET = "invalid_instance_target" + INVALID_PAYLOAD = "invalid_payload" MALFORMED_DELIVERY = "malformed_delivery" + TERMINAL_ROOT = "terminal_root" TOMBSTONED_ROOT = "tombstoned_root" WRONG_ROOT = "wrong_root" @@ -56,8 +65,10 @@ class DispatchRejectionCode(StrEnum): class DispositionCode(StrEnum): + DEFERRED = "deferred" FAULTED = "faulted" HANDLED = "handled" + NOT_RUNNABLE = "not_runnable" REJECTED = "rejected" UNHANDLED = "unhandled" @@ -67,6 +78,7 @@ class EngineFaultCode(StrEnum): BINDING_NOT_EMPTY = "binding_not_empty" CASCADE_FAULT = "cascade_fault" CONTAINED_RUNTIME_FAULT = "contained_runtime_fault" + DEFERRED_EVENT_CAPACITY_EXCEEDED = "deferred_event_capacity_exceeded" GUARD_FAULT = "guard_fault" INACTIVE_COMPONENT_TARGET = "inactive_component_target" INVALID_INSTANCE_TARGET = "invalid_instance_target" diff --git a/src/determa/state/data/aggregate-state-package-v2.schema.json b/src/determa/state/data/aggregate-state-package-v2.schema.json new file mode 100644 index 0000000..65ad286 --- /dev/null +++ b/src/determa/state/data/aggregate-state-package-v2.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/aggregate-state-package-v2.schema.json", + "title": "Determa State self-contained aggregate-state package version 2", + "description": "Closed transport package for one queue-bearing aggregate and its verified attachments.", + "type": "object", + "required": [ + "aggregate_state_package_format", + "aggregate_state_package_schema_version", + "aggregate_state", + "normalized_definitions", + "migration_descriptors", + "migration_route" + ], + "additionalProperties": false, + "properties": { + "aggregate_state_package_format": { + "const": "determa.aggregate_state_package" + }, + "aggregate_state_package_schema_version": { + "const": 2 + }, + "aggregate_state": { + "$ref": "aggregate-state-v2.schema.json" + }, + "normalized_definitions": { + "type": "array", + "items": { + "type": "object", + "required": [ + "validated_bundle_fingerprint", + "normalized_bundle" + ], + "additionalProperties": false, + "properties": { + "validated_bundle_fingerprint": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "normalized_bundle": { + "$ref": "aggregate-state-v2.schema.json#/$defs/typedValue" + } + } + } + }, + "migration_descriptors": { + "type": "array", + "items": { + "$ref": "migration-descriptor-v2.schema.json" + } + }, + "migration_route": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + } + } +} diff --git a/src/determa/state/data/aggregate-state-v2.schema.json b/src/determa/state/data/aggregate-state-v2.schema.json new file mode 100644 index 0000000..3aff5be --- /dev/null +++ b/src/determa/state/data/aggregate-state-v2.schema.json @@ -0,0 +1,447 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/aggregate-state-v2.schema.json", + "title": "Determa State portable aggregate-state envelope version 2", + "description": "Closed wire schema for aggregate_state_schema_version 2 with runtime-local mailboxes.", + "$ref": "#/$defs/aggregateState", + "$defs": { + "nonEmptyString": { + "$ref": "aggregate-state.schema.json#/$defs/nonEmptyString" + }, + "identifier": { + "$ref": "aggregate-state.schema.json#/$defs/identifier" + }, + "namespace": { + "$ref": "aggregate-state.schema.json#/$defs/namespace" + }, + "canonicalDecimal": { + "$ref": "aggregate-state.schema.json#/$defs/canonicalDecimal" + }, + "positiveCanonicalDecimal": { + "$ref": "aggregate-state.schema.json#/$defs/positiveCanonicalDecimal" + }, + "signedCanonicalDecimal": { + "$ref": "aggregate-state.schema.json#/$defs/signedCanonicalDecimal" + }, + "sha256": { + "$ref": "aggregate-state.schema.json#/$defs/sha256" + }, + "jsonPointer": { + "$ref": "aggregate-state.schema.json#/$defs/jsonPointer" + }, + "sourceLocator": { + "$ref": "aggregate-state.schema.json#/$defs/sourceLocator" + }, + "typedValue": { + "$ref": "aggregate-state.schema.json#/$defs/typedValue" + }, + "eventName": { + "oneOf": [ + { + "$ref": "#/$defs/identifier" + }, + { + "enum": [ + "determa.component_completed", + "determa.component_failed", + "determa.spawned_instance_failed" + ] + } + ] + }, + "typedMap": { + "type": "array", + "prefixItems": [ + { + "const": "map" + }, + { + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/typedValue" + } + ], + "minItems": 2, + "maxItems": 2 + } + } + ], + "minItems": 2, + "maxItems": 2 + }, + "machineIdentity": { + "$ref": "aggregate-state.schema.json#/$defs/machineIdentity" + }, + "definitionBinding": { + "$ref": "aggregate-state.schema.json#/$defs/definitionBinding" + }, + "rootIdentityOrigin": { + "$ref": "aggregate-state.schema.json#/$defs/rootIdentityOrigin" + }, + "componentIdentityOrigin": { + "$ref": "aggregate-state.schema.json#/$defs/componentIdentityOrigin" + }, + "spawnedIdentityOrigin": { + "$ref": "aggregate-state.schema.json#/$defs/spawnedIdentityOrigin" + }, + "identityOrigin": { + "$ref": "aggregate-state.schema.json#/$defs/identityOrigin" + }, + "instanceReference": { + "$ref": "aggregate-state.schema.json#/$defs/instanceReference" + }, + "targetIdentity": { + "$ref": "aggregate-state.schema.json#/$defs/targetIdentity" + }, + "envelopeSource": { + "oneOf": [ + { + "type": "object", + "required": [ + "host" + ], + "additionalProperties": false, + "properties": { + "host": { + "const": true + } + } + }, + { + "type": "object", + "required": [ + "runtime" + ], + "additionalProperties": false, + "properties": { + "runtime": { + "$ref": "#/$defs/targetIdentity" + } + } + }, + { + "type": "object", + "required": [ + "system" + ], + "additionalProperties": false, + "properties": { + "system": { + "$ref": "#/$defs/sourceLocator" + } + } + }, + { + "type": "object", + "required": [ + "legacy_v1_internal" + ], + "additionalProperties": false, + "properties": { + "legacy_v1_internal": { + "type": "object", + "required": [ + "producing_receipt_sequence", + "emission_index" + ], + "additionalProperties": false, + "properties": { + "producing_receipt_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "emission_index": { + "$ref": "#/$defs/canonicalDecimal" + } + } + } + } + } + ] + }, + "envelope": { + "type": "object", + "required": [ + "event", + "event_id", + "cause_id", + "source", + "target", + "payload" + ], + "additionalProperties": false, + "properties": { + "event": { + "$ref": "#/$defs/eventName" + }, + "event_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "cause_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "source": { + "$ref": "#/$defs/envelopeSource" + }, + "target": { + "$ref": "#/$defs/targetIdentity" + }, + "payload": { + "$ref": "#/$defs/typedMap" + }, + "correlation_id": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "mailboxEntry": { + "type": "object", + "required": [ + "acceptance_sequence", + "queue_sequence", + "delivery_mode", + "envelope", + "envelope_digest", + "deferral_count" + ], + "additionalProperties": false, + "properties": { + "acceptance_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "queue_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "delivery_mode": { + "enum": [ + "input", + "internal" + ] + }, + "envelope": { + "$ref": "#/$defs/envelope" + }, + "envelope_digest": { + "$ref": "#/$defs/sha256" + }, + "deferral_count": { + "$ref": "#/$defs/canonicalDecimal" + } + } + }, + "lifetimeHolder": { + "$ref": "aggregate-state.schema.json#/$defs/lifetimeHolder" + }, + "relation": { + "$ref": "aggregate-state.schema.json#/$defs/relation" + }, + "stateActivation": { + "$ref": "aggregate-state.schema.json#/$defs/stateActivation" + }, + "variable": { + "$ref": "aggregate-state.schema.json#/$defs/variable" + }, + "history": { + "$ref": "aggregate-state.schema.json#/$defs/history" + }, + "nextCounter": { + "$ref": "aggregate-state.schema.json#/$defs/nextCounter" + }, + "fault": { + "$ref": "aggregate-state.schema.json#/$defs/fault" + }, + "runtime": { + "type": "object", + "required": [ + "runtime_id", + "identity_origin", + "target_identity", + "current_definition", + "relation", + "status", + "active_leaf_state_definition_pointers", + "active_state_activations", + "variables", + "history", + "next_spawn_sequence", + "next_state_activation_sequences", + "next_component_activation_sequences", + "ready_mailbox", + "deferred_mailbox", + "fault" + ], + "additionalProperties": false, + "properties": { + "runtime_id": { + "$ref": "#/$defs/sha256" + }, + "identity_origin": { + "$ref": "#/$defs/identityOrigin" + }, + "target_identity": { + "$ref": "#/$defs/targetIdentity" + }, + "current_definition": { + "$ref": "#/$defs/definitionBinding" + }, + "relation": { + "$ref": "#/$defs/relation" + }, + "status": { + "enum": [ + "running", + "completed", + "faulted" + ] + }, + "active_leaf_state_definition_pointers": { + "type": "array", + "items": { + "$ref": "#/$defs/jsonPointer" + }, + "uniqueItems": true + }, + "active_state_activations": { + "type": "array", + "items": { + "$ref": "#/$defs/stateActivation" + } + }, + "variables": { + "type": "array", + "items": { + "$ref": "#/$defs/variable" + } + }, + "history": { + "type": "array", + "items": { + "$ref": "#/$defs/history" + } + }, + "next_spawn_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "next_state_activation_sequences": { + "type": "array", + "items": { + "$ref": "#/$defs/nextCounter" + } + }, + "next_component_activation_sequences": { + "type": "array", + "items": { + "$ref": "#/$defs/nextCounter" + } + }, + "ready_mailbox": { + "type": "array", + "items": { + "$ref": "#/$defs/mailboxEntry" + } + }, + "deferred_mailbox": { + "type": "array", + "items": { + "$ref": "#/$defs/mailboxEntry" + } + }, + "fault": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/fault" + } + ] + } + } + }, + "aggregateState": { + "type": "object", + "required": [ + "aggregate_state_format", + "aggregate_state_schema_version", + "machine_format", + "validated_bundle_fingerprint", + "namespace", + "root_machine_id", + "root_machine_version", + "root_instance_id", + "creation_id", + "root_runtime_id", + "migration_sequence", + "next_logical_step_sequence", + "next_output_sequence", + "next_acceptance_sequence", + "next_queue_sequence", + "runtimes", + "aggregate_state_digest" + ], + "additionalProperties": false, + "properties": { + "aggregate_state_format": { + "const": "determa.aggregate_state" + }, + "aggregate_state_schema_version": { + "const": 2 + }, + "machine_format": { + "const": 1 + }, + "validated_bundle_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "namespace": { + "$ref": "#/$defs/namespace" + }, + "root_machine_id": { + "$ref": "#/$defs/identifier" + }, + "root_machine_version": { + "$ref": "#/$defs/positiveCanonicalDecimal" + }, + "root_instance_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "creation_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "root_runtime_id": { + "$ref": "#/$defs/sha256" + }, + "migration_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "next_logical_step_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "next_output_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "next_acceptance_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "next_queue_sequence": { + "$ref": "#/$defs/canonicalDecimal" + }, + "runtimes": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/runtime" + } + }, + "aggregate_state_digest": { + "$ref": "#/$defs/sha256" + } + } + } + } +} diff --git a/src/determa/state/data/core-step-result-v2.schema.json b/src/determa/state/data/core-step-result-v2.schema.json new file mode 100644 index 0000000..24309ae --- /dev/null +++ b/src/determa/state/data/core-step-result-v2.schema.json @@ -0,0 +1,187 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/core-step-result-v2.schema.json", + "title": "Determa State queue-bearing core step result version 2", + "description": "Closed result for one explicitly targeted aggregate-state version-2 mailbox step.", + "$ref": "#/$defs/coreStepResult", + "$defs": { + "canonicalDecimal": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)$" + }, + "nonEmptyString": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "internalMailboxEmission": { + "type": "object", + "required": [ + "kind", + "emission_index", + "event_id", + "acceptance_sequence", + "queue_sequence" + ], + "additionalProperties": false, + "properties": { + "kind": { "const": "internal_mailbox" }, + "emission_index": { "$ref": "#/$defs/canonicalDecimal" }, + "event_id": { "$ref": "#/$defs/nonEmptyString" }, + "acceptance_sequence": { "$ref": "#/$defs/canonicalDecimal" }, + "queue_sequence": { "$ref": "#/$defs/canonicalDecimal" } + } + }, + "internalDisposedEmission": { + "type": "object", + "required": [ + "kind", + "emission_index", + "event_id", + "acceptance_sequence", + "lifecycle_disposition_index" + ], + "additionalProperties": false, + "properties": { + "kind": { "const": "internal_disposed" }, + "emission_index": { "$ref": "#/$defs/canonicalDecimal" }, + "event_id": { "$ref": "#/$defs/nonEmptyString" }, + "acceptance_sequence": { "$ref": "#/$defs/canonicalDecimal" }, + "lifecycle_disposition_index": { "$ref": "#/$defs/canonicalDecimal" } + } + }, + "emission": { + "oneOf": [ + { "$ref": "#/$defs/internalMailboxEmission" }, + { "$ref": "#/$defs/internalDisposedEmission" }, + { "$ref": "execution-checkpoint.schema.json#/$defs/outboxIntent" } + ] + }, + "lifecycleDisposition": { + "type": "object", + "required": [ + "event_id", + "request_digest", + "acceptance_sequence", + "final_queue_sequence", + "target_runtime_id", + "reason" + ], + "additionalProperties": false, + "properties": { + "event_id": { "$ref": "#/$defs/nonEmptyString" }, + "request_digest": { "$ref": "#/$defs/sha256" }, + "acceptance_sequence": { "$ref": "#/$defs/canonicalDecimal" }, + "final_queue_sequence": { "$ref": "#/$defs/canonicalDecimal" }, + "target_runtime_id": { "$ref": "#/$defs/sha256" }, + "reason": { + "enum": [ + "runtime_cancelled", + "runtime_completed", + "aggregate_completed" + ] + } + } + }, + "rejection": { + "type": "object", + "required": ["code"], + "additionalProperties": false, + "properties": { + "code": { + "enum": [ + "invalid_instance_target", + "inactive_component_target", + "invalid_prior_state", + "incompatible_bundle" + ] + } + } + }, + "coreStepResult": { + "type": "object", + "required": [ + "core_step_result_format", + "core_step_result_schema_version", + "status", + "disposition", + "state", + "emissions", + "lifecycle_dispositions", + "fault", + "rejection" + ], + "additionalProperties": false, + "properties": { + "core_step_result_format": { + "const": "determa.core_step_result" + }, + "core_step_result_schema_version": { + "const": 2 + }, + "status": { + "enum": ["running", "completed", "faulted"] + }, + "disposition": { + "enum": [ + "handled", + "deferred", + "unhandled", + "not_runnable", + "rejected", + "faulted" + ] + }, + "state": { + "$ref": "aggregate-state-v2.schema.json" + }, + "emissions": { + "type": "array", + "items": { "$ref": "#/$defs/emission" } + }, + "lifecycle_dispositions": { + "type": "array", + "items": { "$ref": "#/$defs/lifecycleDisposition" } + }, + "fault": { + "oneOf": [ + { "type": "null" }, + { "$ref": "aggregate-state-v2.schema.json#/$defs/fault" } + ] + }, + "rejection": { + "oneOf": [ + { "type": "null" }, + { "$ref": "#/$defs/rejection" } + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "disposition": { "const": "rejected" } + }, + "required": ["disposition"] + }, + "then": { + "properties": { + "emissions": { "maxItems": 0 }, + "lifecycle_dispositions": { "maxItems": 0 }, + "fault": { "type": "null" }, + "rejection": { "$ref": "#/$defs/rejection" } + } + }, + "else": { + "properties": { + "rejection": { "type": "null" } + } + } + } + ] + } + } +} diff --git a/src/determa/state/data/execution-checkpoint-v2.schema.json b/src/determa/state/data/execution-checkpoint-v2.schema.json new file mode 100644 index 0000000..d6726a0 --- /dev/null +++ b/src/determa/state/data/execution-checkpoint-v2.schema.json @@ -0,0 +1,658 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/execution-checkpoint-v2.schema.json", + "title": "Determa State portable execution checkpoint version 2", + "description": "Closed durable-host checkpoint whose aggregate owns accepted ready and deferred envelopes.", + "$ref": "#/$defs/executionCheckpoint", + "$defs": { + "internalMailboxEmissionReference": { + "type": "object", + "required": [ + "kind", + "emission_index", + "event_id", + "acceptance_sequence", + "queue_sequence" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal_mailbox" + }, + "emission_index": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "event_id": { + "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + }, + "acceptance_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "queue_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + } + } + }, + "internalTerminalEmissionReference": { + "type": "object", + "required": [ + "kind", + "emission_index", + "event_id", + "acceptance_sequence", + "terminal_receipt_sequence" + ], + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal_terminal" + }, + "emission_index": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "event_id": { + "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + }, + "acceptance_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "terminal_receipt_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + } + } + }, + "emissionReference": { + "oneOf": [ + { + "$ref": "#/$defs/internalMailboxEmissionReference" + }, + { + "$ref": "#/$defs/internalTerminalEmissionReference" + }, + { + "$ref": "execution-checkpoint.schema.json#/$defs/externalOutboxEmissionReference" + } + ] + }, + "creationReceipt": { + "type": "object", + "required": [ + "operation_kind", + "receipt_sequence", + "creation_id", + "request_digest", + "committed_revision", + "resulting_aggregate_state_digest", + "status", + "fault", + "emission_references" + ], + "additionalProperties": false, + "properties": { + "operation_kind": { + "const": "creation" + }, + "receipt_sequence": { + "const": "0" + }, + "creation_id": { + "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + }, + "request_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + }, + "committed_revision": { + "const": "0" + }, + "resulting_aggregate_state_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + }, + "status": { + "enum": [ + "running", + "completed", + "faulted" + ] + }, + "fault": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "aggregate-state-v2.schema.json#/$defs/fault" + } + ] + }, + "emission_references": { + "type": "array", + "items": { + "$ref": "#/$defs/emissionReference" + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "faulted" + } + }, + "required": [ + "status" + ] + }, + "then": { + "properties": { + "fault": { + "$ref": "aggregate-state-v2.schema.json#/$defs/fault" + } + } + }, + "else": { + "properties": { + "fault": { + "type": "null" + } + } + } + } + ] + }, + "acceptanceReceipt": { + "type": "object", + "required": [ + "operation_kind", + "receipt_sequence", + "event_id", + "request_digest", + "acceptance_sequence", + "accepted_revision", + "delivery_mode" + ], + "additionalProperties": false, + "properties": { + "operation_kind": { + "const": "acceptance" + }, + "receipt_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "event_id": { + "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + }, + "request_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + }, + "acceptance_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "accepted_revision": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "delivery_mode": { + "enum": [ + "input", + "internal" + ] + }, + "legacy_v1_delivery": { + "$ref": "#/$defs/legacyV1DeliveryEvidence" + } + }, + "allOf": [ + { + "if": { + "properties": { + "delivery_mode": { + "const": "internal" + } + }, + "required": [ + "delivery_mode" + ] + }, + "then": { + "required": [ + "legacy_v1_delivery" + ] + } + } + ] + }, + "legacyV1DeliveryEvidence": { + "type": "object", + "required": [ + "delivery_sequence", + "envelope_digest", + "origin" + ], + "additionalProperties": false, + "properties": { + "delivery_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "envelope_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + }, + "origin": { + "$ref": "execution-checkpoint.schema.json#/$defs/deliveryOrigin" + } + } + }, + "disposedOutcome": { + "type": "object", + "required": [ + "status", + "disposition", + "reason", + "fault", + "rejection" + ], + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "running", + "completed", + "faulted" + ] + }, + "disposition": { + "const": "disposed" + }, + "reason": { + "enum": [ + "runtime_cancelled", + "runtime_completed", + "aggregate_completed", + "root_tombstoned" + ] + }, + "fault": { + "type": "null" + }, + "rejection": { + "type": "null" + } + } + }, + "migrationDisposedOutcome": { + "type": "object", + "required": [ + "status", + "disposition", + "reason", + "migration_descriptor_digest", + "fault", + "rejection" + ], + "additionalProperties": false, + "properties": { + "status": { + "enum": ["running", "completed", "faulted"] + }, + "disposition": { + "const": "migration_disposed" + }, + "reason": { + "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + }, + "migration_descriptor_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + }, + "fault": { + "type": "null" + }, + "rejection": { + "type": "null" + } + } + }, + "terminalEventOutcome": { + "oneOf": [ + { + "$ref": "execution-checkpoint.schema.json#/$defs/handledOutcome" + }, + { + "$ref": "execution-checkpoint.schema.json#/$defs/unhandledOutcome" + }, + { + "$ref": "execution-checkpoint.schema.json#/$defs/faultedOutcome" + }, + { + "$ref": "#/$defs/disposedOutcome" + }, + { + "$ref": "#/$defs/migrationDisposedOutcome" + } + ] + }, + "legacyV1CreationReceipt": { + "type": "object", + "required": ["operation_kind", "receipt_sequence", "legacy_receipt"], + "additionalProperties": false, + "properties": { + "operation_kind": { + "const": "legacy_v1_creation" + }, + "receipt_sequence": { + "const": "0" + }, + "legacy_receipt": { + "$ref": "execution-checkpoint.schema.json#/$defs/creationReceipt" + } + } + }, + "legacyV1OperationReceipt": { + "type": "object", + "required": ["operation_kind", "receipt_sequence", "legacy_receipt"], + "additionalProperties": false, + "properties": { + "operation_kind": { + "const": "legacy_v1_operation" + }, + "receipt_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "legacy_receipt": { + "oneOf": [ + { + "$ref": "execution-checkpoint.schema.json#/$defs/deliveryReceipt" + }, + { + "$ref": "execution-checkpoint.schema.json#/$defs/maintenanceMigrationReceipt" + } + ] + } + } + }, + "terminalEventReceipt": { + "type": "object", + "required": [ + "operation_kind", + "receipt_sequence", + "event_id", + "request_digest", + "acceptance_sequence", + "final_queue_sequence", + "committed_revision", + "resulting_aggregate_state_digest", + "outcome", + "emission_references" + ], + "additionalProperties": false, + "properties": { + "operation_kind": { + "const": "event_terminal" + }, + "receipt_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "event_id": { + "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + }, + "request_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + }, + "acceptance_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "final_queue_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "committed_revision": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "resulting_aggregate_state_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + }, + "outcome": { + "$ref": "#/$defs/terminalEventOutcome" + }, + "emission_references": { + "type": "array", + "items": { + "$ref": "#/$defs/emissionReference" + } + } + } + }, + "operationReceipt": { + "oneOf": [ + { + "$ref": "#/$defs/creationReceipt" + }, + { + "$ref": "#/$defs/acceptanceReceipt" + }, + { + "$ref": "#/$defs/terminalEventReceipt" + }, + { + "$ref": "#/$defs/legacyV1CreationReceipt" + }, + { + "$ref": "#/$defs/legacyV1OperationReceipt" + } + ] + }, + "eventIdentityTombstone": { + "type": "object", + "required": [ + "event_id", + "request_digest", + "request_digest_domain", + "acceptance_sequence", + "terminal_receipt_sequence", + "terminal_disposition" + ], + "additionalProperties": false, + "properties": { + "event_id": { + "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + }, + "request_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + }, + "request_digest_domain": { + "enum": [ + "determa-inbox-envelope-digest-1", + "determa-inbox-envelope-digest-2" + ] + }, + "acceptance_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "terminal_receipt_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "terminal_disposition": { + "enum": [ + "handled", + "unhandled", + "rejected", + "faulted", + "disposed", + "migration_disposed" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "terminal_disposition": { + "const": "rejected" + } + }, + "required": [ + "terminal_disposition" + ] + }, + "then": { + "properties": { + "request_digest_domain": { + "const": "determa-inbox-envelope-digest-1" + } + } + } + } + ] + }, + "retainedRootRecord": { + "type": "object", + "required": [ + "status", + "aggregate_state" + ], + "additionalProperties": false, + "properties": { + "status": { + "const": "retained" + }, + "aggregate_state": { + "$ref": "aggregate-state-v2.schema.json" + } + } + }, + "rootRecord": { + "oneOf": [ + { + "$ref": "#/$defs/retainedRootRecord" + }, + { + "$ref": "execution-checkpoint.schema.json#/$defs/rootTombstone" + } + ] + }, + "executionCheckpoint": { + "type": "object", + "required": [ + "execution_checkpoint_format", + "execution_checkpoint_schema_version", + "root_instance_id", + "revision", + "root_record", + "replay_retention", + "next_operation_receipt_sequence", + "operation_receipts", + "event_identity_tombstones", + "pending_outbox_intents", + "next_outbox_terminal_sequence", + "terminal_outbox_records", + "outbox_effect_tombstones", + "migration_audit_records", + "execution_checkpoint_digest" + ], + "additionalProperties": false, + "properties": { + "execution_checkpoint_format": { + "const": "determa.execution_checkpoint" + }, + "execution_checkpoint_schema_version": { + "const": 2 + }, + "root_instance_id": { + "$ref": "execution-checkpoint.schema.json#/$defs/nonEmptyString" + }, + "revision": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "root_record": { + "$ref": "#/$defs/rootRecord" + }, + "replay_retention": { + "$ref": "execution-checkpoint.schema.json#/$defs/replayRetention" + }, + "next_operation_receipt_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "operation_receipts": { + "type": "array", + "minItems": 1, + "prefixItems": [ + { + "oneOf": [ + { + "$ref": "#/$defs/creationReceipt" + }, + { + "$ref": "#/$defs/legacyV1CreationReceipt" + } + ] + } + ], + "items": { + "$ref": "#/$defs/operationReceipt" + } + }, + "event_identity_tombstones": { + "type": "array", + "items": { + "$ref": "#/$defs/eventIdentityTombstone" + } + }, + "pending_outbox_intents": { + "type": "array", + "items": { + "$ref": "execution-checkpoint.schema.json#/$defs/pendingOutboxIntent" + } + }, + "next_outbox_terminal_sequence": { + "$ref": "execution-checkpoint.schema.json#/$defs/canonicalDecimal" + }, + "terminal_outbox_records": { + "type": "array", + "items": { + "$ref": "execution-checkpoint.schema.json#/$defs/terminalOutboxRecord" + } + }, + "outbox_effect_tombstones": { + "type": "array", + "items": { + "$ref": "execution-checkpoint.schema.json#/$defs/outboxEffectTombstone" + } + }, + "migration_audit_records": { + "type": "array", + "items": { + "$ref": "execution-checkpoint.schema.json#/$defs/migrationAuditRecord" + } + }, + "execution_checkpoint_digest": { + "$ref": "execution-checkpoint.schema.json#/$defs/sha256" + } + }, + "allOf": [ + { + "if": { + "properties": { + "root_record": { + "properties": { + "status": { + "const": "tombstone" + } + }, + "required": [ + "status" + ] + } + }, + "required": [ + "root_record" + ] + }, + "then": { + "properties": { + "pending_outbox_intents": { + "maxItems": 0 + } + } + } + } + ] + } + } +} diff --git a/src/determa/state/data/machine.schema.json b/src/determa/state/data/machine.schema.json index 21d00d9..f379a20 100644 --- a/src/determa/state/data/machine.schema.json +++ b/src/determa/state/data/machine.schema.json @@ -65,6 +65,21 @@ "type": "string", "pattern": "^(?:[A-Za-z_][A-Za-z0-9_]*|determa\\.(?:component_completed|component_failed|spawned_instance_failed))$" }, + "deferrableEventName": { + "allOf": [ + { + "$ref": "#/$defs/identifier" + }, + { + "not": { + "enum": [ + "done", + "env" + ] + } + } + ] + }, "authorEventName": { "allOf": [ { @@ -908,6 +923,19 @@ "$ref": "#/$defs/transitionOrList" } }, + "deferred_events": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/deferrableEventName" + } + }, + "deferred_event_capacity": { + "type": "integer", + "minimum": 0, + "maximum": 9223372036854775807 + }, "history": { "enum": ["none", "shallow", "deep"], "default": "none" @@ -1017,6 +1045,12 @@ { "required": ["on_events"] }, + { + "required": ["deferred_events"] + }, + { + "required": ["deferred_event_capacity"] + }, { "required": ["history"] }, @@ -1068,6 +1102,12 @@ { "required": ["on_events"] }, + { + "required": ["deferred_events"] + }, + { + "required": ["deferred_event_capacity"] + }, { "required": ["history"] }, diff --git a/src/determa/state/data/migration-descriptor-v2.schema.json b/src/determa/state/data/migration-descriptor-v2.schema.json new file mode 100644 index 0000000..4a328f2 --- /dev/null +++ b/src/determa/state/data/migration-descriptor-v2.schema.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://determa.dev/state/schema/migration-descriptor-v2.schema.json", + "title": "Determa State aggregate migration descriptor version 2", + "description": "Closed migration descriptor for queue-bearing aggregate-state version 2.", + "type": "object", + "required": [ + "migration_descriptor_format", + "migration_descriptor_schema_version", + "base_descriptor", + "queued_event_default", + "queued_event_rules", + "migration_descriptor_digest" + ], + "additionalProperties": false, + "properties": { + "migration_descriptor_format": { + "const": "determa.aggregate_migration" + }, + "migration_descriptor_schema_version": { + "const": 2 + }, + "base_descriptor": { + "$ref": "migration-descriptor.schema.json" + }, + "queued_event_default": { + "const": "preserve_if_compatible" + }, + "queued_event_rules": { + "type": "array", + "items": { + "$ref": "#/$defs/queuedEventRule" + } + }, + "migration_descriptor_digest": { + "$ref": "#/$defs/sha256" + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "eventName": { + "oneOf": [ + { + "$ref": "#/$defs/identifier" + }, + { + "enum": [ + "determa.component_completed", + "determa.component_failed", + "determa.spawned_instance_failed" + ] + } + ] + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "disposeRule": { + "type": "object", + "required": [ + "machine_id", + "event", + "delivery_mode", + "action", + "reason" + ], + "additionalProperties": false, + "properties": { + "machine_id": { + "$ref": "#/$defs/identifier" + }, + "event": { + "$ref": "#/$defs/eventName" + }, + "delivery_mode": { + "enum": ["input", "internal"] + }, + "action": { + "const": "dispose" + }, + "reason": { + "type": "string", + "minLength": 1 + } + } + }, + "queuedEventRule": { + "$ref": "#/$defs/disposeRule" + } + } +} diff --git a/src/determa/state/engine.py b/src/determa/state/engine.py index 1d41e25..0ce8f1d 100644 --- a/src/determa/state/engine.py +++ b/src/determa/state/engine.py @@ -24,10 +24,20 @@ from .codes import ( MachineLoadFailureCode as LoadCode, ) -from .definition import Bundle, BundleSource, _escape_pointer, hash_identity, load_bundle +from .definition import ( + Bundle, + BundleSource, + _escape_pointer, + hash_identity, + load_bundle, +) from .errors import CelError, StepFault, ValidationError from .model import BundleModel, MachineModel, StateNode -from .yaml12 import normalize_portable_values, validate_portable_values, validate_unicode +from .yaml12 import ( + normalize_portable_values, + validate_portable_values, + validate_unicode, +) Result = dict[str, Any] Delivery = dict[str, dict[str, Any]] | None @@ -282,6 +292,8 @@ def create( root_instance_id: str, creation_id: str, bindings: dict[str, dict[str, Any]] | None = None, + *, + _capture_emission_provenance: bool = False, ) -> Result: """Create and synchronously initialize one root ownership aggregate.""" validated = _coerce_bundle(bundle) @@ -322,7 +334,13 @@ def create( "runtimes": {}, "fault": None, } - execution = _Execution(validated, models, state, step_sequence=0) + execution = _Execution( + validated, + models, + state, + step_sequence=0, + capture_emission_provenance=_capture_emission_provenance, + ) runtime = execution.new_runtime( machine, root_id, @@ -372,6 +390,8 @@ def dispatch( bundle: Bundle | BundleSource, prior_state: dict[str, Any], delivery: Delivery = None, + *, + _capture_emission_provenance: bool = False, ) -> Result: """Validate and process at most one envelope against an aggregate copy.""" validated = _coerce_bundle(bundle) @@ -409,7 +429,13 @@ def dispatch( return _rejected(prior_state, rejection) state = _copy_normalized_prior_state(prior_state) step_sequence = int(state["next_logical_step_sequence"]) - execution = _Execution(validated, models, state, step_sequence=step_sequence) + execution = _Execution( + validated, + models, + state, + step_sequence=step_sequence, + capture_emission_provenance=_capture_emission_provenance, + ) runtime = execution.runtime_for_target(envelope["target"]) normalized_envelope = copy.deepcopy(envelope) declaration = execution.event_declaration(runtime, envelope["event"]) @@ -435,11 +461,17 @@ def dispatch( execution.cause_id = str(envelope["event_id"]) before = copy.deepcopy(state) try: - handled = execution.process(runtime, normalized_envelope) + disposition = execution.process(runtime, normalized_envelope) except StepFault as fault: state.clear() state.update(before) - execution = _Execution(validated, models, state, step_sequence=step_sequence) + execution = _Execution( + validated, + models, + state, + step_sequence=step_sequence, + capture_emission_provenance=_capture_emission_provenance, + ) runtime = execution.runtime_for_target(envelope["target"]) execution.finalize_fault(runtime, fault, str(envelope["event_id"])) state["next_logical_step_sequence"] = step_sequence + 1 @@ -454,11 +486,11 @@ def dispatch( result["fault"] = copy.deepcopy(runtime["fault"]) result["emissions"] = execution.emissions return result - if not handled: + if disposition != Disposition.HANDLED.value: result = _empty_result( status=prior_state["status"], state=prior_state, - disposition=Disposition.UNHANDLED.value, + disposition=disposition, ) result["fault"] = copy.deepcopy(prior_state.get("fault")) return result @@ -939,8 +971,7 @@ def _valid_spawned_identity(state: dict[str, Any], runtime: dict[str, Any]) -> b ) or ( not migrated - and runtime["instance_reference"].get("machine_version") - != runtime["machine_version"] + and runtime["instance_reference"].get("machine_version") != runtime["machine_version"] ) ): return False @@ -1073,8 +1104,7 @@ def _valid_spawned_relation( if ( state_path not in owner["active"] or state_path not in owner_machine.states - or owner["state_activation_sequence"].get(state_path) - != holder["state_activation_sequence"] + or owner["state_activation_sequence"].get(state_path) != holder["state_activation_sequence"] ): return False state = owner_machine.states[state_path] @@ -1103,6 +1133,7 @@ def _valid_fault( } system_locators = { FaultCode.CONTAINED_RUNTIME_FAULT.value: "system:unhandled_contained_failure", + FaultCode.DEFERRED_EVENT_CAPACITY_EXCEEDED.value: "system:deferred_event_capacity", FaultCode.CASCADE_FAULT.value: "system:cascade_cleanup", FaultCode.INVARIANT_FAULT.value: "system:invariant", } @@ -1307,9 +1338,7 @@ def _reserved_events() -> set[str]: } -def _validate_reserved_payload( - event: str, envelope: dict[str, Any] -) -> DispatchCode | None: +def _validate_reserved_payload(event: str, envelope: dict[str, Any]) -> DispatchCode | None: payload = envelope.get("payload") if not isinstance(payload, dict): return DispatchCode.INVALID_PAYLOAD @@ -1351,7 +1380,11 @@ def _validate_reserved_payload( else: relationship = payload.get("relationship") if relationship == "parallel": - valid = set(payload) == {"relationship", "state_path", "owner_runtime_id"} and all( + valid = set(payload) == { + "relationship", + "state_path", + "owner_runtime_id", + } and all( isinstance(payload[name], str) and bool(payload[name]) for name in ("state_path", "owner_runtime_id") ) @@ -1437,9 +1470,7 @@ def _locate_target( return DispatchCode.INVALID_INSTANCE_TARGET, None -def _target_eligibility( - state: dict[str, Any], runtime: dict[str, Any] -) -> DispatchCode | None: +def _target_eligibility(state: dict[str, Any], runtime: dict[str, Any]) -> DispatchCode | None: code = ( DispatchCode.INACTIVE_COMPONENT_TARGET if runtime["role"] == "component" @@ -1467,6 +1498,7 @@ class _Execution: emissions: list[dict[str, Any]] cause_id: str event: dict[str, Any] | None + capture_emission_provenance: bool def __init__( self, @@ -1475,6 +1507,7 @@ def __init__( state: dict[str, Any], *, step_sequence: int, + capture_emission_provenance: bool = False, ) -> None: self.bundle = bundle self.models = models @@ -1483,6 +1516,25 @@ def __init__( self.emissions = [] self.cause_id = "" self.event = None + self.capture_emission_provenance = capture_emission_provenance + + def append_emission( + self, + emission: dict[str, Any], + source: dict[str, Any], + *, + system_locator: str | None = None, + ) -> None: + if self.capture_emission_provenance and emission.get("target") != "external": + emission["_determa_v2_provenance"] = { + "cause_id": self.cause_id, + "source": ( + {"system": system_locator} + if system_locator is not None + else {"runtime": self.target_for(source)} + ), + } + self.emissions.append(emission) def new_runtime( self, @@ -1541,9 +1593,7 @@ def runtime_for_target(self, target: dict[str, Any]) -> dict[str, Any]: code, runtime = _locate_target(self.state, target) if code is not None or runtime is None: fault_code = ( - FaultCode(code.value) - if code is not None - else FaultCode.INVALID_INSTANCE_TARGET + FaultCode(code.value) if code is not None else FaultCode.INVALID_INSTANCE_TARGET ) raise StepFault(fault_code, "system:invariant") return runtime @@ -1792,21 +1842,24 @@ def evaluate_author_bindings( for name in sorted(supplied, key=lambda item: item.encode("utf-8")): expression = supplied[name] value = self.evaluate( - expression, activation, f"{pointer}/with/{kind}/{_escape_pointer(name)}" + expression, + activation, + f"{pointer}/with/{kind}/{_escape_pointer(name)}", ) declaration = declarations[name] try: result[kind][name] = _normalize_value(value, str(declaration["type"])) except ValueError as exc: raise StepFault( - FaultCode.ACTION_FAULT, f"{pointer}/with/{kind}/{_escape_pointer(name)}" + FaultCode.ACTION_FAULT, + f"{pointer}/with/{kind}/{_escape_pointer(name)}", ) from exc for name, declaration in declarations.items(): if declaration.get(kind) and name not in result[kind]: result[kind][name] = copy.deepcopy(declaration["init"]) return result - def process(self, runtime: dict[str, Any], envelope: dict[str, Any]) -> bool: + def process(self, runtime: dict[str, Any], envelope: dict[str, Any]) -> str: machine = self.model_for(runtime) active = machine.states[runtime["active"][-1]] if runtime["active"] else machine.root selected: tuple[StateNode, dict[str, Any], str] | None = None @@ -1839,6 +1892,8 @@ def process(self, runtime: dict[str, Any], envelope: dict[str, Any]) -> bool: break if selected is not None: break + if envelope["event"] in (current.raw.get("deferred_events") or []): + return Disposition.DEFERRED.value current = current.parent if selected is None: if envelope["event"] in { @@ -1849,7 +1904,7 @@ def process(self, runtime: dict[str, Any], envelope: dict[str, Any]) -> bool: FaultCode.CONTAINED_RUNTIME_FAULT, "system:unhandled_contained_failure", ) - return False + return Disposition.UNHANDLED.value source, transition, pointer = selected try: target, history = self.resolve_compound_transition( @@ -1861,7 +1916,7 @@ def process(self, runtime: dict[str, Any], envelope: dict[str, Any]) -> bool: event_visible=True, ) if target is None: - return True + return Disposition.HANDLED.value self.apply_transition( runtime, machine, @@ -1872,7 +1927,7 @@ def process(self, runtime: dict[str, Any], envelope: dict[str, Any]) -> bool: ) except _StopRuntime: self.complete_runtime(runtime, machine) - return True + return Disposition.HANDLED.value def resolve_compound_transition( self, @@ -2096,7 +2151,7 @@ def send( } if correlation is not None: emission["correlation_id"] = correlation - self.emissions.append(emission) + self.append_emission(emission, runtime) def resolve_send_target( self, @@ -2265,11 +2320,7 @@ def cancel(self, runtime: dict[str, Any], reference: Any) -> None: if not _is_instance_reference(reference): return child = self.state["runtimes"].get(reference["instance_id"]) - if ( - child is None - or child["role"] != "spawned" - or not self.owns_descendant(runtime, child) - ): + if child is None or child["role"] != "spawned" or not self.owns_descendant(runtime, child): return self.cleanup_descendant(child) @@ -2539,13 +2590,15 @@ def emit_internal_system( locator, ordinal, ) - self.emissions.append( + self.append_emission( { "event": event, "event_id": event_id, "target": target, "payload": copy.deepcopy(payload), - } + }, + source, + system_locator=locator, ) def finalize_fault( diff --git a/src/determa/state/host.py b/src/determa/state/host.py index 3110661..511a5f1 100644 --- a/src/determa/state/host.py +++ b/src/determa/state/host.py @@ -6,7 +6,7 @@ from collections.abc import Callable, Mapping, Sequence from contextlib import nullcontext from dataclasses import dataclass -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast from .checkpoint import ( RestoredExecutionCheckpoint, @@ -49,13 +49,30 @@ aggregate_envelope, decoded_typed_value, hash_value, + strict_json, typed_value, ) +if TYPE_CHECKING: + from .checkpoint_v2 import RestoredExecutionCheckpointV2 + FaultInjector = Callable[[str], None] _MAX_DECIMAL_DIGITS = 4096 +def _bundle_requires_v2(bundle: Bundle) -> bool: + def has_deferral(value: Any) -> bool: + if isinstance(value, Mapping): + return "deferred_events" in value or any( + has_deferral(child) for child in value.values() + ) + if isinstance(value, list): + return any(has_deferral(child) for child in value) + return False + + return has_deferral(bundle.raw["machines"]) + + class ExecutionHostError(DetermaError): """A closed host-layer failure.""" @@ -186,9 +203,7 @@ def maintenance_migration_request_digest( ) -def outbox_intent_digest( - root_instance_id: str, intent: Mapping[str, Any] -) -> str: +def outbox_intent_digest(root_instance_id: str, intent: Mapping[str, Any]) -> str: """Compute the compact evidence digest for one complete outbox intent.""" return hash_value( [ @@ -225,11 +240,15 @@ def validate_host_profile( and PERMANENT_RECEIPT_RETENTION in capabilities ) elif profile == "broker_integrated": - valid = common and atomic and { - "acknowledge_after_checkpoint_commit", - "durable_redelivery", - "outbox_worker", - }.issubset(host_features) + valid = ( + common + and atomic + and { + "acknowledge_after_checkpoint_commit", + "durable_redelivery", + "outbox_worker", + }.issubset(host_features) + ) elif profile == "strict_durable_outbox": valid = ( common @@ -306,9 +325,7 @@ def _project_emission(emission: Mapping[str, Any]) -> dict[str, Any]: return projected -def _project_core_result( - bundle: Bundle, result: Mapping[str, Any] -) -> dict[str, Any]: +def _project_core_result(bundle: Bundle, result: Mapping[str, Any]) -> dict[str, Any]: aggregate = ( aggregate_envelope(bundle, result["state"]) if result["state"] is not None @@ -529,8 +546,19 @@ def _restore( self, source: bytes, root_instance_id: str, - ) -> RestoredExecutionCheckpoint: - restored = restore_execution_checkpoint(source, self.artifact_resolver) + ) -> Any: + document, _ = strict_json(source) + if ( + isinstance(document, Mapping) + and document.get("execution_checkpoint_schema_version") == 2 + ): + from .checkpoint_v2 import restore_execution_checkpoint_v2 + + restored: Any = restore_execution_checkpoint_v2( + source, self.artifact_resolver + ) + else: + restored = restore_execution_checkpoint(source, self.artifact_resolver) if restored.document["root_instance_id"] != root_instance_id: raise ExecutionHostError("transaction_root_mismatch") if ( @@ -593,15 +621,16 @@ def _check_expected( ) -> None: if ( checkpoint["revision"] != expected_revision - or checkpoint["execution_checkpoint_digest"] - != expected_checkpoint_digest + or checkpoint["execution_checkpoint_digest"] != expected_checkpoint_digest ): raise ExecutionHostError(HostCode.CHECKPOINT_REVISION_CONFLICT) def _stage_insert( self, transaction: ExecutionStoreTransaction, candidate: dict[str, Any] ) -> None: - restore_execution_checkpoint(candidate, self.artifact_resolver) + self._restore( + serialize_execution_checkpoint(candidate), candidate["root_instance_id"] + ) self._fault("before_commit") if not transaction.insert(serialize_execution_checkpoint(candidate)): raise ExecutionHostError(HostCode.CHECKPOINT_REVISION_CONFLICT) @@ -612,7 +641,9 @@ def _stage_replace( previous: Mapping[str, Any], candidate: dict[str, Any], ) -> None: - restore_execution_checkpoint(candidate, self.artifact_resolver) + self._restore( + serialize_execution_checkpoint(candidate), candidate["root_instance_id"] + ) self._fault("before_commit") if not transaction.replace( previous["revision"], @@ -624,14 +655,10 @@ def _stage_replace( def read_checkpoint( self, root_instance_id: str, - ) -> RestoredExecutionCheckpoint | None: + ) -> RestoredExecutionCheckpoint | RestoredExecutionCheckpointV2 | None: with self._transaction(root_instance_id) as transaction: source = transaction.load() - return ( - None - if source is None - else self._restore(source, root_instance_id) - ) + return None if source is None else self._restore(source, root_instance_id) def create( self, @@ -643,8 +670,7 @@ def create( ) -> dict[str, Any]: validated = bundle if isinstance(bundle, Bundle) else load_bundle(bundle) normalized_bindings = { - name: copy.deepcopy(dict(value)) - for name, value in (bindings or {}).items() + name: copy.deepcopy(dict(value)) for name, value in (bindings or {}).items() } request_digest = creation_request_digest( validated, @@ -681,6 +707,363 @@ def create( self._after_commit() return {"result": "committed", "receipt": receipt} + def create_v2( + self, + bundle: Bundle | BundleSource, + machine_id: str, + root_instance_id: str, + creation_id: str, + bindings: Mapping[str, Mapping[str, Any]] | None = None, + ) -> dict[str, Any]: + """Create and transactionally retain one queue-bearing checkpoint.""" + from .checkpoint_v2 import create_checkpoint_v2 + + normalized = {name: dict(value) for name, value in (bindings or {}).items()} + with self._transaction(root_instance_id) as transaction: + source = transaction.load() + if source is not None: + restored = self._restore(source, root_instance_id) + receipt = restored.document["operation_receipts"][0] + request_digest = creation_request_digest( + bundle, machine_id, root_instance_id, creation_id, normalized + ) + receipt_value = receipt.get("legacy_receipt", receipt) + if ( + receipt_value["creation_id"] == creation_id + and receipt_value["request_digest"] == request_digest + ): + return {"result": "committed", "receipt": copy.deepcopy(receipt)} + raise ExecutionHostError(HostCode.CREATION_ID_CONFLICT) + candidate = create_checkpoint_v2( + bundle, machine_id, root_instance_id, creation_id, normalized + ) + self._stage_insert(transaction, candidate) + receipt = copy.deepcopy(candidate["operation_receipts"][0]) + self._after_commit() + return {"result": "committed", "receipt": receipt} + + @staticmethod + def _v2_committed_checkpoint(result: Mapping[str, Any]) -> dict[str, Any] | None: + if result.get("execution_checkpoint_schema_version") == 2: + return copy.deepcopy(dict(result)) + checkpoint = result.get("checkpoint") + if ( + isinstance(checkpoint, Mapping) + and checkpoint.get("execution_checkpoint_schema_version") == 2 + ): + return copy.deepcopy(dict(checkpoint)) + return None + + def admit_v2( + self, + root_instance_id: str, + deliveries: Sequence[Mapping[str, Any]], + *, + expected_revision: str, + expected_checkpoint_digest: str, + ) -> dict[str, Any]: + """Admit one v2 batch inside the store transaction.""" + from .checkpoint_v2 import admit_checkpoint_v2 + + with self._transaction(root_instance_id) as transaction: + source = transaction.load() + if source is None: + raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) + prior: dict[str, Any] = self._restore(source, root_instance_id).document + result = admit_checkpoint_v2( + prior, + deliveries, + self.artifact_resolver, + expected_revision=expected_revision, + expected_checkpoint_digest=expected_checkpoint_digest, + ) + candidate = self._v2_committed_checkpoint(result) + if ( + candidate is not None + and candidate["execution_checkpoint_digest"] + != prior["execution_checkpoint_digest"] + ): + self._stage_replace(transaction, prior, candidate) + if ( + candidate is not None + and candidate["execution_checkpoint_digest"] + != prior["execution_checkpoint_digest"] + ): + self._after_commit() + return result + + def process_ready_v2( + self, + root_instance_id: str, + target_runtime_id: str, + *, + expected_revision: str, + expected_checkpoint_digest: str, + ) -> dict[str, Any]: + """Process one v2 ready head inside the store transaction.""" + from .checkpoint_v2 import step_checkpoint_v2 + + with self._transaction(root_instance_id) as transaction: + source = transaction.load() + if source is None: + raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) + prior = self._restore(source, root_instance_id).document + result = step_checkpoint_v2( + prior, + target_runtime_id, + self.artifact_resolver, + expected_revision=expected_revision, + expected_checkpoint_digest=expected_checkpoint_digest, + ) + candidate = self._v2_committed_checkpoint(result) + if ( + candidate is not None + and candidate["execution_checkpoint_digest"] + != prior["execution_checkpoint_digest"] + ): + self._stage_replace(transaction, prior, candidate) + if ( + candidate is not None + and candidate["execution_checkpoint_digest"] + != prior["execution_checkpoint_digest"] + ): + self._after_commit() + return result + + def upgrade_checkpoint_v2( + self, + root_instance_id: str, + *, + expected_revision: str, + expected_checkpoint_digest: str, + ) -> dict[str, Any]: + """Atomically upgrade one retained v1 checkpoint to v2.""" + from .checkpoint_v2 import upgrade_checkpoint_v1_to_v2 + + with self._transaction(root_instance_id) as transaction: + source = transaction.load() + if source is None: + raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) + prior = self._restore(source, root_instance_id).document + self._check_expected(prior, expected_revision, expected_checkpoint_digest) + candidate = upgrade_checkpoint_v1_to_v2(prior, self.artifact_resolver) + self._stage_replace(transaction, prior, candidate) + self._after_commit() + return candidate + + def prune_v2( + self, + root_instance_id: str, + cutoff_receipt_sequence: str, + *, + expected_revision: str, + expected_checkpoint_digest: str, + ) -> dict[str, Any]: + """Atomically prune one dependency-closed bounded v2 checkpoint.""" + from .checkpoint_v2 import prune_checkpoint_v2 + + with self._transaction(root_instance_id) as transaction: + source = transaction.load() + if source is None: + raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) + prior = self._restore(source, root_instance_id).document + candidate = prune_checkpoint_v2( + prior, + cutoff_receipt_sequence, + self.artifact_resolver, + expected_revision=expected_revision, + expected_checkpoint_digest=expected_checkpoint_digest, + ) + if ( + candidate["execution_checkpoint_digest"] + != prior["execution_checkpoint_digest"] + ): + self._stage_replace(transaction, prior, candidate) + if ( + candidate["execution_checkpoint_digest"] + != prior["execution_checkpoint_digest"] + ): + self._after_commit() + return candidate + + @staticmethod + def _terminalize_v2_entries( + checkpoint: dict[str, Any], + entries: Sequence[Mapping[str, Any]], + *, + disposition: str, + reason: str, + resulting_digest: str, + status: str, + migration_descriptor_digest: str | None = None, + ) -> None: + from .checkpoint_v2 import _terminalize_mailbox_reference + + for entry in entries: + receipt_sequence = checkpoint["next_operation_receipt_sequence"] + checkpoint["next_operation_receipt_sequence"] = ( + _increment_checkpoint_number(receipt_sequence) + ) + event_id = entry["envelope"]["event_id"] + _terminalize_mailbox_reference(checkpoint, entry, receipt_sequence) + outcome: dict[str, Any] = { + "status": status, + "disposition": disposition, + "reason": reason, + "fault": None, + "rejection": None, + } + if migration_descriptor_digest is not None: + outcome["migration_descriptor_digest"] = migration_descriptor_digest + checkpoint["operation_receipts"].append( + { + "operation_kind": "event_terminal", + "receipt_sequence": receipt_sequence, + "event_id": event_id, + "request_digest": entry["envelope_digest"], + "acceptance_sequence": entry["acceptance_sequence"], + "final_queue_sequence": entry["queue_sequence"], + "committed_revision": checkpoint["revision"], + "resulting_aggregate_state_digest": resulting_digest, + "outcome": outcome, + "emission_references": [], + } + ) + + def maintenance_migration_v2( + self, + root_instance_id: str, + target_validated_bundle_fingerprint: str, + migration_descriptor_digest_route: Sequence[str], + *, + expected_revision: str, + expected_checkpoint_digest: str, + maintenance_mode: bool = True, + limits: MigrationLimits | None = None, + ) -> dict[str, Any]: + """Migrate a v2 aggregate and all queue ownership in one transaction.""" + from .checkpoint_v2 import _synchronize_mailbox_references + from .queueing import migrate_aggregate_v2 + + with self._transaction(root_instance_id) as transaction: + source = transaction.load() + if source is None: + raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) + prior = self._restore(source, root_instance_id).document + self._check_expected(prior, expected_revision, expected_checkpoint_digest) + aggregate = prior["root_record"].get("aggregate_state") + if aggregate is None: + raise ExecutionHostError(PreAcceptanceCode.TOMBSTONED_ROOT) + migration = migrate_aggregate_v2( + aggregate, + target_validated_bundle_fingerprint, + migration_descriptor_digest_route, + self.artifact_resolver, + maintenance_mode=maintenance_mode, + resource_limits=limits, + _include_host_evidence=True, + ) + migrated = migration["aggregate_state"] + candidate = _mutate(prior) + candidate["root_record"]["aggregate_state"] = migrated + candidate["migration_audit_records"].extend( + copy.deepcopy(migration["audit_records"]) + ) + for entry, disposition in zip( + migration["_disposed_entries"], + migration["dispositions"], + strict=True, + ): + self._terminalize_v2_entries( + candidate, + [entry], + disposition="migration_disposed", + reason=disposition["reason"], + resulting_digest=migrated["aggregate_state_digest"], + status=next( + runtime["status"] + for runtime in migrated["runtimes"] + if runtime["runtime_id"] == migrated["root_runtime_id"] + ), + migration_descriptor_digest=disposition[ + "migration_descriptor_digest" + ], + ) + _synchronize_mailbox_references(candidate) + candidate = seal_execution_checkpoint(candidate) + self._stage_replace(transaction, prior, candidate) + self._after_commit() + return candidate + + def tombstone_root_v2( + self, + root_instance_id: str, + operation_id: str, + *, + expected_revision: str, + expected_checkpoint_digest: str, + ) -> dict[str, Any]: + """Tombstone one v2 root and terminalize all engine-owned work.""" + if not operation_id: + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + with self._transaction(root_instance_id) as transaction: + source = transaction.load() + if source is None: + raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) + prior: dict[str, Any] = self._restore(source, root_instance_id).document + root_record = prior["root_record"] + if root_record["status"] == "tombstone": + if root_record["tombstone_operation_id"] == operation_id: + return prior + raise ExecutionHostError(HostCode.OPERATION_ID_CONFLICT) + self._check_expected(prior, expected_revision, expected_checkpoint_digest) + aggregate = root_record["aggregate_state"] + root_runtime = next( + runtime + for runtime in aggregate["runtimes"] + if runtime["runtime_id"] == aggregate["root_runtime_id"] + ) + if ( + root_runtime["status"] not in {"completed", "faulted"} + or prior["pending_outbox_intents"] + ): + raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) + from .queueing import _lifecycle_runtime_order, restore_aggregate_v2 + + restored_aggregate = restore_aggregate_v2(aggregate, self.artifact_resolver) + runtime_by_id = { + runtime["runtime_id"]: runtime for runtime in aggregate["runtimes"] + } + entries = [ + entry + for runtime_id in _lifecycle_runtime_order( + restored_aggregate.bundle, restored_aggregate.state + ) + for mailbox in ("ready_mailbox", "deferred_mailbox") + for entry in runtime_by_id[runtime_id][mailbox] + ] + candidate = _mutate(prior) + self._terminalize_v2_entries( + candidate, + entries, + disposition="disposed", + reason="root_tombstoned", + resulting_digest=aggregate["aggregate_state_digest"], + status=root_runtime["status"], + ) + candidate["root_record"] = { + "status": "tombstone", + "root_runtime_id": aggregate["root_runtime_id"], + "creation_id": aggregate["creation_id"], + "terminal_status": root_runtime["status"], + "final_aggregate_state_digest": aggregate["aggregate_state_digest"], + "tombstone_operation_id": operation_id, + } + candidate = seal_execution_checkpoint(candidate) + self._stage_replace(transaction, prior, candidate) + self._after_commit() + return candidate + def _delivery_candidate( self, candidate: Any ) -> tuple[str | None, str | None, Any, dict[str, Any] | None, str | None]: @@ -718,9 +1101,7 @@ def _delivery_candidate( supplied_digest, ) - def _not_accepted( - self, code: PreAcceptanceCode - ) -> dict[str, Any]: + def _not_accepted(self, code: PreAcceptanceCode) -> dict[str, Any]: return {"result": "not_accepted", "failure": {"code": code.value}} def _delivery_replay( @@ -740,7 +1121,10 @@ def _delivery_replay( "accepted_revision": pending["accepted_revision"], } for receipt in checkpoint["operation_receipts"]: - if receipt["operation_kind"] == "delivery" and receipt["event_id"] == event_id: + if ( + receipt["operation_kind"] == "delivery" + and receipt["event_id"] == event_id + ): if receipt["request_digest"] != digest: return self._not_accepted(PreAcceptanceCode.EVENT_ID_CONFLICT) return {"result": "committed", "receipt": copy.deepcopy(receipt)} @@ -750,6 +1134,7 @@ def _prepare_acceptance( self, checkpoint: Mapping[str, Any], candidate: Any, + bundle: Bundle | None = None, ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: parsed = self._delivery_candidate(candidate) root_instance_id, mode, origin, envelope, supplied_digest = parsed @@ -759,19 +1144,19 @@ def _prepare_acceptance( return None, self._not_accepted(PreAcceptanceCode.WRONG_ROOT) digest = delivery_request_digest(root_instance_id, mode, envelope) - replay = self._delivery_replay( - checkpoint, envelope["event_id"], digest - ) + replay = self._delivery_replay(checkpoint, envelope["event_id"], digest) if replay is not None: return None, replay if checkpoint["root_record"]["status"] == "tombstone": return None, self._not_accepted(PreAcceptanceCode.TOMBSTONED_ROOT) + if bundle is not None and _bundle_requires_v2(bundle): + return None, self._not_accepted( + PreAcceptanceCode.CHECKPOINT_UPGRADE_REQUIRED + ) valid_mode = mode in {"input", "internal"} valid_origin = validate_execution_checkpoint_member("deliveryOrigin", origin) - valid_pair = ( - mode == "input" and origin == {"kind": "host_input"} - ) or ( + valid_pair = (mode == "input" and origin == {"kind": "host_input"}) or ( mode == "internal" and isinstance(origin, Mapping) and origin.get("kind") == "internal_emission" @@ -781,9 +1166,7 @@ def _prepare_acceptance( if not valid_origin or not valid_pair: return None, self._not_accepted(PreAcceptanceCode.INVALID_DELIVERY_ORIGIN) if supplied_digest is not None and supplied_digest != digest: - return None, self._not_accepted( - PreAcceptanceCode.DELIVERY_DIGEST_MISMATCH - ) + return None, self._not_accepted(PreAcceptanceCode.DELIVERY_DIGEST_MISMATCH) if ( _target_root_instance_id(envelope["target"]) != checkpoint["root_instance_id"] @@ -804,13 +1187,27 @@ def accept_delivery( *, expected_revision: str, expected_checkpoint_digest: str, + selected_bundle: Bundle | BundleSource | None = None, ) -> dict[str, Any]: with self._transaction(root_instance_id) as transaction: source = transaction.load() if source is None: return self._not_accepted(PreAcceptanceCode.WRONG_ROOT) - checkpoint = self._restore(source, root_instance_id).document - prepared, result = self._prepare_acceptance(checkpoint, candidate) + restored = self._restore(source, root_instance_id) + checkpoint = restored.document + prepared, result = self._prepare_acceptance( + checkpoint, + candidate, + ( + selected_bundle + if isinstance(selected_bundle, Bundle) + else load_bundle(selected_bundle) + if selected_bundle is not None + else restored.aggregate.bundle + if restored.aggregate is not None + else None + ), + ) if result is not None: return result assert prepared is not None @@ -875,8 +1272,8 @@ def _commit_delivery( "envelope_digest": pending["envelope_digest"], } receipt_sequence = candidate["next_operation_receipt_sequence"] - candidate["next_operation_receipt_sequence"] = ( - _increment_checkpoint_number(receipt_sequence) + candidate["next_operation_receipt_sequence"] = _increment_checkpoint_number( + receipt_sequence ) aggregate = copy.deepcopy(projected["aggregate_state"]) if aggregate is None or restored.aggregate is None: @@ -892,9 +1289,7 @@ def _commit_delivery( "delivery_mode": request["delivery_mode"], "origin": copy.deepcopy(request["origin"]), "committed_revision": candidate["revision"], - "resulting_aggregate_state_digest": aggregate[ - "aggregate_state_digest" - ], + "resulting_aggregate_state_digest": aggregate["aggregate_state_digest"], "outcome": { "status": projected["status"], "disposition": projected["disposition"], @@ -933,9 +1328,7 @@ def process_pending_delivery( digest = delivery_request_digest(root_instance_id, mode, envelope) if supplied_digest is not None and supplied_digest != digest: raise ExecutionHostError(PreAcceptanceCode.DELIVERY_DIGEST_MISMATCH) - replay = self._delivery_replay( - checkpoint, envelope["event_id"], digest - ) + replay = self._delivery_replay(checkpoint, envelope["event_id"], digest) if replay is not None and replay["result"] == "committed": return replay if replay is not None and replay["result"] == "not_accepted": @@ -958,9 +1351,7 @@ def process_pending_delivery( result = core_dispatch( restored.aggregate.bundle, restored.aggregate.state, - _delivery_from_wire( - pending["delivery_mode"], pending["envelope"] - ), + _delivery_from_wire(pending["delivery_mode"], pending["envelope"]), ) projected = _project_core_result(restored.aggregate.bundle, result) next_checkpoint, receipt = self._commit_delivery( @@ -990,7 +1381,11 @@ def foreground_process_delivery( raise ExecutionHostError(PreAcceptanceCode.WRONG_ROOT) restored = self._restore(source, root_instance_id) checkpoint = restored.document - prepared, replay = self._prepare_acceptance(checkpoint, candidate) + prepared, replay = self._prepare_acceptance( + checkpoint, + candidate, + restored.aggregate.bundle if restored.aggregate is not None else None, + ) if replay is not None: return replay assert prepared is not None @@ -1002,9 +1397,7 @@ def foreground_process_delivery( result = core_dispatch( restored.aggregate.bundle, restored.aggregate.state, - _delivery_from_wire( - prepared["delivery_mode"], prepared["envelope"] - ), + _delivery_from_wire(prepared["delivery_mode"], prepared["envelope"]), ) projected = _project_core_result(restored.aggregate.bundle, result) next_checkpoint, receipt = self._commit_delivery( @@ -1032,11 +1425,8 @@ def maintenance_migration( maintenance_mode: bool = True, limits: MigrationLimits | None = None, ) -> dict[str, Any]: - if ( - not operation_id - or not validate_execution_checkpoint_member( - "sha256", source_aggregate_state_digest - ) + if not operation_id or not validate_execution_checkpoint_member( + "sha256", source_aggregate_state_digest ): raise ExecutionHostError(PersistenceCode.INVALID_MIGRATION_REQUEST) request_digest = maintenance_migration_request_digest( @@ -1091,8 +1481,8 @@ def maintenance_migration( raise ExecutionHostError(code) candidate = _mutate(checkpoint) receipt_sequence = candidate["next_operation_receipt_sequence"] - candidate["next_operation_receipt_sequence"] = ( - _increment_checkpoint_number(receipt_sequence) + candidate["next_operation_receipt_sequence"] = _increment_checkpoint_number( + receipt_sequence ) migration_sequences = [ item["migration_sequence"] for item in result.audit_records @@ -1227,8 +1617,8 @@ def terminalize_outbox( ) candidate["pending_outbox_intents"].remove(candidate_pending) terminal_sequence = candidate["next_outbox_terminal_sequence"] - candidate["next_outbox_terminal_sequence"] = ( - _increment_checkpoint_number(terminal_sequence) + candidate["next_outbox_terminal_sequence"] = _increment_checkpoint_number( + terminal_sequence ) record = { "terminal_sequence": terminal_sequence, @@ -1381,7 +1771,10 @@ def update_replay_retention( checkpoint = self._restore(source, root_instance_id).document current = checkpoint["replay_retention"] if current == target: - return {"result": "committed", "replay_retention": copy.deepcopy(current)} + return { + "result": "committed", + "replay_retention": copy.deepcopy(current), + } if current["mode"] == "bounded" and target["mode"] == "permanent": raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) current_cutoff = current["pruned_through_receipt_sequence"] @@ -1389,26 +1782,18 @@ def update_replay_retention( if target["mode"] == "bounded": if ( current["mode"] == "bounded" - and current["policy_identifier"] - != target["policy_identifier"] + and current["policy_identifier"] != target["policy_identifier"] ): raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) - if ( - current_cutoff is not None - and ( - target_cutoff is None - or _checkpoint_number(target_cutoff) - < _checkpoint_number(current_cutoff) - ) + if current_cutoff is not None and ( + target_cutoff is None + or _checkpoint_number(target_cutoff) + < _checkpoint_number(current_cutoff) ): raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) - if ( - target_cutoff is not None - and _checkpoint_number(target_cutoff) - >= _checkpoint_number( - checkpoint["next_operation_receipt_sequence"] - ) - ): + if target_cutoff is not None and _checkpoint_number( + target_cutoff + ) >= _checkpoint_number(checkpoint["next_operation_receipt_sequence"]): raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) self._check_expected( checkpoint, expected_revision, expected_checkpoint_digest @@ -1455,7 +1840,9 @@ def update_replay_retention( self._stage_replace(transaction, checkpoint, candidate) except Exception as exc: if getattr(exc, "code", None) == HostCode.INVALID_EXECUTION_CHECKPOINT: - raise ExecutionHostError(HostCode.INVALID_EXECUTION_CHECKPOINT) from exc + raise ExecutionHostError( + HostCode.INVALID_EXECUTION_CHECKPOINT + ) from exc raise response = { "result": "committed", @@ -1509,9 +1896,7 @@ def tombstone_root( "root_runtime_id": aggregate["root_runtime_id"], "creation_id": aggregate["creation_id"], "terminal_status": root_runtime["status"], - "final_aggregate_state_digest": aggregate[ - "aggregate_state_digest" - ], + "final_aggregate_state_digest": aggregate["aggregate_state_digest"], "tombstone_operation_id": operation_id, } candidate["root_record"] = tombstone diff --git a/src/determa/state/queueing.py b/src/determa/state/queueing.py new file mode 100644 index 0000000..f330a73 --- /dev/null +++ b/src/determa/state/queueing.py @@ -0,0 +1,1300 @@ +"""Pure queue-bearing aggregate-state version 2 operations.""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping, Sequence +from functools import cache +from typing import Any, Literal, cast + +from .codes import ( + CheckpointPreAcceptanceFailureCode as AdmissionCode, +) +from .codes import ( + DispatchRejectionCode, + DispositionCode, + EngineFaultCode, + PersistenceFailureCode, +) +from .definition import Bundle, BundleSource, load_bundle +from .engine import ( + _Execution, + _normalize_payload, + _normalize_value, + _pointer_get, + _runtime_model, + _validate_envelope, + _validate_reserved_payload, + create, + dispatch, +) +from .errors import ArtifactError, StepFault +from .migration import MigrationLimits, migrate_aggregate +from .model import BundleModel, StateNode +from .wire import ( + ArtifactSource, + DefinitionResolver, + MemoryArtifactResolver, + RestoredAggregate, + _schema_registry, + aggregate_envelope, + aggregate_state_digest, + artifact_schema, + canonical_bytes, + decimal, + decoded_typed_value, + hash_value, + load_json_artifact, + migration_descriptor_digest, + restore_aggregate, + typed_value, +) + + +def seal_aggregate_v2(document: Mapping[str, Any]) -> dict[str, Any]: + """Copy, canonically order, and seal one version-2 aggregate.""" + result = copy.deepcopy(dict(document)) + for runtime in result.get("runtimes", []): + runtime["active_leaf_state_definition_pointers"].sort(key=_utf8) + runtime["active_state_activations"].sort( + key=lambda item: ( + _utf8(item["state_definition_pointer"]), + int(item["activation_sequence"]), + ) + ) + runtime["variables"].sort( + key=lambda item: ( + _utf8(item["variable_declaration_pointer"]), + int(item["declaring_state_activation_sequence"]), + ) + ) + runtime["history"].sort(key=lambda item: _utf8(item["history_declaration_pointer"])) + for name in ( + "next_state_activation_sequences", + "next_component_activation_sequences", + ): + runtime[name].sort(key=lambda item: _utf8(item["definition_pointer"])) + for name in ("ready_mailbox", "deferred_mailbox"): + runtime[name].sort(key=lambda item: int(item["queue_sequence"])) + result["runtimes"].sort(key=lambda item: _utf8(item["runtime_id"])) + result.pop("aggregate_state_digest", None) + result["aggregate_state_digest"] = aggregate_state_digest(result) + return result + + +def _utf8(value: str) -> bytes: + return value.encode("utf-8", errors="strict") + + +def _project_v1(document: Mapping[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(dict(document)) + result["aggregate_state_schema_version"] = 1 + result.pop("next_acceptance_sequence", None) + result.pop("next_queue_sequence", None) + for runtime in result["runtimes"]: + runtime.pop("ready_mailbox", None) + runtime.pop("deferred_mailbox", None) + result.pop("aggregate_state_digest", None) + result["aggregate_state_digest"] = aggregate_state_digest(result) + return result + + +def _upgrade_document(document: Mapping[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(dict(document)) + result["aggregate_state_schema_version"] = 2 + result["next_acceptance_sequence"] = "0" + result["next_queue_sequence"] = "0" + for runtime in result["runtimes"]: + runtime["ready_mailbox"] = [] + runtime["deferred_mailbox"] = [] + return seal_aggregate_v2(result) + + +def upgrade_aggregate_v1_to_v2( + source: ArtifactSource, definition_resolver: DefinitionResolver +) -> dict[str, Any]: + """Explicitly convert a valid version-1 aggregate to empty version-2 mailboxes.""" + del definition_resolver + document, _ = load_json_artifact(source, "aggregate_state") + if aggregate_state_digest(document) != document["aggregate_state_digest"]: + raise ArtifactError(PersistenceFailureCode.AGGREGATE_STATE_DIGEST_MISMATCH) + return _upgrade_document(document) + + +def downgrade_aggregate_v2_to_v1( + source: ArtifactSource, definition_resolver: DefinitionResolver +) -> dict[str, Any]: + """Explicitly convert a version-2 aggregate only when no queued work exists.""" + del definition_resolver + document, _ = load_json_artifact(source, "aggregate_state_v2") + if aggregate_state_digest(document) != document["aggregate_state_digest"]: + raise ArtifactError(PersistenceFailureCode.AGGREGATE_STATE_DIGEST_MISMATCH) + _validate_mailboxes(document) + if document["next_acceptance_sequence"] != "0" or document["next_queue_sequence"] != "0": + raise ArtifactError(PersistenceFailureCode.MIGRATION_TOTALITY_FAILURE) + if any( + runtime[mailbox] + for runtime in document["runtimes"] + for mailbox in ("ready_mailbox", "deferred_mailbox") + ): + raise ArtifactError(PersistenceFailureCode.MIGRATION_TOTALITY_FAILURE) + return {"result": "success", "aggregate_state": _project_v1(document)} + + +def create_aggregate_v2( + bundle: Bundle | BundleSource, + machine_id: str, + root_instance_id: str, + creation_id: str, + bindings: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Create one queue-bearing aggregate and route initialization emissions.""" + validated = bundle if isinstance(bundle, Bundle) else load_bundle(bundle) + result = create( + validated, + machine_id, + root_instance_id, + creation_id, + bindings, + _capture_emission_provenance=True, + ) + if result["state"] is None: + return {**result, "lifecycle_dispositions": []} + encoded = _upgrade_document(aggregate_envelope(validated, result["state"])) + root_runtime = next( + runtime + for runtime in encoded["runtimes"] + if runtime["runtime_id"] == encoded["root_runtime_id"] + ) + lifecycle: list[dict[str, Any]] = [] + emissions = _enqueue_emissions( + encoded, + encoded, + root_runtime, + cast(list[Mapping[str, Any]], result["emissions"]), + lifecycle, + ) + encoded = seal_aggregate_v2(encoded) + restore_aggregate_v2(encoded, resolver_for_bundle(validated)) + result_fault = copy.deepcopy(root_runtime["fault"]) if result["status"] == "faulted" else None + return { + "status": result["status"], + "disposition": None, + "state": encoded, + "emissions": emissions, + "lifecycle_dispositions": lifecycle, + "fault": result_fault, + "rejection": copy.deepcopy(result["rejection"]), + } + + +@cache +def _envelope_validator() -> Any: + import jsonschema + + schema = artifact_schema("aggregate_state_v2") + return jsonschema.Draft202012Validator( + {"$ref": f"{schema['$id']}#/$defs/envelope"}, registry=_schema_registry() + ) + + +def _valid_envelope_shape(value: Any) -> bool: + return ( + isinstance(value, Mapping) and next(_envelope_validator().iter_errors(value), None) is None + ) + + +def _entry_digest(root_instance_id: str, mode: str, envelope: Mapping[str, Any]) -> str: + return hash_value(["determa-inbox-envelope-digest-2", "2", root_instance_id, mode, envelope]) + + +def _validate_mailboxes(document: dict[str, Any]) -> None: + next_acceptance = decimal(document["next_acceptance_sequence"]) + next_queue = decimal(document["next_queue_sequence"]) + acceptance_sequences: list[int] = [] + queue_sequences: list[int] = [] + event_ids: set[str] = set() + for runtime in document["runtimes"]: + for mailbox in ("ready_mailbox", "deferred_mailbox"): + prior_queue = -1 + for entry in runtime[mailbox]: + acceptance = decimal(entry["acceptance_sequence"]) + queue = decimal(entry["queue_sequence"]) + if queue <= prior_queue: + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) + prior_queue = queue + if acceptance >= next_acceptance or queue >= next_queue: + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) + envelope = entry["envelope"] + event_id = envelope["event_id"] + if ( + envelope["target"] != runtime["target_identity"] + or event_id in event_ids + or entry["envelope_digest"] + != _entry_digest(document["root_instance_id"], entry["delivery_mode"], envelope) + or not isinstance(decoded_typed_value(envelope["payload"]), dict) + ): + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) + event_ids.add(event_id) + acceptance_sequences.append(acceptance) + queue_sequences.append(queue) + if len(acceptance_sequences) != len(set(acceptance_sequences)) or len(queue_sequences) != len( + set(queue_sequences) + ): + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) + + +def _valid_envelope_source( + document: Mapping[str, Any], + entry: Mapping[str, Any], + declaration: Mapping[str, Any] | None, + payload: Mapping[str, Any], + *, + allow_legacy: bool, +) -> bool: + envelope = entry["envelope"] + source = envelope["source"] + mode = entry["delivery_mode"] + if mode == "input": + return bool(source == {"host": True} and envelope["cause_id"] == envelope["event_id"]) + if "legacy_v1_internal" in source: + return allow_legacy + event = envelope["event"] + if declaration is not None or event == "env": + if "runtime" not in source: + return False + return any( + runtime["target_identity"] == source["runtime"] for runtime in document["runtimes"] + ) + expected_system = { + "determa.component_completed": "system:component_completion", + "determa.component_failed": "system:component_failure", + "determa.spawned_instance_failed": "system:spawned_failure", + }.get(event) + if event == "done": + relationship = payload.get("relationship") + expected_system = { + "parallel": "system:component_completion", + "spawned_instance": "system:spawned_completion", + }.get(relationship if isinstance(relationship, str) else "") + return expected_system is not None and source == {"system": expected_system} + + +def _mailbox_payload_is_normalized( + restored: RestoredAggregate, + runtime: Mapping[str, Any], + entry: Mapping[str, Any], + declaration: Mapping[str, Any] | None, + payload: Mapping[str, Any], +) -> bool: + envelope = entry["envelope"] + event = envelope["event"] + if declaration is not None: + normalized = _normalize_payload(dict(declaration), dict(payload)) + return normalized is not None and typed_value(normalized) == envelope["payload"] + if event == "env": + if set(payload) != {"changed"} or not isinstance(payload["changed"], Mapping): + return False + native_runtime = next( + ( + item + for item in restored.state["runtimes"] + if item["runtime_id"] == runtime["runtime_id"] + ), + None, + ) + if native_runtime is None: + return False + root = _pointer_get(restored.bundle.raw, native_runtime["root_pointer"]) + external = { + name: variable + for name, variable in (root.get("variables") or {}).items() + if variable.get("external") is True + } + changed = payload["changed"] + if not changed or set(changed) - set(external): + return False + try: + normalized_changed = { + name: _normalize_value(value, str(external[name]["type"])) + for name, value in changed.items() + } + except (KeyError, TypeError, ValueError): + return False + return bool(typed_value({"changed": normalized_changed}) == envelope["payload"]) + return _validate_reserved_payload(event, _native_envelope(entry)) is None + + +def _validate_mailbox_semantics(document: Mapping[str, Any], restored: RestoredAggregate) -> None: + for runtime in document["runtimes"]: + for mailbox in ("ready_mailbox", "deferred_mailbox"): + for entry in runtime[mailbox]: + try: + mode = entry["delivery_mode"] + envelope = entry["envelope"] + relation = runtime["relation"]["kind"] + machine_id = runtime["current_definition"]["machine"]["machine_id"] + machine = restored.bundle.machine(machine_id) + declarations = dict(restored.bundle.raw.get("events") or {}) + declarations.update((machine or {}).get("events") or {}) + event = envelope["event"] + declaration = declarations.get(event) + payload = decoded_typed_value(envelope["payload"]) + if not isinstance(payload, Mapping): + raise ValueError + if mode == "input" and relation == "component": + raise ValueError + if event == "env": + valid_event = (mode == "input" and relation != "component") or ( + mode == "internal" and relation == "component" + ) + elif declaration is None: + valid_event = mode == "internal" and event in { + "done", + "determa.component_completed", + "determa.component_failed", + "determa.spawned_instance_failed", + } + else: + expected = "input" if mode == "input" else "internal" + valid_event = declaration["direction"] == expected + correlation = envelope.get("correlation_id") + valid_event = valid_event and bool(declaration.get("correlates_to")) == ( + correlation is not None + ) + valid_event = ( + valid_event + and _valid_envelope_source( + document, + entry, + declaration, + payload, + allow_legacy=True, + ) + and _mailbox_payload_is_normalized( + restored, runtime, entry, declaration, payload + ) + ) + except (ArtifactError, KeyError, TypeError, ValueError) as error: + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) from error + if not valid_event: + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) + + +def restore_aggregate_v2( + source: ArtifactSource, definition_resolver: DefinitionResolver +) -> RestoredAggregate: + """Structurally and semantically restore one queue-bearing aggregate.""" + document, raw = load_json_artifact(source, "aggregate_state_v2") + if aggregate_state_digest(document) != document["aggregate_state_digest"]: + raise ArtifactError(PersistenceFailureCode.AGGREGATE_STATE_DIGEST_MISMATCH) + if document["runtimes"] != sorted( + document["runtimes"], key=lambda item: _utf8(item["runtime_id"]) + ): + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) + _validate_mailboxes(document) + restored = restore_aggregate(_project_v1(document), definition_resolver) + _validate_mailbox_semantics(document, restored) + return RestoredAggregate( + bundle=restored.bundle, + state=restored.state, + aggregate_envelope=copy.deepcopy(document), + canonical_bytes=canonical_bytes(document), + source_bytes=raw, + ) + + +def _wire_target_to_native(target: Mapping[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(dict(target)) + if "component" in result: + result["component"]["activation_sequence"] = decimal( + result["component"]["activation_sequence"] + ) + elif "spawned_instance" in result: + result["spawned_instance"]["machine_version"] = decimal( + result["spawned_instance"]["machine_version"], positive=True + ) + return result + + +def _native_envelope(entry: Mapping[str, Any]) -> dict[str, Any]: + wire = entry["envelope"] + result = { + "event": wire["event"], + "event_id": wire["event_id"], + "target": _wire_target_to_native(wire["target"]), + "payload": decoded_typed_value(wire["payload"]), + } + if "correlation_id" in wire: + result["correlation_id"] = wire["correlation_id"] + return result + + +def _runtime_id_for_target(target: Mapping[str, Any]) -> str: + if "root" in target: + return str(target["root"]["root_runtime_id"]) + if "component" in target: + return str(target["component"]["component_runtime_id"]) + return str(target["spawned_instance"]["instance_id"]) + + +def _admission_rejection(code: str, state: dict[str, Any]) -> dict[str, Any]: + return { + "result": "rejected", + "status": _root_status(state), + "accepted": [], + "state": state, + "rejection": {"code": code}, + } + + +def _root_status(document: Mapping[str, Any]) -> str: + return str( + next( + runtime["status"] + for runtime in document["runtimes"] + if runtime["runtime_id"] == document["root_runtime_id"] + ) + ) + + +def _dispatch_code_to_admission(code: DispatchRejectionCode) -> str: + return { + DispatchRejectionCode.INVALID_EVENT: AdmissionCode.INVALID_EVENT.value, + DispatchRejectionCode.INVALID_PAYLOAD: AdmissionCode.INVALID_PAYLOAD.value, + DispatchRejectionCode.INVALID_CORRELATION: AdmissionCode.INVALID_CORRELATION.value, + DispatchRejectionCode.INVALID_INSTANCE_TARGET: AdmissionCode.INVALID_INSTANCE_TARGET.value, + DispatchRejectionCode.INACTIVE_COMPONENT_TARGET: ( + AdmissionCode.INACTIVE_COMPONENT_TARGET.value + ), + DispatchRejectionCode.INVALID_PRIOR_STATE: AdmissionCode.INVALID_INSTANCE_TARGET.value, + DispatchRejectionCode.INCOMPATIBLE_BUNDLE: AdmissionCode.INVALID_INSTANCE_TARGET.value, + }[code] + + +def admit_aggregate_v2( + source: ArtifactSource, + deliveries: Sequence[Mapping[str, Any]], + definition_resolver: DefinitionResolver, +) -> dict[str, Any]: + """Atomically admit or replay an ordered delivery batch.""" + restored = restore_aggregate_v2(source, definition_resolver) + document = restored.aggregate_envelope + if ( + not isinstance(deliveries, Sequence) + or isinstance(deliveries, str | bytes) + or not deliveries + ): + return _admission_rejection(AdmissionCode.MALFORMED_DELIVERY.value, document) + event_ids: list[str] = [] + target_roots: list[str] = [] + for delivery in deliveries: + if ( + not isinstance(delivery, Mapping) + or set(delivery) != {"delivery_mode", "envelope", "envelope_digest"} + or not isinstance(delivery.get("delivery_mode"), str) + or not isinstance(delivery.get("envelope_digest"), str) + or not _valid_envelope_shape(delivery.get("envelope")) + ): + return _admission_rejection(AdmissionCode.MALFORMED_DELIVERY.value, document) + event_id = delivery["envelope"].get("event_id") + if not isinstance(event_id, str) or not event_id: + return _admission_rejection(AdmissionCode.MALFORMED_DELIVERY.value, document) + try: + target_root = next(iter(delivery["envelope"]["target"].values()))["root_instance_id"] + except (KeyError, StopIteration, TypeError): + return _admission_rejection(AdmissionCode.MALFORMED_DELIVERY.value, document) + event_ids.append(event_id) + target_roots.append(target_root) + if any(target_root != document["root_instance_id"] for target_root in target_roots): + return _admission_rejection(AdmissionCode.WRONG_ROOT.value, document) + if len(event_ids) != len(set(event_ids)): + return _admission_rejection(AdmissionCode.DUPLICATE_EVENT_ID_IN_BATCH.value, document) + + existing: dict[str, tuple[str, dict[str, Any]]] = {} + for runtime in document["runtimes"]: + for mailbox, location in ( + ("ready_mailbox", "ready"), + ("deferred_mailbox", "deferred"), + ): + for entry in runtime[mailbox]: + existing[entry["envelope"]["event_id"]] = (location, entry) + + replayed: list[tuple[str, dict[str, Any]]] = [] + new_deliveries: list[Mapping[str, Any]] = [] + for delivery in deliveries: + mode = delivery.get("delivery_mode") + envelope = delivery["envelope"] + event_id = envelope["event_id"] + prior = existing.get(event_id) + candidate_digest = _entry_digest(document["root_instance_id"], str(mode), envelope) + if prior is not None: + if candidate_digest != prior[1]["envelope_digest"]: + return _admission_rejection(AdmissionCode.EVENT_ID_CONFLICT.value, document) + replayed.append(prior) + else: + new_deliveries.append(delivery) + if replayed and not new_deliveries and len(replayed) == 1: + location, entry = replayed[0] + return { + "result": "replay", + "status": _root_status(document), + "event_id": entry["envelope"]["event_id"], + "acceptance_sequence": entry["acceptance_sequence"], + "location": location, + "state": document, + "rejection": None, + } + + models = BundleModel(restored.bundle) + accepted: list[dict[str, str]] = [] + candidate = copy.deepcopy(document) + runtime_by_id = {runtime["runtime_id"]: runtime for runtime in candidate["runtimes"]} + for delivery in new_deliveries: + mode = delivery.get("delivery_mode") + envelope = delivery["envelope"] + if mode not in {"input", "internal"}: + return _admission_rejection(AdmissionCode.INVALID_DELIVERY_MODE.value, document) + source_value = envelope.get("source") + if ( + (mode == "input" and source_value != {"host": True}) + or (mode == "internal" and not isinstance(source_value, Mapping)) + or (mode == "input" and envelope.get("cause_id") != envelope.get("event_id")) + ): + return _admission_rejection(AdmissionCode.INVALID_DELIVERY_SOURCE.value, document) + if mode == "internal": + assert isinstance(source_value, Mapping) + source_runtime = source_value.get("runtime") + source_system = source_value.get("system") + valid_runtime_source = source_runtime is not None and any( + runtime["target_identity"] == source_runtime for runtime in document["runtimes"] + ) + valid_system_source = source_system in { + "system:component_completion", + "system:spawned_completion", + "system:component_failure", + "system:spawned_failure", + } + if not valid_runtime_source and not valid_system_source: + return _admission_rejection(AdmissionCode.INVALID_DELIVERY_SOURCE.value, document) + supplied_digest = delivery.get("envelope_digest") + expected_digest = _entry_digest(document["root_instance_id"], str(mode), envelope) + if supplied_digest != expected_digest: + return _admission_rejection(AdmissionCode.DELIVERY_DIGEST_MISMATCH.value, document) + try: + native = _native_envelope(delivery) + except (ArtifactError, KeyError, TypeError): + return _admission_rejection(AdmissionCode.INVALID_PAYLOAD.value, document) + try: + runtime_id = _runtime_id_for_target(envelope["target"]) + except (KeyError, TypeError): + return _admission_rejection(AdmissionCode.INVALID_INSTANCE_TARGET.value, document) + target_runtime = runtime_by_id.get(runtime_id) + if target_runtime is None or target_runtime["target_identity"] != envelope["target"]: + return _admission_rejection(AdmissionCode.INVALID_INSTANCE_TARGET.value, document) + rejection = _validate_envelope( + restored.bundle, + models, + restored.state, + native, + cast(Literal["input", "internal"], mode), + ) + if rejection is not None: + return _admission_rejection(_dispatch_code_to_admission(rejection), document) + machine_id = target_runtime["current_definition"]["machine"]["machine_id"] + machine = restored.bundle.machine(machine_id) + declarations = dict(restored.bundle.raw.get("events") or {}) + declarations.update((machine or {}).get("events") or {}) + declaration = declarations.get(envelope["event"]) + payload = decoded_typed_value(envelope["payload"]) + if not isinstance(payload, Mapping) or not _valid_envelope_source( + document, + delivery, + declaration, + payload, + allow_legacy=False, + ): + return _admission_rejection(AdmissionCode.INVALID_DELIVERY_SOURCE.value, document) + if not _mailbox_payload_is_normalized( + restored, target_runtime, delivery, declaration, payload + ): + return _admission_rejection(AdmissionCode.INVALID_PAYLOAD.value, document) + acceptance = candidate["next_acceptance_sequence"] + queue = candidate["next_queue_sequence"] + entry = { + "acceptance_sequence": acceptance, + "queue_sequence": queue, + "delivery_mode": mode, + "envelope": copy.deepcopy(envelope), + "envelope_digest": expected_digest, + "deferral_count": "0", + } + runtime_by_id[runtime_id]["ready_mailbox"].append(entry) + accepted.append( + { + "event_id": envelope["event_id"], + "acceptance_sequence": acceptance, + "queue_sequence": queue, + } + ) + candidate["next_acceptance_sequence"] = str(int(acceptance) + 1) + candidate["next_queue_sequence"] = str(int(queue) + 1) + candidate = seal_aggregate_v2(candidate) + return { + "result": "accepted", + "status": _root_status(candidate), + "accepted": accepted, + "state": candidate, + "rejection": None, + } + + +def _mailbox_maps( + document: Mapping[str, Any], +) -> dict[str, tuple[list[Any], list[Any]]]: + return { + runtime["runtime_id"]: ( + copy.deepcopy(runtime["ready_mailbox"]), + copy.deepcopy(runtime["deferred_mailbox"]), + ) + for runtime in document["runtimes"] + } + + +def _wire_target(target: Mapping[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(dict(target)) + if "component" in result: + result["component"]["activation_sequence"] = str(result["component"]["activation_sequence"]) + elif "spawned_instance" in result: + result["spawned_instance"]["machine_version"] = str( + result["spawned_instance"]["machine_version"] + ) + return result + + +def _wire_source(source: Mapping[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(dict(source)) + runtime = result.get("runtime") + if isinstance(runtime, Mapping): + result["runtime"] = _wire_target(runtime) + return result + + +def _lifecycle_disposition( + entry: Mapping[str, Any], runtime_id: str, reason: str +) -> dict[str, Any]: + return { + "event_id": entry["envelope"]["event_id"], + "request_digest": entry["envelope_digest"], + "acceptance_sequence": entry["acceptance_sequence"], + "final_queue_sequence": entry["queue_sequence"], + "target_runtime_id": runtime_id, + "reason": reason, + } + + +def _enqueue_emissions( + encoded: dict[str, Any], + before: Mapping[str, Any], + selected_runtime: Mapping[str, Any], + native_emissions: Sequence[Mapping[str, Any]], + lifecycle: list[dict[str, Any]], +) -> list[dict[str, Any]]: + projected: list[dict[str, Any]] = [] + runtimes = {runtime["runtime_id"]: runtime for runtime in encoded["runtimes"]} + root_completed = _root_status(encoded) == "completed" + for index, emission in enumerate(native_emissions): + if emission.get("target") == "external": + projected.append( + { + "effect_id": emission["effect_id"], + "sequence": str(emission["sequence"]), + "event": emission["event"], + "payload": typed_value(emission["payload"]), + "correlation_id": emission["correlation_id"], + } + ) + continue + target = cast(Mapping[str, Any], emission["target"]) + target_wire = _wire_target(target) + target_runtime_id = _runtime_id_for_target(target_wire) + acceptance = encoded["next_acceptance_sequence"] + queue = encoded["next_queue_sequence"] + encoded["next_acceptance_sequence"] = str(int(acceptance) + 1) + encoded["next_queue_sequence"] = str(int(queue) + 1) + provenance = emission.get("_determa_v2_provenance") + if not isinstance(provenance, Mapping): + raise ArtifactError(PersistenceFailureCode.INVALID_AGGREGATE_STATE) + envelope = { + "event": emission["event"], + "event_id": emission["event_id"], + "cause_id": provenance["cause_id"], + "source": _wire_source(cast(Mapping[str, Any], provenance["source"])), + "target": target_wire, + "payload": typed_value(emission["payload"]), + } + if emission.get("correlation_id") is not None: + envelope["correlation_id"] = emission["correlation_id"] + entry = { + "acceptance_sequence": acceptance, + "queue_sequence": queue, + "delivery_mode": "internal", + "envelope": envelope, + "envelope_digest": _entry_digest(encoded["root_instance_id"], "internal", envelope), + "deferral_count": "0", + } + target_runtime = runtimes.get(target_runtime_id) + reason: str | None = None + if target_runtime is None: + reason = "runtime_cancelled" + elif root_completed: + reason = "aggregate_completed" + elif target_runtime["status"] == "completed": + reason = "runtime_completed" + if reason is None: + assert target_runtime is not None + target_runtime["ready_mailbox"].append(entry) + projected.append( + { + "kind": "internal_mailbox", + "emission_index": str(index), + "event_id": emission["event_id"], + "acceptance_sequence": acceptance, + "queue_sequence": queue, + } + ) + else: + disposition_index = str(len(lifecycle)) + lifecycle.append(_lifecycle_disposition(entry, target_runtime_id, reason)) + projected.append( + { + "kind": "internal_disposed", + "emission_index": str(index), + "event_id": emission["event_id"], + "acceptance_sequence": acceptance, + "lifecycle_disposition_index": disposition_index, + } + ) + return projected + + +def _encode_after_dispatch( + bundle: Bundle, + state: dict[str, Any], + before: Mapping[str, Any], + mailboxes: Mapping[str, tuple[list[Any], list[Any]]], +) -> dict[str, Any]: + result = _upgrade_document(aggregate_envelope(bundle, state)) + result["next_acceptance_sequence"] = before["next_acceptance_sequence"] + result["next_queue_sequence"] = before["next_queue_sequence"] + for runtime in result["runtimes"]: + ready, deferred = mailboxes.get(runtime["runtime_id"], ([], [])) + runtime["ready_mailbox"] = ready + runtime["deferred_mailbox"] = deferred + return seal_aggregate_v2(result) + + +def _runtime_capacity(bundle: Bundle, state: dict[str, Any], runtime_id: str) -> int | None: + runtime = state["runtimes"][runtime_id] + machine = _runtime_model(bundle, BundleModel(bundle), runtime) + value = machine.root.raw.get("deferred_event_capacity") + return int(value) if value is not None else None + + +def _structurally_deferred( + bundle: Bundle, state: dict[str, Any], runtime_id: str, event: str +) -> bool: + runtime = state["runtimes"][runtime_id] + machine = _runtime_model(bundle, BundleModel(bundle), runtime) + current: StateNode | None = ( + machine.states[runtime["active"][-1]] if runtime["active"] else machine.root + ) + while current is not None: + if event in (current.raw.get("on_events") or {}): + return False + if event in (current.raw.get("deferred_events") or []): + return True + current = current.parent + return False + + +def _runtime_is_runnable(state: Mapping[str, Any], runtime_id: str) -> bool: + runtime = state["runtimes"].get(runtime_id) + while isinstance(runtime, Mapping): + if runtime["status"] != "running": + return False + owner_id = runtime.get("owner_runtime_id") + if owner_id is None: + return True + runtime = state["runtimes"].get(owner_id) + return False + + +def _lifecycle_runtime_order(bundle: Bundle, state: dict[str, Any]) -> list[str]: + execution = _Execution( + bundle, + BundleModel(bundle), + state, + step_sequence=state["next_logical_step_sequence"], + ) + ordered: list[str] = [] + + def visit(runtime: dict[str, Any]) -> None: + for child in execution.ordered_children(runtime): + visit(child) + ordered.append(runtime["runtime_id"]) + + visit(state["runtimes"][state["root_runtime_id"]]) + return ordered + + +def _step_result( + state: dict[str, Any], + disposition: str, + *, + emissions: list[dict[str, Any]] | None = None, + lifecycle: list[dict[str, Any]] | None = None, + fault: dict[str, Any] | None = None, + rejection: str | None = None, +) -> dict[str, Any]: + return { + "core_step_result_format": "determa.core_step_result", + "core_step_result_schema_version": 2, + "status": _root_status(state), + "disposition": disposition, + "state": state, + "emissions": emissions or [], + "lifecycle_dispositions": lifecycle or [], + "fault": fault, + "rejection": None if rejection is None else {"code": rejection}, + } + + +def step_aggregate_v2( + source: ArtifactSource, + target_runtime_id: str, + definition_resolver: DefinitionResolver, +) -> dict[str, Any]: + """Process at most the selected runtime's ready-mailbox head.""" + restored = restore_aggregate_v2(source, definition_resolver) + before = restored.aggregate_envelope + wire_runtime = next( + (runtime for runtime in before["runtimes"] if runtime["runtime_id"] == target_runtime_id), + None, + ) + if wire_runtime is None: + return _step_result( + before, DispositionCode.REJECTED.value, rejection="invalid_instance_target" + ) + if _root_status(before) != "running": + return _step_result( + before, + DispositionCode.REJECTED.value, + rejection="invalid_instance_target", + ) + if wire_runtime["status"] != "running": + code = ( + "inactive_component_target" + if wire_runtime["relation"]["kind"] == "component" + else "invalid_instance_target" + ) + return _step_result(before, DispositionCode.REJECTED.value, rejection=code) + if not wire_runtime["ready_mailbox"]: + return _step_result(before, DispositionCode.NOT_RUNNABLE.value) + + selected = wire_runtime["ready_mailbox"][0] + mailboxes = _mailbox_maps(before) + ready, deferred = mailboxes[target_runtime_id] + native = _native_envelope(selected) + result = dispatch( + restored.bundle, + restored.state, + {selected["delivery_mode"]: native}, + _capture_emission_provenance=True, + ) + disposition = result["disposition"] + if disposition == DispositionCode.REJECTED.value: + return _step_result( + before, + disposition, + fault=copy.deepcopy(result.get("fault")), + rejection=(result.get("rejection") or {}).get("code"), + ) + ready.pop(0) + if disposition == DispositionCode.DEFERRED.value: + capacity = _runtime_capacity(restored.bundle, restored.state, target_runtime_id) + if capacity is not None and len(deferred) >= capacity: + state = copy.deepcopy(restored.state) + runtime = state["runtimes"][target_runtime_id] + execution = _Execution( + restored.bundle, + BundleModel(restored.bundle), + state, + step_sequence=state["next_logical_step_sequence"], + capture_emission_provenance=True, + ) + cause_id = selected["envelope"]["cause_id"] + execution.cause_id = cause_id + execution.finalize_fault( + runtime, + StepFault( + EngineFaultCode.DEFERRED_EVENT_CAPACITY_EXCEEDED, + "system:deferred_event_capacity", + ), + cause_id, + ) + state["next_logical_step_sequence"] += 1 + if runtime["role"] == "root": + state["status"] = "faulted" + state["fault"] = copy.deepcopy(runtime["fault"]) + else: + execution.emit_failure(runtime, cause_id) + encoded = _encode_after_dispatch(restored.bundle, state, before, mailboxes) + overflow_lifecycle: list[dict[str, Any]] = [] + emissions = _enqueue_emissions( + encoded, before, wire_runtime, execution.emissions, overflow_lifecycle + ) + encoded_fault = copy.deepcopy(runtime["fault"]) + encoded_fault["step_sequence"] = str(encoded_fault["step_sequence"]) + encoded_fault["definition_fingerprint"] = restored.bundle.fingerprint + return _step_result( + seal_aggregate_v2(encoded), + DispositionCode.FAULTED.value, + emissions=emissions, + lifecycle=overflow_lifecycle, + fault=encoded_fault, + ) + moved = copy.deepcopy(selected) + moved["queue_sequence"] = before["next_queue_sequence"] + moved["deferral_count"] = str(int(moved["deferral_count"]) + 1) + deferred.append(moved) + candidate = copy.deepcopy(before) + candidate["next_logical_step_sequence"] = str( + int(candidate["next_logical_step_sequence"]) + 1 + ) + candidate["next_queue_sequence"] = str(int(candidate["next_queue_sequence"]) + 1) + for runtime in candidate["runtimes"]: + if runtime["runtime_id"] == target_runtime_id: + runtime["ready_mailbox"] = ready + runtime["deferred_mailbox"] = deferred + return _step_result(seal_aggregate_v2(candidate), disposition) + + state = cast(dict[str, Any], result["state"]) + encoded = _encode_after_dispatch(restored.bundle, state, before, mailboxes) + lifecycle: list[dict[str, Any]] = [] + live = {runtime["runtime_id"]: runtime for runtime in encoded["runtimes"]} + root_completed = state["status"] == "completed" + before_by_id = {runtime["runtime_id"]: runtime for runtime in before["runtimes"]} + for runtime_id in _lifecycle_runtime_order(restored.bundle, restored.state): + runtime = before_by_id[runtime_id] + current = live.get(runtime["runtime_id"]) + if not root_completed and current is not None and current["status"] != "completed": + continue + reason = ( + "aggregate_completed" + if root_completed + else "runtime_completed" + if current is not None + else "runtime_cancelled" + ) + entries = [ + *mailboxes[runtime["runtime_id"]][0], + *mailboxes[runtime["runtime_id"]][1], + ] + for entry in entries: + lifecycle.append(_lifecycle_disposition(entry, runtime["runtime_id"], reason)) + if current is not None: + current["ready_mailbox"] = [] + current["deferred_mailbox"] = [] + emissions = _enqueue_emissions( + encoded, + before, + wire_runtime, + cast(list[Mapping[str, Any]], result["emissions"]), + lifecycle, + ) + if ( + disposition == DispositionCode.HANDLED.value + and target_runtime_id in live + and live[target_runtime_id]["status"] == "running" + ): + runtime = next( + item for item in encoded["runtimes"] if item["runtime_id"] == target_runtime_id + ) + current_ready = runtime["ready_mailbox"] + current_deferred = runtime["deferred_mailbox"] + for deferred_entry in list(current_deferred): + if not _structurally_deferred( + restored.bundle, + state, + target_runtime_id, + deferred_entry["envelope"]["event"], + ): + current_deferred.remove(deferred_entry) + deferred_entry["queue_sequence"] = encoded["next_queue_sequence"] + encoded["next_queue_sequence"] = str(int(encoded["next_queue_sequence"]) + 1) + current_ready.append(deferred_entry) + encoded = seal_aggregate_v2(encoded) + result_fault = result.get("fault") + if isinstance(result_fault, dict): + result_fault = copy.deepcopy(result_fault) + result_fault["step_sequence"] = str(result_fault["step_sequence"]) + result_fault["definition_fingerprint"] = restored.bundle.fingerprint + return _step_result( + encoded, + disposition, + emissions=emissions, + lifecycle=lifecycle, + fault=result_fault, + rejection=(result.get("rejection") or {}).get("code"), + ) + + +def resolver_for_bundle(bundle: Bundle | BundleSource) -> MemoryArtifactResolver: + """Build a trusted resolver for a single normalized bundle.""" + validated = bundle if isinstance(bundle, Bundle) else load_bundle(bundle) + return MemoryArtifactResolver(definitions={validated.fingerprint: validated}) + + +class _V1MigrationResolver: + def __init__( + self, + parent: DefinitionResolver, + descriptors: Mapping[str, Mapping[str, Any]], + ) -> None: + self.parent = parent + self.descriptors = descriptors + + def resolve_definition(self, fingerprint: str) -> Bundle | BundleSource | None: + return self.parent.resolve_definition(fingerprint) + + def definition_is_trusted(self, fingerprint: str) -> bool: + return self.parent.definition_is_trusted(fingerprint) + + def resolve_migration_descriptor(self, digest: str) -> ArtifactSource | None: + return self.descriptors.get(digest) + + def migration_descriptor_is_trusted(self, digest: str) -> bool: + return digest in self.descriptors + + +def _queue_rule( + descriptor: Mapping[str, Any], runtime: Mapping[str, Any], entry: Mapping[str, Any] +) -> Mapping[str, Any] | None: + machine_id = runtime["current_definition"]["machine"]["machine_id"] + for rule in descriptor["queued_event_rules"]: + if ( + rule["machine_id"] == machine_id + and rule["delivery_mode"] == entry["delivery_mode"] + and rule["event"] == entry["envelope"]["event"] + ): + return cast(Mapping[str, Any], rule) + return None + + +def _queue_compatible( + bundle: Bundle, + runtime: Mapping[str, Any], + entry: Mapping[str, Any], +) -> bool: + machine_id = runtime["current_definition"]["machine"]["machine_id"] + machine = bundle.machine(machine_id) + if machine is None: + return False + event = entry["envelope"]["event"] + declarations = dict(bundle.raw.get("events") or {}) + declarations.update(machine.get("events") or {}) + declaration = declarations.get(event) + if declaration is None: + return event in { + "done", + "determa.component_completed", + "determa.component_failed", + "determa.spawned_instance_failed", + "env", + } + expected_direction = "input" if entry["delivery_mode"] == "input" else "internal" + if declaration["direction"] != expected_direction: + return False + envelope = entry["envelope"] + correlation = envelope.get("correlation_id") + if bool(declaration.get("correlates_to")) != (correlation is not None): + return False + try: + payload = decoded_typed_value(envelope["payload"]) + except ArtifactError: + return False + normalized = _normalize_payload(declaration, payload) + return normalized is not None and typed_value(normalized) == envelope["payload"] + + +def migrate_aggregate_v2( + aggregate: ArtifactSource, + target_validated_bundle_fingerprint: str, + migration_route: Sequence[str], + artifact_resolver: Any, + *, + maintenance_mode: bool, + resource_limits: MigrationLimits | None = None, + _include_host_evidence: bool = False, +) -> dict[str, Any]: + """Migrate one queue-bearing aggregate through exact version-2 descriptors.""" + restored = restore_aggregate_v2(aggregate, artifact_resolver) + limits = resource_limits or MigrationLimits() + if ( + not isinstance(migration_route, Sequence) + or isinstance(migration_route, str | bytes) + or not all(isinstance(item, str) and item for item in migration_route) + or not isinstance(target_validated_bundle_fingerprint, str) + or not target_validated_bundle_fingerprint + or not isinstance(maintenance_mode, bool) + ): + raise ArtifactError(PersistenceFailureCode.INVALID_MIGRATION_REQUEST) + if len(migration_route) > limits.maximum_chain_length: + raise ArtifactError(PersistenceFailureCode.MIGRATION_RESOURCE_LIMIT_EXCEEDED) + descriptors: list[dict[str, Any]] = [] + base_descriptors: dict[str, Mapping[str, Any]] = {} + for digest in migration_route: + source = artifact_resolver.resolve_migration_descriptor(digest) + if source is None or not artifact_resolver.migration_descriptor_is_trusted(digest): + raise ArtifactError(PersistenceFailureCode.MIGRATION_DESCRIPTOR_UNTRUSTED) + descriptor, _ = load_json_artifact(source, "migration_descriptor_v2") + if migration_descriptor_digest(descriptor) != digest: + raise ArtifactError(PersistenceFailureCode.INVALID_MIGRATION_DESCRIPTOR) + base = descriptor["base_descriptor"] + base_digest = base["migration_descriptor_digest"] + base_descriptors[base_digest] = base + descriptors.append(descriptor) + if not descriptors: + if ( + restored.aggregate_envelope["validated_bundle_fingerprint"] + != target_validated_bundle_fingerprint + ): + raise ArtifactError(PersistenceFailureCode.MIGRATION_ROUTE_MISSING) + empty_result = { + "result": "success", + "aggregate_state": copy.deepcopy(restored.aggregate_envelope), + "dispositions": [], + "audit_records": [], + } + if _include_host_evidence: + empty_result["_disposed_entries"] = [] + return empty_result + if len({item["migration_descriptor_digest"] for item in descriptors}) != len(descriptors): + raise ArtifactError(PersistenceFailureCode.MIGRATION_ROUTE_MISMATCH) + fingerprints = [descriptors[0]["base_descriptor"]["source_validated_bundle_fingerprint"]] + [ + descriptor["base_descriptor"]["target_validated_bundle_fingerprint"] + for descriptor in descriptors + ] + if ( + fingerprints[0] != restored.aggregate_envelope["validated_bundle_fingerprint"] + or fingerprints[-1] != target_validated_bundle_fingerprint + or len(set(fingerprints)) != len(fingerprints) + or any( + left["base_descriptor"]["target_validated_bundle_fingerprint"] + != right["base_descriptor"]["source_validated_bundle_fingerprint"] + for left, right in zip(descriptors, descriptors[1:], strict=False) + ) + ): + raise ArtifactError(PersistenceFailureCode.MIGRATION_ROUTE_MISMATCH) + + resolver = cast(Any, _V1MigrationResolver(artifact_resolver, base_descriptors)) + candidate = copy.deepcopy(restored.aggregate_envelope) + dispositions: list[dict[str, Any]] = [] + disposed_entries: list[dict[str, Any]] = [] + audits: list[dict[str, Any]] = [] + for descriptor in descriptors: + before_digest = candidate["aggregate_state_digest"] + source_runtimes = candidate["runtimes"] + base = descriptor["base_descriptor"] + migration = migrate_aggregate( + _project_v1(candidate), + base["target_validated_bundle_fingerprint"], + [base["migration_descriptor_digest"]], + resolver, + maintenance_mode=maintenance_mode, + resource_limits=limits, + ) + if not migration.succeeded or migration.aggregate_envelope is None: + assert migration.failure is not None + raise ArtifactError(migration.failure.code) + hop = _upgrade_document(migration.aggregate_envelope) + hop["next_acceptance_sequence"] = candidate["next_acceptance_sequence"] + hop["next_queue_sequence"] = candidate["next_queue_sequence"] + target_runtimes = {runtime["runtime_id"]: runtime for runtime in hop["runtimes"]} + target_bundle_source = artifact_resolver.resolve_definition( + base["target_validated_bundle_fingerprint"] + ) + if target_bundle_source is None: + raise ArtifactError(PersistenceFailureCode.TARGET_DEFINITION_UNAVAILABLE) + target_bundle = ( + target_bundle_source + if isinstance(target_bundle_source, Bundle) + else load_bundle(target_bundle_source) + ) + target_state = restore_aggregate(_project_v1(hop), artifact_resolver).state + for source_runtime in source_runtimes: + runtime = target_runtimes.get(source_runtime["runtime_id"]) + for mailbox_name in ("ready_mailbox", "deferred_mailbox"): + output = runtime[mailbox_name] if runtime is not None else None + for entry in source_runtime[mailbox_name]: + rule = _queue_rule(descriptor, source_runtime, entry) + if rule is not None and rule["action"] == "dispose": + dispositions.append( + { + "disposition": "migration_disposed", + "reason": rule["reason"], + "migration_descriptor_digest": descriptor[ + "migration_descriptor_digest" + ], + } + ) + disposed_entries.append(copy.deepcopy(entry)) + continue + if runtime is None or not _queue_compatible(target_bundle, runtime, entry): + raise ArtifactError(PersistenceFailureCode.MIGRATION_TOTALITY_FAILURE) + assert output is not None + output.append(copy.deepcopy(entry)) + for runtime in hop["runtimes"]: + ready = runtime["ready_mailbox"] + deferred = runtime["deferred_mailbox"] + if _runtime_is_runnable(target_state, runtime["runtime_id"]): + for entry in list(deferred): + if not _structurally_deferred( + target_bundle, + target_state, + runtime["runtime_id"], + entry["envelope"]["event"], + ): + deferred.remove(entry) + entry["queue_sequence"] = hop["next_queue_sequence"] + hop["next_queue_sequence"] = str(int(hop["next_queue_sequence"]) + 1) + ready.append(entry) + capacity = _runtime_capacity(target_bundle, target_state, runtime["runtime_id"]) + if capacity is not None and len(deferred) > capacity: + raise ArtifactError(PersistenceFailureCode.MIGRATION_TOTALITY_FAILURE) + candidate = seal_aggregate_v2(hop) + restore_aggregate_v2(candidate, artifact_resolver) + audits.append( + { + "migration_audit_record_schema_version": 1, + "root_instance_id": candidate["root_instance_id"], + "root_runtime_id": candidate["root_runtime_id"], + "migration_sequence": candidate["migration_sequence"], + "source_validated_bundle_fingerprint": base["source_validated_bundle_fingerprint"], + "target_validated_bundle_fingerprint": base["target_validated_bundle_fingerprint"], + "migration_descriptor_digest": descriptor["migration_descriptor_digest"], + "source_aggregate_state_digest": before_digest, + "target_aggregate_state_digest": candidate["aggregate_state_digest"], + "result_code": "migration_applied", + } + ) + result: dict[str, Any] = { + "result": "success", + "aggregate_state": candidate, + "dispositions": dispositions, + "audit_records": audits, + } + if _include_host_evidence: + result["_disposed_entries"] = disposed_entries + return result diff --git a/src/determa/state/validator.py b/src/determa/state/validator.py index 6f43c8d..54aeaf8 100644 --- a/src/determa/state/validator.py +++ b/src/determa/state/validator.py @@ -265,6 +265,16 @@ def _validate_state_structure( graph: dict[str, set[str]], ) -> None: scope, scope_declarations = _scope(machine, state) + if "deferred_event_capacity" in state.raw and state is not machine.root: + raise ValidationError(LoadCode.SEMANTIC_VALIDATION) + for event_name in state.raw.get("deferred_events") or []: + declaration = events.get(event_name) + if ( + declaration is None + or declaration["direction"] == "output" + or event_name in _RESERVED_EVENTS + ): + raise ValidationError(LoadCode.SEMANTIC_VALIDATION) if state is machine.root: for declaration in (bundle.raw.get("events") or {}).values(): _validate_payload_literals(declaration) diff --git a/src/determa/state/wire.py b/src/determa/state/wire.py index 2baa2bd..410009e 100644 --- a/src/determa/state/wire.py +++ b/src/determa/state/wire.py @@ -112,7 +112,11 @@ def migration_descriptor_is_trusted(self, digest: str) -> bool: return digest in self._trusted_migration_descriptors def put_definition( - self, fingerprint: str, definition: Bundle | BundleSource, *, trusted: bool = True + self, + fingerprint: str, + definition: Bundle | BundleSource, + *, + trusted: bool = True, ) -> None: bundle = definition if isinstance(definition, Bundle) else load_bundle(definition) if bundle.fingerprint != fingerprint: @@ -132,12 +136,19 @@ def put_definition( def put_migration_descriptor( self, digest: str, descriptor: ArtifactSource, *, trusted: bool = True ) -> None: - document, _ = load_json_artifact(descriptor, "migration_descriptor") + candidate, _ = strict_json(descriptor) + kind = ( + "migration_descriptor_v2" + if isinstance(candidate, dict) + and candidate.get("migration_descriptor_schema_version") == 2 + else "migration_descriptor" + ) + document, _ = load_json_artifact(candidate, kind) if migration_descriptor_digest(document) != digest: raise ArtifactError(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) existing = self._migration_descriptors.get(digest) if existing is not None: - current, _ = load_json_artifact(existing, "migration_descriptor") + current, _ = load_json_artifact(existing, kind) if canonical_bytes(current) != canonical_bytes(document): raise ArtifactError(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) else: @@ -328,13 +339,16 @@ def decimal(value: Any, *, positive: bool = False) -> int: def artifact_schema(kind: str) -> dict[str, Any]: filename = { "aggregate_state": "aggregate-state.schema.json", + "aggregate_state_v2": "aggregate-state-v2.schema.json", "migration_descriptor": "migration-descriptor.schema.json", + "migration_descriptor_v2": "migration-descriptor-v2.schema.json", "aggregate_state_package": "aggregate-state-package.schema.json", + "aggregate_state_package_v2": "aggregate-state-package-v2.schema.json", "execution_checkpoint": "execution-checkpoint.schema.json", + "execution_checkpoint_v2": "execution-checkpoint-v2.schema.json", + "core_step_result_v2": "core-step-result-v2.schema.json", }[kind] - return cast( - dict[str, Any], json.loads((_DATA / filename).read_text(encoding="utf-8")) - ) + return cast(dict[str, Any], json.loads((_DATA / filename).read_text(encoding="utf-8"))) @lru_cache(maxsize=1) @@ -344,14 +358,17 @@ def _schema_registry() -> Any: registry = Registry() for kind in ( "aggregate_state", + "aggregate_state_v2", "migration_descriptor", + "migration_descriptor_v2", "aggregate_state_package", + "aggregate_state_package_v2", "execution_checkpoint", + "execution_checkpoint_v2", + "core_step_result_v2", ): document = artifact_schema(kind) - registry = registry.with_resource( - document["$id"], Resource.from_contents(document) - ) + registry = registry.with_resource(document["$id"], Resource.from_contents(document)) return registry @@ -367,6 +384,14 @@ def _format_code(document: Any, kind: str) -> str | None: PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_FORMAT, PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_SCHEMA_VERSION, ), + "aggregate_state_v2": ( + "aggregate_state_format", + "determa.aggregate_state", + "aggregate_state_schema_version", + 2, + PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_FORMAT, + PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_SCHEMA_VERSION, + ), "migration_descriptor": ( "migration_descriptor_format", "determa.aggregate_migration", @@ -375,6 +400,14 @@ def _format_code(document: Any, kind: str) -> str | None: PersistenceCode.UNSUPPORTED_MIGRATION_DESCRIPTOR_FORMAT, PersistenceCode.UNSUPPORTED_MIGRATION_DESCRIPTOR_SCHEMA_VERSION, ), + "migration_descriptor_v2": ( + "migration_descriptor_format", + "determa.aggregate_migration", + "migration_descriptor_schema_version", + 2, + PersistenceCode.UNSUPPORTED_MIGRATION_DESCRIPTOR_FORMAT, + PersistenceCode.UNSUPPORTED_MIGRATION_DESCRIPTOR_SCHEMA_VERSION, + ), "aggregate_state_package": ( "aggregate_state_package_format", "determa.aggregate_state_package", @@ -383,6 +416,14 @@ def _format_code(document: Any, kind: str) -> str | None: PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_PACKAGE_FORMAT, PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_PACKAGE_SCHEMA_VERSION, ), + "aggregate_state_package_v2": ( + "aggregate_state_package_format", + "determa.aggregate_state_package", + "aggregate_state_package_schema_version", + 2, + PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_PACKAGE_FORMAT, + PersistenceCode.UNSUPPORTED_AGGREGATE_STATE_PACKAGE_SCHEMA_VERSION, + ), "execution_checkpoint": ( "execution_checkpoint_format", "determa.execution_checkpoint", @@ -391,10 +432,31 @@ def _format_code(document: Any, kind: str) -> str | None: CheckpointCode.UNSUPPORTED_EXECUTION_CHECKPOINT_FORMAT, CheckpointCode.UNSUPPORTED_EXECUTION_CHECKPOINT_SCHEMA_VERSION, ), + "execution_checkpoint_v2": ( + "execution_checkpoint_format", + "determa.execution_checkpoint", + "execution_checkpoint_schema_version", + 2, + CheckpointCode.UNSUPPORTED_EXECUTION_CHECKPOINT_FORMAT, + CheckpointCode.UNSUPPORTED_EXECUTION_CHECKPOINT_SCHEMA_VERSION, + ), + "core_step_result_v2": ( + "core_step_result_format", + "determa.core_step_result", + "core_step_result_schema_version", + 2, + "invalid_core_step_result", + "invalid_core_step_result", + ), } - format_member, expected_format, version_member, expected_version, format_code, version_code = ( - definitions[kind] - ) + ( + format_member, + expected_format, + version_member, + expected_version, + format_code, + version_code, + ) = definitions[kind] if format_member not in document or document[format_member] != expected_format: return format_code if version_member in document and document[version_member] != expected_version: @@ -402,18 +464,21 @@ def _format_code(document: Any, kind: str) -> str | None: return None -def load_json_artifact( - source: ArtifactSource, kind: str -) -> tuple[dict[str, Any], bytes]: +def load_json_artifact(source: ArtifactSource, kind: str) -> tuple[dict[str, Any], bytes]: """Parse and structurally validate one recognized persistence artifact.""" try: document, raw = strict_json(source) except ArtifactError as exc: code = { "aggregate_state": PersistenceCode.INVALID_AGGREGATE_STATE, + "aggregate_state_v2": PersistenceCode.INVALID_AGGREGATE_STATE, "migration_descriptor": PersistenceCode.INVALID_MIGRATION_DESCRIPTOR, + "migration_descriptor_v2": PersistenceCode.INVALID_MIGRATION_DESCRIPTOR, "aggregate_state_package": PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE, + "aggregate_state_package_v2": PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE, "execution_checkpoint": CheckpointCode.INVALID_EXECUTION_CHECKPOINT, + "execution_checkpoint_v2": CheckpointCode.INVALID_EXECUTION_CHECKPOINT, + "core_step_result_v2": "invalid_core_step_result", }[kind] raise ArtifactError(code) from exc unsupported = _format_code(document, kind) @@ -421,30 +486,39 @@ def load_json_artifact( raise ArtifactError(unsupported) import jsonschema - validator = jsonschema.Draft202012Validator( - artifact_schema(kind), registry=_schema_registry() - ) + validator = jsonschema.Draft202012Validator(artifact_schema(kind), registry=_schema_registry()) if not isinstance(document, dict) or next(validator.iter_errors(document), None) is not None: code = { "aggregate_state": PersistenceCode.INVALID_AGGREGATE_STATE, + "aggregate_state_v2": PersistenceCode.INVALID_AGGREGATE_STATE, "migration_descriptor": PersistenceCode.INVALID_MIGRATION_DESCRIPTOR, + "migration_descriptor_v2": PersistenceCode.INVALID_MIGRATION_DESCRIPTOR, "aggregate_state_package": PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE, + "aggregate_state_package_v2": PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE, "execution_checkpoint": CheckpointCode.INVALID_EXECUTION_CHECKPOINT, + "execution_checkpoint_v2": CheckpointCode.INVALID_EXECUTION_CHECKPOINT, + "core_step_result_v2": "invalid_core_step_result", }[kind] raise ArtifactError(code) return document, raw def aggregate_state_digest(document: Mapping[str, Any]) -> str: - body = dict(document) + body = copy.deepcopy(dict(document)) body.pop("aggregate_state_digest", None) - return hash_value(["determa-aggregate-state-digest-1", body]) + version = body.get("aggregate_state_schema_version") + domain = ( + "determa-aggregate-state-digest-2" if version == 2 else "determa-aggregate-state-digest-1" + ) + return hash_value([domain, body]) def migration_descriptor_digest(document: Mapping[str, Any]) -> str: - body = dict(document) + body = copy.deepcopy(dict(document)) body.pop("migration_descriptor_digest", None) - return hash_value(["determa-migration-descriptor-1", body]) + version = body.get("migration_descriptor_schema_version") + domain = "determa-migration-descriptor-2" if version == 2 else "determa-migration-descriptor-1" + return hash_value([domain, body]) def normalized_definition_attachment(bundle: Bundle) -> dict[str, Any]: @@ -508,9 +582,7 @@ def _definition_binding(bundle: Bundle, runtime: Mapping[str, Any]) -> dict[str, def _wire_target(target: Mapping[str, Any]) -> dict[str, Any]: result = copy.deepcopy(dict(target)) if "component" in result: - result["component"]["activation_sequence"] = str( - result["component"]["activation_sequence"] - ) + result["component"]["activation_sequence"] = str(result["component"]["activation_sequence"]) elif "spawned_instance" in result: result["spawned_instance"]["machine_version"] = str( result["spawned_instance"]["machine_version"] @@ -518,9 +590,7 @@ def _wire_target(target: Mapping[str, Any]) -> dict[str, Any]: return result -def _runtime_identity_origin( - bundle: Bundle, runtime: Mapping[str, Any] -) -> dict[str, Any]: +def _runtime_identity_origin(bundle: Bundle, runtime: Mapping[str, Any]) -> dict[str, Any]: stored = runtime.get("_identity_origin") if isinstance(stored, dict): return copy.deepcopy(stored) @@ -561,9 +631,7 @@ def _runtime_relation(runtime: Mapping[str, Any]) -> dict[str, Any]: "kind": "component", "owner_runtime_id": runtime["owner_runtime_id"], "component_id": runtime["component_id"], - "current_component_definition_pointer": runtime[ - "component_definition_pointer" - ], + "current_component_definition_pointer": runtime["component_definition_pointer"], "activation_sequence": str(runtime["component_activation_sequence"]), "declaration_index": str(runtime["component_declaration_index"]), } @@ -574,9 +642,7 @@ def _runtime_relation(runtime: Mapping[str, Any]) -> dict[str, Any]: else { "holder_runtime_id": runtime["owner_runtime_id"], "variable_declaration_pointer": holder["pointer"], - "holder_state_activation_sequence": str( - holder["state_activation_sequence"] - ), + "holder_state_activation_sequence": str(holder["state_activation_sequence"]), } ) return { @@ -605,11 +671,7 @@ def _runtime_wire( machine = _runtime_model(bundle, models, cast(dict[str, Any], runtime)) active_nodes = [_node_for_runtime(machine, path) for path in runtime["active"]] - leaves = ( - [] - if not active_nodes - else [active_nodes[-1].pointer] - ) + leaves = [] if not active_nodes else [active_nodes[-1].pointer] activations = sorted( ( { @@ -734,8 +796,7 @@ def aggregate_envelope(bundle: Bundle | BundleSource, state: dict[str, Any]) -> restored_order = state.get("_wire_runtime_order") order_rank = ( {runtime_id: index for index, runtime_id in enumerate(restored_order)} - if isinstance(restored_order, list) - and set(restored_order) == set(state["runtimes"]) + if isinstance(restored_order, list) and set(restored_order) == set(state["runtimes"]) else None ) document: dict[str, Any] = { @@ -768,9 +829,7 @@ def aggregate_envelope(bundle: Bundle | BundleSource, state: dict[str, Any]) -> return document -def serialize_aggregate( - bundle: Bundle | BundleSource, state: dict[str, Any] -) -> bytes: +def serialize_aggregate(bundle: Bundle | BundleSource, state: dict[str, Any]) -> bytes: """Serialize one engine aggregate to exact canonical portable bytes.""" return canonical_bytes(aggregate_envelope(bundle, state)) @@ -782,9 +841,7 @@ def _state_path_for_pointer(machine: MachineModel, pointer: str) -> str: raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) -def _variable_for_pointer( - machine: MachineModel, pointer: str -) -> tuple[str, str, dict[str, Any]]: +def _variable_for_pointer(machine: MachineModel, pointer: str) -> tuple[str, str, dict[str, Any]]: for path, node in machine.states.items(): prefix = f"{node.pointer}/variables/" if pointer.startswith(prefix): @@ -849,26 +906,26 @@ def _validate_immutable_identity( target: Mapping[str, Any], ) -> None: origin = document["identity_origin"] - if not isinstance(origin, Mapping) or origin.get("kind") != { - "root": "root", - "component": "component", - "spawned": "owned_spawned_instance", - }[role]: + if ( + not isinstance(origin, Mapping) + or origin.get("kind") + != { + "root": "root", + "component": "component", + "spawned": "owned_spawned_instance", + }[role] + ): raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) origin_bundle, origin_machine = _origin_machine(resolver, origin) root_instance_id = aggregate["root_instance_id"] runtime_id = document["runtime_id"] if role == "root": - if ( - origin.get("root_instance_id") != root_instance_id - or target - != { - "root": { - "root_instance_id": root_instance_id, - "root_runtime_id": runtime_id, - } + if origin.get("root_instance_id") != root_instance_id or target != { + "root": { + "root_instance_id": root_instance_id, + "root_runtime_id": runtime_id, } - ): + }: raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) return if role == "component": @@ -900,16 +957,12 @@ def _validate_immutable_identity( raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) return spawned = target.get("spawned_instance") - if ( - not isinstance(spawned, Mapping) - or spawned - != { - "root_instance_id": root_instance_id, - "instance_id": runtime_id, - "machine_id": origin_machine.machine_id, - "machine_version": origin_machine.version, - } - ): + if not isinstance(spawned, Mapping) or spawned != { + "root_instance_id": root_instance_id, + "instance_id": runtime_id, + "machine_id": origin_machine.machine_id, + "machine_version": origin_machine.version, + }: raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) @@ -1024,9 +1077,7 @@ def _runtime_from_wire( { "component_id": relation["component_id"], "component_runtime_id": document["runtime_id"], - "component_definition_pointer": relation[ - "current_component_definition_pointer" - ], + "component_definition_pointer": relation["current_component_definition_pointer"], "component_declaration_index": decimal(relation["declaration_index"]), "component_activation_sequence": decimal(relation["activation_sequence"]), "owning_state_path": "", @@ -1066,9 +1117,7 @@ def _runtime_from_wire( return runtime -def _finish_relationships( - bundle: Bundle, state: dict[str, Any], models: BundleModel -) -> None: +def _finish_relationships(bundle: Bundle, state: dict[str, Any], models: BundleModel) -> None: from .engine import _runtime_model for runtime in state["runtimes"].values(): @@ -1090,9 +1139,9 @@ def _finish_relationships( if owning is None or owning.path not in owner["state_activation_sequence"]: raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) runtime["owning_state_path"] = owning.path - runtime["owning_state_activation_sequence"] = owner[ - "state_activation_sequence" - ][owning.path] + runtime["owning_state_activation_sequence"] = owner["state_activation_sequence"][ + owning.path + ] if runtime["component_id"] in owner["components"]: raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) owner["components"][runtime["component_id"]] = runtime["runtime_id"] @@ -1102,9 +1151,7 @@ def _finish_relationships( raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) owner_machine = _runtime_model(bundle, models, owner) holder_pointer = runtime["holder"]["pointer"] - path, _name, _declaration = _variable_for_pointer( - owner_machine, holder_pointer - ) + path, _name, _declaration = _variable_for_pointer(owner_machine, holder_pointer) if path not in owner["state_activation_sequence"]: raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) runtime["holder"]["state_path"] = path @@ -1154,10 +1201,8 @@ def restore_aggregate( raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) if ( document["root_machine_id"] != root["machine_id"] - or decimal(document["root_machine_version"], positive=True) - != root["machine_version"] - or root["_current_definition"]["machine"]["root_definition_pointer"] - != root["root_pointer"] + or decimal(document["root_machine_version"], positive=True) != root["machine_version"] + or root["_current_definition"]["machine"]["root_definition_pointer"] != root["root_pointer"] ): raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE) state["status"] = root["status"] @@ -1180,7 +1225,16 @@ def restore_aggregate_package( source: ArtifactSource, artifact_resolver: ArtifactResolver ) -> RestoredAggregatePackage: """Verify a transport package and seed one mutable resolver atomically.""" - document, _raw = load_json_artifact(source, "aggregate_state_package") + candidate, _raw = strict_json(source) + aggregate_version = ( + candidate.get("aggregate_state", {}).get("aggregate_state_schema_version") + if isinstance(candidate, dict) and isinstance(candidate.get("aggregate_state"), dict) + else None + ) + version = 2 if aggregate_version == 2 else 1 + package_kind = "aggregate_state_package_v2" if version == 2 else "aggregate_state_package" + descriptor_kind = "migration_descriptor_v2" if version == 2 else "migration_descriptor" + document, _raw = load_json_artifact(candidate, package_kind) definitions: dict[str, Bundle] = {} descriptors: dict[str, dict[str, Any]] = {} try: @@ -1205,18 +1259,14 @@ def restore_aggregate_package( if isinstance(existing_definition, Bundle) else load_bundle(existing_definition) ) - if ( - current_bundle.fingerprint != fingerprint - or canonical_bytes(typed_value(current_bundle.raw)) - != canonical_bytes(typed_value(bundle.raw)) - ): + if current_bundle.fingerprint != fingerprint or canonical_bytes( + typed_value(current_bundle.raw) + ) != canonical_bytes(typed_value(bundle.raw)): raise ArtifactError(PersistenceCode.DEFINITION_FINGERPRINT_MISMATCH) for digest, descriptor in descriptors.items(): existing_descriptor = artifact_resolver.resolve_migration_descriptor(digest) if existing_descriptor is not None: - current_descriptor, _ = load_json_artifact( - existing_descriptor, "migration_descriptor" - ) + current_descriptor, _ = load_json_artifact(existing_descriptor, descriptor_kind) if canonical_bytes(current_descriptor) != canonical_bytes(descriptor): raise ArtifactError(PersistenceCode.INVALID_MIGRATION_DESCRIPTOR) except (ArtifactError, KeyError, TypeError, ValidationError) as exc: @@ -1238,11 +1288,14 @@ def restore_aggregate_package( raise ArtifactError(PersistenceCode.INVALID_AGGREGATE_STATE_PACKAGE) overlay = _PackageResolver(artifact_resolver, definitions, descriptors) try: - aggregate = restore_aggregate(document["aggregate_state"], overlay) + if version == 2: + from .queueing import restore_aggregate_v2 + + aggregate = restore_aggregate_v2(document["aggregate_state"], overlay) + else: + aggregate = restore_aggregate(document["aggregate_state"], overlay) store_definition = cast(Callable[[str, Bundle], None], put_definition) - store_descriptor = cast( - Callable[[str, Mapping[str, Any]], None], put_descriptor - ) + store_descriptor = cast(Callable[[str, Mapping[str, Any]], None], put_descriptor) for fingerprint, bundle in definitions.items(): store_definition(fingerprint, bundle) for digest, descriptor in descriptors.items(): @@ -1268,33 +1321,23 @@ def __init__( self.descriptors = descriptors def resolve_definition(self, fingerprint: str) -> Bundle | BundleSource | None: - return self.definitions.get(fingerprint) or self.parent.resolve_definition( - fingerprint - ) + return self.definitions.get(fingerprint) or self.parent.resolve_definition(fingerprint) def definition_is_trusted(self, fingerprint: str) -> bool: - return fingerprint in self.definitions or self.parent.definition_is_trusted( - fingerprint - ) + return fingerprint in self.definitions or self.parent.definition_is_trusted(fingerprint) def resolve_migration_descriptor(self, digest: str) -> ArtifactSource | None: - return self.descriptors.get(digest) or self.parent.resolve_migration_descriptor( - digest - ) + return self.descriptors.get(digest) or self.parent.resolve_migration_descriptor(digest) def migration_descriptor_is_trusted(self, digest: str) -> bool: - return digest in self.descriptors or self.parent.migration_descriptor_is_trusted( - digest - ) + return digest in self.descriptors or self.parent.migration_descriptor_is_trusted(digest) def aggregate_shape_fingerprint(bundle: Bundle | BundleSource) -> str: """Compute the exact state-bearing definition fingerprint from SPEC §16.6.""" validated = bundle if isinstance(bundle, Bundle) else load_bundle(bundle) - def variable_projection( - declaration: Mapping[str, Any], pointer: str - ) -> dict[str, Any]: + def variable_projection(declaration: Mapping[str, Any], pointer: str) -> dict[str, Any]: result: dict[str, Any] = { "declaration_pointer": pointer, "type": declaration["type"], @@ -1323,9 +1366,7 @@ def action_spawn_sites( spawn = action["spawn"] holder_pointer = None if "bind_to" in spawn: - holder_pointer = _resolve_variable_pointer( - machine, state, spawn["bind_to"] - ) + holder_pointer = _resolve_variable_pointer(machine, state, spawn["bind_to"]) sites.append( { "action_pointer": f"{pointer}/{index}/spawn", @@ -1335,9 +1376,7 @@ def action_spawn_sites( ) return sites - def state_projection( - machine: MachineModel, state: StateNode - ) -> dict[str, Any]: + def state_projection(machine: MachineModel, state: StateNode) -> dict[str, Any]: result: dict[str, Any] = { "definition_pointer": state.pointer, "type": state.type, @@ -1345,9 +1384,7 @@ def state_projection( if state.type == "composite": result["history"] = state.raw.get("history", "none") variables = [ - variable_projection( - declaration, f"{state.pointer}/variables/{_escape_pointer(name)}" - ) + variable_projection(declaration, f"{state.pointer}/variables/{_escape_pointer(name)}") for name, declaration in (state.raw.get("variables") or {}).items() ] variables.sort(key=lambda item: item["declaration_pointer"].encode("utf-8")) @@ -1378,30 +1415,19 @@ def state_projection( components.append(item) if components: result["components"] = components - sites = action_spawn_sites( - machine, state, state.raw.get("entry"), f"{state.pointer}/entry" - ) - sites += action_spawn_sites( - machine, state, state.raw.get("exit"), f"{state.pointer}/exit" - ) + sites = action_spawn_sites(machine, state, state.raw.get("entry"), f"{state.pointer}/entry") + sites += action_spawn_sites(machine, state, state.raw.get("exit"), f"{state.pointer}/exit") for event_name, transition_or_list in (state.raw.get("on_events") or {}).items(): transitions = ( - transition_or_list - if isinstance(transition_or_list, list) - else [transition_or_list] + transition_or_list if isinstance(transition_or_list, list) else [transition_or_list] ) for transition_index, transition in enumerate(transitions): - suffix = ( - f"/{transition_index}" if isinstance(transition_or_list, list) else "" - ) + suffix = f"/{transition_index}" if isinstance(transition_or_list, list) else "" sites += action_spawn_sites( machine, state, transition.get("action"), - ( - f"{state.pointer}/on_events/{_escape_pointer(event_name)}" - f"{suffix}/action" - ), + (f"{state.pointer}/on_events/{_escape_pointer(event_name)}{suffix}/action"), ) sites.sort(key=lambda item: item["action_pointer"].encode("utf-8")) if sites: @@ -1422,14 +1448,10 @@ def state_projection( "namespace": validated.namespace, "machines": machine_values, } - return hash_value( - ["determa-aggregate-shape-fingerprint-1", typed_value(tree)] - ) + return hash_value(["determa-aggregate-shape-fingerprint-1", typed_value(tree)]) -def _resolve_variable_pointer( - machine: MachineModel, state: StateNode, name: str -) -> str: +def _resolve_variable_pointer(machine: MachineModel, state: StateNode, name: str) -> str: current: StateNode | None = state while current is not None: if name in (current.raw.get("variables") or {}): diff --git a/tests/test_version2_review_findings.py b/tests/test_version2_review_findings.py new file mode 100644 index 0000000..e22d172 --- /dev/null +++ b/tests/test_version2_review_findings.py @@ -0,0 +1,1766 @@ +from __future__ import annotations + +import copy +from typing import Any + +import pytest + +from determa.state import ( + ArtifactError, + ExecutionHost, + MemoryArtifactResolver, + MemoryExecutionStore, + admit_aggregate_v2, + admit_checkpoint_v2, + aggregate_shape_fingerprint, + create_aggregate_v2, + create_checkpoint_v2, + downgrade_aggregate_v2_to_v1, + load_bundle, + migrate_aggregate_v2, + restore_aggregate_package, + restore_aggregate_v2, + restore_execution_checkpoint_v2, + seal_aggregate_v2, + seal_execution_checkpoint, + serialize_execution_checkpoint, + step_aggregate_v2, + step_checkpoint_v2, + upgrade_checkpoint_v1_to_v2, +) +from determa.state.queueing import _entry_digest +from determa.state.wire import ( + aggregate_state_digest, + load_json_artifact, + migration_descriptor_digest, + typed_value, +) + + +def _bundle(*, deferred: bool = False, external: bool = False) -> Any: + events: dict[str, Any] = { + "go": {"direction": "input"}, + "hold": {"direction": "input"}, + "tail": {"direction": "input"}, + "work": {"direction": "internal"}, + } + action: list[dict[str, Any]] = [] + if external: + events["outside"] = {"direction": "output"} + action = [ + { + "send": { + "event": "outside", + "to": {"external": True}, + "correlation_id": "'effect'", + } + } + ] + root: dict[str, Any] = { + "type": "simple", + "on_events": {"go": {"action": action}, "tail": {}, "work": {}}, + } + if deferred: + root["deferred_events"] = ["hold"] + return load_bundle( + { + "format": 1, + "namespace": f"tests.review82.{deferred}.{external}", + "events": events, + "machines": [{"machine_id": "machine", "root": root}], + } + ) + + +def _resolver(bundle: Any) -> MemoryArtifactResolver: + return MemoryArtifactResolver(definitions={bundle.fingerprint: bundle}) + + +def _created(bundle: Any, root: str = "root") -> dict[str, Any]: + result = create_aggregate_v2(bundle, "machine", root, "create", {}) + assert result["state"] is not None + return result["state"] + + +def _delivery(aggregate: dict[str, Any], event_id: str, event: str = "go") -> dict[str, Any]: + runtime = next( + item for item in aggregate["runtimes"] if item["runtime_id"] == aggregate["root_runtime_id"] + ) + envelope = { + "event": event, + "event_id": event_id, + "cause_id": event_id, + "source": {"host": True}, + "target": copy.deepcopy(runtime["target_identity"]), + "payload": ["map", []], + } + return { + "delivery_mode": "input", + "envelope": envelope, + "envelope_digest": _entry_digest(aggregate["root_instance_id"], "input", envelope), + } + + +def _admit(aggregate: dict[str, Any], bundle: Any, *events: tuple[str, str]) -> dict[str, Any]: + result = admit_aggregate_v2( + aggregate, + [_delivery(aggregate, event_id, event) for event_id, event in events], + _resolver(bundle), + ) + assert result["result"] == "accepted" + return result["state"] + + +def _compatible_v2_descriptor( + source: Any, + target: Any, + *, + queued_event_rules: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + base = { + "migration_descriptor_format": "determa.aggregate_migration", + "migration_descriptor_schema_version": 1, + "source_machine_format": 1, + "target_machine_format": 1, + "source_validated_bundle_fingerprint": source.fingerprint, + "target_validated_bundle_fingerprint": target.fingerprint, + "source_aggregate_shape_fingerprint": aggregate_shape_fingerprint(source), + "target_aggregate_shape_fingerprint": aggregate_shape_fingerprint(target), + "mode": "compatible", + "mappings": { + name: [] + for name in ( + "machines", + "active_states", + "variables", + "history", + "components", + "owned_runtimes", + "lifetime_holders", + "counters", + ) + }, + "terminal_policy": {"completed": "preserve", "faulted": "preserve"}, + "resource_requirements": { + "maximum_transformed_output_bytes": "0", + "maximum_cel_expression_length": "0", + "maximum_cel_ast_nodes": "0", + "maximum_cel_evaluation_steps": "0", + }, + } + base["migration_descriptor_digest"] = migration_descriptor_digest(base) + descriptor = { + "migration_descriptor_format": "determa.aggregate_migration", + "migration_descriptor_schema_version": 2, + "base_descriptor": base, + "queued_event_default": "preserve_if_compatible", + "queued_event_rules": queued_event_rules or [], + } + descriptor["migration_descriptor_digest"] = migration_descriptor_digest(descriptor) + return descriptor + + +def _upgraded_legacy_internal_checkpoint() -> tuple[Any, MemoryArtifactResolver, dict[str, Any]]: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.legacy_provenance", + "events": { + "go": {"direction": "input"}, + "work": {"direction": "internal"}, + }, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "simple", + "entry": [{"send": {"event": "work"}}], + "on_events": {"work": {}}, + }, + } + ], + } + ) + resolver = _resolver(bundle) + host = ExecutionHost(MemoryExecutionStore(), resolver) + host.create(bundle, "machine", "root", "create", {}) + upgraded = upgrade_checkpoint_v1_to_v2(host.read_checkpoint("root").document, resolver) + return bundle, resolver, upgraded + + +def _upgraded_legacy_terminal_with_native_allocations() -> tuple[ + MemoryArtifactResolver, dict[str, Any] +]: + bundle, resolver, _checkpoint = _upgraded_legacy_internal_checkpoint() + host = ExecutionHost(MemoryExecutionStore(), resolver) + host.create(bundle, "machine", "root", "create", {}) + created = host.read_checkpoint("root") + assert created is not None + pending = created.document["pending_deliveries"][0] + host.process_pending_delivery( + "root", + { + "root_instance_id": "root", + "delivery_mode": pending["delivery_mode"], + "origin": copy.deepcopy(pending["origin"]), + "envelope": copy.deepcopy(pending["envelope"]), + "envelope_digest": pending["envelope_digest"], + }, + expected_revision=created.document["revision"], + expected_checkpoint_digest=created.document["execution_checkpoint_digest"], + ) + processed = host.read_checkpoint("root") + assert processed is not None + checkpoint = upgrade_checkpoint_v1_to_v2(processed.document, resolver) + checkpoint = admit_checkpoint_v2( + checkpoint, + [ + _delivery(checkpoint["root_record"]["aggregate_state"], "native-1"), + _delivery(checkpoint["root_record"]["aggregate_state"], "native-2"), + ], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + assert checkpoint["root_record"]["aggregate_state"]["next_acceptance_sequence"] == "3" + return resolver, checkpoint + + +def test_create_v2_routes_initial_internal_and_external_work_with_provenance() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review82.creation", + "events": { + "inside": {"direction": "internal"}, + "outside": {"direction": "output"}, + }, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "simple", + "entry": [ + {"send": {"event": "inside"}}, + { + "send": { + "event": "outside", + "to": {"external": True}, + "correlation_id": "'creation'", + } + }, + ], + "on_events": {"inside": {}}, + }, + } + ], + } + ) + result = create_aggregate_v2(bundle, "machine", "root", "create", {}) + state = result["state"] + assert state["next_acceptance_sequence"] == "1" + entry = state["runtimes"][0]["ready_mailbox"][0] + assert entry["envelope"]["cause_id"] != entry["envelope"]["event_id"] + assert entry["envelope"]["source"] == {"runtime": state["runtimes"][0]["target_identity"]} + assert ["kind" not in item for item in result["emissions"]] == [False, True] + checkpoint = create_checkpoint_v2(bundle, "machine", "other", "create", {}) + assert len(checkpoint["pending_outbox_intents"]) == 1 + assert len(checkpoint["operation_receipts"][0]["emission_references"]) == 2 + + +def test_pre_step_rejection_preserves_ready_head( + monkeypatch: pytest.MonkeyPatch, +) -> None: + bundle = _bundle() + aggregate = _admit(_created(bundle), bundle, ("event", "go")) + before = copy.deepcopy(aggregate) + + def rejected(*args: Any, **kwargs: Any) -> dict[str, Any]: + return { + "status": "running", + "disposition": "rejected", + "state": args[1], + "emissions": [], + "fault": None, + "rejection": {"code": "invalid_instance_target"}, + } + + monkeypatch.setattr("determa.state.queueing.dispatch", rejected) + result = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) + assert result["state"] == before + + +def test_pending_replay_uses_canonical_digest_not_supplied_digest() -> None: + bundle = _bundle() + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "same") + admitted = admit_checkpoint_v2( + checkpoint, + [delivery], + _resolver(bundle), + expected_revision="0", + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + changed = copy.deepcopy(delivery) + changed["envelope"]["event"] = "tail" + with pytest.raises(ArtifactError, match="event_id_conflict"): + admit_checkpoint_v2( + admitted, + [changed], + _resolver(bundle), + expected_revision=admitted["revision"], + expected_checkpoint_digest=admitted["execution_checkpoint_digest"], + ) + + +def test_checkpoint_step_creates_external_outbox_reference() -> None: + bundle = _bundle(external=True) + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "go") + admitted = admit_checkpoint_v2( + checkpoint, + [delivery], + _resolver(bundle), + expected_revision="0", + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + stepped = step_checkpoint_v2( + admitted, + admitted["root_record"]["aggregate_state"]["root_runtime_id"], + _resolver(bundle), + expected_revision=admitted["revision"], + expected_checkpoint_digest=admitted["execution_checkpoint_digest"], + ) + assert len(stepped["pending_outbox_intents"]) == 1 + assert stepped["operation_receipts"][-1]["emission_references"][0]["kind"] == ( + "external_outbox" + ) + + +def test_v1_host_gate_is_real_and_store_is_unchanged() -> None: + bundle = _bundle(deferred=True) + host = ExecutionHost(MemoryExecutionStore(), _resolver(bundle)) + host.create(bundle, "machine", "root", "create", {}) + before = host.read_checkpoint("root") + assert before is not None + aggregate = before.document["root_record"]["aggregate_state"] + candidate = { + "root_instance_id": "root", + "delivery_mode": "input", + "origin": {"kind": "host_input"}, + "envelope": { + key: value + for key, value in _delivery(aggregate, "go")["envelope"].items() + if key not in {"cause_id", "source"} + }, + } + result = host.accept_delivery( + "root", + candidate, + expected_revision=before.document["revision"], + expected_checkpoint_digest=before.document["execution_checkpoint_digest"], + selected_bundle=bundle, + ) + assert result["failure"]["code"] == "checkpoint_upgrade_required" + assert host.read_checkpoint("root").source_bytes == before.source_bytes + + +def test_execution_host_round_trips_v2_transactions() -> None: + bundle = _bundle() + host = ExecutionHost(MemoryExecutionStore(), _resolver(bundle)) + host.create_v2(bundle, "machine", "root", "create", {}) + restored = host.read_checkpoint("root") + assert restored is not None + assert restored.document["execution_checkpoint_schema_version"] == 2 + delivery = _delivery(restored.document["root_record"]["aggregate_state"], "go") + host.admit_v2( + "root", + [delivery], + expected_revision=restored.document["revision"], + expected_checkpoint_digest=restored.document["execution_checkpoint_digest"], + ) + admitted = host.read_checkpoint("root") + host.process_ready_v2( + "root", + admitted.document["root_record"]["aggregate_state"]["root_runtime_id"], + expected_revision=admitted.document["revision"], + expected_checkpoint_digest=admitted.document["execution_checkpoint_digest"], + ) + assert ( + host.read_checkpoint("root").document["operation_receipts"][-1]["operation_kind"] + == "event_terminal" + ) + + +def test_execution_host_prunes_and_tombstones_v2_transactionally() -> None: + bundle = _bundle() + host = ExecutionHost(MemoryExecutionStore(), _resolver(bundle)) + host.create_v2(bundle, "machine", "root", "create", {}) + created = host.read_checkpoint("root") + delivery = _delivery(created.document["root_record"]["aggregate_state"], "go") + host.admit_v2( + "root", + [delivery], + expected_revision=created.document["revision"], + expected_checkpoint_digest=created.document["execution_checkpoint_digest"], + ) + admitted = host.read_checkpoint("root") + host.process_ready_v2( + "root", + admitted.document["root_record"]["aggregate_state"]["root_runtime_id"], + expected_revision=admitted.document["revision"], + expected_checkpoint_digest=admitted.document["execution_checkpoint_digest"], + ) + terminal = host.read_checkpoint("root") + host.update_replay_retention( + "root", + { + "mode": "bounded", + "permanent_replay_eligible": False, + "pruned_through_receipt_sequence": None, + "policy_identifier": "test-policy", + }, + expected_revision=terminal.document["revision"], + expected_checkpoint_digest=terminal.document["execution_checkpoint_digest"], + ) + bounded = host.read_checkpoint("root") + host.prune_v2( + "root", + "2", + expected_revision=bounded.document["revision"], + expected_checkpoint_digest=bounded.document["execution_checkpoint_digest"], + ) + assert host.read_checkpoint("root").document["event_identity_tombstones"][0]["event_id"] == "go" + + terminal_bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review82.tombstone", + "machines": [ + { + "machine_id": "machine", + "root": {"type": "simple", "entry": [{"stop": {}}]}, + } + ], + } + ) + terminal_host = ExecutionHost(MemoryExecutionStore(), _resolver(terminal_bundle)) + terminal_host.create_v2(terminal_bundle, "machine", "terminal", "create", {}) + before = terminal_host.read_checkpoint("terminal") + terminal_host.tombstone_root_v2( + "terminal", + "delete", + expected_revision=before.document["revision"], + expected_checkpoint_digest=before.document["execution_checkpoint_digest"], + ) + assert terminal_host.read_checkpoint("terminal").document["root_record"]["status"] == ( + "tombstone" + ) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda delivery: delivery["envelope"].update({"extra": True}), + lambda delivery: delivery["envelope"].update({"source": {"host": False}}), + lambda delivery: delivery.update({"extra": True}), + lambda delivery: delivery["envelope"].update({"target": {}}), + ], +) +def test_admission_rejects_closed_envelope_shapes_without_leaking( + mutate: Any, +) -> None: + bundle = _bundle() + aggregate = _created(bundle) + delivery = _delivery(aggregate, "event") + mutate(delivery) + result = admit_aggregate_v2(aggregate, [delivery], _resolver(bundle)) + assert result["result"] == "rejected" + assert result["state"] == aggregate + + +def _terminal_checkpoint() -> tuple[Any, MemoryArtifactResolver, dict[str, Any]]: + bundle = _bundle() + resolver = _resolver(bundle) + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "go") + admitted = admit_checkpoint_v2( + checkpoint, + [delivery], + resolver, + expected_revision="0", + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + terminal = step_checkpoint_v2( + admitted, + admitted["root_record"]["aggregate_state"]["root_runtime_id"], + resolver, + expected_revision=admitted["revision"], + expected_checkpoint_digest=admitted["execution_checkpoint_digest"], + ) + return bundle, resolver, terminal + + +@pytest.mark.parametrize("case", ["counter", "digest", "sequence", "overlap", "evidence"]) +def test_checkpoint_restore_rejects_identity_allocation_corruption(case: str) -> None: + _bundle_value, resolver, checkpoint = _terminal_checkpoint() + candidate = copy.deepcopy(checkpoint) + aggregate = candidate["root_record"]["aggregate_state"] + terminal = candidate["operation_receipts"][-1] + if case == "counter": + aggregate["next_acceptance_sequence"] = "0" + elif case == "digest": + terminal["request_digest"] = "sha256:" + "0" * 64 + elif case == "sequence": + aggregate["next_acceptance_sequence"] = "2" + terminal["acceptance_sequence"] = "1" + elif case == "overlap": + candidate["event_identity_tombstones"].append( + { + "event_id": terminal["event_id"], + "request_digest": terminal["request_digest"], + "request_digest_domain": "determa-inbox-envelope-digest-2", + "acceptance_sequence": terminal["acceptance_sequence"], + "terminal_receipt_sequence": terminal["receipt_sequence"], + "terminal_disposition": terminal["outcome"]["disposition"], + } + ) + else: + terminal["event_id"] = "unowned-terminal" + candidate["root_record"]["aggregate_state"] = seal_aggregate_v2(aggregate) + candidate = seal_execution_checkpoint(candidate) + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(candidate, resolver) + + +def test_permanent_prune_rejects_without_mutation() -> None: + _bundle_value, resolver, checkpoint = _terminal_checkpoint() + before = copy.deepcopy(checkpoint) + from determa.state import prune_checkpoint_v2 + + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + prune_checkpoint_v2( + checkpoint, + "1", + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + assert checkpoint == before + + +def test_unhandled_delivery_allocates_no_logical_step() -> None: + bundle = _bundle() + aggregate = _admit(_created(bundle), bundle, ("unhandled", "hold")) + before = aggregate["next_logical_step_sequence"] + result = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) + assert result["disposition"] == "unhandled" + assert result["state"]["next_logical_step_sequence"] == before + + +def test_downgrade_requires_zero_mailbox_counters() -> None: + bundle = _bundle() + aggregate = _created(bundle) + aggregate["next_queue_sequence"] = "1" + aggregate = seal_aggregate_v2(aggregate) + with pytest.raises(ArtifactError, match="migration_totality_failure"): + downgrade_aggregate_v2_to_v1(aggregate, _resolver(bundle)) + + +def test_resource_bound_counter_validation_does_not_expand_counter() -> None: + bundle = _bundle() + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + checkpoint["next_operation_receipt_sequence"] = "9" * 4097 + checkpoint = seal_execution_checkpoint(checkpoint) + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(checkpoint, _resolver(bundle)) + + +def test_lifecycle_dispositions_are_ready_then_deferred() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review82.lifecycle", + "events": { + "finish": {"direction": "input"}, + "hold": {"direction": "input"}, + "tail": {"direction": "input"}, + }, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "simple", + "deferred_events": ["hold"], + "on_events": { + "finish": {"action": [{"stop": {}}]}, + "tail": {}, + }, + }, + } + ], + } + ) + aggregate = _admit( + _created(bundle), + bundle, + ("hold", "hold"), + ("finish", "finish"), + ("tail", "tail"), + ) + first = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) + result = step_aggregate_v2(first["state"], aggregate["root_runtime_id"], _resolver(bundle)) + assert [item["event_id"] for item in result["lifecycle_dispositions"]] == [ + "tail", + "hold", + ] + + +def test_current_step_emissions_precede_structural_recall() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review82.recall_order", + "events": { + "go": {"direction": "input"}, + "hold": {"direction": "input"}, + "work": {"direction": "internal"}, + }, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "composite", + "initial": {"transition_to": "waiting"}, + "states": { + "waiting": { + "deferred_events": ["hold"], + "on_events": { + "go": { + "action": [{"send": {"event": "work"}}], + "transition_to": "active", + } + }, + }, + "active": {"on_events": {"work": {}}}, + }, + }, + } + ], + } + ) + aggregate = _admit(_created(bundle), bundle, ("hold", "hold"), ("go", "go")) + aggregate = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle))[ + "state" + ] + result = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) + ready = result["state"]["runtimes"][0]["ready_mailbox"] + assert [entry["envelope"]["event"] for entry in ready] == ["work", "hold"], result + assert int(ready[0]["queue_sequence"]) < int(ready[1]["queue_sequence"]) + + +def test_contained_capacity_fault_notifies_owner_and_freezes_other_work() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review82.contained_capacity", + "events": { + "start": {"direction": "input"}, + "work": {"direction": "internal"}, + }, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "parallel", + "on_events": { + "start": { + "action": [ + { + "send": { + "event": "work", + "to": {"component": "worker"}, + } + }, + { + "send": { + "event": "work", + "to": {"component": "worker"}, + } + }, + ] + }, + "determa.component_failed": {}, + }, + "components": [ + { + "component_id": "worker", + "root": { + "type": "simple", + "deferred_event_capacity": 0, + "deferred_events": ["work"], + }, + }, + {"component_id": "peer", "root": {"type": "simple"}}, + ], + }, + } + ], + } + ) + aggregate = _admit(_created(bundle), bundle, ("start", "start")) + emitted = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle))["state"] + child = next( + runtime for runtime in emitted["runtimes"] if runtime["relation"]["kind"] == "component" + ) + result = step_aggregate_v2(emitted, child["runtime_id"], _resolver(bundle)) + child_after = next( + runtime + for runtime in result["state"]["runtimes"] + if runtime["runtime_id"] == child["runtime_id"] + ) + assert child_after["status"] == "faulted" + assert len(child_after["ready_mailbox"]) == 1 + owner = next( + runtime + for runtime in result["state"]["runtimes"] + if runtime["runtime_id"] == result["state"]["root_runtime_id"] + ) + notification = owner["ready_mailbox"][-1]["envelope"] + assert notification["event"] == "determa.component_failed" + assert notification["source"] == {"system": "system:component_failure"} + + +def test_migration_structurally_recalls_with_fresh_queue_sequence() -> None: + source = _bundle(deferred=True) + target_document = copy.deepcopy(source.raw) + target_document["machines"][0]["root"].pop("deferred_events") + target = load_bundle(target_document) + aggregate = _admit(_created(source), source, ("hold", "hold")) + aggregate = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(source))[ + "state" + ] + shape = aggregate_shape_fingerprint(source) + base = { + "migration_descriptor_format": "determa.aggregate_migration", + "migration_descriptor_schema_version": 1, + "source_machine_format": 1, + "target_machine_format": 1, + "source_validated_bundle_fingerprint": source.fingerprint, + "target_validated_bundle_fingerprint": target.fingerprint, + "source_aggregate_shape_fingerprint": shape, + "target_aggregate_shape_fingerprint": aggregate_shape_fingerprint(target), + "mode": "compatible", + "mappings": { + name: [] + for name in ( + "machines", + "active_states", + "variables", + "history", + "components", + "owned_runtimes", + "lifetime_holders", + "counters", + ) + }, + "terminal_policy": {"completed": "preserve", "faulted": "preserve"}, + "resource_requirements": { + "maximum_transformed_output_bytes": "0", + "maximum_cel_expression_length": "0", + "maximum_cel_ast_nodes": "0", + "maximum_cel_evaluation_steps": "0", + }, + } + base["migration_descriptor_digest"] = migration_descriptor_digest(base) + descriptor = { + "migration_descriptor_format": "determa.aggregate_migration", + "migration_descriptor_schema_version": 2, + "base_descriptor": base, + "queued_event_default": "preserve_if_compatible", + "queued_event_rules": [], + } + descriptor["migration_descriptor_digest"] = migration_descriptor_digest(descriptor) + resolver = MemoryArtifactResolver( + definitions={source.fingerprint: source, target.fingerprint: target}, + migration_descriptors={descriptor["migration_descriptor_digest"]: descriptor}, + ) + prior_next = int(aggregate["next_queue_sequence"]) + migrated = migrate_aggregate_v2( + aggregate, + target.fingerprint, + [descriptor["migration_descriptor_digest"]], + resolver, + maintenance_mode=True, + )["aggregate_state"] + runtime = migrated["runtimes"][0] + assert not runtime["deferred_mailbox"] + assert runtime["ready_mailbox"][0]["queue_sequence"] == str(prior_next) + + checkpoint = create_checkpoint_v2(source, "machine", "host-root", "create", {}) + host_delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "host-hold", "hold") + checkpoint = admit_checkpoint_v2( + checkpoint, + [host_delivery], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v2( + checkpoint, + checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + host = ExecutionHost( + MemoryExecutionStore({"host-root": serialize_execution_checkpoint(checkpoint)}), + resolver, + ) + host.maintenance_migration_v2( + "host-root", + target.fingerprint, + [descriptor["migration_descriptor_digest"]], + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + restored = host.read_checkpoint("host-root") + assert ( + restored.document["root_record"]["aggregate_state"]["runtimes"][0]["ready_mailbox"][0][ + "envelope" + ]["event"] + == "hold" + ) + + +def test_restore_aggregate_package_accepts_strict_v2_package() -> None: + bundle = _bundle() + aggregate = _created(bundle) + package = { + "aggregate_state_package_format": "determa.aggregate_state_package", + "aggregate_state_package_schema_version": 2, + "aggregate_state": aggregate, + "normalized_definitions": [ + { + "validated_bundle_fingerprint": bundle.fingerprint, + "normalized_bundle": typed_value(bundle.raw), + } + ], + "migration_descriptors": [], + "migration_route": [], + } + restored = restore_aggregate_package(package, MemoryArtifactResolver()) + assert restored.aggregate.aggregate_envelope == aggregate + + +def test_created_component_emission_uses_canonical_wire_source_and_restores() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.component_source", + "events": {"notice": {"direction": "internal"}}, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "parallel", + "on_events": {"notice": {}}, + "components": [ + { + "component_id": "sender", + "root": { + "type": "simple", + "entry": [ + { + "send": { + "event": "notice", + "to": {"owner": True}, + } + }, + {"stop": {}}, + ], + }, + }, + {"component_id": "peer", "root": {"type": "simple"}}, + ], + }, + } + ], + } + ) + result = create_aggregate_v2(bundle, "machine", "root", "create", {}) + aggregate = result["state"] + entry = next( + entry + for runtime in aggregate["runtimes"] + for entry in runtime["ready_mailbox"] + if entry["envelope"]["event"] == "notice" + ) + source = entry["envelope"]["source"]["runtime"]["component"] + assert isinstance(source["activation_sequence"], str) + assert aggregate_state_digest(aggregate) == aggregate["aggregate_state_digest"] + assert restore_aggregate_v2(aggregate, _resolver(bundle)).aggregate_envelope == aggregate + + +def test_checkpoint_updates_internal_reference_on_deferral_and_recall() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.reference_recall", + "events": { + "go": {"direction": "input"}, + "work": {"direction": "internal"}, + }, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "composite", + "entry": [{"send": {"event": "work"}}], + "initial": {"transition_to": "waiting"}, + "states": { + "waiting": { + "deferred_events": ["work"], + "on_events": {"go": {"transition_to": "active"}}, + }, + "active": {"on_events": {"work": {}}}, + }, + }, + } + ], + } + ) + resolver = _resolver(bundle) + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + runtime_id = checkpoint["root_record"]["aggregate_state"]["root_runtime_id"] + checkpoint = step_checkpoint_v2( + checkpoint, + runtime_id, + resolver, + expected_revision="0", + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + deferred = checkpoint["root_record"]["aggregate_state"]["runtimes"][0]["deferred_mailbox"][0] + reference = checkpoint["operation_receipts"][0]["emission_references"][0] + assert reference["queue_sequence"] == deferred["queue_sequence"] + delivery = _delivery(checkpoint["root_record"]["aggregate_state"], "go") + checkpoint = admit_checkpoint_v2( + checkpoint, + [delivery], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v2( + checkpoint, + runtime_id, + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + recalled = checkpoint["root_record"]["aggregate_state"]["runtimes"][0]["ready_mailbox"][0] + reference = checkpoint["operation_receipts"][0]["emission_references"][0] + assert reference["queue_sequence"] == recalled["queue_sequence"] + restore_execution_checkpoint_v2(checkpoint, resolver) + + +def test_checkpoint_terminalizes_selected_and_lifecycle_disposed_references() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.reference_terminal", + "events": { + "start": {"direction": "input"}, + "finish": {"direction": "internal"}, + "work": {"direction": "internal"}, + }, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "parallel", + "on_events": { + "start": { + "action": [ + { + "send": { + "event": "finish", + "to": {"component": "worker"}, + } + }, + { + "send": { + "event": "work", + "to": {"component": "worker"}, + } + }, + ] + } + }, + "components": [ + { + "component_id": "worker", + "root": { + "type": "simple", + "on_events": { + "finish": {"action": [{"stop": {}}]}, + "work": {}, + }, + }, + }, + {"component_id": "peer", "root": {"type": "simple"}}, + ], + }, + } + ], + } + ) + resolver = _resolver(bundle) + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + checkpoint = admit_checkpoint_v2( + checkpoint, + [_delivery(checkpoint["root_record"]["aggregate_state"], "start", "start")], + resolver, + expected_revision="0", + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v2( + checkpoint, + checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + worker = next( + runtime + for runtime in checkpoint["root_record"]["aggregate_state"]["runtimes"] + if runtime["relation"].get("component_id") == "worker" + ) + producer = checkpoint["operation_receipts"][-1] + checkpoint = step_checkpoint_v2( + checkpoint, + worker["runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + assert [reference["kind"] for reference in producer["emission_references"]] == [ + "internal_mailbox", + "internal_mailbox", + ] + committed_producer = next( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt.get("event_id") == "start" and receipt["operation_kind"] == "event_terminal" + ) + assert [reference["kind"] for reference in committed_producer["emission_references"]] == [ + "internal_terminal", + "internal_terminal", + ] + restore_execution_checkpoint_v2(checkpoint, resolver) + + +@pytest.mark.parametrize("case", ["orphan", "receipt_order", "revision_order"]) +def test_checkpoint_rejects_reverse_orphan_and_terminal_chronology(case: str) -> None: + _bundle_value, resolver, checkpoint = _terminal_checkpoint() + candidate = copy.deepcopy(checkpoint) + acceptance = candidate["operation_receipts"][1] + terminal = candidate["operation_receipts"][2] + if case == "orphan": + candidate["operation_receipts"].pop() + candidate["next_operation_receipt_sequence"] = "2" + elif case == "receipt_order": + acceptance["receipt_sequence"] = "2" + terminal["receipt_sequence"] = "1" + candidate["operation_receipts"] = [ + candidate["operation_receipts"][0], + terminal, + acceptance, + ] + else: + acceptance["accepted_revision"] = "2" + terminal["committed_revision"] = "1" + candidate = seal_execution_checkpoint(candidate) + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(candidate, resolver) + + +@pytest.mark.parametrize("case", ["cause", "source", "event", "direction"]) +def test_aggregate_restore_validates_mailbox_envelope_semantics(case: str) -> None: + bundle = _bundle() + aggregate = _admit(_created(bundle), bundle, ("event", "go")) + entry = aggregate["runtimes"][0]["ready_mailbox"][0] + if case == "cause": + entry["envelope"]["cause_id"] = "other" + elif case == "source": + entry["envelope"]["source"] = { + "runtime": copy.deepcopy(aggregate["runtimes"][0]["target_identity"]) + } + elif case == "event": + entry["envelope"]["event"] = "missing" + else: + entry["envelope"]["event"] = "work" + entry["envelope_digest"] = _entry_digest( + aggregate["root_instance_id"], entry["delivery_mode"], entry["envelope"] + ) + aggregate = seal_aggregate_v2(aggregate) + with pytest.raises(ArtifactError, match="invalid_aggregate_state"): + restore_aggregate_v2(aggregate, _resolver(bundle)) + + +@pytest.mark.parametrize("forgery", ["wrong_type", "missing_default"]) +def test_restore_aggregate_v2_rejects_resealed_noncanonical_payload(forgery: str) -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.payload_restore", + "events": { + "go": { + "direction": "input", + "payload": { + "count": {"type": "int", "required": True}, + "label": {"type": "string", "default": "defaulted"}, + }, + } + }, + "machines": [ + {"machine_id": "machine", "root": {"type": "simple", "on_events": {"go": {}}}} + ], + } + ) + aggregate = _created(bundle) + delivery = _delivery(aggregate, "event") + delivery["envelope"]["payload"] = typed_value({"count": 1, "label": "defaulted"}) + delivery["envelope_digest"] = _entry_digest( + aggregate["root_instance_id"], "input", delivery["envelope"] + ) + admitted = admit_aggregate_v2(aggregate, [delivery], _resolver(bundle)) + assert admitted["result"] == "accepted" + candidate = admitted["state"] + entry = candidate["runtimes"][0]["ready_mailbox"][0] + entry["envelope"]["payload"] = typed_value( + {"count": "1", "label": "defaulted"} + if forgery == "wrong_type" + else {"count": 1} + ) + entry["envelope_digest"] = _entry_digest( + candidate["root_instance_id"], entry["delivery_mode"], entry["envelope"] + ) + candidate = seal_aggregate_v2(candidate) + + with pytest.raises(ArtifactError, match="invalid_aggregate_state"): + restore_aggregate_v2(candidate, _resolver(bundle)) + + +def test_admission_rejects_payload_that_would_materialize_a_default() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.payload_admission", + "events": { + "go": { + "direction": "input", + "payload": {"label": {"type": "string", "default": "defaulted"}}, + } + }, + "machines": [ + {"machine_id": "machine", "root": {"type": "simple", "on_events": {"go": {}}}} + ], + } + ) + aggregate = _created(bundle) + delivery = _delivery(aggregate, "event") + result = admit_aggregate_v2(aggregate, [delivery], _resolver(bundle)) + + assert result["result"] == "rejected" + assert result["rejection"]["code"] == "invalid_payload" + assert result["state"] == aggregate + + +def test_restore_rejects_resealed_author_event_with_system_source() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.author_source", + "events": {"work": {"direction": "internal"}}, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "simple", + "entry": [{"send": {"event": "work"}}], + "on_events": {"work": {}}, + }, + } + ], + } + ) + aggregate = _created(bundle) + entry = aggregate["runtimes"][0]["ready_mailbox"][0] + entry["envelope"]["source"] = {"system": "system:component_completion"} + entry["envelope_digest"] = _entry_digest( + aggregate["root_instance_id"], entry["delivery_mode"], entry["envelope"] + ) + aggregate = seal_aggregate_v2(aggregate) + + with pytest.raises(ArtifactError, match="invalid_aggregate_state"): + restore_aggregate_v2(aggregate, _resolver(bundle)) + + +def test_restore_rejects_resealed_reserved_event_with_wrong_system_source() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.reserved_source", + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "parallel", + "components": [ + { + "component_id": "worker", + "root": {"type": "simple", "entry": [{"stop": {}}]}, + }, + {"component_id": "peer", "root": {"type": "simple"}}, + ], + }, + } + ], + } + ) + aggregate = _created(bundle) + entry = next( + entry + for runtime in aggregate["runtimes"] + for entry in runtime["ready_mailbox"] + if entry["envelope"]["event"] == "determa.component_completed" + ) + entry["envelope"]["source"] = {"system": "system:component_failure"} + entry["envelope_digest"] = _entry_digest( + aggregate["root_instance_id"], entry["delivery_mode"], entry["envelope"] + ) + aggregate = seal_aggregate_v2(aggregate) + + with pytest.raises(ArtifactError, match="invalid_aggregate_state"): + restore_aggregate_v2(aggregate, _resolver(bundle)) + + +def test_checkpoint_restore_rejects_resealed_false_creation_result_digest() -> None: + bundle = _bundle() + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + checkpoint["operation_receipts"][0]["resulting_aggregate_state_digest"] = ( + "sha256:" + "0" * 64 + ) + checkpoint = seal_execution_checkpoint(checkpoint) + + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(checkpoint, _resolver(bundle)) + + +@pytest.mark.parametrize("field", ["producing_receipt_sequence", "emission_index"]) +def test_checkpoint_restore_rejects_resealed_legacy_origin_forgery(field: str) -> None: + _bundle_value, resolver, checkpoint = _upgraded_legacy_internal_checkpoint() + acceptance = next( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["operation_kind"] == "acceptance" + ) + acceptance["legacy_v1_delivery"]["origin"][field] = "999" + checkpoint = seal_execution_checkpoint(checkpoint) + + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(checkpoint, resolver) + + +def test_checkpoint_restore_rejects_resealed_legacy_producer_reference_forgery() -> None: + _bundle_value, resolver, checkpoint = _upgraded_legacy_internal_checkpoint() + producer = next( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["operation_kind"] == "legacy_v1_creation" + ) + producer["legacy_receipt"]["emission_references"][0]["emission_index"] = "999" + checkpoint = seal_execution_checkpoint(checkpoint) + + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(checkpoint, resolver) + + +def test_checkpoint_restore_rejects_resealed_legacy_mailbox_relabelled_native() -> None: + _bundle_value, resolver, checkpoint = _upgraded_legacy_internal_checkpoint() + aggregate = checkpoint["root_record"]["aggregate_state"] + entry = aggregate["runtimes"][0]["ready_mailbox"][0] + entry["envelope"]["source"] = { + "runtime": copy.deepcopy(aggregate["runtimes"][0]["target_identity"]) + } + entry["envelope_digest"] = _entry_digest( + aggregate["root_instance_id"], entry["delivery_mode"], entry["envelope"] + ) + acceptance = next( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["operation_kind"] == "acceptance" + ) + acceptance["request_digest"] = entry["envelope_digest"] + checkpoint["root_record"]["aggregate_state"] = seal_aggregate_v2(aggregate) + checkpoint = seal_execution_checkpoint(checkpoint) + + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(checkpoint, resolver) + + +def test_checkpoint_restore_rejects_resealed_processed_legacy_terminal_origin() -> None: + _bundle_value, resolver, checkpoint = _upgraded_legacy_internal_checkpoint() + checkpoint = step_checkpoint_v2( + checkpoint, + checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + acceptance = next( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["operation_kind"] == "acceptance" + ) + acceptance["legacy_v1_delivery"]["origin"]["emission_index"] = "999" + checkpoint = seal_execution_checkpoint(checkpoint) + + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(checkpoint, resolver) + + +def _set_wrapped_legacy_terminal_allocation( + checkpoint: dict[str, Any], sequence: str +) -> None: + wrapper = next( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["operation_kind"] == "legacy_v1_operation" + and receipt["legacy_receipt"]["operation_kind"] == "delivery" + ) + legacy = wrapper["legacy_receipt"] + legacy["accepted_delivery_sequence"] = sequence + producer_sequence = legacy["origin"]["producing_receipt_sequence"] + producer = next( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["receipt_sequence"] == producer_sequence + ) + reference = next( + item + for item in producer["legacy_receipt"]["emission_references"] + if item["event_id"] == legacy["event_id"] + ) + reference["delivery_sequence"] = sequence + + +def test_checkpoint_restore_rejects_legacy_allocation_collision_with_native() -> None: + resolver, checkpoint = _upgraded_legacy_terminal_with_native_allocations() + native_acceptance = next( + receipt + for receipt in checkpoint["operation_receipts"] + if receipt["operation_kind"] == "acceptance" + ) + _set_wrapped_legacy_terminal_allocation( + checkpoint, native_acceptance["acceptance_sequence"] + ) + checkpoint = seal_execution_checkpoint(checkpoint) + + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(checkpoint, resolver) + + +def test_checkpoint_restore_rejects_legacy_allocation_at_or_above_next_counter() -> None: + resolver, checkpoint = _upgraded_legacy_terminal_with_native_allocations() + assert checkpoint["root_record"]["aggregate_state"]["next_acceptance_sequence"] == "3" + _set_wrapped_legacy_terminal_allocation(checkpoint, "999") + checkpoint = seal_execution_checkpoint(checkpoint) + + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + restore_execution_checkpoint_v2(checkpoint, resolver) + + +def test_v1_upgrade_gate_finds_nested_inline_component_deferral() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.inline_gate", + "events": {"hold": {"direction": "internal"}}, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "parallel", + "components": [ + { + "component_id": "nested", + "root": { + "type": "composite", + "initial": {"transition_to": "waiting"}, + "states": {"waiting": {"deferred_events": ["hold"]}}, + }, + }, + {"component_id": "peer", "root": {"type": "simple"}}, + ], + }, + } + ], + } + ) + host = ExecutionHost(MemoryExecutionStore(), _resolver(bundle)) + host.create(bundle, "machine", "root", "create", {}) + before = host.read_checkpoint("root") + aggregate = before.document["root_record"]["aggregate_state"] + result = host.accept_delivery( + "root", + { + "root_instance_id": "root", + "delivery_mode": "input", + "origin": {"kind": "host_input"}, + "envelope": { + key: value + for key, value in _delivery(aggregate, "event")["envelope"].items() + if key not in {"cause_id", "source"} + }, + }, + expected_revision=before.document["revision"], + expected_checkpoint_digest=before.document["execution_checkpoint_digest"], + selected_bundle=bundle, + ) + assert result["failure"]["code"] == "checkpoint_upgrade_required" + assert host.read_checkpoint("root").source_bytes == before.source_bytes + + +def test_pruning_rejects_pending_outbox_producer_removal() -> None: + bundle = _bundle(external=True) + resolver = _resolver(bundle) + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + checkpoint = admit_checkpoint_v2( + checkpoint, + [_delivery(checkpoint["root_record"]["aggregate_state"], "go")], + resolver, + expected_revision="0", + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v2( + checkpoint, + checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint["replay_retention"] = { + "mode": "bounded", + "permanent_replay_eligible": False, + "pruned_through_receipt_sequence": None, + "policy_identifier": "test", + } + checkpoint = seal_execution_checkpoint(checkpoint) + with pytest.raises(ArtifactError, match="invalid_execution_checkpoint"): + from determa.state import prune_checkpoint_v2 + + prune_checkpoint_v2( + checkpoint, + "2", + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + + +def test_migration_rejects_payload_default_materialization() -> None: + source = _bundle(deferred=True) + target_document = copy.deepcopy(source.raw) + target_document["events"]["hold"]["payload"] = { + "added": {"type": "string", "default": "defaulted"} + } + target = load_bundle(target_document) + descriptor = _compatible_v2_descriptor(source, target) + aggregate = _admit(_created(source), source, ("hold", "hold")) + aggregate = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(source))[ + "state" + ] + resolver = MemoryArtifactResolver( + definitions={source.fingerprint: source, target.fingerprint: target}, + migration_descriptors={descriptor["migration_descriptor_digest"]: descriptor}, + ) + with pytest.raises(ArtifactError, match="migration_totality_failure"): + migrate_aggregate_v2( + aggregate, + target.fingerprint, + [descriptor["migration_descriptor_digest"]], + resolver, + maintenance_mode=True, + ) + + +def test_public_v2_migration_returns_exact_ordered_audit_records() -> None: + source = _bundle() + target_document = copy.deepcopy(source.raw) + target_document["events"]["extra"] = {"direction": "input"} + target = load_bundle(target_document) + descriptor = _compatible_v2_descriptor(source, target) + aggregate = _created(source) + resolver = MemoryArtifactResolver( + definitions={source.fingerprint: source, target.fingerprint: target}, + migration_descriptors={descriptor["migration_descriptor_digest"]: descriptor}, + ) + + result = migrate_aggregate_v2( + aggregate, + target.fingerprint, + [descriptor["migration_descriptor_digest"]], + resolver, + maintenance_mode=True, + ) + + assert set(result) == {"result", "aggregate_state", "dispositions", "audit_records"} + assert result["dispositions"] == [] + assert result["audit_records"] == [ + { + "migration_audit_record_schema_version": 1, + "root_instance_id": aggregate["root_instance_id"], + "root_runtime_id": aggregate["root_runtime_id"], + "migration_sequence": "1", + "source_validated_bundle_fingerprint": source.fingerprint, + "target_validated_bundle_fingerprint": target.fingerprint, + "migration_descriptor_digest": descriptor["migration_descriptor_digest"], + "source_aggregate_state_digest": aggregate["aggregate_state_digest"], + "target_aggregate_state_digest": result["aggregate_state"][ + "aggregate_state_digest" + ], + "result_code": "migration_applied", + } + ] + + no_operation = migrate_aggregate_v2( + aggregate, + source.fingerprint, + [], + resolver, + maintenance_mode=False, + ) + assert no_operation == { + "result": "success", + "aggregate_state": aggregate, + "dispositions": [], + "audit_records": [], + } + + +def test_host_v2_empty_route_maintenance_migration_commits_no_operation() -> None: + bundle = _bundle() + resolver = _resolver(bundle) + checkpoint = create_checkpoint_v2(bundle, "machine", "root", "create", {}) + host = ExecutionHost( + MemoryExecutionStore({"root": serialize_execution_checkpoint(checkpoint)}), + resolver, + ) + + result = host.maintenance_migration_v2( + "root", + bundle.fingerprint, + [], + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + maintenance_mode=False, + ) + + assert int(result["revision"]) == int(checkpoint["revision"]) + 1 + assert result["root_record"]["aggregate_state"] == checkpoint["root_record"][ + "aggregate_state" + ] + assert result["migration_audit_records"] == checkpoint["migration_audit_records"] + assert host.read_checkpoint("root").document == result + + +def test_multihop_migration_recalls_each_hop_and_host_appends_exact_audit() -> None: + source = _bundle(deferred=True) + middle_document = copy.deepcopy(source.raw) + middle_document["machines"][0]["root"].pop("deferred_events") + middle = load_bundle(middle_document) + target_document = copy.deepcopy(source.raw) + target_document["events"]["extra"] = {"direction": "input"} + target = load_bundle(target_document) + first = _compatible_v2_descriptor(source, middle) + second = _compatible_v2_descriptor(middle, target) + resolver = MemoryArtifactResolver( + definitions={ + source.fingerprint: source, + middle.fingerprint: middle, + target.fingerprint: target, + }, + migration_descriptors={ + first["migration_descriptor_digest"]: first, + second["migration_descriptor_digest"]: second, + }, + ) + checkpoint = create_checkpoint_v2(source, "machine", "root", "create", {}) + checkpoint = admit_checkpoint_v2( + checkpoint, + [_delivery(checkpoint["root_record"]["aggregate_state"], "hold", "hold")], + resolver, + expected_revision="0", + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + checkpoint = step_checkpoint_v2( + checkpoint, + checkpoint["root_record"]["aggregate_state"]["root_runtime_id"], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + public_result = migrate_aggregate_v2( + checkpoint["root_record"]["aggregate_state"], + target.fingerprint, + [first["migration_descriptor_digest"], second["migration_descriptor_digest"]], + resolver, + maintenance_mode=True, + ) + assert [ + record["migration_descriptor_digest"] for record in public_result["audit_records"] + ] == [first["migration_descriptor_digest"], second["migration_descriptor_digest"]] + old_queue = checkpoint["root_record"]["aggregate_state"]["runtimes"][0]["deferred_mailbox"][0][ + "queue_sequence" + ] + host = ExecutionHost( + MemoryExecutionStore({"root": serialize_execution_checkpoint(checkpoint)}), + resolver, + ) + migrated = host.maintenance_migration_v2( + "root", + target.fingerprint, + [first["migration_descriptor_digest"], second["migration_descriptor_digest"]], + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + runtime = migrated["root_record"]["aggregate_state"]["runtimes"][0] + assert not runtime["deferred_mailbox"] + assert int(runtime["ready_mailbox"][0]["queue_sequence"]) > int(old_queue) + audits = migrated["migration_audit_records"] + assert len(audits) == 2 + assert ( + audits[0]["source_aggregate_state_digest"] + == checkpoint["root_record"]["aggregate_state"]["aggregate_state_digest"] + ) + assert audits[0]["target_aggregate_state_digest"] == audits[1]["source_aggregate_state_digest"] + assert ( + audits[1]["target_aggregate_state_digest"] + == migrated["root_record"]["aggregate_state"]["aggregate_state_digest"] + ) + restore_execution_checkpoint_v2(migrated, resolver) + + +def test_lifecycle_cleanup_orders_component_mailboxes_reverse_declaration() -> None: + bundle = load_bundle( + { + "format": 1, + "namespace": "tests.review83.cleanup_order", + "events": { + "start": {"direction": "input"}, + "finish": {"direction": "input"}, + "work": {"direction": "internal"}, + }, + "machines": [ + { + "machine_id": "machine", + "root": { + "type": "parallel", + "on_events": { + "start": { + "action": [ + { + "send": { + "event": "work", + "to": {"component": "first"}, + } + }, + { + "send": { + "event": "work", + "to": {"component": "second"}, + } + }, + ] + }, + "finish": {"action": [{"stop": {}}]}, + }, + "components": [ + {"component_id": "first", "root": {"type": "simple"}}, + {"component_id": "second", "root": {"type": "simple"}}, + ], + }, + } + ], + } + ) + aggregate = _admit(_created(bundle), bundle, ("start", "start")) + aggregate = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle))[ + "state" + ] + aggregate = _admit(aggregate, bundle, ("finish", "finish")) + result = step_aggregate_v2(aggregate, aggregate["root_runtime_id"], _resolver(bundle)) + component_by_runtime = { + runtime["runtime_id"]: runtime["relation"].get("component_id") + for runtime in aggregate["runtimes"] + } + assert [ + component_by_runtime[item["target_runtime_id"]] for item in result["lifecycle_dispositions"] + ] == ["second", "first"] + + +@pytest.mark.parametrize("first_failure", ["wrong_root", "conflict"]) +def test_batch_malformed_precedes_wrong_root_and_conflict(first_failure: str) -> None: + bundle = _bundle() + resolver = _resolver(bundle) + aggregate = _created(bundle) + if first_failure == "conflict": + aggregate = _admit(aggregate, bundle, ("same", "go")) + first = _delivery(aggregate, "same", "tail") + else: + first = _delivery(aggregate, "first") + first["envelope"]["target"]["root"]["root_instance_id"] = "wrong" + first["envelope_digest"] = _entry_digest("root", "input", first["envelope"]) + malformed = _delivery(aggregate, "malformed") + malformed["envelope"]["extra"] = True + result = admit_aggregate_v2(aggregate, [first, malformed], resolver) + assert result["rejection"]["code"] == "malformed_delivery" + + checkpoint = create_checkpoint_v2(bundle, "machine", "checkpoint", "create", {}) + if first_failure == "conflict": + original = _delivery(checkpoint["root_record"]["aggregate_state"], "same") + checkpoint = admit_checkpoint_v2( + checkpoint, + [original], + resolver, + expected_revision="0", + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + first = _delivery(checkpoint["root_record"]["aggregate_state"], "same", "tail") + else: + first = _delivery(checkpoint["root_record"]["aggregate_state"], "first") + first["envelope"]["target"]["root"]["root_instance_id"] = "wrong" + first["envelope_digest"] = _entry_digest("checkpoint", "input", first["envelope"]) + malformed = _delivery(checkpoint["root_record"]["aggregate_state"], "malformed") + malformed["envelope"]["extra"] = True + with pytest.raises(ArtifactError, match="malformed_delivery"): + admit_checkpoint_v2( + checkpoint, + [first, malformed], + resolver, + expected_revision=checkpoint["revision"], + expected_checkpoint_digest=checkpoint["execution_checkpoint_digest"], + ) + + +def test_invalid_core_step_result_reports_its_public_artifact_code() -> None: + with pytest.raises(ArtifactError, match="invalid_core_step_result"): + load_json_artifact( + { + "core_step_result_format": "determa.core_step_result", + "core_step_result_schema_version": 2, + }, + "core_step_result_v2", + )