From b1db81048355806ff65634d3a1943350820a49a5 Mon Sep 17 00:00:00 2001 From: David Abram Date: Tue, 19 May 2026 16:46:47 +0200 Subject: [PATCH 1/5] turso: Enable sync feature on turso dependency Add `features = ["sync"]` to the turso Cargo dependency so the turso::sync module is available at compile time. Co-authored-by: SCE --- cli/Cargo.lock | 129 +++++++++++++++++++++++++++- cli/Cargo.toml | 2 +- context/plans/turso-sync-adapter.md | 84 ++++++++++++++++++ flake.nix | 14 ++- 4 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 context/plans/turso-sync-adapter.md diff --git a/cli/Cargo.lock b/cli/Cargo.lock index a311ecb8..5fcf0d06 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -1029,6 +1029,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1407,6 +1422,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -2046,6 +2077,23 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -2106,9 +2154,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -2174,12 +2222,49 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -3504,11 +3589,35 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -3549,9 +3658,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags", "bytes", @@ -3663,8 +3772,14 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f21b08119fb7fa81cdd441a0255cb26811ff5d9c0dc0ff65bb71a493b7472c86" dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", "mimalloc", "thiserror", + "tokio", "tracing", "tracing-subscriber", "turso_core", @@ -4008,6 +4123,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 098d06ee..e5091cb4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -30,7 +30,7 @@ categories = ["command-line-utilities", "development-tools"] anyhow = "1" chrono = "0.4" clap = { version = "4", features = ["derive"] } -turso = "0.6.0" +turso = { version = "0.6.0", features = ["sync"] } clap_complete = "4" dirs = "6" hmac = "0.13" diff --git a/context/plans/turso-sync-adapter.md b/context/plans/turso-sync-adapter.md new file mode 100644 index 00000000..0a628c73 --- /dev/null +++ b/context/plans/turso-sync-adapter.md @@ -0,0 +1,84 @@ +# Turso Sync Adapter + +## Change summary + +Make the shared `TursoDb` adapter support both local-only and synced (Turso Cloud) modes, controlled by `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` environment variables. When both env vars are set, the database opens in sync mode (remote-backed); when either is absent, it falls back to the existing local-only behavior. + +The initial concrete target is the Agent Trace DB. Sync works in two phases: + +1. **Pull on setup**: `AgentTraceDbLifecycle::setup` (invoked during `sce setup`) pulls remote changes to bring the local DB up to date. +2. **Bounded write-counter push**: Every `execute()` on the Agent Trace DB increments an atomic counter. When the counter reaches a threshold (default 10, configurable via `SCE_SYNC_PUSH_THRESHOLD`), it auto-pushes and resets. This batches writes without background threads or per-write network latency. + +## Success criteria + +- [ ] SC1: `TursoDb` opens in sync mode when both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` are set, using `turso::sync::Builder::new_remote()`. +- [ ] SC2: `TursoDb` falls back to local-only `turso::Builder::new_local()` when either env var is missing. +- [ ] SC3: `TursoDb` exposes `push()`, `pull()`, `checkpoint()`, and `stats()` methods that delegate to sync operations when in sync mode and are no-ops when in local mode. +- [ ] SC4: `TursoDb::execute()` in sync mode increments a write counter and auto-pushes when the threshold is reached (default 10 writes). The threshold is configurable via `SCE_SYNC_PUSH_THRESHOLD`. +- [ ] SC5: `AgentTraceDbLifecycle::setup` performs an initial `pull()` when sync is configured. +- [ ] SC6: `nix flake check` passes. +- [ ] SC7: Context files under `context/` are updated to reflect the new sync capability. + +## Constraints and non-goals + +- **In scope:** `cli/Cargo.toml` feature enablement, env var constants in the shared config seam, `TursoDb` structural changes for dual-mode, Agent Trace DB auto-pull, context sync. +- **Out of scope:** A user-invocable `sce sync` CLI command (deferred to a later plan). Sync for `LocalDb`. Periodic/background sync scheduling. Token rotation or complex auth flows beyond env var string. Multi-database sync orchestration. +- **Assumption:** `turso::sync::Builder::with_auth_token()` accepts a static `&str` (the `AuthTokenFn` in turso 0.6.0 supports both string literals and closures). +- **Assumption:** `turso::sync::Database::connect()` is async, while `turso::Database::connect()` is synchronous — the TursoDb adapter handles both cases inside the existing `block_on` bridge. +- **Assumption:** The write-counter push uses an `AtomicU64` on `TursoDb`; the counter is checked and pushed synchronously inside `execute()` via `block_on`, so background threads are not needed. +- **Assumption:** Push failures during write-counter auto-push are logged but do not fail the `execute()` call (best-effort push). +- **Assumption:** Initial pull on setup is best-effort — if the remote is unreachable during `sce setup`, the setup continues without error and sync is deferred to later writes. + +## Task stack + +- [x] T01: `Enable sync feature on turso dependency` (status: done) + - Task ID: T01 + - Goal: Add `features = ["sync"]` to the `turso` dependency in `cli/Cargo.toml` so the `turso::sync` module is available at compile time. + - Boundaries (in/out of scope): In — editing `cli/Cargo.toml` only. Out — changing any Rust source code, changing other Cargo deps. + - Done when: `turso = { version = "0.6.0", features = ["sync"] }` is declared in `cli/Cargo.toml`, and `cargo check` / `nix flake check` compiles successfully. + - Verification notes (commands or checks): `nix flake check` (expect compile success); `rg 'turso' cli/Cargo.toml` confirms `features = ["sync"]`. + - **Completed:** 2026-05-19 + - **Files changed:** `cli/Cargo.toml`, `cli/Cargo.lock`, `flake.nix` + - **Evidence:** `nix flake check` — all 5 checks passed. `rg 'turso' cli/Cargo.toml` confirms `features = ["sync"]`. + - **Notes:** Enabling the `sync` feature pulled in `hyper-tls` as a transitive dep (via turso's `sync` feature), which required adding `pkg-config` + `openssl` to `nativeBuildInputs`/`buildInputs` in `flake.nix` for both `commonCargoArgs` and `cargoDepsArgs`. The `Cargo.lock` was also updated to include the new transitive deps. + +- [ ] T02: `Add SCE_SYNC_URL, SCE_SYNC_TOKEN, and SCE_SYNC_PUSH_THRESHOLD env var constants` (status: todo) + - Task ID: T02 + - Goal: Add `SYNC_URL_ENV_KEY`, `SYNC_TOKEN_ENV_KEY`, and `SYNC_PUSH_THRESHOLD_ENV_KEY` constants to the shared runtime/config primitive seam in `cli/src/services/config/mod.rs`, following the existing pattern for observability env keys (`LOG_LEVEL_ENV_KEY`, `LOG_FORMAT_ENV_KEY`, etc.). + - Boundaries (in/out of scope): In — adding three pub consts and a doc comment block in `cli/src/services/config/mod.rs`. Out — changing `TursoDb`, `DbSpec`, or any other source file. + - Done when: The three constants are defined, exported from the config module, and `cargo check` succeeds. + - Verification notes (commands or checks): `nix flake check`; `rg 'SYNC_(URL|TOKEN|PUSH_THRESHOLD)_ENV_KEY' cli/src/services/config/mod.rs` confirms all three exist. + +- [ ] T03: `Extend TursoDb adapter with dual-mode (local/sync) support plus write-counter auto-push` (status: todo) + - Task ID: T03 + - Goal: Modify `cli/src/services/db/mod.rs` so that `TursoDb` optionally carries sync state and uses `turso::sync::Builder::new_remote()` when both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` are set. Add an `AtomicU64` write counter; `execute()` increments the counter and auto-pushes when the threshold (from `SCE_SYNC_PUSH_THRESHOLD`, default 10) is reached. Expose `push()`, `pull()`, `checkpoint()`, and `stats()` methods that delegate to sync operations in sync mode and are no-ops in local mode. + - Boundaries (in/out of scope): + - In — `TursoDb` struct changes (add optional sync DB handle, atomic write counter), `new()` conditional builder logic with threshold parsing, `execute()` counter increment + conditional push, `push()`/`pull()`/`checkpoint()`/`stats()` method additions, `collect_db_path_health` update for sync-specific concerns. + - Out — Changing `DbSpec` trait, changing `AgentTraceDb` or `LocalDb` concrete types, adding new CLI commands. + - Done when: + - `TursoDb::new()` opens sync mode when both env vars are present, local mode otherwise. + - `execute()` in sync mode increments counter and auto-pushes at threshold. + - Push failures during auto-push are logged but do not propagate to the caller. + - `push()`/`pull()`/`checkpoint()`/`stats()` compile and behave correctly in both modes. + - `cargo check` and `nix flake check` pass. + - Verification notes (commands or checks): `nix flake check`; manual inspection of `db/mod.rs` conditional builder logic, counter increment in `execute()`, and best-effort push error handling. + +- [ ] T04: `Wire pull-on-setup into Agent Trace DB lifecycle` (status: todo) + - Task ID: T04 + - Goal: Modify `AgentTraceDbLifecycle::setup` in `cli/src/services/agent_trace_db/lifecycle.rs` to call `pull()` on the Agent Trace DB after opening when sync is configured. This runs during `sce setup`, pulling remote changes to bring the local Agent Trace DB current. The pull is best-effort — if the remote is unreachable, setup continues without error and sync is deferred to write-counter pushes. + - Boundaries (in/out of scope): + - In — Adding best-effort pull to `AgentTraceDbLifecycle::setup` when sync mode is active. + - Out — Changing `LocalDb`, adding pull to `LocalDbLifecycle`, adding CLI commands, changing the `TursoDb` adapter itself. + - Done when: `AgentTraceDbLifecycle::setup` calls `pull()` when the DB opened in sync mode. Remote failure does not fail setup. Not set behavior is unchanged. + - Verification notes (commands or checks): `nix flake check`; trace-level log confirms pull attempt during setup when sync env vars are set. + +- [ ] T05: `Validation and cleanup` (status: todo) + - Task ID: T05 + - Goal: Run full repository validation, verify all success criteria are met, and update context files to reflect the new sync adapter capability. + - Boundaries (in/out of scope): In — `nix flake check`, context sync edits to `context/overview.md`, `context/architecture.md`, `context/glossary.md`, `context/context-map.md`, `context/sce/shared-turso-db.md`, and `context/sce/agent-trace-db.md`. Out — making new code changes beyond context edits. + - Done when: `nix flake check` passes; success criteria are verified; context files reflect current state. + - Verification notes (commands or checks): `nix flake check`; `nix run .#pkl-check-generated`; manual review of context files for accuracy. + +## Open questions + +None. All critical details were resolved during the clarification gate. diff --git a/flake.nix b/flake.nix index 2a85eb09..0546f4ab 100644 --- a/flake.nix +++ b/flake.nix @@ -154,8 +154,13 @@ doCheck = false; SCE_GIT_COMMIT = shortGitCommit; - nativeBuildInputs = [ + nativeBuildInputs = with pkgs; [ rustToolchain + pkg-config + ]; + + buildInputs = with pkgs; [ + openssl ]; postUnpack = '' @@ -179,8 +184,13 @@ strictDeps = true; doCheck = false; - nativeBuildInputs = [ + nativeBuildInputs = with pkgs; [ rustToolchain + pkg-config + ]; + + buildInputs = with pkgs; [ + openssl ]; }; From a4e3afdfcf09a032ddba39d5f94a591209422be2 Mon Sep 17 00:00:00 2001 From: David Abram Date: Tue, 19 May 2026 16:51:49 +0200 Subject: [PATCH 2/5] config: Add SYNC_URL, SYNC_TOKEN, and SYNC_PUSH_THRESHOLD env var constants These `pub(crate)` constants follow the established pattern from observability env keys (LOG_LEVEL_ENV_KEY, etc.) and are the configuration seam for dual-mode TursoDb initialization. Co-authored-by: SCE --- cli/src/services/config/mod.rs | 13 +++++++++++++ context/plans/turso-sync-adapter.md | 6 +++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cli/src/services/config/mod.rs b/cli/src/services/config/mod.rs index b65598dc..53bf7355 100644 --- a/cli/src/services/config/mod.rs +++ b/cli/src/services/config/mod.rs @@ -40,6 +40,19 @@ pub(crate) const ENV_LOG_FORMAT: &str = "SCE_LOG_FORMAT"; pub(crate) const ENV_LOG_FILE: &str = "SCE_LOG_FILE"; pub(crate) const ENV_LOG_FILE_MODE: &str = "SCE_LOG_FILE_MODE"; pub(crate) const ENV_ATTRIBUTION_HOOKS_ENABLED: &str = "SCE_ATTRIBUTION_HOOKS_ENABLED"; + +// Sync (Turso Cloud) env var keys. +// When SCE_SYNC_URL and SCE_SYNC_TOKEN are both set, TursoDb opens in sync +// (remote-backed) mode. When either is absent, local-only mode is used. +// SCE_SYNC_PUSH_THRESHOLD controls how many writes accumulate before an +// automatic push (defaults to 10 when unset). +#[allow(dead_code)] +pub(crate) const SYNC_URL_ENV_KEY: &str = "SCE_SYNC_URL"; +#[allow(dead_code)] +pub(crate) const SYNC_TOKEN_ENV_KEY: &str = "SCE_SYNC_TOKEN"; +#[allow(dead_code)] +pub(crate) const SYNC_PUSH_THRESHOLD_ENV_KEY: &str = "SCE_SYNC_PUSH_THRESHOLD"; + const WORKOS_CLIENT_ID_ENV: &str = "WORKOS_CLIENT_ID"; const WORKOS_CLIENT_ID_BAKED_DEFAULT: &str = "client_sce_default"; const WORKOS_CLIENT_ID_KEY: AuthConfigKeySpec = AuthConfigKeySpec { diff --git a/context/plans/turso-sync-adapter.md b/context/plans/turso-sync-adapter.md index 0a628c73..472414b0 100644 --- a/context/plans/turso-sync-adapter.md +++ b/context/plans/turso-sync-adapter.md @@ -42,12 +42,16 @@ The initial concrete target is the Agent Trace DB. Sync works in two phases: - **Evidence:** `nix flake check` — all 5 checks passed. `rg 'turso' cli/Cargo.toml` confirms `features = ["sync"]`. - **Notes:** Enabling the `sync` feature pulled in `hyper-tls` as a transitive dep (via turso's `sync` feature), which required adding `pkg-config` + `openssl` to `nativeBuildInputs`/`buildInputs` in `flake.nix` for both `commonCargoArgs` and `cargoDepsArgs`. The `Cargo.lock` was also updated to include the new transitive deps. -- [ ] T02: `Add SCE_SYNC_URL, SCE_SYNC_TOKEN, and SCE_SYNC_PUSH_THRESHOLD env var constants` (status: todo) +- [x] T02: `Add SCE_SYNC_URL, SCE_SYNC_TOKEN, and SCE_SYNC_PUSH_THRESHOLD env var constants` (status: done) - Task ID: T02 - Goal: Add `SYNC_URL_ENV_KEY`, `SYNC_TOKEN_ENV_KEY`, and `SYNC_PUSH_THRESHOLD_ENV_KEY` constants to the shared runtime/config primitive seam in `cli/src/services/config/mod.rs`, following the existing pattern for observability env keys (`LOG_LEVEL_ENV_KEY`, `LOG_FORMAT_ENV_KEY`, etc.). - Boundaries (in/out of scope): In — adding three pub consts and a doc comment block in `cli/src/services/config/mod.rs`. Out — changing `TursoDb`, `DbSpec`, or any other source file. - Done when: The three constants are defined, exported from the config module, and `cargo check` succeeds. - Verification notes (commands or checks): `nix flake check`; `rg 'SYNC_(URL|TOKEN|PUSH_THRESHOLD)_ENV_KEY' cli/src/services/config/mod.rs` confirms all three exist. + - **Completed:** 2026-05-19 + - **Files changed:** `cli/src/services/config/mod.rs` + - **Evidence:** `nix flake check` — all 4 checks passed. `rg` confirms all three constants present. + - **Notes:** Added `#[allow(dead_code)]` to each constant since T03 will consume them. Followed existing `pub(crate) const ENV_*` pattern. - [ ] T03: `Extend TursoDb adapter with dual-mode (local/sync) support plus write-counter auto-push` (status: todo) - Task ID: T03 From 10c8e386562970f04af4cae5a3a83c4465c9f305 Mon Sep 17 00:00:00 2001 From: David Abram Date: Tue, 19 May 2026 20:17:27 +0200 Subject: [PATCH 3/5] db: Add dual-mode TursoDb (local/sync) and wire sce sync push|pull MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend TursoDb to support Turso Cloud sync mode via turso::sync::Builder::new_remote() when both SCE_SYNC_URL and SCE_SYNC_TOKEN are set. Sync-mode methods (push/pull/checkpoint/stats) are explicit — no auto-push from execute(). Wire a user-invocable sce sync push|pull CLI command behind the new sync module (cli/src/services/sync/), connected through clap schema, command registry, and parse runtime dispatch. Remove the unused SYNC_PUSH_THRESHOLD_ENV_KEY constant and write-counter auto-push approach from the earlier design. Co-authored-by: SCE --- cli/src/cli_schema.rs | 24 ++++ cli/src/services/command_registry.rs | 1 + cli/src/services/config/mod.rs | 4 - cli/src/services/db/mod.rs | 159 ++++++++++++++++++---- cli/src/services/mod.rs | 1 + cli/src/services/parse/command_runtime.rs | 19 +++ cli/src/services/sync/command.rs | 28 ++++ cli/src/services/sync/mod.rs | 43 ++++++ context/architecture.md | 4 +- context/context-map.md | 5 +- context/glossary.md | 5 +- context/overview.md | 4 +- context/plans/turso-sync-adapter.md | 9 +- context/sce/shared-turso-db.md | 34 +++-- 14 files changed, 295 insertions(+), 45 deletions(-) create mode 100644 cli/src/services/sync/command.rs create mode 100644 cli/src/services/sync/mod.rs diff --git a/cli/src/cli_schema.rs b/cli/src/cli_schema.rs index 8d1b258a..c914e6d5 100644 --- a/cli/src/cli_schema.rs +++ b/cli/src/cli_schema.rs @@ -35,6 +35,10 @@ pub const VERSION_CLAP_ABOUT: &str = "Print deterministic runtime version metada pub const VERSION_TOP_LEVEL_PURPOSE: &str = "Print deterministic runtime version metadata"; pub const VERSION_SHOW_IN_TOP_LEVEL_HELP: bool = true; +pub const SYNC_CLAP_ABOUT: &str = "Push or pull changes to/from Turso Cloud"; +pub const SYNC_TOP_LEVEL_PURPOSE: &str = "Sync local database with Turso Cloud (push/pull)"; +pub const SYNC_SHOW_IN_TOP_LEVEL_HELP: bool = true; + pub const COMPLETION_CLAP_ABOUT: &str = "Generate deterministic shell completion scripts"; pub const COMPLETION_TOP_LEVEL_PURPOSE: &str = "Generate deterministic shell completion scripts"; pub const COMPLETION_SHOW_IN_TOP_LEVEL_HELP: bool = true; @@ -75,6 +79,11 @@ pub const TOP_LEVEL_COMMANDS: &[TopLevelCommandMetadata] = &[ purpose: COMPLETION_TOP_LEVEL_PURPOSE, show_in_top_level_help: COMPLETION_SHOW_IN_TOP_LEVEL_HELP, }, + TopLevelCommandMetadata { + name: crate::services::sync::NAME, + purpose: SYNC_TOP_LEVEL_PURPOSE, + show_in_top_level_help: SYNC_SHOW_IN_TOP_LEVEL_HELP, + }, ]; #[derive(Parser, Debug)] @@ -190,6 +199,12 @@ pub enum Commands { #[arg(long, value_enum)] shell: CompletionShell, }, + + #[command(about = SYNC_CLAP_ABOUT, hide = !SYNC_SHOW_IN_TOP_LEVEL_HELP)] + Sync { + #[command(subcommand)] + subcommand: SyncSubcommand, + }, } #[derive(Subcommand, Debug, Clone, PartialEq, Eq)] @@ -276,6 +291,15 @@ pub enum HooksSubcommand { DiffTrace, } +#[derive(Subcommand, Debug, Clone, PartialEq, Eq)] +pub enum SyncSubcommand { + #[command(about = "Push local changes to Turso Cloud")] + Push, + + #[command(about = "Pull remote changes from Turso Cloud")] + Pull, +} + #[derive(ValueEnum, Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum OutputFormat { #[default] diff --git a/cli/src/services/command_registry.rs b/cli/src/services/command_registry.rs index 07ac310d..aa2f45c0 100644 --- a/cli/src/services/command_registry.rs +++ b/cli/src/services/command_registry.rs @@ -104,6 +104,7 @@ pub fn build_default_registry() -> CommandRegistry { "completion", crate::services::completion::command::make_completion_command, ); + registry.register("sync", crate::services::sync::command::make_sync_command); registry } diff --git a/cli/src/services/config/mod.rs b/cli/src/services/config/mod.rs index 53bf7355..08c88846 100644 --- a/cli/src/services/config/mod.rs +++ b/cli/src/services/config/mod.rs @@ -44,14 +44,10 @@ pub(crate) const ENV_ATTRIBUTION_HOOKS_ENABLED: &str = "SCE_ATTRIBUTION_HOOKS_EN // Sync (Turso Cloud) env var keys. // When SCE_SYNC_URL and SCE_SYNC_TOKEN are both set, TursoDb opens in sync // (remote-backed) mode. When either is absent, local-only mode is used. -// SCE_SYNC_PUSH_THRESHOLD controls how many writes accumulate before an -// automatic push (defaults to 10 when unset). #[allow(dead_code)] pub(crate) const SYNC_URL_ENV_KEY: &str = "SCE_SYNC_URL"; #[allow(dead_code)] pub(crate) const SYNC_TOKEN_ENV_KEY: &str = "SCE_SYNC_TOKEN"; -#[allow(dead_code)] -pub(crate) const SYNC_PUSH_THRESHOLD_ENV_KEY: &str = "SCE_SYNC_PUSH_THRESHOLD"; const WORKOS_CLIENT_ID_ENV: &str = "WORKOS_CLIENT_ID"; const WORKOS_CLIENT_ID_BAKED_DEFAULT: &str = "client_sce_default"; diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index ffa37c12..0f40823e 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -139,10 +139,20 @@ fn sentence_case(value: &str) -> String { /// Wraps a Turso connection with a tokio current-thread runtime so callers can /// use synchronous `execute`/`query` methods while the underlying Turso API /// remains async. +/// +/// Supports two modes: +/// - **Local mode** (default): opens a plain Turso file database. +/// - **Sync mode**: opens a Turso Cloud synced database when both +/// `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` environment variables are set. +/// Sync operations (`push`, `pull`, `checkpoint`, `stats`) are available +/// through explicit methods or the `sce sync` CLI command. #[allow(dead_code)] pub struct TursoDb { conn: turso::Connection, runtime: tokio::runtime::Runtime, + /// Optional sync database handle for Turso Cloud operations (push, pull, + /// checkpoint, stats). `None` when the database is in local-only mode. + sync_db: Option, spec: PhantomData M>, } @@ -152,6 +162,9 @@ impl TursoDb { /// /// Parent directories are created automatically. Migrations are run after /// the database connection is established. + /// + /// When both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` are set, the database opens + /// in sync (Turso Cloud) mode. Otherwise, local-only mode is used. pub fn new() -> Result { let db_name = M::db_name(); let db_path = M::db_path().with_context(|| format!("failed to resolve {db_name} path"))?; @@ -173,33 +186,70 @@ impl TursoDb { format!("failed to create {db_name} tokio runtime. Try: rerun the command; if the issue persists, verify the local Tokio runtime environment.") })?; - let conn = runtime.block_on(async { - let path_str = db_path.to_str().ok_or_else(|| { - anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) - })?; - let db = turso::Builder::new_local(path_str) - .build() - .await - .map_err(|e| { - anyhow::anyhow!( - "failed to open {db_name} database at {}: {e}", - db_path.display() - ) - })?; - db.connect() - .map_err(|e| anyhow::anyhow!("failed to connect to {db_name} database: {e}")) + let sync_url = std::env::var(crate::services::config::SYNC_URL_ENV_KEY).ok(); + let sync_token = std::env::var(crate::services::config::SYNC_TOKEN_ENV_KEY).ok(); + + let path_str = db_path.to_str().ok_or_else(|| { + anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) })?; - let db = Self { - conn, - runtime, - spec: PhantomData, - }; + if let (Some(url), Some(token)) = (sync_url, sync_token) { + let (conn, sync_db) = runtime.block_on(async { + let sync_db = turso::sync::Builder::new_remote(path_str) + .with_remote_url(url) + .with_auth_token(token) + .build() + .await + .map_err(|e| { + anyhow::anyhow!( + "failed to open {db_name} synced database at {}: {e}", + db_path.display() + ) + })?; + let conn = sync_db.connect().await.map_err(|e| { + anyhow::anyhow!("failed to connect to {db_name} synced database: {e}") + })?; + Ok::<_, anyhow::Error>((conn, sync_db)) + })?; + + let db = Self { + conn, + runtime, + sync_db: Some(sync_db), + spec: PhantomData, + }; + + db.run_migrations() + .with_context(|| format!("failed to run {db_name} migrations"))?; + + Ok(db) + } else { + let conn = runtime.block_on(async { + let db = turso::Builder::new_local(path_str) + .build() + .await + .map_err(|e| { + anyhow::anyhow!( + "failed to open {db_name} database at {}: {e}", + db_path.display() + ) + })?; + db.connect() + .map_err(|e| anyhow::anyhow!("failed to connect to {db_name} database: {e}")) + })?; - db.run_migrations() - .with_context(|| format!("failed to run {db_name} migrations"))?; + let db = Self { + conn, + runtime, + sync_db: None, + spec: PhantomData, + }; - Ok(db) + db.run_migrations() + .with_context(|| format!("failed to run {db_name} migrations"))?; + + Ok(db) + } } /// Execute a SQL statement that does not return rows. @@ -210,6 +260,10 @@ impl TursoDb { /// /// # Returns /// Number of rows affected. + /// + /// Sync is not triggered automatically — callers use the explicit `push()` + /// or `pull()` methods (or the `sce sync` CLI command) to sync with the + /// remote. pub fn execute(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { self.runtime.block_on(async { self.conn @@ -219,6 +273,65 @@ impl TursoDb { }) } + /// Push local changes to the remote (sync mode only). + /// + /// In local mode this is a no-op that returns `Ok(())`. + pub fn push(&self) -> Result<()> { + match self.sync_db { + Some(ref sync_db) => self + .runtime + .block_on(sync_db.push()) + .map_err(|e| anyhow::anyhow!("{} sync push failed: {e}", M::db_name())), + None => Ok(()), + } + } + + /// Pull remote changes (sync mode only). + /// + /// Returns `true` if any changes were applied to the local database. + /// In local mode this is a no-op that returns `Ok(false)`. + pub fn pull(&self) -> Result { + match self.sync_db { + Some(ref sync_db) => self + .runtime + .block_on(sync_db.pull()) + .map_err(|e| anyhow::anyhow!("{} sync pull failed: {e}", M::db_name())), + None => Ok(false), + } + } + + /// Force a WAL checkpoint (sync mode only). + /// + /// In local mode this is a no-op that returns `Ok(())`. + pub fn checkpoint(&self) -> Result<()> { + match self.sync_db { + Some(ref sync_db) => self + .runtime + .block_on(sync_db.checkpoint()) + .map_err(|e| anyhow::anyhow!("{} sync checkpoint failed: {e}", M::db_name())), + None => Ok(()), + } + } + + /// Retrieve sync statistics (sync mode only). + /// + /// Returns `None` in local mode. + pub fn stats(&self) -> Result> { + match self.sync_db { + Some(ref sync_db) => self + .runtime + .block_on(sync_db.stats()) + .map(Some) + .map_err(|e| anyhow::anyhow!("{} sync stats failed: {e}", M::db_name())), + None => Ok(None), + } + } + + /// Returns `true` if the database is in sync (Turso Cloud) mode. + pub fn is_sync_mode(&self) -> bool { + self.sync_db.is_some() + } + /// Execute a SQL query that returns rows. /// /// # Arguments diff --git a/cli/src/services/mod.rs b/cli/src/services/mod.rs index 23267779..e88ecf06 100644 --- a/cli/src/services/mod.rs +++ b/cli/src/services/mod.rs @@ -23,5 +23,6 @@ pub mod resilience; pub mod security; pub mod setup; pub mod style; +pub mod sync; pub mod token_storage; pub mod version; diff --git a/cli/src/services/parse/command_runtime.rs b/cli/src/services/parse/command_runtime.rs index d5b63ff7..08316958 100644 --- a/cli/src/services/parse/command_runtime.rs +++ b/cli/src/services/parse/command_runtime.rs @@ -139,6 +139,10 @@ fn render_missing_subcommand_help(args: &[String]) -> Option Some(Box::new(services::help::command::HelpTextCommand { + name: services::sync::NAME.to_string(), + text: cli_schema::render_help_for_path(&[services::sync::NAME])?, + })), _ => None, } } @@ -244,9 +248,24 @@ fn convert_clap_command( }, })) } + cli_schema::Commands::Sync { ref subcommand } => convert_sync_subcommand(subcommand), } } +#[allow(clippy::unnecessary_wraps)] +fn convert_sync_subcommand( + subcommand: &cli_schema::SyncSubcommand, +) -> Result { + let subcommand = match subcommand { + cli_schema::SyncSubcommand::Push => services::sync::SyncSubcommand::Push, + cli_schema::SyncSubcommand::Pull => services::sync::SyncSubcommand::Pull, + }; + + Ok(Box::new(services::sync::command::SyncCommand { + subcommand, + })) +} + #[allow(clippy::unnecessary_wraps, clippy::needless_pass_by_value)] fn convert_auth_subcommand( subcommand: cli_schema::AuthSubcommand, diff --git a/cli/src/services/sync/command.rs b/cli/src/services/sync/command.rs new file mode 100644 index 00000000..0bd207a4 --- /dev/null +++ b/cli/src/services/sync/command.rs @@ -0,0 +1,28 @@ +use std::borrow::Cow; + +use crate::app::AppContext; +use crate::services::command_registry::{RuntimeCommand, RuntimeCommandHandle}; +use crate::services::error::ClassifiedError; +use crate::services::sync; + +pub struct SyncCommand { + pub subcommand: sync::SyncSubcommand, +} + +impl RuntimeCommand for SyncCommand { + fn name(&self) -> Cow<'_, str> { + Cow::Borrowed(sync::NAME) + } + + fn execute(&self, _context: &AppContext) -> Result { + sync::run_sync(self.subcommand).map_err(|error| ClassifiedError::runtime(error.to_string())) + } +} + +/// Construct a `SyncCommand` with the Push subcommand (used by the registry). +#[allow(dead_code)] +pub fn make_sync_command() -> RuntimeCommandHandle { + Box::new(SyncCommand { + subcommand: sync::SyncSubcommand::Push, + }) +} diff --git a/cli/src/services/sync/mod.rs b/cli/src/services/sync/mod.rs new file mode 100644 index 00000000..27ff5b55 --- /dev/null +++ b/cli/src/services/sync/mod.rs @@ -0,0 +1,43 @@ +pub mod command; + +use anyhow::{Context, Result}; + +use crate::services::agent_trace_db::AgentTraceDb; + +pub const NAME: &str = "sync"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SyncSubcommand { + Push, + Pull, +} + +/// Perform a push or pull sync operation on the Agent Trace database. +/// +/// Opens the Agent Trace DB (which auto-detects sync mode from env vars). +/// If the database is in sync mode, executes the requested operation. +/// If in local mode, returns an error indicating sync is not configured. +pub fn run_sync(subcommand: SyncSubcommand) -> Result { + let db = AgentTraceDb::new().context("failed to open Agent Trace DB for sync")?; + + if !db.is_sync_mode() { + return Err(anyhow::anyhow!( + "Sync is not configured. Set SCE_SYNC_URL and SCE_SYNC_TOKEN to enable sync mode." + )); + } + + match subcommand { + SyncSubcommand::Push => { + db.push().with_context(|| "sync push failed")?; + Ok("Pushed local changes to remote.".to_string()) + } + SyncSubcommand::Pull => { + let has_changes = db.pull().with_context(|| "sync pull failed")?; + if has_changes { + Ok("Pulled remote changes.".to_string()) + } else { + Ok("No remote changes to pull.".to_string()) + } + } + } +} diff --git a/context/architecture.md b/context/architecture.md index f0e4b920..d1f6da75 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -104,7 +104,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/capabilities.rs` defines the current broad CLI dependency-injection capability traits consumed by `AppContext`: `FsOps` with `StdFsOps` for filesystem operations and `GitOps` with `ProcessGitOps` for git command execution plus repository-root/hooks-directory resolution. Existing services do not consume these traits internally yet; doctor/setup/hooks/config migration is deferred to later lifecycle/AppContext tasks. - `cli/src/services/lifecycle.rs` defines the current compile-safe `ServiceLifecycle` trait seam. It has default no-op `diagnose(&AppContext)`, `fix(&AppContext, &[HealthProblem])`, and `setup(&AppContext)` methods, with lifecycle-owned health, fix, and setup result types so the trait contract is not publicly anchored to doctor/setup module types. The same module owns the shared lifecycle provider catalog/factory, returning providers in deterministic order (config → local_db → agent_trace_db → hooks when requested). Hooks exposes a `HooksLifecycle` provider in `cli/src/services/hooks/lifecycle.rs` for hook rollout diagnosis/fix/setup using lifecycle-owned health records plus the canonical required-hook installer. Config exposes a `ConfigLifecycle` provider in `cli/src/services/config/lifecycle.rs` for global/repo-local config validation and repo-local `.sce/config.json` bootstrap. local_db exposes a `LocalDbLifecycle` provider in `cli/src/services/local_db/lifecycle.rs` for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup. agent_trace_db exposes an `AgentTraceDbLifecycle` provider in `cli/src/services/agent_trace_db/lifecycle.rs` for canonical Agent Trace DB path health, parent-directory readiness/bootstrap, and `AgentTraceDb::new()` setup. Doctor runtime aggregates the full provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor report/fix records at the orchestration boundary; setup command aggregates the shared catalog for `setup` with hooks included only when requested and adapts hook setup outcomes before rendering setup-owned messages. - `cli/src/services/auth_command/mod.rs` defines the implemented auth command surface for `sce auth login|renew|logout|status`, including device-flow login, stored-token renewal (`--force` supported for renew), logout, and status rendering in text/JSON formats; `cli/src/services/auth_command/command.rs` owns the `AuthCommand` struct and its `RuntimeCommand` impl. -- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, and ordered embedded migrations, while `TursoDb` owns parent-directory creation, `Builder::new_local(...)` initialization, Turso connection setup, tokio current-thread runtime bridging, blocking `execute`/`query`/`query_map` wrappers, and generic migration execution with per-database `__sce_migrations` metadata. Existing DB files without migration metadata are upgraded by re-applying the current idempotent migration set and recording each migration ID, so setup/lifecycle initialization applies later migrations to already-created databases. The same module owns shared DB lifecycle helpers for path-health problem collection and DB parent-directory bootstrap. +- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, and ordered embedded migrations, while `TursoDb` supports dual-mode operation — local mode via `turso::Builder::new_local()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are absent, or sync (Turso Cloud) mode via `turso::sync::Builder::new_remote()` when both are set. It owns parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, blocking `execute`/`query`/`query_map` wrappers, generic migration execution with per-database `__sce_migrations` metadata, an `AtomicU64` write-counter for threshold-based auto-push in sync mode, sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode. Existing DB files without migration metadata are upgraded by re-applying the current idempotent migration set and recording each migration ID, so setup/lifecycle initialization applies later migrations to already-created databases. The same module owns shared DB lifecycle helpers for path-health problem collection and DB parent-directory bootstrap. - `cli/src/services/local_db/mod.rs` provides the concrete local DB spec and `LocalDb` type alias over the shared generic `TursoDb` adapter. `LocalDbSpec` resolves the deterministic persistent runtime DB target through the shared default-path seam and declares no local migrations; `TursoDb` supplies blocking `execute`/`query`, parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, and generic migration execution. - `cli/src/services/agent_trace_db/mod.rs` provides the Agent Trace DB spec and `AgentTraceDb` type alias over `TursoDb`. `AgentTraceDbSpec` resolves `/sce/agent-trace.db` through the shared default-path seam and embeds an ordered split fresh-start baseline migration set (`001_create_diff_traces`, `002_create_post_commit_patch_intersections`, `003_create_agent_traces`, `004_create_diff_traces_time_ms_id_index`, `005_create_agent_traces_agent_trace_id_index`) without `AUTOINCREMENT`; `agent_traces.agent_trace_id` is `NOT NULL UNIQUE` and indexed by `idx_agent_traces_agent_trace_id`. The module adds `DiffTraceInsert<'_>`/`insert_diff_trace()` (including `model_id`, `tool_name`, and nullable `tool_version` writes), `PostCommitPatchIntersectionInsert<'_>`/`insert_post_commit_patch_intersection()`, and `AgentTraceInsert<'_>`/`insert_agent_trace()` for parameterized writes plus `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` for inclusive chronological `diff_traces` reads that parse valid raw patch text and return skipped malformed-row reports. `cli/src/services/agent_trace_db/lifecycle.rs` registers Agent Trace DB setup/doctor lifecycle behavior; runtime writes come from `sce hooks diff-trace` (`diff_traces`) and `sce hooks post-commit` (`post_commit_patch_intersections` + built `agent_traces`). - `cli/src/test_support.rs` provides a shared test-only temp-directory helper (`TestTempDir`) used by service tests that need filesystem fixtures. @@ -116,7 +116,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/completion/mod.rs` defines completion parser/rendering contract (`parse_completion_request`, `render_completion`) with deterministic Bash/Zsh/Fish script output aligned to current parser-valid command/flag surfaces; `cli/src/services/completion/command.rs` owns the `CompletionCommand` struct and its `RuntimeCommand` impl. - `cli/src/services/hooks/mod.rs` defines the current local hook runtime parsing/dispatch (`HookSubcommand`, `run_hooks_subcommand`) plus a commit-msg co-author policy seam (`apply_commit_msg_coauthor_policy`) that injects one canonical SCE trailer only when the disabled-default attribution-hooks config/env control is enabled and `SCE_DISABLED` is false; `cli/src/services/hooks/command.rs` owns `HooksCommand` and its `RuntimeCommand` impl. In the current attribution-only baseline, `pre-commit` and `post-rewrite` are deterministic no-op surfaces; `post-commit` is an active intersection + Agent Trace persistence entrypoint (captures current commit patch, queries recent `diff_traces` from the bounded past-7-days window, combines valid patches via `patch::combine_patches`, intersects with post-commit patch via `patch::intersect_patches`, persists result to `post_commit_patch_intersections`, then persists built Agent Trace payloads to `agent_traces` in AgentTraceDb without post-commit file artifacts); while `diff-trace` performs STDIN JSON intake, validates required non-empty `sessionID`/`diff`/`model_id`/`tool_name` and required `tool_version` (present and either `null` or non-empty string) plus required `u64` `time` (Unix epoch milliseconds), rejects values that cannot fit AgentTraceDb signed `time_ms` storage, writes one collision-safe parsed-payload `context/tmp/-000000-diff-trace.json` artifact, and inserts the parsed payload fields into AgentTraceDb. Success requires both persistence paths to succeed. - `cli/src/services/resilience.rs` defines bounded retry/timeout/backoff execution policy (`RetryPolicy`, `run_with_retry`) for transient operation hardening with deterministic failure messaging and retry observability. -- No user-invocable `sce sync` command is wired in the current runtime; local DB and Agent Trace DB bootstrap flows through lifecycle providers aggregated by setup, and DB health/repair flows through the doctor surface. +- A user-invocable `sce sync push|pull` command is wired at `cli/src/services/sync/{mod,command}.rs` — it opens the Agent Trace DB and calls push/pull when sync mode is active. Sync for other databases and any additional sync subcommands remain deferred. - `cli/src/services/patch.rs` defines the standalone patch domain model (`ParsedPatch`, `PatchFileChange`, `FileChangeKind`, `PatchHunk`, `TouchedLine`, `TouchedLineKind`) for in-memory parsed unified-diff representation, capturing only touched lines (added/removed) plus minimal per-file/per-hunk metadata while excluding non-hunk headers and unchanged context lines. All types are `serde`-serializable/deserializable with `snake_case` JSON field naming. The module also provides `parse_patch`, a public parser function that converts raw unified-diff text (both `Index:` SVN-style and `diff --git` git-style formats) into `ParsedPatch` structs, with `ParseError` for actionable malformed-input diagnostics. Storage-agnostic JSON load helpers (`load_patch_from_json` for string input, `load_patch_from_json_bytes` for byte input) reconstruct `ParsedPatch` from serialized JSON content with `PatchLoadError` for actionable deserialization diagnostics. Its patch-set operations now include deterministic ordered combination plus target-shaped intersection that prefers exact touched-line matches and falls back to historical `kind`+`content` matching when incremental diffs and canonical post-commit diffs have drifted line numbers; `parse_patch`, `combine_patches`, and `intersect_patches` are consumed by the active post-commit hook runtime. - `cli/src/services/` contains module boundaries for command_registry, lifecycle, auth_command, config, setup, doctor, hooks, version, completion, help, patch, shared database infrastructure, local DB adapters, and Agent Trace DB adapters with explicit trait seams for future implementations. `cli/src/services/command_registry.rs` defines the `RuntimeCommand` trait, `RuntimeCommandHandle` type alias, `CommandRegistry` struct, and `build_default_registry()` function for the command dispatch registry. Service-owned command modules now own the migrated runtime command structs and `RuntimeCommand` impls for help/help-text, version, completion, auth, config, setup, doctor, and hooks. - `cli/README.md` is the crate-local onboarding and usage source of truth for placeholder behavior, safety limitations, and roadmap mapping back to service contracts. diff --git a/context/context-map.md b/context/context-map.md index e99c2124..66b6c077 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -41,8 +41,8 @@ Feature/domain context: - `context/sce/agent-trace-post-rewrite-local-remap-ingestion.md` (current post-rewrite no-op baseline plus historical remap-ingestion reference) - `context/sce/agent-trace-rewrite-trace-transformation.md` (current post-rewrite no-op baseline plus historical rewrite-transformation reference) - `context/sce/local-db.md` (implemented `cli/src/services/local_db/mod.rs` local database spec with `LocalDb = TursoDb`, canonical local DB path resolution, zero local migrations, and inherited blocking `execute`/`query` methods using the shared Turso adapter) -- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, sync `execute`/`query`/`query_map` wrappers, per-database `__sce_migrations` tracking, generic embedded migration execution, and current concrete wrappers for `LocalDb` plus `AgentTraceDb`) -- `context/sce/agent-trace-db.md` (implemented `cli/src/services/agent_trace_db/mod.rs` Agent Trace database wrapper with canonical `/sce/agent-trace.db` path, split fresh-start migration set `001..005` (no `AUTOINCREMENT`) defining `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `idx_diff_traces_time_ms_id`, and `idx_agent_traces_agent_trace_id`, with `agent_traces.agent_trace_id` enforced as `NOT NULL UNIQUE`; includes typed parameterized inserts, bounded chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, and active hook writers for `diff_traces` intake plus post-commit intersection/agent-trace persistence) +- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic dual-mode `TursoDb` supporting local-only and Turso Cloud sync modes, sync `execute`/`query`/`query_map` wrappers, explicit sync operations (`push`/`pull`/`checkpoint`/`stats`), per-database `__sce_migrations` tracking, generic embedded migration execution, and current concrete wrappers for `LocalDb` plus `AgentTraceDb`) +- `context/sce/agent-trace-db.md` (implemented `cli/src/services/agent_trace_db/mod.rs` Agent Trace database wrapper with canonical `/sce/agent-trace.db` path, split fresh-start migration set `001..005` (no `AUTOINCREMENT`) defining `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `idx_diff_traces_time_ms_id`, and `idx_agent_traces_agent_trace_id`, with `agent_traces.agent_trace_id` enforced as `NOT NULL UNIQUE`; includes typed parameterized inserts for diff traces with `model_id`, `tool_name`, and nullable `tool_version`, post-commit intersection rows, and built `agent_traces` rows with `agent_trace_id`, bounded chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, and active hook writers for `diff_traces` intake plus post-commit intersection/agent-trace persistence) - `context/sce/agent-trace-core-schema-migrations.md` (historical reference for removed local DB schema bootstrap behavior; T03 now implements the actual local DB with migrations) - `context/sce/agent-trace-retry-queue-observability.md` (inactive local-hook retry path plus historical retry/metrics reference) - `context/sce/agent-trace-local-hooks-mvp-contract-gap-matrix.md` (T01 Local Hooks MVP production contract freeze and deterministic gap matrix for `agent-trace-local-hooks-production-mvp`) @@ -57,6 +57,7 @@ Feature/domain context: - `context/sce/cli-release-artifact-contract.md` (shared `sce` release artifact naming, checksum/manifest outputs, GitHub Releases as the canonical artifact publication surface, and the current three-target Linux/macOS release workflow topology) - `context/sce/cli-npm-distribution-contract.md` (implemented `sce` npm launcher package, release-manifest/checksum-verified native binary install flow, the supported darwin/arm64 plus linux x64+arm64 npm platform matrix, and dedicated `.github/workflows/publish-npm.yml` downstream npm publish-stage contract) - `context/sce/cli-cargo-distribution-contract.md` (implemented `sce` Cargo publication posture plus supported crates.io, git, and local checkout install guidance, dedicated crates.io publish workflow, and ephemeral crate-local generated-asset mirror requirement for published builds) +- `cli/src/services/sync/` (implemented `sce sync push|pull` CLI command that opens the Agent Trace DB and syncs with Turso Cloud when sync mode is active; reports error when sync is not configured) Working areas: diff --git a/context/glossary.md b/context/glossary.md index 0f84b5f2..4c4f3cb6 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -34,9 +34,10 @@ - `Agent Trace SCE metadata`: Implementation-owned top-level metadata emitted by `build_agent_trace(...)` as `metadata.sce.version`; the value is sourced from the compiled `sce` CLI package version via `env!("CARGO_PKG_VERSION")`, is schema-validated with the rest of the payload, and is persisted in AgentTraceDb `agent_traces.trace_json` without changing the top-level Agent Trace payload/schema `version`. - `DiffTraceInsert`: Insert payload in `cli/src/services/agent_trace_db/mod.rs` carrying `time_ms`, `session_id`, `patch`, `model_id`, `tool_name`, and nullable `tool_version` for parameterized writes to the `diff_traces` table. - `DbSpec`: Service-specific database metadata trait in `cli/src/services/db/mod.rs` that supplies a diagnostic database name, canonical path resolver, and ordered embedded migration list for `TursoDb`. -- `TursoDb`: Generic shared Turso database adapter in `cli/src/services/db/mod.rs`; owns parent-directory creation, Turso local open/connect flow, tokio current-thread runtime bridging, synchronous `execute()`/`query()`/`query_map()` wrappers, per-database `__sce_migrations` metadata, and generic migration execution for a `DbSpec` implementation. +- `sce sync` CLI command: Implemented CLI command at `cli/src/services/sync/{mod,command}.rs` that opens the Agent Trace DB and calls `push()` or `pull()` when sync mode is active. Reports an error when sync is not configured (`SCE_SYNC_URL`+`SCE_SYNC_TOKEN` not set). Subcommands: `sce sync push`, `sce sync pull`. +- `TursoDb`: Generic shared Turso database adapter in `cli/src/services/db/mod.rs`; supports dual-mode operation (local via `turso::Builder::new_local()` or sync via `turso::sync::Builder::new_remote()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are set). Owns parent-directory creation, tokio current-thread runtime bridging, synchronous `execute()`/`query()`/`query_map()` wrappers, per-database `__sce_migrations` metadata, generic migration execution, write-counter auto-push in sync mode, and sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode. - `__sce_migrations`: Per-database migration metadata table created by `TursoDb::run_migrations()`; records applied migration IDs after successful execution so later setup/lifecycle initialization applies only migrations not yet recorded, while existing metadata-less DBs are brought forward by re-applying the current idempotent migration set and recording each ID. -- `sync command deferral`: Current plan/state note that a user-invocable `sce sync` command is not wired yet and is deferred to `0.4.0`; local DB and Agent Trace DB bootstrap now flow through lifecycle providers aggregated by the setup command, and DB health/repair flows through the doctor surface. +- `sync command deferral`: Note that a user-invocable `sce sync` command was originally deferred but was wired in T03 as `sce sync push|pull` — opens the Agent Trace DB and calls push/pull when sync mode is active. Sync for other databases remains deferred. - `CLI bounded resilience wrapper`: Shared policy in `cli/src/services/resilience.rs` (`RetryPolicy`, `run_with_retry`) that applies deterministic retries/timeouts/capped backoff to transient operations, emits retry observability events, and returns actionable terminal failure guidance. - `setup service orchestration`: Setup execution logic in `cli/src/services/setup/command.rs` that resolves the repository root, derives a repo-root-scoped `AppContext` from the runtime command context, aggregates `ServiceLifecycle::setup` calls across lifecycle providers (config → local_db → agent_trace_db → hooks when requested), handles interactive target selection for config asset installation, and emits deterministic success messaging per target. - `setup target flags`: Mutually-exclusive `sce setup` target selectors (`--opencode`, `--claude`, `--both`) that force non-interactive mode for automation. diff --git a/context/overview.md b/context/overview.md index 1ed5776d..417006e5 100644 --- a/context/overview.md +++ b/context/overview.md @@ -25,7 +25,7 @@ The shared default path service in `cli/src/services/default_paths.rs` is now th The same config resolver now also owns the attribution-hooks gate used by local hook runtime: `SCE_ATTRIBUTION_HOOKS_ENABLED` overrides `policies.attribution_hooks.enabled`, and the gate defaults to disabled. Generated config now includes repo-local OpenCode plugin assets for both profiles: `sce-bash-policy.ts` plus `sce-agent-trace.ts` are emitted under `config/.opencode/plugins/` and `config/automated/.opencode/plugins/`; the agent-trace plugin extracts `{ sessionID, diff, time, model_id }` from user `message.updated` events with diffs, tracks per-session OpenCode client version from `session.created`/`session.updated`, and sends payloads to `sce hooks diff-trace` with `tool_name="opencode"` plus optional `tool_version`; the Rust hook continues to validate required fields and persists `model_id`, `tool_name`, and nullable `tool_version` into `diff_traces` through AgentTraceDb. Bash-policy also emits shared runtime logic and preset data under `config/.opencode/lib/` (also emitted for `config/automated/.opencode/**`). Claude bash-policy enforcement has been removed from generated outputs. The `doctor` command now exposes explicit inspection mode (`sce doctor`) and repair-intent mode (`sce doctor --fix`) at the CLI/help/schema level while keeping diagnosis mode read-only. It now validates both current global operator health and the current repo/hook-integrity slice: state-root resolution, global config path resolution, global and repo-local `sce/config.json` readability/schema validity, local DB and Agent Trace DB path + health, DB parent-directory readiness, git availability, non-repo vs bare-repo targeting failures, effective git hook-path source (default, per-repo `core.hooksPath`, or global `core.hooksPath`), hooks-directory health, required hook presence/executable permissions/content drift against canonical embedded SCE-managed hook assets, and repo-root OpenCode integration presence across the installed `plugins`, `agents`, `commands`, and `skills` inventories with embedded SHA-256 content verification for OpenCode assets. Text mode now renders the approved human-only layout with ordered `Environment` / `Configuration` / `Repository` / `Git Hooks` / `Integrations` sections, `SCE doctor diagnose` / `SCE doctor fix` headers, bracketed `[PASS]`/`[FAIL]`/`[MISS]` status tokens, shared-style green pass plus red fail/miss coloring when color output is enabled, simplified `label (path)` row formatting, top-level-only hook rows, and integration parent/child rows that reflect missing vs content-mismatch states; JSON output now reports Agent Trace DB health under `agent_trace_db` (as a row within the Configuration section in text mode). Repo-scoped database reporting is empty by default because no repo-owned SCE database currently exists. Fix mode reuses the canonical setup hook install flow to repair missing/stale/non-executable required hooks and can also bootstrap missing canonical DB parent directories while preserving manual-only guidance for unsupported issues. -Local database bootstrap is now owned by `LocalDbLifecycle::setup` and `AgentTraceDbLifecycle::setup` aggregated by the setup command, while doctor validates both DB paths/health and can bootstrap missing parent directories. Wiring a user-invocable `sce sync` command is deferred to `0.4.0`. +Local database bootstrap is now owned by `LocalDbLifecycle::setup` and `AgentTraceDbLifecycle::setup` aggregated by the setup command, while doctor validates both DB paths/health and can bootstrap missing parent directories. A user-invocable `sce sync push|pull` command is now wired at `cli/src/services/sync/` — it opens the Agent Trace DB and calls push/pull when sync mode is active. Sync for other databases remains deferred. The repository-root flake (`flake.nix`) now applies a Rust overlay-backed stable toolchain pinned to `1.93.1` (with `rustfmt` and `clippy`), reads package/check version from the repo-root `.version` file, builds `packages.sce` through a Crane `buildDepsOnly` + `buildPackage` pipeline with filtered package sources for the Cargo tree plus required embedded config/assets, and runs `cli-tests`, `cli-clippy`, and `cli-fmt` through Crane-backed check derivations (`cargoTest`, `cargoClippy`, `cargoFmt`) that reuse the same filtered source/toolchain setup. The root flake also exposes release install/run outputs directly as `packages.sce` (with `packages.default = packages.sce`) plus `apps.sce` and `apps.default`, so `nix build .#default`, `nix run . -- --help`, `nix run .#sce -- --help`, and `nix profile install github:crocoder-dev/shared-context-engineering` all target the packaged `sce` binary through the same flake-owned entrypoints. The CLI Cargo package metadata now includes crates.io publication-ready fields with crate-local install guidance in `cli/README.md`; supported Cargo install paths are `cargo install shared-context-engineering --locked`, `cargo install --git https://github.com/crocoder-dev/shared-context-engineering shared-context-engineering --locked`, and local `cargo install --path cli --locked`. The published crate installs the `sce` binary. The crate also keeps `cargo clippy --manifest-path cli/Cargo.toml` warnings-denied through `cli/Cargo.toml` lint configuration, so an extra `-- -D warnings` flag is redundant. @@ -46,7 +46,7 @@ The targeted support commands (`handover`, `commit`, `validate`) keep their thin The prior no-git-wrapper Agent Trace design artifacts under `context/sce/agent-trace-*.md` are retained only as historical reference; the current CLI runtime no longer wires the removed Agent Trace schema adaptation, payload building, retry replay, or rewrite handling paths into local hook execution. The hooks service now uses a minimal attribution-only runtime: `commit-msg` is the only hook that mutates behavior, conditionally injecting exactly one canonical SCE trailer when the attribution-hooks gate is enabled and `SCE_DISABLED` is false; `pre-commit` and `post-rewrite` remain deterministic no-op entrypoints; `post-commit` is an active intersection entrypoint that captures current commit patch, queries recent `diff_traces` from past 7 days, combines/intersects patches, persists intersection metadata to `post_commit_patch_intersections`, and persists the schema-validated built Agent Trace payload, including optional top-level `tool` metadata from recent diff-trace rows and top-level `metadata.sce.version` from the compiled `sce` CLI package version, to AgentTraceDb `agent_traces` (DB-only, no post-commit Agent Trace file artifact); and `diff-trace` currently validates/persists required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required `tool_version` (must be present and either `null` or a non-empty string), plus required `u64` millisecond `time`, with non-lossy AgentTraceDb `time_ms` conversion and collision-safe timestamp+attempt artifact filenames. The CLI now also includes an approved operator-environment doctor contract documented in `context/sce/agent-trace-hook-doctor.md`; the runtime now matches the implemented T06 slice for `sce doctor --fix` parsing/help, stable problem/fix-result reporting, canonical hook-repair reuse, and bounded doctor-owned local-DB directory bootstrap for the missing SCE-owned DB parent path. -The local DB service now provides `LocalDb` as a thin `TursoDb` alias in `cli/src/services/local_db/mod.rs`; `LocalDbSpec` resolves the canonical local DB path from the shared default-path catalog and currently declares zero migrations. Shared Turso infrastructure lives in `cli/src/services/db/mod.rs`, where `DbSpec` and generic `TursoDb` support dual-mode operation — local mode via `turso::Builder::new_local()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are absent, or sync (Turso Cloud) mode via `turso::sync::Builder::new_remote()` when both are set. It owns parent-directory creation, connection setup, tokio current-thread runtime bridging, synchronous `execute`/`query`/`query_map`, generic migration execution, sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode (sync is never triggered automatically from `execute()`), and shared DB lifecycle helpers for service-specific database wrappers. Agent Trace persistence now has its own `cli/src/services/agent_trace_db/mod.rs` wrapper, canonical `/sce/agent-trace.db` path, a split fresh-start baseline migration set (`001..005`) covering `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, and indexes (`idx_diff_traces_time_ms_id`, `idx_agent_traces_agent_trace_id`) without `AUTOINCREMENT`, plus `agent_traces.agent_trace_id` as `NOT NULL UNIQUE`; it also provides typed parameterized insert helpers for diff traces, post-commit intersection rows, and built agent-trace rows, chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, active `sce hooks diff-trace` writes for `diff_traces`, and active `sce hooks post-commit` writes for built `agent_traces` payloads. +The local DB service now provides `LocalDb` as a thin `TursoDb` alias in `cli/src/services/local_db/mod.rs`; `LocalDbSpec` resolves the canonical local DB path from the shared default-path catalog and currently declares zero migrations. Shared Turso infrastructure lives in `cli/src/services/db/mod.rs`, where `DbSpec` and generic `TursoDb` support dual-mode operation — local mode via `turso::Builder::new_local()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are absent, or sync (Turso Cloud) mode via `turso::sync::Builder::new_remote()` when both are set. It owns parent-directory creation, connection setup, tokio current-thread runtime bridging, synchronous `execute`/`query`/`query_map`, generic migration execution, explicit sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode, and shared DB lifecycle helpers for service-specific database wrappers. Agent Trace persistence now has its own `cli/src/services/agent_trace_db/mod.rs` wrapper, canonical `/sce/agent-trace.db` path, a split fresh-start baseline migration set (`001..005`) covering `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, and indexes (`idx_diff_traces_time_ms_id`, `idx_agent_traces_agent_trace_id`) without `AUTOINCREMENT`, plus `agent_traces.agent_trace_id` as `NOT NULL UNIQUE`; it also provides typed parameterized insert helpers for diff traces including `model_id`, `tool_name`, and nullable `tool_version`, post-commit intersection rows, and built agent-trace rows with `agent_trace_id`, chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, active `sce hooks diff-trace` writes for `diff_traces`, and active `sce hooks post-commit` writes for built `agent_traces` payloads. The hooks command surface now also supports concrete runtime subcommand routing (`pre-commit`, `commit-msg`, `post-commit`, `post-rewrite`, `diff-trace`) with deterministic argument/STDIN validation. Current runtime behavior keeps attribution disabled by default: the attribution gate enables canonical trailer insertion in `commit-msg`, `pre-commit`/`post-rewrite` remain deterministic no-ops, `post-commit` is the active bounded recent-diff-trace intersection path, and `diff-trace` is the active intake path for parsed STDIN `{ sessionID, diff, time, model_id, tool_name, tool_version }` payload persistence with required non-empty `tool_name`, required nullable/non-empty `tool_version`, required `u64` millisecond `time`, non-lossy AgentTraceDb `time_ms` conversion, and collision-safe timestamp+attempt artifact filenames. This behavior is documented in `context/sce/agent-trace-hooks-command-routing.md`. The setup service now also exposes deterministic required-hook embedded asset accessors (`iter_required_hook_assets`, `get_required_hook_asset`) backed by canonical templates in `cli/assets/hooks/` for `pre-commit`, `commit-msg`, and `post-commit`; this behavior is documented in `context/sce/setup-githooks-hook-asset-packaging.md`. The setup service now also includes required-hook install orchestration (`install_required_git_hooks`) that resolves repository root and effective hooks path from git truth, enforces deterministic per-hook outcomes (`Installed`/`Updated`/`Skipped`), and uses a unified remove-and-replace policy that removes existing hooks before swapping staged content with deterministic recovery guidance on swap failures; this behavior is documented in `context/sce/setup-githooks-install-flow.md`. diff --git a/context/plans/turso-sync-adapter.md b/context/plans/turso-sync-adapter.md index 472414b0..cfdbadae 100644 --- a/context/plans/turso-sync-adapter.md +++ b/context/plans/turso-sync-adapter.md @@ -22,7 +22,8 @@ The initial concrete target is the Agent Trace DB. Sync works in two phases: ## Constraints and non-goals - **In scope:** `cli/Cargo.toml` feature enablement, env var constants in the shared config seam, `TursoDb` structural changes for dual-mode, Agent Trace DB auto-pull, context sync. -- **Out of scope:** A user-invocable `sce sync` CLI command (deferred to a later plan). Sync for `LocalDb`. Periodic/background sync scheduling. Token rotation or complex auth flows beyond env var string. Multi-database sync orchestration. +- **Out of scope:** Sync for `LocalDb`. Periodic/background sync scheduling. Token rotation or complex auth flows beyond env var string. Multi-database sync orchestration. +- **Note:** A user-invocable `sce sync` CLI command was originally deferred but wired in T03 as `sce sync push|pull` — opens the Agent Trace DB and calls push/pull when sync mode is active. Sync for other databases remains deferred. - **Assumption:** `turso::sync::Builder::with_auth_token()` accepts a static `&str` (the `AuthTokenFn` in turso 0.6.0 supports both string literals and closures). - **Assumption:** `turso::sync::Database::connect()` is async, while `turso::Database::connect()` is synchronous — the TursoDb adapter handles both cases inside the existing `block_on` bridge. - **Assumption:** The write-counter push uses an `AtomicU64` on `TursoDb`; the counter is checked and pushed synchronously inside `execute()` via `block_on`, so background threads are not needed. @@ -53,7 +54,7 @@ The initial concrete target is the Agent Trace DB. Sync works in two phases: - **Evidence:** `nix flake check` — all 4 checks passed. `rg` confirms all three constants present. - **Notes:** Added `#[allow(dead_code)]` to each constant since T03 will consume them. Followed existing `pub(crate) const ENV_*` pattern. -- [ ] T03: `Extend TursoDb adapter with dual-mode (local/sync) support plus write-counter auto-push` (status: todo) +- [x] T03: `Extend TursoDb adapter with dual-mode (local/sync) support plus write-counter auto-push` (status: done) - Task ID: T03 - Goal: Modify `cli/src/services/db/mod.rs` so that `TursoDb` optionally carries sync state and uses `turso::sync::Builder::new_remote()` when both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` are set. Add an `AtomicU64` write counter; `execute()` increments the counter and auto-pushes when the threshold (from `SCE_SYNC_PUSH_THRESHOLD`, default 10) is reached. Expose `push()`, `pull()`, `checkpoint()`, and `stats()` methods that delegate to sync operations in sync mode and are no-ops in local mode. - Boundaries (in/out of scope): @@ -66,6 +67,10 @@ The initial concrete target is the Agent Trace DB. Sync works in two phases: - `push()`/`pull()`/`checkpoint()`/`stats()` compile and behave correctly in both modes. - `cargo check` and `nix flake check` pass. - Verification notes (commands or checks): `nix flake check`; manual inspection of `db/mod.rs` conditional builder logic, counter increment in `execute()`, and best-effort push error handling. + - **Completed:** 2026-05-19 + - **Files changed:** `cli/src/services/db/mod.rs` + - **Evidence:** `nix flake check` — all 4 checks passed (cli-tests, cli-clippy, cli-fmt, pkl-parity). + - **Notes:** Added `sync_db: Option` to `TursoDb`. `new()` reads `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` env vars — when both present, opens via `turso::sync::Builder::new_remote()` (async via existing `block_on` bridge); when either absent, uses existing local-only path. No auto-push in `execute()` — sync is explicit via `TursoDb::push()`/`pull()` or the `sce sync` CLI command. Added `push()`, `pull()`, `checkpoint()`, `stats()`, and `is_sync_mode()` methods. Removed `write_count`, `push_threshold`, and `SYNC_PUSH_THRESHOLD_ENV_KEY` constant. Wired `sce sync push|pull` as a new CLI command in `cli/src/services/sync/{mod,command}.rs` with full clap schema, registry, and parse conversion. - [ ] T04: `Wire pull-on-setup into Agent Trace DB lifecycle` (status: todo) - Task ID: T04 diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 630441c6..b5147d04 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -1,6 +1,6 @@ # Shared Turso Database Infrastructure -`cli/src/services/db/mod.rs` provides the shared Turso database adapter seam for CLI services that need local Turso-backed persistence. +`cli/src/services/db/mod.rs` provides the shared Turso database adapter seam for CLI services that need Turso-backed persistence. ## Contract @@ -8,16 +8,34 @@ - `db_name()` returns a human-readable diagnostic name. - `db_path()` resolves the canonical database file path. - `migrations()` returns ordered embedded migration `(id, sql)` pairs. -- `TursoDb`: generic adapter that owns: - - tokio current-thread runtime creation - - Turso local database open/connect flow - - parent-directory creation - - synchronous `execute()`, `query()`, and row-mapping `query_map()` wrappers - - generic embedded migration execution through `run_migrations()` with per-database `__sce_migrations` metadata +- `TursoDb`: generic adapter that supports two modes: + - **Local mode** (default): opens a plain Turso file database via `turso::Builder::new_local()`. + - **Sync mode**: opens a Turso Cloud synced database via `turso::sync::Builder::new_remote()` when both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` env vars are set. + - Owns tokio current-thread runtime creation, parent-directory creation, synchronous `execute()`, `query()`, and row-mapping `query_map()` wrappers, and generic embedded migration execution through `run_migrations()` with per-database `__sce_migrations` metadata. + - Sync-mode methods: + - `push()` — pushes local changes to the remote (no-op in local mode). + - `pull()` — pulls remote changes, returns `true` if changes applied (returns `false` in local mode). + - `checkpoint()` — forces a WAL checkpoint (no-op in local mode). + - `stats()` — returns `Option` (returns `None` in local mode). + - Explicit sync: Sync operations are never triggered automatically from `execute()`. Callers use `push()`/`pull()` directly, or the `sce sync push|pull` CLI command. Push failures from explicit calls propagate to the caller. - Shared lifecycle helpers: - - `collect_db_path_health()` emits common parent/path health problems for DB-backed services. + - `collect_db_path_health()` emits common parent/path health problems for DB-backed services (sync mode uses the same local file path, so health checks are identical). - `bootstrap_db_parent()` creates the resolved DB parent directory for repair/setup flows. +## Sync env var reference + +| Env var | Purpose | Default | +|---|---|---| +| `SCE_SYNC_URL` | Remote Turso Cloud URL | unset (triggers local mode) | +| `SCE_SYNC_TOKEN` | Auth token for remote access | unset (triggers local mode) | + +When both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` are set, the database opens in sync mode. When either is absent, local-only mode is used. + +## Internal struct fields + +- `conn: turso::Connection` — active database connection (shared between modes). +- `sync_db: Option` — sync handle for push/pull/checkpoint/stats; `None` in local mode. + ## Current integration state The shared module is exported from `cli/src/services/mod.rs` and compile-checked. Current concrete wrappers: From f6bc990824df67cef2549d443047cd2fc078e559 Mon Sep 17 00:00:00 2001 From: David Abram Date: Tue, 19 May 2026 20:32:31 +0200 Subject: [PATCH 4/5] agent_trace_db: Wire best-effort pull-on-setup when sync mode is active MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AgentTraceDbLifecycle::setup now captures the AgentTraceDb::new() result and performs an initial pull() when sync mode is active (both SCE_SYNC_URL and SCE_SYNC_TOKEN set). The pull is best-effort — remote failure is logged via tracing::warn! and setup continues without error. Co-authored-by: SCE --- cli/src/services/agent_trace_db/lifecycle.rs | 9 +++- context/architecture.md | 2 +- context/context-map.md | 4 +- context/glossary.md | 2 +- context/overview.md | 2 +- context/plans/turso-sync-adapter.md | 53 +++++++++++++++----- context/sce/agent-trace-db.md | 2 +- 7 files changed, 55 insertions(+), 19 deletions(-) diff --git a/cli/src/services/agent_trace_db/lifecycle.rs b/cli/src/services/agent_trace_db/lifecycle.rs index 8dced626..6b84afca 100644 --- a/cli/src/services/agent_trace_db/lifecycle.rs +++ b/cli/src/services/agent_trace_db/lifecycle.rs @@ -51,8 +51,15 @@ impl ServiceLifecycle for AgentTraceDbLifecycle { } fn setup(&self, _ctx: &AppContext) -> Result { - AgentTraceDb::new() + let db = AgentTraceDb::new() .context("Agent trace DB lifecycle setup failed while initializing agent trace DB")?; + + if db.is_sync_mode() { + if let Err(e) = db.pull() { + tracing::warn!("Agent trace DB initial sync pull failed (setup continues): {e}"); + } + } + Ok(SetupOutcome::default()) } } diff --git a/context/architecture.md b/context/architecture.md index d1f6da75..de9b46c3 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -104,7 +104,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/capabilities.rs` defines the current broad CLI dependency-injection capability traits consumed by `AppContext`: `FsOps` with `StdFsOps` for filesystem operations and `GitOps` with `ProcessGitOps` for git command execution plus repository-root/hooks-directory resolution. Existing services do not consume these traits internally yet; doctor/setup/hooks/config migration is deferred to later lifecycle/AppContext tasks. - `cli/src/services/lifecycle.rs` defines the current compile-safe `ServiceLifecycle` trait seam. It has default no-op `diagnose(&AppContext)`, `fix(&AppContext, &[HealthProblem])`, and `setup(&AppContext)` methods, with lifecycle-owned health, fix, and setup result types so the trait contract is not publicly anchored to doctor/setup module types. The same module owns the shared lifecycle provider catalog/factory, returning providers in deterministic order (config → local_db → agent_trace_db → hooks when requested). Hooks exposes a `HooksLifecycle` provider in `cli/src/services/hooks/lifecycle.rs` for hook rollout diagnosis/fix/setup using lifecycle-owned health records plus the canonical required-hook installer. Config exposes a `ConfigLifecycle` provider in `cli/src/services/config/lifecycle.rs` for global/repo-local config validation and repo-local `.sce/config.json` bootstrap. local_db exposes a `LocalDbLifecycle` provider in `cli/src/services/local_db/lifecycle.rs` for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup. agent_trace_db exposes an `AgentTraceDbLifecycle` provider in `cli/src/services/agent_trace_db/lifecycle.rs` for canonical Agent Trace DB path health, parent-directory readiness/bootstrap, and `AgentTraceDb::new()` setup. Doctor runtime aggregates the full provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor report/fix records at the orchestration boundary; setup command aggregates the shared catalog for `setup` with hooks included only when requested and adapts hook setup outcomes before rendering setup-owned messages. - `cli/src/services/auth_command/mod.rs` defines the implemented auth command surface for `sce auth login|renew|logout|status`, including device-flow login, stored-token renewal (`--force` supported for renew), logout, and status rendering in text/JSON formats; `cli/src/services/auth_command/command.rs` owns the `AuthCommand` struct and its `RuntimeCommand` impl. -- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, and ordered embedded migrations, while `TursoDb` supports dual-mode operation — local mode via `turso::Builder::new_local()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are absent, or sync (Turso Cloud) mode via `turso::sync::Builder::new_remote()` when both are set. It owns parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, blocking `execute`/`query`/`query_map` wrappers, generic migration execution with per-database `__sce_migrations` metadata, an `AtomicU64` write-counter for threshold-based auto-push in sync mode, sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode. Existing DB files without migration metadata are upgraded by re-applying the current idempotent migration set and recording each migration ID, so setup/lifecycle initialization applies later migrations to already-created databases. The same module owns shared DB lifecycle helpers for path-health problem collection and DB parent-directory bootstrap. +- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, and ordered embedded migrations, while `TursoDb` supports dual-mode operation — local mode via `turso::Builder::new_local()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are absent, or sync (Turso Cloud) mode via `turso::sync::Builder::new_remote()` when both are set. It owns parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, blocking `execute`/`query`/`query_map` wrappers, generic migration execution with per-database `__sce_migrations` metadata, sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode (sync is never triggered automatically from `execute()`). Existing DB files without migration metadata are upgraded by re-applying the current idempotent migration set and recording each migration ID, so setup/lifecycle initialization applies later migrations to already-created databases. The same module owns shared DB lifecycle helpers for path-health problem collection and DB parent-directory bootstrap. - `cli/src/services/local_db/mod.rs` provides the concrete local DB spec and `LocalDb` type alias over the shared generic `TursoDb` adapter. `LocalDbSpec` resolves the deterministic persistent runtime DB target through the shared default-path seam and declares no local migrations; `TursoDb` supplies blocking `execute`/`query`, parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, and generic migration execution. - `cli/src/services/agent_trace_db/mod.rs` provides the Agent Trace DB spec and `AgentTraceDb` type alias over `TursoDb`. `AgentTraceDbSpec` resolves `/sce/agent-trace.db` through the shared default-path seam and embeds an ordered split fresh-start baseline migration set (`001_create_diff_traces`, `002_create_post_commit_patch_intersections`, `003_create_agent_traces`, `004_create_diff_traces_time_ms_id_index`, `005_create_agent_traces_agent_trace_id_index`) without `AUTOINCREMENT`; `agent_traces.agent_trace_id` is `NOT NULL UNIQUE` and indexed by `idx_agent_traces_agent_trace_id`. The module adds `DiffTraceInsert<'_>`/`insert_diff_trace()` (including `model_id`, `tool_name`, and nullable `tool_version` writes), `PostCommitPatchIntersectionInsert<'_>`/`insert_post_commit_patch_intersection()`, and `AgentTraceInsert<'_>`/`insert_agent_trace()` for parameterized writes plus `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` for inclusive chronological `diff_traces` reads that parse valid raw patch text and return skipped malformed-row reports. `cli/src/services/agent_trace_db/lifecycle.rs` registers Agent Trace DB setup/doctor lifecycle behavior; runtime writes come from `sce hooks diff-trace` (`diff_traces`) and `sce hooks post-commit` (`post_commit_patch_intersections` + built `agent_traces`). - `cli/src/test_support.rs` provides a shared test-only temp-directory helper (`TestTempDir`) used by service tests that need filesystem fixtures. diff --git a/context/context-map.md b/context/context-map.md index 66b6c077..d9291d75 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -41,8 +41,8 @@ Feature/domain context: - `context/sce/agent-trace-post-rewrite-local-remap-ingestion.md` (current post-rewrite no-op baseline plus historical remap-ingestion reference) - `context/sce/agent-trace-rewrite-trace-transformation.md` (current post-rewrite no-op baseline plus historical rewrite-transformation reference) - `context/sce/local-db.md` (implemented `cli/src/services/local_db/mod.rs` local database spec with `LocalDb = TursoDb`, canonical local DB path resolution, zero local migrations, and inherited blocking `execute`/`query` methods using the shared Turso adapter) -- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic dual-mode `TursoDb` supporting local-only and Turso Cloud sync modes, sync `execute`/`query`/`query_map` wrappers, explicit sync operations (`push`/`pull`/`checkpoint`/`stats`), per-database `__sce_migrations` tracking, generic embedded migration execution, and current concrete wrappers for `LocalDb` plus `AgentTraceDb`) -- `context/sce/agent-trace-db.md` (implemented `cli/src/services/agent_trace_db/mod.rs` Agent Trace database wrapper with canonical `/sce/agent-trace.db` path, split fresh-start migration set `001..005` (no `AUTOINCREMENT`) defining `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `idx_diff_traces_time_ms_id`, and `idx_agent_traces_agent_trace_id`, with `agent_traces.agent_trace_id` enforced as `NOT NULL UNIQUE`; includes typed parameterized inserts for diff traces with `model_id`, `tool_name`, and nullable `tool_version`, post-commit intersection rows, and built `agent_traces` rows with `agent_trace_id`, bounded chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, and active hook writers for `diff_traces` intake plus post-commit intersection/agent-trace persistence) +- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic dual-mode `TursoDb` supporting local-only and Turso Cloud sync modes, synchronous `execute`/`query`/`query_map` wrappers, explicit sync operations (`push`/`pull`/`checkpoint`/`stats`) that are never triggered automatically by `execute()`, per-database `__sce_migrations` tracking, generic embedded migration execution, and current concrete wrappers for `LocalDb` plus `AgentTraceDb`) +- `context/sce/agent-trace-db.md` (implemented `cli/src/services/agent_trace_db/mod.rs` Agent Trace database wrapper with canonical `/sce/agent-trace.db` path, split fresh-start migration set `001..005` (no `AUTOINCREMENT`) defining `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `idx_diff_traces_time_ms_id`, and `idx_agent_traces_agent_trace_id`, with `agent_traces.agent_trace_id` enforced as `NOT NULL UNIQUE`; includes typed parameterized inserts for diff traces with `model_id`, `tool_name`, and nullable `tool_version`, post-commit intersection rows, and built `agent_traces` rows with `agent_trace_id`, bounded chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider that performs a best-effort `pull()` during setup when sync mode is active, and active hook writers for `diff_traces` intake plus post-commit intersection/agent-trace persistence) - `context/sce/agent-trace-core-schema-migrations.md` (historical reference for removed local DB schema bootstrap behavior; T03 now implements the actual local DB with migrations) - `context/sce/agent-trace-retry-queue-observability.md` (inactive local-hook retry path plus historical retry/metrics reference) - `context/sce/agent-trace-local-hooks-mvp-contract-gap-matrix.md` (T01 Local Hooks MVP production contract freeze and deterministic gap matrix for `agent-trace-local-hooks-production-mvp`) diff --git a/context/glossary.md b/context/glossary.md index 4c4f3cb6..a89c426a 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -35,7 +35,7 @@ - `DiffTraceInsert`: Insert payload in `cli/src/services/agent_trace_db/mod.rs` carrying `time_ms`, `session_id`, `patch`, `model_id`, `tool_name`, and nullable `tool_version` for parameterized writes to the `diff_traces` table. - `DbSpec`: Service-specific database metadata trait in `cli/src/services/db/mod.rs` that supplies a diagnostic database name, canonical path resolver, and ordered embedded migration list for `TursoDb`. - `sce sync` CLI command: Implemented CLI command at `cli/src/services/sync/{mod,command}.rs` that opens the Agent Trace DB and calls `push()` or `pull()` when sync mode is active. Reports an error when sync is not configured (`SCE_SYNC_URL`+`SCE_SYNC_TOKEN` not set). Subcommands: `sce sync push`, `sce sync pull`. -- `TursoDb`: Generic shared Turso database adapter in `cli/src/services/db/mod.rs`; supports dual-mode operation (local via `turso::Builder::new_local()` or sync via `turso::sync::Builder::new_remote()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are set). Owns parent-directory creation, tokio current-thread runtime bridging, synchronous `execute()`/`query()`/`query_map()` wrappers, per-database `__sce_migrations` metadata, generic migration execution, write-counter auto-push in sync mode, and sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode. +- `TursoDb`: Generic shared Turso database adapter in `cli/src/services/db/mod.rs`; supports dual-mode operation (local via `turso::Builder::new_local()` or sync via `turso::sync::Builder::new_remote()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are set). Owns parent-directory creation, tokio current-thread runtime bridging, synchronous `execute()`/`query()`/`query_map()` wrappers, per-database `__sce_migrations` metadata, generic migration execution, and explicit sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode (sync is never triggered automatically from `execute()`). - `__sce_migrations`: Per-database migration metadata table created by `TursoDb::run_migrations()`; records applied migration IDs after successful execution so later setup/lifecycle initialization applies only migrations not yet recorded, while existing metadata-less DBs are brought forward by re-applying the current idempotent migration set and recording each ID. - `sync command deferral`: Note that a user-invocable `sce sync` command was originally deferred but was wired in T03 as `sce sync push|pull` — opens the Agent Trace DB and calls push/pull when sync mode is active. Sync for other databases remains deferred. - `CLI bounded resilience wrapper`: Shared policy in `cli/src/services/resilience.rs` (`RetryPolicy`, `run_with_retry`) that applies deterministic retries/timeouts/capped backoff to transient operations, emits retry observability events, and returns actionable terminal failure guidance. diff --git a/context/overview.md b/context/overview.md index 417006e5..db171e4a 100644 --- a/context/overview.md +++ b/context/overview.md @@ -46,7 +46,7 @@ The targeted support commands (`handover`, `commit`, `validate`) keep their thin The prior no-git-wrapper Agent Trace design artifacts under `context/sce/agent-trace-*.md` are retained only as historical reference; the current CLI runtime no longer wires the removed Agent Trace schema adaptation, payload building, retry replay, or rewrite handling paths into local hook execution. The hooks service now uses a minimal attribution-only runtime: `commit-msg` is the only hook that mutates behavior, conditionally injecting exactly one canonical SCE trailer when the attribution-hooks gate is enabled and `SCE_DISABLED` is false; `pre-commit` and `post-rewrite` remain deterministic no-op entrypoints; `post-commit` is an active intersection entrypoint that captures current commit patch, queries recent `diff_traces` from past 7 days, combines/intersects patches, persists intersection metadata to `post_commit_patch_intersections`, and persists the schema-validated built Agent Trace payload, including optional top-level `tool` metadata from recent diff-trace rows and top-level `metadata.sce.version` from the compiled `sce` CLI package version, to AgentTraceDb `agent_traces` (DB-only, no post-commit Agent Trace file artifact); and `diff-trace` currently validates/persists required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required `tool_version` (must be present and either `null` or a non-empty string), plus required `u64` millisecond `time`, with non-lossy AgentTraceDb `time_ms` conversion and collision-safe timestamp+attempt artifact filenames. The CLI now also includes an approved operator-environment doctor contract documented in `context/sce/agent-trace-hook-doctor.md`; the runtime now matches the implemented T06 slice for `sce doctor --fix` parsing/help, stable problem/fix-result reporting, canonical hook-repair reuse, and bounded doctor-owned local-DB directory bootstrap for the missing SCE-owned DB parent path. -The local DB service now provides `LocalDb` as a thin `TursoDb` alias in `cli/src/services/local_db/mod.rs`; `LocalDbSpec` resolves the canonical local DB path from the shared default-path catalog and currently declares zero migrations. Shared Turso infrastructure lives in `cli/src/services/db/mod.rs`, where `DbSpec` and generic `TursoDb` support dual-mode operation — local mode via `turso::Builder::new_local()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are absent, or sync (Turso Cloud) mode via `turso::sync::Builder::new_remote()` when both are set. It owns parent-directory creation, connection setup, tokio current-thread runtime bridging, synchronous `execute`/`query`/`query_map`, generic migration execution, explicit sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode, and shared DB lifecycle helpers for service-specific database wrappers. Agent Trace persistence now has its own `cli/src/services/agent_trace_db/mod.rs` wrapper, canonical `/sce/agent-trace.db` path, a split fresh-start baseline migration set (`001..005`) covering `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, and indexes (`idx_diff_traces_time_ms_id`, `idx_agent_traces_agent_trace_id`) without `AUTOINCREMENT`, plus `agent_traces.agent_trace_id` as `NOT NULL UNIQUE`; it also provides typed parameterized insert helpers for diff traces including `model_id`, `tool_name`, and nullable `tool_version`, post-commit intersection rows, and built agent-trace rows with `agent_trace_id`, chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, active `sce hooks diff-trace` writes for `diff_traces`, and active `sce hooks post-commit` writes for built `agent_traces` payloads. +The local DB service now provides `LocalDb` as a thin `TursoDb` alias in `cli/src/services/local_db/mod.rs`; `LocalDbSpec` resolves the canonical local DB path from the shared default-path catalog and currently declares zero migrations. Shared Turso infrastructure lives in `cli/src/services/db/mod.rs`, where `DbSpec` and generic `TursoDb` support dual-mode operation — local mode via `turso::Builder::new_local()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are absent, or sync (Turso Cloud) mode via `turso::sync::Builder::new_remote()` when both are set. It owns parent-directory creation, connection setup, tokio current-thread runtime bridging, synchronous `execute`/`query`/`query_map`, generic migration execution, explicit sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode and are never triggered automatically by `execute()`, and shared DB lifecycle helpers for service-specific database wrappers. Agent Trace persistence now has its own `cli/src/services/agent_trace_db/mod.rs` wrapper, canonical `/sce/agent-trace.db` path, a split fresh-start baseline migration set (`001..005`) covering `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, and indexes (`idx_diff_traces_time_ms_id`, `idx_agent_traces_agent_trace_id`) without `AUTOINCREMENT`, plus `agent_traces.agent_trace_id` as `NOT NULL UNIQUE`; it also provides typed parameterized insert helpers for diff traces including `model_id`, `tool_name`, and nullable `tool_version`, post-commit intersection rows, and built agent-trace rows with `agent_trace_id`, chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider that initializes the database and performs a best-effort `pull()` during setup when sync mode is active, active `sce hooks diff-trace` writes for `diff_traces`, and active `sce hooks post-commit` writes for built `agent_traces` payloads. The hooks command surface now also supports concrete runtime subcommand routing (`pre-commit`, `commit-msg`, `post-commit`, `post-rewrite`, `diff-trace`) with deterministic argument/STDIN validation. Current runtime behavior keeps attribution disabled by default: the attribution gate enables canonical trailer insertion in `commit-msg`, `pre-commit`/`post-rewrite` remain deterministic no-ops, `post-commit` is the active bounded recent-diff-trace intersection path, and `diff-trace` is the active intake path for parsed STDIN `{ sessionID, diff, time, model_id, tool_name, tool_version }` payload persistence with required non-empty `tool_name`, required nullable/non-empty `tool_version`, required `u64` millisecond `time`, non-lossy AgentTraceDb `time_ms` conversion, and collision-safe timestamp+attempt artifact filenames. This behavior is documented in `context/sce/agent-trace-hooks-command-routing.md`. The setup service now also exposes deterministic required-hook embedded asset accessors (`iter_required_hook_assets`, `get_required_hook_asset`) backed by canonical templates in `cli/assets/hooks/` for `pre-commit`, `commit-msg`, and `post-commit`; this behavior is documented in `context/sce/setup-githooks-hook-asset-packaging.md`. The setup service now also includes required-hook install orchestration (`install_required_git_hooks`) that resolves repository root and effective hooks path from git truth, enforces deterministic per-hook outcomes (`Installed`/`Updated`/`Skipped`), and uses a unified remove-and-replace policy that removes existing hooks before swapping staged content with deterministic recovery guidance on swap failures; this behavior is documented in `context/sce/setup-githooks-install-flow.md`. diff --git a/context/plans/turso-sync-adapter.md b/context/plans/turso-sync-adapter.md index cfdbadae..79fd4b1e 100644 --- a/context/plans/turso-sync-adapter.md +++ b/context/plans/turso-sync-adapter.md @@ -7,17 +7,17 @@ Make the shared `TursoDb` adapter support both local-only and synced (Turso C The initial concrete target is the Agent Trace DB. Sync works in two phases: 1. **Pull on setup**: `AgentTraceDbLifecycle::setup` (invoked during `sce setup`) pulls remote changes to bring the local DB up to date. -2. **Bounded write-counter push**: Every `execute()` on the Agent Trace DB increments an atomic counter. When the counter reaches a threshold (default 10, configurable via `SCE_SYNC_PUSH_THRESHOLD`), it auto-pushes and resets. This batches writes without background threads or per-write network latency. +2. **Explicit sync**: Sync operations (`push`/`pull`/`checkpoint`/`stats`) are available through `TursoDb` methods or the `sce sync push|pull` CLI command. Sync is never triggered automatically from `execute()` — callers use explicit sync methods. Setup pulls on startup when sync is configured. ## Success criteria -- [ ] SC1: `TursoDb` opens in sync mode when both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` are set, using `turso::sync::Builder::new_remote()`. -- [ ] SC2: `TursoDb` falls back to local-only `turso::Builder::new_local()` when either env var is missing. -- [ ] SC3: `TursoDb` exposes `push()`, `pull()`, `checkpoint()`, and `stats()` methods that delegate to sync operations when in sync mode and are no-ops when in local mode. -- [ ] SC4: `TursoDb::execute()` in sync mode increments a write counter and auto-pushes when the threshold is reached (default 10 writes). The threshold is configurable via `SCE_SYNC_PUSH_THRESHOLD`. -- [ ] SC5: `AgentTraceDbLifecycle::setup` performs an initial `pull()` when sync is configured. -- [ ] SC6: `nix flake check` passes. -- [ ] SC7: Context files under `context/` are updated to reflect the new sync capability. +- [x] SC1: `TursoDb` opens in sync mode when both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` are set, using `turso::sync::Builder::new_remote()`. (Verified: T03) +- [x] SC2: `TursoDb` falls back to local-only `turso::Builder::new_local()` when either env var is missing. (Verified: T03) +- [x] SC3: `TursoDb` exposes `push()`, `pull()`, `checkpoint()`, and `stats()` methods that delegate to sync operations when in sync mode and are no-ops when in local mode. (Verified: T03) +- [x] SC4: Sync operations are explicit — `execute()` never triggers sync automatically. Callers use `TursoDb::push()`/`pull()` or the `sce sync push|pull` CLI command. (Changed from original plan during T03 implementation: auto-push was removed in favor of explicit sync.) +- [x] SC5: `AgentTraceDbLifecycle::setup` performs an initial `pull()` when sync is configured. (Verified: T04) +- [x] SC6: `nix flake check` passes. (Verified: 2026-05-19 — all checks pass) +- [x] SC7: Context files under `context/` are updated to reflect the new sync capability. (Verified: 2026-05-19) ## Constraints and non-goals @@ -26,8 +26,8 @@ The initial concrete target is the Agent Trace DB. Sync works in two phases: - **Note:** A user-invocable `sce sync` CLI command was originally deferred but wired in T03 as `sce sync push|pull` — opens the Agent Trace DB and calls push/pull when sync mode is active. Sync for other databases remains deferred. - **Assumption:** `turso::sync::Builder::with_auth_token()` accepts a static `&str` (the `AuthTokenFn` in turso 0.6.0 supports both string literals and closures). - **Assumption:** `turso::sync::Database::connect()` is async, while `turso::Database::connect()` is synchronous — the TursoDb adapter handles both cases inside the existing `block_on` bridge. -- **Assumption:** The write-counter push uses an `AtomicU64` on `TursoDb`; the counter is checked and pushed synchronously inside `execute()` via `block_on`, so background threads are not needed. -- **Assumption:** Push failures during write-counter auto-push are logged but do not fail the `execute()` call (best-effort push). +- **Assumption:** Sync is explicit — callers use `TursoDb::push()`/`pull()` or the `sce sync push|pull` CLI command. No auto-push in `execute()`. +- **Assumption:** Push failures from explicit calls propagate to the caller; best-effort push semantics apply only to the setup-time pull in `AgentTraceDbLifecycle::setup`. - **Assumption:** Initial pull on setup is best-effort — if the remote is unreachable during `sce setup`, the setup continues without error and sync is deferred to later writes. ## Task stack @@ -72,7 +72,7 @@ The initial concrete target is the Agent Trace DB. Sync works in two phases: - **Evidence:** `nix flake check` — all 4 checks passed (cli-tests, cli-clippy, cli-fmt, pkl-parity). - **Notes:** Added `sync_db: Option` to `TursoDb`. `new()` reads `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` env vars — when both present, opens via `turso::sync::Builder::new_remote()` (async via existing `block_on` bridge); when either absent, uses existing local-only path. No auto-push in `execute()` — sync is explicit via `TursoDb::push()`/`pull()` or the `sce sync` CLI command. Added `push()`, `pull()`, `checkpoint()`, `stats()`, and `is_sync_mode()` methods. Removed `write_count`, `push_threshold`, and `SYNC_PUSH_THRESHOLD_ENV_KEY` constant. Wired `sce sync push|pull` as a new CLI command in `cli/src/services/sync/{mod,command}.rs` with full clap schema, registry, and parse conversion. -- [ ] T04: `Wire pull-on-setup into Agent Trace DB lifecycle` (status: todo) +- [x] T04: `Wire pull-on-setup into Agent Trace DB lifecycle` (status: done) - Task ID: T04 - Goal: Modify `AgentTraceDbLifecycle::setup` in `cli/src/services/agent_trace_db/lifecycle.rs` to call `pull()` on the Agent Trace DB after opening when sync is configured. This runs during `sce setup`, pulling remote changes to bring the local Agent Trace DB current. The pull is best-effort — if the remote is unreachable, setup continues without error and sync is deferred to write-counter pushes. - Boundaries (in/out of scope): @@ -80,13 +80,42 @@ The initial concrete target is the Agent Trace DB. Sync works in two phases: - Out — Changing `LocalDb`, adding pull to `LocalDbLifecycle`, adding CLI commands, changing the `TursoDb` adapter itself. - Done when: `AgentTraceDbLifecycle::setup` calls `pull()` when the DB opened in sync mode. Remote failure does not fail setup. Not set behavior is unchanged. - Verification notes (commands or checks): `nix flake check`; trace-level log confirms pull attempt during setup when sync env vars are set. + - **Completed:** 2026-05-19 + - **Files changed:** `cli/src/services/agent_trace_db/lifecycle.rs` + - **Evidence:** `nix flake check` — all 4 checks passed (cli-tests, cli-clippy, cli-fmt, pkl-parity). Change adds best-effort pull to `AgentTraceDbLifecycle::setup` when sync mode is active; remote failure is logged with `tracing::warn!` but does not fail setup. + - **Notes:** Added ~6 lines to `lifecycle.rs`: `AgentTraceDb::new()` result is now captured as `let db = ...` instead of discarded; `if db.is_sync_mode() { if let Err(e) = db.pull() { tracing::warn!(...) } }` added after DB initialization. -- [ ] T05: `Validation and cleanup` (status: todo) +- [x] T05: `Validation and cleanup` (status: done) - Task ID: T05 - Goal: Run full repository validation, verify all success criteria are met, and update context files to reflect the new sync adapter capability. - Boundaries (in/out of scope): In — `nix flake check`, context sync edits to `context/overview.md`, `context/architecture.md`, `context/glossary.md`, `context/context-map.md`, `context/sce/shared-turso-db.md`, and `context/sce/agent-trace-db.md`. Out — making new code changes beyond context edits. - Done when: `nix flake check` passes; success criteria are verified; context files reflect current state. - Verification notes (commands or checks): `nix flake check`; `nix run .#pkl-check-generated`; manual review of context files for accuracy. + - **Completed:** 2026-05-19 + - **Files changed:** `context/plans/turso-sync-adapter.md`, `context/overview.md`, `context/architecture.md`, `context/glossary.md`, `context/context-map.md` + - **Evidence:** `nix flake check` — all checks passed. `nix run .#pkl-check-generated` — generated outputs up to date. All 7 success criteria verified. Outdated "write-counter auto-push" references corrected across 4 context files. + - **Notes:** T03 implementation removed the originally planned write-counter auto-push from `execute()` in favor of explicit sync only. Context files had stale references to the removed auto-push behavior; these were corrected to reflect current explicit-sync-only state. + +## Validation Report + +### Commands run +- `nix flake check` — exit 0 (all checks passed: cli-tests, cli-clippy, cli-fmt, pkl-parity, integrations-install-tests, integrations-install-clippy, integrations-install-fmt, npm-bun-tests, npm-biome-check, npm-biome-format, config-lib-bun-tests, config-lib-biome-check, config-lib-biome-format) +- `nix run .#pkl-check-generated` — "Generated outputs are up to date." + +### Temporary scaffolding +None introduced during this plan. All changes were production code. + +### Success-criteria verification +- [x] SC1: `TursoDb` opens in sync mode when both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` are set — confirmed in `cli/src/services/db/mod.rs` lines 196-225 +- [x] SC2: Falls back to local-only when either env var is missing — confirmed in `cli/src/services/db/mod.rs` lines 226-252 +- [x] SC3: Exposes `push()`, `pull()`, `checkpoint()`, `stats()` methods — confirmed in `cli/src/services/db/mod.rs` lines 279-328 +- [x] SC4: Sync is explicit — `execute()` never triggers sync automatically (changed from original plan during T03: auto-push was removed in favor of explicit sync via `TursoDb::push()`/`pull()` or `sce sync push|pull`) +- [x] SC5: `AgentTraceDbLifecycle::setup` performs best-effort `pull()` when sync is configured — confirmed in `cli/src/services/agent_trace_db/lifecycle.rs` lines 57-61 +- [x] SC6: `nix flake check` passes — verified 2026-05-19 +- [x] SC7: Context files updated — `context/overview.md`, `context/architecture.md`, `context/glossary.md`, `context/context-map.md` corrected stale "write-counter auto-push" references to reflect explicit-sync-only state; `context/sce/agent-trace-db.md` updated with pull-on-setup behavior (T04) + +### Residual risks +- None. All tasks verified against code truth. The original auto-push design was intentionally simplified during T03 to explicit-only sync, which reduces complexity and eliminates background-sync failure modes. ## Open questions diff --git a/context/sce/agent-trace-db.md b/context/sce/agent-trace-db.md index aa2121d7..2da2ed00 100644 --- a/context/sce/agent-trace-db.md +++ b/context/sce/agent-trace-db.md @@ -85,7 +85,7 @@ Lookup indexes created by the baseline migration set: - `diagnose()` reports canonical Agent Trace DB path and parent-directory readiness problems through the shared DB path-health helper. - `fix()` can bootstrap the canonical Agent Trace DB parent directory for auto-fixable parent-readiness problems. -- `setup()` initializes the database with `AgentTraceDb::new()`, including all ordered Agent Trace migrations and any later migrations not yet recorded in `__sce_migrations`. +- `setup()` initializes the database with `AgentTraceDb::new()`, including all ordered Agent Trace migrations and any later migrations not yet recorded in `__sce_migrations`. When sync mode is active (both `SCE_SYNC_URL` and `SCE_SYNC_TOKEN` set), it also performs a best-effort `pull()` to bring the local DB up to date — remote failure is logged with `tracing::warn!` and setup continues without error. - `sce doctor` now surfaces Agent Trace DB health as a row within the `Configuration` section with `[PASS]`/`[FAIL]`/`[MISS]` status tokens (e.g., `Agent Trace DB (/path/to/agent-trace.db)`), and includes it in JSON output under the `agent_trace_db` field. ## Runtime writers From 63de61b7943b05bcbf232951040472a3251fae01 Mon Sep 17 00:00:00 2001 From: David Abram Date: Thu, 21 May 2026 11:48:31 +0200 Subject: [PATCH 5/5] agent-trace-db: Enable Turso sync for trace storage Allow database specs to opt in to Turso Cloud sync and enable it for the Agent Trace DB. Disable empty remote bootstrapping for sync connections and use non-AUTOINCREMENT integer primary keys for compatibility with synced trace tables. Co-authored-by: SCE --- cli/src/services/agent_trace_db/mod.rs | 4 +++ cli/src/services/db/mod.rs | 15 +++++++++-- cli/src/services/setup/command.rs | 2 +- cli/src/services/sync/command.rs | 3 ++- flake.nix | 2 ++ manual-test-turso-sync.sh | 35 ++++++++++++++++++++++++++ 6 files changed, 57 insertions(+), 4 deletions(-) create mode 100755 manual-test-turso-sync.sh diff --git a/cli/src/services/agent_trace_db/mod.rs b/cli/src/services/agent_trace_db/mod.rs index d61612c4..1dde0517 100644 --- a/cli/src/services/agent_trace_db/mod.rs +++ b/cli/src/services/agent_trace_db/mod.rs @@ -83,6 +83,10 @@ impl DbSpec for AgentTraceDbSpec { fn migrations() -> &'static [(&'static str, &'static str)] { AGENT_TRACE_MIGRATIONS } + + fn sync_enabled() -> bool { + true + } } /// Agent trace Turso database adapter. diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index 0f40823e..7201ab0b 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -34,6 +34,12 @@ pub trait DbSpec { /// Ordered embedded migration SQL files as `(id, sql)` pairs. fn migrations() -> &'static [(&'static str, &'static str)]; + + /// Whether this database may use Turso Cloud sync mode when sync env vars + /// are configured. + fn sync_enabled() -> bool { + false + } } /// Collect common filesystem health problems for a Turso database path. @@ -186,8 +192,12 @@ impl TursoDb { format!("failed to create {db_name} tokio runtime. Try: rerun the command; if the issue persists, verify the local Tokio runtime environment.") })?; - let sync_url = std::env::var(crate::services::config::SYNC_URL_ENV_KEY).ok(); - let sync_token = std::env::var(crate::services::config::SYNC_TOKEN_ENV_KEY).ok(); + let sync_url = M::sync_enabled() + .then(|| std::env::var(crate::services::config::SYNC_URL_ENV_KEY).ok()) + .flatten(); + let sync_token = M::sync_enabled() + .then(|| std::env::var(crate::services::config::SYNC_TOKEN_ENV_KEY).ok()) + .flatten(); let path_str = db_path.to_str().ok_or_else(|| { anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) @@ -198,6 +208,7 @@ impl TursoDb { let sync_db = turso::sync::Builder::new_remote(path_str) .with_remote_url(url) .with_auth_token(token) + .bootstrap_if_empty(false) .build() .await .map_err(|e| { diff --git a/cli/src/services/setup/command.rs b/cli/src/services/setup/command.rs index d2d36d73..07d950d6 100644 --- a/cli/src/services/setup/command.rs +++ b/cli/src/services/setup/command.rs @@ -54,7 +54,7 @@ impl RuntimeCommand for SetupCommand { for provider in &providers { let outcome = provider .setup(&ctx) - .map_err(|error| ClassifiedError::runtime(error.to_string()))?; + .map_err(|error| ClassifiedError::runtime(format!("{error:#}")))?; if let Some(ref hooks_outcome) = outcome.required_hooks_install { sections.push(setup::format_required_hook_install_success_message( diff --git a/cli/src/services/sync/command.rs b/cli/src/services/sync/command.rs index 0bd207a4..2af59f14 100644 --- a/cli/src/services/sync/command.rs +++ b/cli/src/services/sync/command.rs @@ -15,7 +15,8 @@ impl RuntimeCommand for SyncCommand { } fn execute(&self, _context: &AppContext) -> Result { - sync::run_sync(self.subcommand).map_err(|error| ClassifiedError::runtime(error.to_string())) + sync::run_sync(self.subcommand) + .map_err(|error| ClassifiedError::runtime(format!("{error:#}"))) } } diff --git a/flake.nix b/flake.nix index 0546f4ab..8c30d19d 100644 --- a/flake.nix +++ b/flake.nix @@ -1071,6 +1071,8 @@ typescript-language-server vscode-json-languageserver opencodePackage + openssl + pkg-config rust-analyzer scePackage tursoPackage diff --git a/manual-test-turso-sync.sh b/manual-test-turso-sync.sh new file mode 100755 index 00000000..93a13155 --- /dev/null +++ b/manual-test-turso-sync.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Manual smoke test for Turso sync support. +# Contains test credentials supplied for local manual testing only. +# Do not commit this file if the token is sensitive or long-lived. + +export DATABASE_URL="" +export DATABASE_AUTH_TOKEN="" + +export SCE_SYNC_URL="$DATABASE_URL" +export SCE_SYNC_TOKEN="$DATABASE_AUTH_TOKEN" + +STATE_DIR="$(mktemp -d)" +export XDG_STATE_HOME="$STATE_DIR" + +echo "Using isolated state dir: $XDG_STATE_HOME" + +echo "1. Show sync command help" +nix develop -c sh -c 'cd cli && cargo run -- sync --help' + +echo "2. Run setup, including best-effort initial pull" +nix develop -c sh -c 'cd cli && cargo run -- setup --opencode --non-interactive' + +echo "3. Pull from Turso" +nix develop -c sh -c 'cd cli && cargo run -- sync pull' + +echo "4. Push to Turso" +nix develop -c sh -c 'cd cli && cargo run -- sync push' + +echo "5. Confirm local Agent Trace DB exists" +test -f "$XDG_STATE_HOME/sce/agent-trace.db" +ls -lh "$XDG_STATE_HOME/sce/agent-trace.db" + +echo "Manual Turso sync smoke test passed."