diff --git a/Cargo.lock b/Cargo.lock index a46a744a0..8bb7bec2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3678,7 +3678,6 @@ dependencies = [ "unicode-width", "ureq", "windows-native-keyring-store", - "windows-sys 0.61.2", "zbus-secret-service-keyring-store", ] @@ -3853,7 +3852,6 @@ dependencies = [ "sha2", "tokio", "ureq", - "windows-sys 0.61.2", ] [[package]] diff --git a/README.md b/README.md index 28536363f..25e7df416 100644 --- a/README.md +++ b/README.md @@ -970,6 +970,25 @@ first: if it cannot be, nothing is removed, so you can fix the cause and run `rocm uninstall` again. Each step is reported as it finishes, so if a later step fails, the output still shows what was already removed. +`rocm uninstall` stops any managed model server that is still running before it +removes anything, and also stops the background helper (`rocmd`), which is +force-stopped (on Linux its whole process tree, on Windows only the recorded +process) because it would otherwise restart the servers. +If a server cannot be stopped (or its service record cannot be read), the +command exits non-zero, leaves every file in place, and says what to repair or +stop by hand, so the tooling needed to stop it is never deleted out from under +a running server. A live server or helper recorded without a start time (written +by an older version) cannot be proven to be ours, so it is refused rather than +killed: stop it yourself, then re-run. The same refusal applies when the +helper's runtime state file cannot be read. The plan shows the refusal before +you confirm it. The stop pass is not atomic: if it aborts after stopping some +servers (or the helper), those stay stopped and their endpoint keys are gone; +the error names each one and the `rocm serve` command that brings it back. A +run that keeps the binaries and data (`--keep-binaries --keep-data`) leaves +running servers and the helper alone. +If a removal fails partway, the error lists what was removed and what is left, +because the binaries and service records may already be gone. + On Linux, if a config, data, or cache folder is a link to somewhere else (for example a cache moved to a bigger disk), `rocm uninstall` removes the link and leaves the folder it points to alone, whether or not the setting ends in a `/`. diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index a319b11c6..5324dd4db 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -56,7 +56,6 @@ libc.workspace = true [target.'cfg(windows)'.dependencies] self-replace = "1.5" windows-native-keyring-store = "1.1" -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [target.'cfg(target_os = "macos")'.dependencies] apple-native-keyring-store = { version = "1.0", features = ["keychain"] } diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 6b8ee14dc..0ee1cd6a1 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -38,6 +38,7 @@ mod storage; mod test_support; mod therock; mod uninstall; +mod uninstall_gate; // Per-command handlers mechanically relocated into modules — fns, plus the // types (e.g. `ServeArgs`, driver-plan types) that moved with their cluster. @@ -653,6 +654,21 @@ rocm logs --search error timeout")] /// PATH change the installer made (its marked shell-profile block, or the /// user PATH entry it recorded adding on Windows), unless other programs /// share the install folder. + /// + /// Managed model servers that are still running are stopped first, and so + /// is the background helper (`rocmd`), which is force-stopped (on Linux its + /// whole process tree; on Windows only the recorded process, as + /// descendants are not enumerated there) because it would otherwise + /// restart them. If one cannot be + /// stopped, nothing is removed, the command exits non-zero, and it names + /// what to repair or stop by hand. A server or helper that was recorded by + /// an older version, with no start time to prove its pid, is refused rather + /// than killed: stop it yourself and re-run, as when the helper's runtime + /// state cannot be read. If the stop pass aborts after stopping some + /// services (or the helper), those stay stopped with their keys dropped; + /// the error lists the commands that bring them back. A run that keeps the + /// binaries and data (`--keep-binaries --keep-data`) leaves running servers + /// alone. Uninstall { /// Do not ask for interactive confirmation. #[arg(long)] @@ -8482,6 +8498,19 @@ fn run_chat_port_status_tool( })) } +/// Whether `host` resolves to any address at all. +/// +/// Split out from [`loopback_tcp_port_is_reachable`], which answers `false` both +/// for "resolved, nothing listening" and for "did not resolve". Those mean +/// opposite things to the uninstall gate, and only the caller that has to +/// distinguish them pays for the second lookup. +fn host_port_resolves(host: &str, port: u16) -> bool { + use std::net::ToSocketAddrs as _; + (host, port) + .to_socket_addrs() + .is_ok_and(|mut addresses| addresses.next().is_some()) +} + const fn chat_read_only_tool_status_label(is_error: bool) -> &'static str { if is_error { "reported an error" @@ -10539,24 +10568,8 @@ const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); /// known token. Returns the PIDs actually signalled and whether every recorded /// process is confirmed no longer running. fn terminate_recorded_service_pids(record: &ManagedServiceRecord) -> (Vec, bool) { - // Build the (pid, own-token) work list, de-duplicating on PID and preferring - // an entry that carries a verifiable start-time. - let mut entries: Vec<(u32, Option)> = Vec::new(); - for (pid, ticks) in [ - (Some(record.supervisor_pid), record.supervisor_start_ticks), - (record.engine_pid, record.engine_start_ticks), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0 && *pid != std::process::id()) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _)| *seen == pid) { - if existing.1.is_none() { - existing.1 = ticks; - } - } else { - entries.push((pid, ticks)); - } - } + // The same (pid, own-token) list the uninstall gate pre-checks. + let entries = uninstall_gate::recorded_pid_identities(record); let mut signaled_pids = Vec::new(); let mut all_stopped = true; @@ -10593,6 +10606,17 @@ fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result Result<(serde_json::Value, bool)> { + stop_internal_managed_service_confirmed_with(paths, service_id, terminate_recorded_service_pids) +} + /// [`stop_internal_managed_service`], with the termination of the recorded /// processes supplied by the caller. Production passes /// [`terminate_recorded_service_pids`]; a test passes a stand-in that observes @@ -13170,14 +13194,14 @@ fn build_uninstall_plan(paths: &AppPaths, options: &UninstallOptions) -> Result< plan.warnings.push(note); } - let managed_services = load_managed_services(paths).unwrap_or_default(); - if !managed_services.is_empty() { - plan.warnings.push(format!( - "{} managed service record(s) exist under {}; background processes are not stopped automatically in this pass", - managed_services.len(), - paths.services_dir().display() - )); - } + // The gate propagates this error while the plan does not, so a plan that + // silently read zero records would tell the operator "no managed services" + // about a run that is going to abort on exactly that failure. Keep the plan + // non-fatal (it is also the read-only dry-run path) but say what happened. + let (managed_services, services_read_error) = match load_managed_services(paths) { + Ok(records) => (records, None), + Err(error) => (Vec::new(), Some(format!("{error:#}"))), + }; // Remote sessions are worse than local ones to drop silently. The model runs // on someone else's machine and its endpoint is published there, so removing @@ -13221,9 +13245,124 @@ fn build_uninstall_plan(paths: &AppPaths, options: &UninstallOptions) -> Result< plan.actions .sort_by(|left, right| left.path.cmp(&right.path)); plan.actions.dedup_by(|left, right| left.path == right.path); + + if let Some(error) = &services_read_error { + // The refusal is real only where the stop pass runs, and that pass is + // skipped when the tooling is kept; promising it there would be wrong. + let consequence = if plan_removes_recovery_tooling(&plan, paths) { + "Uninstall will refuse to remove anything until they can be read" + } else { + "This run keeps the tooling, so it does not check them" + }; + plan.warnings.push(format!( + "could not read the managed service records under {}: {error}. {consequence}", + paths.services_dir().display() + )); + } + + // What the stop pass would refuse, decided by the very function it runs first + // (`uninstall_preflight`), so the lines below describe the run that will + // happen instead of a promise the gate then breaks. Only meaningful where the + // stop pass runs at all: a run that keeps the tooling does not check. + let removes_recovery_tooling = plan_removes_recovery_tooling(&plan, paths); + let mut preflight_refusal: Option = None; + let mut preflight_warnings: Vec = Vec::new(); + if removes_recovery_tooling && services_read_error.is_none() { + match uninstall_gate::uninstall_preflight(paths) { + Ok(report) => { + preflight_refusal = uninstall_gate::preflight_refusal_line(&report); + preflight_warnings = report.warnings; + } + Err(error) => { + preflight_refusal = Some(format!( + "uninstall will refuse to remove anything, and will stop nothing, until the \ + managed service records under {} can be checked: {error:#}", + paths.services_dir().display() + )); + } + } + } + let refusal_line_present = preflight_refusal.is_some(); + let run_refuses = refusal_line_present || services_read_error.is_some(); + if let Some(line) = preflight_refusal { + plan.warnings.push(line); + } + // The fail-open disclosures the stop pass would make, shown while the + // operator can still decline rather than only after they have confirmed. + plan.warnings.append(&mut preflight_warnings); + + if !managed_services.is_empty() { + // Two independent conditions decide whether a server actually gets + // stopped, and the warning must reflect BOTH or it describes work + // uninstall never does: the record has to need stopping, and the plan has + // to be removing the tooling that stops it (`uninstall()` skips the whole + // stop pass otherwise). A run that will refuse stops nothing at all. + let in_use = managed_services + .iter() + .filter(|record| uninstall_gate::record_needs_stopping(record)) + .count(); + let services_dir = paths.services_dir().display().to_string(); + let total = managed_services.len(); + plan.warnings.push(if in_use == 0 { + // "none is in use", not "none has a running server": this counts + // statuses and recorded pids, and a record reads "stopped" the moment + // its supervisor is killed even if an engine grandchild kept the + // port. The stop pass probes for exactly that, so the plan must not + // promise it away — this line is the last thing read before + // confirming. + format!("{total} managed service record(s) exist under {services_dir}; none is recorded as running") + } else if !removes_recovery_tooling { + // Cache-only and other tooling-preserving runs: say what is true — + // the servers keep running, and they remain stoppable afterwards. + format!( + "{in_use} of {total} managed service record(s) under {services_dir} are in use (recorded as running, or with a recorded process still alive); this removal keeps `rocm services stop` and the service records, so those servers are left running" + ) + } else if run_refuses { + format!( + "{in_use} of {total} managed service record(s) under {services_dir} are in use (recorded as running, or with a recorded process still alive); none of them will be stopped, because uninstall will refuse (see above)" + ) + } else { + format!( + "{in_use} of {total} managed service record(s) under {services_dir} are in use (recorded as running, or with a recorded process still alive); those servers will be stopped before removal" + ) + }); + } + + // The helper is reported whether or not any service record exists: a live + // `rocmd` with an empty services directory still gets force-stopped. + if let Some(note) = uninstall_gate::background_helper_plan_note( + paths, + removes_recovery_tooling, + run_refuses, + refusal_line_present, + ) { + plan.warnings.push(note); + } + Ok(plan) } +/// Whether this plan removes what an operator would need to stop a managed +/// server afterwards — the `rocm`/`rocmd` binaries, or the service records under +/// the data directory. +/// +/// The stop-before-remove gate exists only for that case. A `--keep-binaries +/// --keep-data` run removes the cache alone: the binaries and every service +/// record survive, so `rocm services stop` still works and force-stopping live +/// GPU servers (or hard-aborting a cache cleanup because one will not stop) +/// would be pure collateral damage. +fn plan_removes_recovery_tooling(plan: &UninstallPlan, paths: &AppPaths) -> bool { + let services_dir = paths.services_dir(); + // The running executable is not an action on Windows (it is deleted as the + // command exits), but it is still the `rocm` that stops servers. + plan.self_delete.is_some() + || plan.actions.iter().any(|entry| { + entry.kind == "binary" + || entry.path.starts_with(&services_dir) + || services_dir.starts_with(&entry.path) + }) +} + /// Plan removing the installed binaries beside the running one, which /// `current_exe` names, and undoing the installer's PATH change for their /// folder. `--keep-binaries` keeps both, and so does a cargo build's folder. @@ -15579,6 +15718,8 @@ pub(crate) mod tests { } use super::*; + use crate::service_launch::tests::runtime_state; + use crate::uninstall_gate::*; use serde_json::json; use crate::runtime_install_finalize::tests::{ @@ -25489,6 +25630,7 @@ install therock"; running: true, automations_enabled: true, daemon_pid: 123, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 2, local_webhook_endpoint: Some("http://127.0.0.1:19191/automation-events".to_owned()), @@ -26569,20 +26711,66 @@ install therock"; ); } + /// Assert `child` exits within a bounded grace, and reap it either way. + /// + /// NOT `wait()`. These tests spawn a stand-in that exits on its own after a + /// minute, so an unbounded `wait()` blocks until that happens and then finds + /// the process gone — passing whether or not anything killed it. The only + /// symptom of a total regression would be the suite taking a minute. Polling + /// against a deadline asserts it died *now*, which is the actual claim. + /// + /// The grace exceeds the stop path's own per-process wait but stays far + /// below the stand-in's lifetime, or this goes back to measuring nothing. + /// + /// Shared with `crate::uninstall`'s tests rather than duplicated there: two + /// near-identical copies of this, with near-identical comments, is what made + /// it easy to believe both process-based tests were bounded when only one + /// of them was. + #[cfg(unix)] + pub(crate) const STOP_ASSERTION_GRACE: Duration = Duration::from_secs(30); + + #[cfg(unix)] + pub(crate) fn assert_stopped_within_grace(child: &mut std::process::Child, message: &str) { + let deadline = std::time::Instant::now() + STOP_ASSERTION_GRACE; + let exited = loop { + match child.try_wait().expect("poll the stand-in") { + Some(_) => break true, + None if std::time::Instant::now() >= deadline => break false, + None => thread::sleep(Duration::from_millis(50)), + } + }; + if !exited { + let _ = child.kill(); + let _ = child.wait(); + } + assert!(exited, "{message}"); + } + #[cfg(unix)] fn managed_record_for_pid( paths: &AppPaths, pid: u32, start_ticks: Option, + ) -> ManagedServiceRecord { + managed_record_named(paths, "svc-managed-stop", pid, start_ticks) + } + + #[cfg(unix)] + fn managed_record_named( + paths: &AppPaths, + service_id: &str, + pid: u32, + start_ticks: Option, ) -> ManagedServiceRecord { let mut record = ManagedServiceRecord::new( paths, - "svc-managed-stop", + service_id, "vllm", "m", "m", "127.0.0.1", - 9, + // Port 0 is never served, so the gate's probe cannot meet a listener. + 0, "managed", pid, None, @@ -26675,10 +26863,10 @@ install therock"; assert_eq!(signaled, vec![pid]); assert!(all_stopped); - // Reap our own child (a detached managed process would be reaped by init) - // so the liveness check does not observe a not-yet-reaped zombie. + // Bounded, and our own child, so this pins that the terminate killed it + // rather than that it outlived the test. let mut child = child; - let _ = child.wait(); + assert_stopped_within_grace(&mut child, "the verified process must be terminated"); assert!( !rocm_core::process_is_running(pid), "the verified process must be terminated" @@ -26686,177 +26874,3660 @@ install therock"; let _ = fs::remove_dir_all(root); } - pub(crate) fn test_paths(name: &str) -> (PathBuf, AppPaths) { - let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join("..") - .join(".rocm-work") - .join("tests") - .join("main") - .join(format!( - "rocm-cli-main-test-{name}-{}", - crate::test_support::unique_suffix() - )); - let _ = fs::remove_dir_all(&root); - ( - root.clone(), - AppPaths { - config_dir: root.join("config"), - data_dir: root.join("data"), - cache_dir: root.join("cache"), - }, - ) - } - + // Linux-only, not merely unix: these rely on `process_start_ticks` (Some on + // Linux, None elsewhere) and zombie-state detection in `terminate_verified`. + #[cfg(target_os = "linux")] #[test] - fn test_paths_gives_each_call_its_own_root_even_for_one_label() { - crate::test_support::assert_each_call_gets_its_own_root(|| test_paths("same-label").0); - } - - // ---- Phase 9: reroute dispatch (bare `rocm` + interactive `rocm chat`) ---- - // - // The interactive branches require a real TTY (`interactive_terminal()`), - // which is unavailable in CI, and the dash visuals are trust-prior. These - // tests instead PROVE the dispatch TARGET changed: the two interactive - // handlers now call `dash::run_chat` and no longer call `tui::run`. We read - // this source file at test time and assert on the handler bodies. - fn main_rs_source() -> String { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("src") - .join("main.rs"); - fs::read_to_string(path).expect("read apps/rocm/src/main.rs") - } - - /// Extract the body of `fn launch_default()` (brace-balanced). - fn launch_default_body(src: &str) -> String { - slice_braced_block(src, "fn launch_default() -> Result<()> {") - } - - /// Extract the whole `Some(Command::Chat { .. }) => { .. }` handler arm, - /// brace-balanced from the destructure pattern through the block body, so - /// both the destructured fields (e.g. `chat_mock`) and the dispatch code are - /// visible. Restricts to the production handler (the `=> {` form), not the - /// variant declaration which has no `=>`. - fn command_chat_handler_body(src: &str) -> String { - // Anchor on the handler arm specifically: the destructure that is - // immediately followed (after the closing `}` + `)`) by `=> {`. - let arm = "Some(Command::Chat {"; - let arm_at = src.find(arm).expect("Command::Chat handler present"); - // Balance from the `{` of the destructure to capture the full arm, - // including the `=> { ... }` block that follows. - slice_braced_to_arm_end(&src[arm_at..]) - } - - /// Given a slice beginning at `Some(Command::Chat {`, return the text from - /// that point through the end of the arm's `=> { .. }` block. - fn slice_braced_to_arm_end(src: &str) -> String { - // 1. balance the destructure `{ .. }`. - let pat_open = src.find('{').expect("destructure open brace"); - let bytes = src.as_bytes(); - let mut depth = 0usize; - let mut pat_end = pat_open; - for (i, &b) in bytes.iter().enumerate().skip(pat_open) { - match b { - b'{' => depth += 1, - b'}' => { - depth -= 1; - if depth == 0 { - pat_end = i + 1; - break; - } - } - _ => {} - } - } - // 2. find the `=> {` block opener after the destructure and balance it. - let after = &src[pat_end..]; - let block_rel = after.find("=> {").expect("arm block opener `=> {`"); - let block_body = slice_braced_block(after, &after[block_rel..block_rel + 4]); - // Return destructure + block body together. - format!("{}{}", &src[pat_open..pat_end], block_body) - } - - /// Return the brace-balanced block that follows `opener` (which must end in - /// `{`), including the contents but excluding the trailing brace's tail. - fn slice_braced_block(src: &str, opener: &str) -> String { - let start = src.find(opener).unwrap_or_else(|| { - panic!("opener not found: {opener}"); - }); - let brace_start = start + opener.len() - 1; // index of the `{` - let bytes = src.as_bytes(); - let mut depth = 0usize; - let mut end = brace_start; - for (i, &b) in bytes.iter().enumerate().skip(brace_start) { - match b { - b'{' => depth += 1, - b'}' => { - depth -= 1; - if depth == 0 { - end = i + 1; - break; - } - } - _ => {} - } - } - src[brace_start..end].to_string() - } - - /// Strip `//` line comments so assertions key on real code, not the prose - /// comments that still mention `tui::run` for documentation. - fn strip_line_comments(block: &str) -> String { - block - .lines() - .map(|line| match line.find("//") { - Some(idx) => &line[..idx], - None => line, - }) - .collect::>() - .join("\n") - } + fn uninstall_stops_live_managed_service_and_reports_it() { + // EAI-8014: a live managed server must be stopped before uninstall + // removes the tooling that stops it, and reported so the operator knows. + let child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-stops-live"); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = managed_record_for_pid(&paths, pid, Some(real)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); - #[test] - fn dash_run_chat_is_the_bare_and_chat_dispatch_target() { - // Symbol exists with the expected signature (bare `rocm` / `rocm chat` - // interactive target). Coercing to the fn pointer fails to compile if - // the entrypoint is removed or its signature drifts; casting the pointer - // to an address gives the binding a real effect (no underscore-bind). - let target: fn(bool, ChatInferenceParams) -> Result<()> = dash::run_chat; - assert_ne!(target as usize, 0); - } + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); - #[test] - fn launch_default_routes_to_dash_run_chat_not_tui() { - let src = main_rs_source(); - let body = strip_line_comments(&launch_default_body(&src)); - assert!( - body.contains("dash::run_launcher(false)"), - "bare `rocm` interactive branch must route to the launcher; body:\n{body}" + assert!(report.failed.is_empty(), "nothing should fail: {report:?}"); + assert_eq!(report.stopped, vec![record.service_id]); + let mut child = child; + assert_stopped_within_grace( + &mut child, + "the managed server must be stopped before uninstall proceeds", ); assert!( - !body.contains("tui::run"), - "launch_default must NOT call tui::run after the reroute; body:\n{body}" + !rocm_core::process_is_running(pid), + "the managed server must be stopped before uninstall proceeds" ); + let _ = fs::remove_dir_all(root); } + #[cfg(target_os = "linux")] #[test] - fn command_chat_interactive_routes_to_dash_run_chat_not_tui() { - let src = main_rs_source(); - let body = strip_line_comments(&command_chat_handler_body(&src)); + fn uninstall_skips_already_dead_managed_service() { + // A service whose process already crashed is not live; it must neither be + // counted as stopped by us nor fail the abort gate that keeps the tooling. + // + // A live service shares the directory so the assertion cannot be + // satisfied by a gate that simply does nothing: the dead one must be + // skipped WHILE the live one is stopped. + let mut dead = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn"); + let dead_pid = dead.id(); + let _ = dead.kill(); + let _ = dead.wait(); assert!( - body.contains("dash::run_chat("), - "interactive `rocm chat` must route to dash::run_chat(...); body:\n{body}" + !rocm_core::process_is_running(dead_pid), + "the process must be gone before the record is loaded" ); + let (root, paths) = test_paths("uninstall-skips-dead"); + let mut crashed = managed_record_for_pid(&paths, dead_pid, None); + crashed.status = "ready".to_owned(); + crashed.write().expect("write service record"); + + let live = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn live managed server"); + let live_pid = live.id(); + let mut running = managed_record_for_pid(&paths, live_pid, None); + running.service_id = "svc-managed-live".to_owned(); + running.manifest_path = paths.service_manifest_path(&running.service_id); + running.supervisor_start_ticks = rocm_core::process_start_ticks(live_pid); + running.status = "ready".to_owned(); + running.write().expect("write live service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + assert!( - !body.contains("tui::run"), - "Command::Chat handler must NOT call tui::run after the reroute; body:\n{body}" + report.failed.is_empty(), + "a crashed service must not abort uninstall: {report:?}" + ); + assert_eq!( + report.stopped, + vec!["svc-managed-live".to_owned()], + "only the live service is stopped and reported: {report:?}" ); + let mut live = live; + assert_stopped_within_grace(&mut live, "the live service must really have been stopped"); + let _ = fs::remove_dir_all(root); } + #[cfg(target_os = "linux")] #[test] - fn command_chat_interactive_notes_dropped_provider_flag() { - // Phase-9 polish: --provider on interactive `rocm chat` is no longer - // silently ignored — the handler emits a one-line note when provider is + fn uninstall_stops_a_live_background_helper() { + // This pins that the daemon stop still happens at all — the + // feature-removal guard. It does NOT pin identity verification: an + // unrecorded start-time also classifies as `Matches`, so this passes + // against the pre-fix `ProcessIdentity::new(pid, None)` too. The + // verification itself is pinned by + // `uninstall_never_kills_a_daemon_pid_that_was_recycled` and + // `uninstall_never_kills_a_daemon_pid_from_a_state_file_that_predates_start_ticks`, + // both of which fail if the identity check is dropped. + // + // Why the daemon goes first: it restarts a managed service whose + // endpoint stops answering, which is exactly the state the stop pass + // creates before writing the record back. + let child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn daemon stand-in"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-stops-verified-daemon"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + assert!( + state.daemon_start_ticks.is_some(), + "the recorded identity is the point of this test" + ); + state.write(&paths).expect("write runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!(report.failed.is_empty(), "nothing should fail: {report:?}"); + assert_eq!( + report.stopped, + Vec::::new(), + "the helper is not a service: {report:?}" + ); + assert!( + report.helper_stopped, + "a verified daemon is stopped and reported: {report:?}" + ); + let mut child = child; + assert_stopped_within_grace( + &mut child, + "the background helper must be stopped before uninstall proceeds", + ); + assert!( + !rocm_core::process_is_running(pid), + "the background helper must be stopped before uninstall proceeds" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn uninstall_never_kills_a_daemon_pid_from_a_state_file_that_predates_start_ticks() { + // The ordinary upgrade path: `runtime-state.json` written by a pre-upgrade + // `rocmd` carries no `daemon_start_ticks`. `identity_state` calls that + // `Matches` (its legacy best-effort arm), so signalling on that verdict + // would force-kill a whole tree at a pid nothing has verified. On a + // platform that CAN read start-times, an unrecorded one means the record + // is stale, not that the pid is ours. + let stranger = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn unrelated process"); + let pid = stranger.id(); + let (root, paths) = test_paths("uninstall-daemon-legacy-record"); + let state = runtime_state(true, pid); + assert!( + state.daemon_start_ticks.is_none(), + "this test is about the unrecorded-identity path" + ); + state.write(&paths).expect("write runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let mut stranger = stranger; + assert!( + stranger + .try_wait() + .expect("poll the unrelated process") + .is_none(), + "an unverifiable pid must never be signalled: uninstall killed an unrelated process" + ); + assert!( + report.stopped.is_empty() && !report.helper_stopped, + "nothing was confirmed stopped: {report:?}" + ); + assert!( + report.failed.iter().any(|failure| failure.remedy + == StopFailureRemedy::ConfirmTheDaemonPid + && failure.service_id.contains(&pid.to_string())), + "an unverifiable helper must abort the uninstall and name its pid: {report:?}" + ); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn uninstall_refuses_when_the_daemon_runtime_state_cannot_be_read() { + // `load` returns `Err` for a corrupt or unreadable state file and + // `Ok(None)` only when there is none. Discarding that `Err` would skip the + // daemon stop silently and remove the tooling while a live `rocmd` can + // still respawn what the service pass just stopped. Reachable in practice: + // the file is rewritten on every tick. + let (root, paths) = test_paths("uninstall-daemon-state-corrupt"); + paths.ensure().expect("create the app directories"); + fs::write(paths.automation_state_path(), b"{ not json") + .expect("seed a corrupt runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::RepairTheDaemonState), + "an unreadable runtime state must abort the uninstall: {report:?}" + ); + let error = uninstall_removal_gate(&report) + .expect_err("the gate must refuse to remove anything") + .to_string(); + // The abort has to be followable. No pid was ever parsed out of this + // file, so "kill that pid" would be a dead end — uninstall has no + // `--force`, and repairing or deleting the named file is the only way + // out. + let state_path = paths.automation_state_path(); + assert!( + error.contains(&state_path.display().to_string()), + "the abort must name the file to repair or delete: {error}" + ); + assert!( + error.contains("repair or delete"), + "the abort must say what to do with it: {error}" + ); + assert!( + !error.contains("kill that pid"), + "no pid was ever read from this file, so that advice cannot be followed: {error}" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn uninstall_never_kills_a_daemon_pid_that_was_recycled() { + // `runtime-state.json` survives a crash, OOM-kill or reboot with + // `running` still true, so the recorded pid can belong to a stranger by + // the time uninstall runs. Killing on a bare pid would make uninstall + // destroy something the user never installed — with `KillScope::Tree` + // and `force`, an unrelated process AND all its children. + // + // The stand-in plays the recycled process: a real live pid recorded with + // a start-time that is not its own. It must come through untouched. + let stranger = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn unrelated process"); + let pid = stranger.id(); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let (root, paths) = test_paths("uninstall-recycled-daemon-pid"); + let mut state = runtime_state(true, pid); + // The daemon that recorded this pid started at a different time; this pid + // has since been reissued to `stranger`. + state.daemon_start_ticks = Some(real.wrapping_add(1)); + state.write(&paths).expect("write runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + // `try_wait` on our own child, not `process_is_running`: a killed but + // unreaped child is a zombie, which a pid probe still calls running. + let mut stranger = stranger; + assert!( + stranger + .try_wait() + .expect("poll the unrelated process") + .is_none(), + "a recycled pid must never be signalled: uninstall killed an unrelated process" + ); + assert!( + !report.helper_stopped, + "a recycled pid is not a helper this pass stopped: {report:?}" + ); + assert!( + report.stopped.is_empty(), + "nothing of ours was running, so nothing was stopped: {report:?}" + ); + assert!( + report.failed.is_empty(), + "a recycled pid means the daemon is already gone, not that it is stuck: {report:?}" + ); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn uninstall_refuses_when_a_service_record_cannot_be_parsed() { + // `load_managed_services` skips a manifest it cannot parse, which is + // right for listing and wrong here: the unreadable record may describe a + // live, GPU-holding server, so uninstall must not proceed blind to it. + let (root, paths) = test_paths("uninstall-corrupt-record"); + let services_dir = paths.services_dir(); + fs::create_dir_all(&services_dir).expect("create services dir"); + fs::write(services_dir.join("svc-corrupt.json"), b"{ not json") + .expect("write corrupt service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failed: Vec<&str> = report + .failed + .iter() + .map(|failure| failure.service_id.as_str()) + .collect(); + let corrupt = services_dir.join("svc-corrupt.json"); + assert_eq!( + failed, + vec![corrupt.display().to_string().as_str()], + "an unparseable record must fail the gate: {report:?}" + ); + let error = uninstall_removal_gate(&report) + .expect_err("an unparseable record must abort uninstall") + .to_string(); + // The abort has to be escapable. `rocm services stop` loads the same + // file and fails the same way, so prescribing it here would make every + // retry abort identically — a permanent block with no way out. The only + // remedy is on disk, so the message must name the file and say so. + assert!( + error.contains(&corrupt.display().to_string()), + "names the file to act on: {error}" + ); + assert!( + error.contains("repair or delete the file"), + "states the remedy that can actually clear it: {error}" + ); + assert!( + !error.contains("rocm services stop"), + "must not prescribe a command that fails on the same unparseable file: {error}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn uninstall_removal_gate_aborts_when_a_service_cannot_be_stopped() { + // The core safety guarantee: a service that could not be confirmed stopped + // aborts uninstall (Err, so the removal loop never runs — nothing removed), + // and the message names the offending service and points at the recovery + // path. Guards against a regression that would delete the tooling while a + // GPU-holding endpoint keeps serving (EAI-8014). + let report = ManagedServiceStopReport { + stopped: vec!["svc-stopped".to_owned()], + failed: vec![FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\" after the stop attempt".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let error = uninstall_removal_gate(&report) + .expect_err("a non-empty `failed` must abort uninstall") + .to_string(); + assert!( + error.contains("svc-stuck"), + "names the stuck service: {error}" + ); + assert!( + error.contains("still \"ready\" after the stop attempt"), + "says why the stop could not be confirmed: {error}" + ); + assert!( + error.contains("rocm services stop"), + "points at the recovery path: {error}" + ); + assert!( + error.contains("No files were removed"), + "states nothing was deleted: {error}" + ); + } + + #[test] + fn uninstall_abort_says_which_services_it_already_stopped() { + // The stop pass is not atomic: services stopped before the failing one + // stay stopped and have lost their endpoint keys. An abort that only + // said "No files were removed" would read as "nothing happened". + let report = ManagedServiceStopReport { + stopped: vec!["svc-public".to_owned()], + relaunch: vec![format!( + "svc-public: `{}`", + rocm_engine_protocol::relaunch_command("0.0.0.0", true) + )], + failed: vec![FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\" after the stop attempt".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + }; + let error = uninstall_removal_gate(&report) + .expect_err("a non-empty `failed` must abort uninstall") + .to_string(); + assert!( + error.contains("svc-public"), + "names what it already stopped: {error}" + ); + assert!( + error.contains("--allow-public-bind"), + "says how a public service comes back after losing its key: {error}" + ); + } + + #[test] + fn every_failure_class_carries_its_own_recovery_advice() { + // This gate refuses to remove anything while a stop is unconfirmed, so + // the advice it prints is the operator's only way out; a class that + // reaches the abort with no advice turns the refusal into a dead end. + // `StopFailureRemedy::advice` and `advice_rank` are exhaustive matches, + // so a new variant cannot compile without being given text and a + // place in the order — but the compiler cannot check that the text is + // the *right* text, or that the three id-carrying classes still name + // their ids. That is what this asserts, across every class at once. + // Listed in REVERSE `advice_rank` order, and that is load-bearing + // rather than arbitrary. Listing them in rank order makes insertion + // order and rank order coincide, so the ordering assertion below holds + // whether or not the code sorts at all — deleting the sort left the + // whole suite green. Reversed, the assertion can only pass because + // `advice_rank` put them back. + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![ + FailedManagedServiceStop { + service_id: "svc-corrupt.json".to_owned(), + reason: "does not parse".to_owned(), + remedy: StopFailureRemedy::RepairTheRecord, + }, + FailedManagedServiceStop { + service_id: "runtime.json".to_owned(), + reason: "does not parse".to_owned(), + remedy: StopFailureRemedy::RepairTheDaemonState, + }, + FailedManagedServiceStop { + service_id: "rocmd (pid 4321)".to_owned(), + reason: "identity unverified".to_owned(), + remedy: StopFailureRemedy::StopTheDaemon, + }, + FailedManagedServiceStop { + service_id: "svc-orphaned".to_owned(), + reason: "endpoint still answers".to_owned(), + remedy: StopFailureRemedy::StopWhatHoldsThePort, + }, + FailedManagedServiceStop { + service_id: "svc-wedged".to_owned(), + reason: "still ready".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }, + FailedManagedServiceStop { + service_id: "svc-oldport".to_owned(), + reason: "an authenticated server answers".to_owned(), + remedy: StopFailureRemedy::RemoveTheStoppedRecord, + }, + FailedManagedServiceStop { + service_id: "svc-tickless".to_owned(), + reason: "no start time".to_owned(), + remedy: StopFailureRemedy::ConfirmTheServicePid, + }, + FailedManagedServiceStop { + service_id: "rocmd (pid 4321; state file /x/runtime-state.json)".to_owned(), + reason: "identity unverified".to_owned(), + remedy: StopFailureRemedy::ConfirmTheDaemonPid, + }, + ], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + + let error = uninstall_removal_gate(&report) + .expect_err("a non-empty `failed` must abort uninstall") + .to_string(); + + // One distinctive phrase per class, none of them supplied by this test. + for expected in [ + "rocm services stop --yes", + "Find what holds that port", + "restarts managed services on its own", + "cannot tell whether the helper is running", + "No `rocm` command can act on an unparseable record", + "rocm services remove --yes", + "No start time was recorded for a live pid", + "delete the runtime state file named there", + ] { + assert!( + error.contains(expected), + "every failure class must carry its own remedy; missing {expected:?} in: {error}" + ); + } + // Most actionable first, hand-repair last: a dead-end-avoidance + // ordering, not cosmetics. + let position = |needle: &str| error.find(needle).expect("asserted present above"); + // The whole `advice_rank` chain, one phrase per class: moving any one + // class out of its place breaks a neighbouring comparison. + let in_rank_order = [ + "rocm services stop --yes", + "Find what holds that port", + "rocm services remove --yes", + "No start time was recorded for a live pid", + "restarts managed services on its own", + "delete the runtime state file named there", + "cannot tell whether the helper is running", + "No `rocm` command can act on an unparseable record", + ]; + for pair in in_rank_order.windows(2) { + assert!( + position(pair[0]) < position(pair[1]), + "remedies must stay ordered most-actionable-first ({:?} before {:?}): {error}", + pair[0], + pair[1] + ); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn uninstall_refuses_while_a_recorded_endpoint_still_accepts_connections() { + // The Windows grandchild case, modelled faithfully: the recorded + // processes really do die, so the stop reports "stopped" — but an engine + // child that outlived them still holds the port and the GPU. Only the + // port probe can catch that; PID bookkeeping alone says success. + let listener = + std::net::TcpListener::bind("127.0.0.1:0").expect("bind the surviving engine's socket"); + let port = listener.local_addr().expect("socket address").port(); + // The recorded supervisor: alive now, so the record stays live through + // the liveness refresh, and killable, so the stop confirms. + let supervisor = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the recorded supervisor"); + let supervisor_pid = supervisor.id(); + let (root, paths) = test_paths("uninstall-endpoint-still-serving"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-orphaned-engine", + "vllm", + "m", + "m", + "127.0.0.1", + port, + "managed", + supervisor_pid, + None, + None, + None, + ); + record.supervisor_start_ticks = rocm_core::process_start_ticks(supervisor_pid); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let mut supervisor = supervisor; + assert_stopped_within_grace( + &mut supervisor, + "the recorded process must really have been stopped, so only the port probe \ + can catch the survivor", + ); + assert!( + !rocm_core::process_is_running(supervisor_pid), + "the recorded process must really have been stopped, so only the \ + port probe can catch the survivor" + ); + let failed: Vec<&str> = report + .failed + .iter() + .map(|failure| failure.service_id.as_str()) + .collect(); + assert_eq!( + failed, + vec!["svc-orphaned-engine"], + "a reachable endpoint must fail the gate: {report:?}" + ); + assert!( + !report.stopped.iter().any(|id| id == "svc-orphaned-engine"), + "a service that is still serving must not also be counted stopped: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "uninstall must not remove the tooling while something still serves" + ); + drop(listener); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stale_record_whose_old_port_was_reused_does_not_block_uninstall() { + // Nothing prunes a stopped record automatically, so an old one stays on + // disk until the operator runs `rocm services remove --yes`. If the gate judged + // those records by their recorded port, any unrelated process that later + // bound it would fail uninstall deterministically, with no override and + // no recovery — `rocm services stop` cannot help an already-stopped + // record, so every retry would fail identically. + // + // Unlike its neighbours this one does NOT fail if the port probe is + // deleted, and that is deliberate rather than an oversight: it guards + // the opposite direction. The others pin that a live server blocks; this + // pins that a *stranger* does not, so it fails against a naive "any + // listener blocks" gate — the over-strict implementation the rest of + // this file's pressure pushes toward — and passes against no gate at + // all. It is a false-positive guard, so read it as one. + let listener = std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind an unrelated process on a recycled port"); + let port = listener.local_addr().expect("socket address").port(); + let (root, paths) = test_paths("uninstall-stale-record-recycled-port"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-long-stopped", + "vllm", + "m", + "m", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.failed.is_empty(), + "a stale record must not be judged by whoever holds its old port now: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_ok(), + "uninstall must not be permanently blocked by a recycled port" + ); + drop(listener); + let _ = fs::remove_dir_all(root); + } + + /// A fake OpenAI `/v1/models` endpoint on loopback that shuts itself down. + /// + /// `drop`ping a bare `JoinHandle` only detaches it: the thread stays parked + /// in `accept()` holding an ephemeral port for the life of the test binary, + /// and a failing assertion panics before any manual cleanup line. Owning the + /// shutdown in `Drop` means the port is released even when the test fails, + /// which is when it matters. + struct ServingEndpoint { + bind_host: &'static str, + port: u16, + shutdown: std::sync::Arc, + thread: Option>, + } + + impl ServingEndpoint { + /// Serve `model_id` from `/v1/models` on IPv4 loopback until dropped. + fn serving(model_id: &str) -> Self { + Self::bind(model_id, "127.0.0.1", false).expect("bind the surviving engine on IPv4") + } + + /// Serve `model_id` only to a request carrying an `Authorization` header, + /// answering 401 otherwise — a public service as the retry run meets it. + fn serving_with_authorization(model_id: &str) -> Self { + Self::bind(model_id, "127.0.0.1", true).expect("bind the authenticated engine") + } + + /// Answer `/v1/models` with an empty list — an engine that is up and + /// holding the port but has not populated its models, or is mid-unload. + fn listing_nothing() -> Self { + Self::bind("", "127.0.0.1", false).expect("bind the empty-listing engine") + } + + /// Serve `model_id` on `bind_host`, or `None` when the host's address + /// family is unavailable (IPv6 is absent in some containers, and a test + /// that needs it has to skip rather than fail). + fn bind(model_id: &str, bind_host: &'static str, require_auth: bool) -> Option { + Self::bind_refusing_with(model_id, bind_host, require_auth, 401) + } + + /// As [`Self::bind`], answering `refusal_status` to a request the + /// authorization requirement rejects (401 and 403 are both "guarded"). + fn bind_refusing_with( + model_id: &str, + bind_host: &'static str, + require_auth: bool, + refusal_status: u16, + ) -> Option { + Self::bind_at(model_id, bind_host, 0, require_auth, refusal_status) + } + + /// Both loopback families on ONE port: `v6_model` on `::1` and + /// `v4_model` on `127.0.0.1`, so a test about a wildcard record owns + /// both halves and no concurrent test can grab the v4 side of its port. + /// `None` when this host has no IPv6 loopback. + fn bind_loopback_pair(v6_model: &str, v4_model: &str) -> Option<(Self, Self)> { + for _ in 0..50 { + let v4 = Self::bind(v4_model, "127.0.0.1", false)?; + if let Some(v6) = Self::bind_at(v6_model, "::1", v4.port, false, 401) { + return Some((v6, v4)); + } + // The port was free on v4 but taken on v6: try another. + } + None + } + + fn bind_at( + model_id: &str, + bind_host: &'static str, + port: u16, + require_auth: bool, + refusal_status: u16, + ) -> Option { + use std::sync::atomic::Ordering; + + let listener = std::net::TcpListener::bind((bind_host, port)).ok()?; + let port = listener.local_addr().ok()?.port(); + let shutdown = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stopping = std::sync::Arc::clone(&shutdown); + // An empty `model_id` means "list nothing at all", not "list a model + // whose id is the empty string" — the two are different answers and + // only the first is the engine-still-loading case. + let body = if model_id.is_empty() { + r#"{"data":[]}"#.to_owned() + } else { + format!(r#"{{"data":[{{"id":"{model_id}"}}]}}"#) + }; + let thread = thread::spawn(move || { + use std::io::{Read, Write}; + while let Ok((mut stream, _)) = listener.accept() { + if stopping.load(Ordering::SeqCst) { + break; + } + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + continue; + }; + let authorized = !require_auth + || String::from_utf8_lossy(&buffer[..read]) + .to_ascii_lowercase() + .contains("authorization: bearer "); + if authorized { + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + } else { + let _ = write!( + stream, + "HTTP/1.1 {refusal_status} Refused\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{{}}" + ); + } + } + }); + Some(Self { + bind_host, + port, + shutdown, + thread: Some(thread), + }) + } + } + + impl Drop for ServingEndpoint { + fn drop(&mut self) { + self.shutdown + .store(true, std::sync::atomic::Ordering::SeqCst); + // Unblock the parked `accept()` so the thread observes the flag. + let _ = std::net::TcpStream::connect((self.bind_host, self.port)); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } + } + + #[test] + fn a_stopped_record_still_serving_its_own_model_blocks_uninstall() { + // The retry the abort message asks for must not be the hole. A stop + // persists `status = "stopped"` BEFORE the gate probes the port, so the + // record that failed run 1 reads as not-live on run 2 — and if the gate + // only probed what it had just tried to stop, run 2 would sail through + // and remove the tooling while the survivor kept serving and holding the + // GPU. That is EAI-8014 reached by following the gate's own + // instructions, so the evidence has to survive the retry: an endpoint + // serving this record's own model blocks, whoever it belongs to. + let endpoint = ServingEndpoint::serving("amd/orphaned-model"); + let port = endpoint.port; + + let (root, paths) = test_paths("uninstall-stopped-record-still-serving"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-orphaned-engine", + "vllm", + "amd/orphaned-model", + "amd/orphaned-model", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + // Exactly the state run 1 leaves behind: supervisor killed, record + // written as stopped, engine grandchild still on the port. + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-orphaned-engine") + .unwrap_or_else(|| panic!("a still-serving engine must fail the gate: {report:?}")); + assert_eq!( + failure.remedy, + StopFailureRemedy::StopWhatHoldsThePort, + "the recorded processes are gone, so `rocm services stop` is not the remedy" + ); + // This record was already marked stopped before the run, so this pass + // never attempted a stop on it. The reason must not say the endpoint + // survived one — that describes something that did not happen, on the + // one output its operator has to reason from. Collapsing the two arms + // back into the old single sentence fails here. + assert!( + failure + .reason + .contains("is recorded stopped, but something there is still serving"), + "a record nothing was attempted on must not be reported as surviving a stop: {}", + failure.reason + ); + assert!( + !failure.reason.contains("after the stop"), + "no stop ran for this record, so the reason must not claim one did: {}", + failure.reason + ); + let error = uninstall_removal_gate(&report) + .expect_err("a still-serving engine must abort uninstall") + .to_string(); + assert!( + error.contains("outlived its supervisor"), + "says why `rocm services stop` will not help: {error}" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_wildcard_bound_engine_is_identified_instead_of_waved_through() { + // `probe_host` normalizes a wildcard bind to loopback for the + // reachability check, but the identity probe used to be handed the raw + // record, whose `endpoint_url` still spells the wildcard. That address + // does not resolve, so the probe errored and took the fail-open branch: + // uninstall removed the tooling while a wildcard-bound engine was still + // serving — the exact outcome this gate exists to prevent. + // + // `*` is one of the spellings `probe_host` documents a record can carry, + // and unlike `0.0.0.0` it fails to resolve on every platform, so this + // pins the behaviour rather than a host quirk. + let endpoint = ServingEndpoint::serving("amd/wildcard-model"); + let (root, paths) = test_paths("uninstall-wildcard-bound-engine"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-wildcard-engine", + "vllm", + "amd/wildcard-model", + "amd/wildcard-model", + "*", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-wildcard-engine") + .unwrap_or_else(|| { + panic!("a wildcard-bound engine still serving must fail the gate: {report:?}") + }); + assert_eq!( + failure.remedy, + StopFailureRemedy::StopWhatHoldsThePort, + "its recorded processes are gone, so the port holder is the remedy" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse to remove anything" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn an_ipv6_only_wildcard_engine_is_not_waved_through() { + // The IPv6 half of the wildcard case. `::` with `IPV6_V6ONLY` — the + // default on Windows, and what `TcpListener::bind("::1", 0)` gives here + // — answers on `::1` and refuses `127.0.0.1`. Probing IPv4 alone reads + // that live, port-holding engine as gone and removes the tooling anyway, + // which is EAI-8014 reached through the address family. + // The v4 half of the same port is held by a listener that names no + // models, so no concurrent test can bind it and answer for the engine. + let Some((endpoint, v4_holder)) = ServingEndpoint::bind_loopback_pair("amd/v6-model", "") + else { + // No IPv6 on this host: the thing under test cannot be staged. Say + // so on stderr rather than returning green and silent — a skip that + // looks identical to a pass is how a lane stops covering something + // without anyone noticing. `every_wildcard_bind_spelling_is_probed_\ + // on_both_loopback_families` still pins both families here, with no + // socket required. + eprintln!( + "SKIPPED an_ipv6_only_wildcard_engine_is_not_waved_through: no IPv6 loopback \ + on this host" + ); + return; + }; + let (root, paths) = test_paths("uninstall-ipv6-wildcard-engine"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-v6-engine", + "vllm", + "amd/v6-model", + "amd/v6-model", + // Recorded as the wildcard it was launched on, which is all the + // record ever says — not which family the listener chose. + "::", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-v6-engine") + .unwrap_or_else(|| { + panic!("an IPv6-only engine still serving must fail the gate: {report:?}") + }); + assert_eq!( + failure.remedy, + StopFailureRemedy::StopWhatHoldsThePort, + "its recorded processes are gone, so the port holder is the remedy" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse to remove anything" + ); + drop(endpoint); + drop(v4_holder); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stranger_on_one_loopback_family_does_not_hide_the_record_serving_on_the_other() { + // `probe_records_for_survivors` used to identity-probe only the FIRST + // loopback family that accepted a connection. Here `127.0.0.1` answers + // first with another model (proceed), while the record's own model still + // serves on `::1`: a first-hit probe waved the removal through. + let Some((v6_engine, v4_stranger)) = + ServingEndpoint::bind_loopback_pair("amd/dual-model", "amd/someone-elses-model") + else { + eprintln!( + "SKIPPED a_stranger_on_one_loopback_family_does_not_hide_the_record_serving_on_\ + the_other: no IPv6 loopback on this host" + ); + return; + }; + let (root, paths) = test_paths("uninstall-dual-family"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-dual", + "vllm", + "amd/dual-model", + "amd/dual-model", + "::", + v6_engine.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report + .failed + .iter() + .any(|failure| failure.service_id == "svc-dual" + && failure.remedy == StopFailureRemedy::StopWhatHoldsThePort), + "the record's own model still serving on ::1 must block: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse to remove anything" + ); + drop(v6_engine); + drop(v4_stranger); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn every_identity_answer_maps_to_exactly_one_gate_outcome() { + // The five arms, asserted directly. Reached through the stop loop each + // one needs a server that answers a particular way plus a reading of + // stderr, which is why two of them were previously pinned by nothing: + // mutating either "proceed" arm into a block, or a block into a proceed, + // left the whole suite green while changing what a destructive command + // does to a live endpoint. + use rocm_core::EndpointIdentity::{ListsNoModels, ServesExpectedModel, ServesOtherModels}; + + assert_eq!( + stopped_record_verdict(Some(ServesExpectedModel), false), + StoppedRecordVerdict::BlockServingOurModel, + "an engine serving this record's own model outlived its supervisor" + ); + // Only this one is allowed to be silent: naming models and not naming + // ours is the single answer that is real evidence of a stranger. + assert_eq!( + stopped_record_verdict(Some(ServesOtherModels), false), + StoppedRecordVerdict::ProceedUnrelated, + "a listener naming other models is somebody else on a recycled port" + ); + // Blocking here would turn any JSON listener that lists nothing into an + // unescapable abort; proceeding silently would remove the tooling from + // under an engine that is merely still loading. Hence a third outcome. + assert_eq!( + stopped_record_verdict(Some(ListsNoModels), false), + StoppedRecordVerdict::ProceedListingNothing, + "an empty listing is not evidence of a stranger, and not silent" + ); + assert_eq!( + stopped_record_verdict(None, true), + StoppedRecordVerdict::BlockAuthRefused, + "a refusal is a live server stating it guards the path" + ); + assert_eq!( + stopped_record_verdict(None, false), + StoppedRecordVerdict::ProceedUnidentified, + "no usable answer is the documented fail-open" + ); + + assert!(stopped_record_verdict(Some(ServesExpectedModel), false).blocks()); + assert!(stopped_record_verdict(None, true).blocks()); + assert!(!stopped_record_verdict(Some(ServesOtherModels), false).blocks()); + assert!(!stopped_record_verdict(Some(ListsNoModels), false).blocks()); + assert!(!stopped_record_verdict(None, false).blocks()); + } + + #[test] + fn an_endpoint_listing_nothing_does_not_block_uninstall() { + // The end-to-end half of the `ListsNoModels` arm: it must not abort. + // Blocking would make any listener that answers `/v1/models` with an + // empty list — including something unrelated on a recycled port — an + // abort with no override, which is the dead end this gate must not + // create. Sensitive to exactly one mutation: turning that arm into a + // block. + // + // What keeps this a tradeoff rather than a silent removal is the + // warning, a value on the report that this test asserts below. The + // verdict mapping is pinned separately, so emptying the warning body or + // flipping the verdict each turn a test red. + let endpoint = ServingEndpoint::listing_nothing(); + let (root, paths) = test_paths("uninstall-endpoint-listing-nothing"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-empty-listing", + "vllm", + "amd/loading-model", + "amd/loading-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.failed.is_empty(), + "an empty model list must not abort uninstall: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_ok(), + "the gate must let the removal proceed" + ); + // The disclosure, not just the decision. Proceeding here is only + // defensible because the operator is told the port was never proven + // free, so emptying that message is as much a regression as flipping + // the verdict — and until the warnings became values on the report, + // nothing could say so. + assert!( + report + .warnings + .iter() + .any(|warning| warning.contains("empty model list") + && warning.contains("svc-empty-listing")), + "proceeding past an empty listing must disclose itself: {:?}", + report.warnings + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_record_whose_host_no_longer_resolves_says_so_before_proceeding() { + // The third fail-open, and the one that used to pass in silence. A + // refused connect and an unresolvable name both come back `false` from + // `loopback_tcp_port_is_reachable`, but they are opposite evidence: the + // first says the port is free, the second says the question was never + // asked. Proceeding on the second is still the right call — a record + // written on another machine must not brick uninstall — but it has to + // be disclosed, and `.invalid` is reserved by RFC 2606 precisely so it + // never resolves anywhere. + let (root, paths) = test_paths("uninstall-host-unresolvable"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-foreign-host", + "vllm", + "amd/our-model", + "amd/our-model", + "no-such-host.invalid", + 8123, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.failed.is_empty(), + "an unresolvable host must not abort uninstall: {report:?}" + ); + assert!( + report + .warnings + .iter() + .any(|warning| warning.contains("does not resolve here") + && warning.contains("svc-foreign-host")), + "skipping a record whose host does not resolve must disclose itself: {:?}", + report.warnings + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_listener_naming_another_model_does_not_block_uninstall() { + // The end-to-end half of the `ServesOtherModels` arm, and the only test + // that drives it through the real call site rather than through the + // pure helper. A listener that names its models and does not name ours + // is the one answer that is positive evidence of a stranger on a + // recycled port, so it must proceed — blocking here would let any + // unrelated OpenAI-shaped server on a reused port wedge uninstall with + // no override. Sensitive to exactly one mutation: turning that arm into + // a block. + let endpoint = ServingEndpoint::serving("amd/somebody-elses-model"); + let (root, paths) = test_paths("uninstall-endpoint-other-model"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-recycled-port", + "vllm", + "amd/our-model", + "amd/our-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.failed.is_empty(), + "a listener naming only other models must not abort uninstall: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_ok(), + "the gate must let the removal proceed" + ); + // The one proceed-arm that is allowed to be silent, and the assertion + // that keeps it distinguishable from the two that are not: a listener + // that named its models and did not name ours is positive evidence of a + // stranger, so there is nothing to disclose. + assert!( + report.warnings.is_empty(), + "a named stranger is evidence, not a fail-open, so it warns about nothing: {:?}", + report.warnings + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn an_authenticated_survivor_blocks_the_retry_instead_of_failing_open() { + // The retry run is where a public service loses its own evidence: + // stopping the recorded processes clears the stored endpoint key, so run + // 2 probes the still-serving endpoint with no credentials and is + // answered 401. `managed_service_endpoint_model_ready` turns any non-200 + // into `Err`, which used to take the fail-open branch — deleting the + // tooling while a publicly reachable, GPU-holding server kept answering. + // + // A refusal is not a failure to reach: it is a live HTTP server saying + // it guards this path, and that has to block. + let endpoint = ServingEndpoint::serving_with_authorization("amd/guarded-model"); + let (root, paths) = test_paths("uninstall-authenticated-survivor"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-guarded-engine", + "vllm", + "amd/guarded-model", + "amd/guarded-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + // Exactly run 2's state: recorded stopped, and no key on disk for it. + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + assert!( + endpoint_keys::endpoint_api_key(&paths, "svc-guarded-engine").is_none(), + "the retry has no credentials left — that is the whole scenario" + ); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-guarded-engine") + .unwrap_or_else(|| { + panic!( + "an authenticated survivor must fail the gate, not be waved through: {report:?}" + ) + }); + assert_eq!( + failure.remedy, + StopFailureRemedy::RemoveTheStoppedRecord, + "an authenticated answer on an old record's port is cleared by removing the record" + ); + assert!( + failure.reason.contains("authenticated server answers"), + "the abort has to say an authenticated server answers there: {}", + failure.reason + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse to remove anything" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn a_forbidden_answer_on_a_stopped_records_port_blocks_like_an_unauthorized_one() { + // A server that answers 403 is as much a live, guarded HTTP server as one + // that answers 401; narrowing the refusal check to 401 alone would wave + // it through as an unrelated listener. + let endpoint = + ServingEndpoint::bind_refusing_with("amd/guarded-model", "127.0.0.1", true, 403) + .expect("bind the forbidding engine"); + let (root, paths) = test_paths("uninstall-forbidden-survivor"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-forbidden-engine", + "vllm", + "amd/guarded-model", + "amd/guarded-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-forbidden-engine") + .unwrap_or_else(|| panic!("a 403 survivor must fail the gate: {report:?}")); + assert_eq!(failure.remedy, StopFailureRemedy::RemoveTheStoppedRecord); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn the_stored_endpoint_key_reaches_the_identity_probe() { + // A keyed server answers 401 without its key and lists its model with it. + // The record's key, when one is still stored, has to be what the probe + // sends: passing no key reads the server as a stranger-or-refusal, and + // here that changes the outcome from "serving our model" to "auth refused". + let endpoint = ServingEndpoint::serving_with_authorization("amd/keyed-model"); + let (root, paths) = test_paths("uninstall-stored-key-probe"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-keyed-engine", + "vllm", + "amd/keyed-model", + "amd/keyed-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + endpoint_keys::store_endpoint_api_key(&paths, "svc-keyed-engine", "sekret") + .expect("store the endpoint key"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-keyed-engine") + .unwrap_or_else(|| panic!("a keyed survivor must fail the gate: {report:?}")); + assert_eq!( + failure.remedy, + StopFailureRemedy::StopWhatHoldsThePort, + "with its key the probe sees the record's own model, not a refusal: {report:?}" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn only_json_files_in_the_services_dir_are_service_manifests() { + // The directory also holds non-record files (editor backups, logs); a + // non-.json file that does not parse must not read as a corrupt record. + let (root, paths) = test_paths("uninstall-non-json-manifest"); + fs::create_dir_all(paths.services_dir()).expect("create services dir"); + fs::write(paths.services_dir().join("svc-x.txt"), b"{ not json").expect("seed"); + fs::write(paths.services_dir().join("svc-y.json.bak"), b"{ not json").expect("seed"); + assert!( + unreadable_service_manifests(&paths) + .expect("scan") + .is_empty() + ); + fs::write(paths.services_dir().join("svc-z.json"), b"{ not json").expect("seed"); + assert_eq!( + unreadable_service_manifests(&paths).expect("scan"), + vec![paths.services_dir().join("svc-z.json")] + ); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux: drives a real spawned stand-in verified through its start time and zombie-state detection. + #[cfg(target_os = "linux")] + #[test] + fn a_doomed_run_stops_nothing_on_its_way_to_the_abort() { + // Stopping is not free and not undoable: each confirmed stop drops that + // service's endpoint key, and a publicly bound service has to be served + // again with an explicit flag to get one back. Once the gate is certain + // to abort — nothing removed, tooling intact — every stop performed + // first is pure cost for a removal that will not happen. + // + // The daemon failure used here is the unreadable-state one, which this + // file already documents at length as the reason to leave the helper + // alone. That same reasoning has to cover the services: it made no sense + // to spare the daemon and then stop everything else for no gain. + let server = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn managed server"); + let pid = server.id(); + let (root, paths) = test_paths("uninstall-doomed-run-stops-nothing"); + paths.ensure().expect("create the app directories"); + fs::write(paths.automation_state_path(), b"{ not json") + .expect("seed a corrupt runtime state"); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = managed_record_for_pid(&paths, pid, Some(real)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.stopped.is_empty(), + "a run that is already going to abort must stop nothing: {report:?}" + ); + let mut server = server; + assert!( + server.try_wait().expect("poll the server").is_none(), + "the managed server must be left running by a run that aborts anyway" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the unreadable runtime state must still abort the uninstall" + ); + let _ = server.kill(); + let _ = server.wait(); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn serving_the_same_model_again_after_a_stop_does_not_doom_the_uninstall() { + // serve M -> stop -> serve M: record A "stopped", record B "ready", same + // host:port, and B's own server answers there. A must not block the run; + // B gets stopped (by the injected stop) and only then is A judged. + let endpoint = std::sync::Mutex::new(Some(ServingEndpoint::serving("amd/reused-model"))); + let port = endpoint.lock().unwrap().as_ref().unwrap().port; + let (root, paths) = test_paths("uninstall-reused-port"); + let make = |id: &str, status: &str| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/reused-model", + "amd/reused-model", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = status.to_owned(); + record.write().expect("write record"); + }; + make("svc-a", "stopped"); + make("svc-b", "ready"); + let calls = std::sync::Mutex::new(Vec::::new()); + + let report = stop_managed_services_with(&paths, |_, id| { + calls.lock().unwrap().push(id.to_owned()); + // Stopping B is what frees the port. + drop(endpoint.lock().unwrap().take()); + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("the pass succeeds"); + + assert_eq!(*calls.lock().unwrap(), vec!["svc-b".to_owned()]); + assert!( + !report.failed.iter().any(|f| f.service_id == "svc-a"), + "the stopped record shares the live record's port and must not doom the run first: {report:?}" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn the_plan_read_error_warning_only_promises_a_refusal_where_the_stop_pass_runs() { + use std::os::unix::fs::PermissionsExt; + + let (root, paths) = test_paths("uninstall-plan-read-error"); + paths.ensure().expect("create the app directories"); + // A services directory that exists but cannot be listed makes the read fail. + fs::create_dir_all(paths.services_dir()).expect("create services dir"); + fs::set_permissions(paths.services_dir(), fs::Permissions::from_mode(0o000)) + .expect("make the services dir unreadable"); + if fs::read_dir(paths.services_dir()).is_ok() { + // Running as root: permissions do not bind, so this cannot stage a failure. + fs::set_permissions(paths.services_dir(), fs::Permissions::from_mode(0o755)).ok(); + let _ = fs::remove_dir_all(root); + eprintln!("SKIPPED the_plan_read_error_warning: permissions do not bind this user"); + return; + } + + let full = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("plan"); + let kept = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + keep_binaries: true, + keep_config: true, + keep_data: true, + ..UninstallOptions::default() + }, + ) + .expect("plan"); + let find = |plan: &UninstallPlan| { + plan.warnings + .iter() + .find(|w| w.contains("could not read the managed service records")) + .cloned() + .unwrap_or_default() + }; + assert!( + find(&full).contains("will refuse to remove anything"), + "{}", + find(&full) + ); + assert!( + find(&kept).contains("keeps the tooling") && !find(&kept).contains("will refuse"), + "{}", + find(&kept) + ); + // And the behaviour the warning promises: the real command, against the + // same unreadable directory, errors out and removes nothing. + let marker = paths.config_dir.join("keep-me"); + fs::create_dir_all(&paths.config_dir).expect("config dir"); + fs::write(&marker, b"x").expect("seed a planned file"); + // `keep_binaries` keeps this off the real executable-discovery path: with + // the dev-binary override a regression here would delete the shared build + // output. The services dir lives under the data dir, so it still refuses. + let outcome = crate::uninstall::uninstall_with_paths( + &paths, + &UninstallOptions { + yes: true, + keep_binaries: true, + ..UninstallOptions::default() + }, + ); + assert!( + outcome.is_err(), + "an unreadable services dir must abort uninstall" + ); + assert!(marker.exists(), "an aborted uninstall must remove nothing"); + fs::set_permissions(paths.services_dir(), fs::Permissions::from_mode(0o755)).ok(); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn a_stopped_record_spelled_with_a_wildcard_host_still_waits_for_its_live_partner() { + // Same shape as the port-reuse case, but the stopped record says + // `0.0.0.0` and the live one `127.0.0.1`: both name the one listener. + let endpoint = std::sync::Mutex::new(Some(ServingEndpoint::serving("amd/reused-model"))); + let port = endpoint.lock().unwrap().as_ref().unwrap().port; + let (root, paths) = test_paths("uninstall-reused-port-wildcard"); + let make = |id: &str, host: &str, status: &str| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/reused-model", + "amd/reused-model", + host, + port, + "managed", + 0, + None, + None, + None, + ); + record.status = status.to_owned(); + record.write().expect("write record"); + }; + make("svc-a", "0.0.0.0", "stopped"); + make("svc-b", "127.0.0.1", "ready"); + let report = stop_managed_services_with(&paths, |_, _| { + drop(endpoint.lock().unwrap().take()); + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("the pass succeeds"); + // Only the stopped record is judged here, as in the sibling test above. + // The live record's own post-stop probe asks whether the freed port is + // quiet, and a concurrent test binding an ephemeral port can answer it, + // which says nothing about the host spelling this test pins. + assert!( + !report.failed.iter().any(|f| f.service_id == "svc-a"), + "a differently spelled host for the same listener must not doom the run: {report:?}" + ); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux: drives a real spawned stand-in verified through its start time and zombie-state detection. + #[cfg(target_os = "linux")] + #[test] + fn a_deferred_stopped_record_is_judged_once_its_live_partner_has_stopped() { + // The stopped record shares `::1` with the live one, so it waits for the + // live stop. The live record's own probe looks at `::1` and finds + // nothing, so it is confirmed stopped; but the stopped record's wildcard + // host also answers on IPv4 loopback, where a survivor still serves its + // model. Only the post-stop judgement of the deferred record can see it, + // so dropping that judgement lets the removal through. + let endpoint = ServingEndpoint::serving("amd/reused-model"); + let port = endpoint.port; + let (root, paths) = test_paths("uninstall-deferred-survivor"); + // The live record is held live by a real process, not by an answering + // endpoint: nothing answers on `::1`, and a record whose process is gone + // and whose endpoint is silent is demoted to stopped on load. + let supervisor = KillOnDrop( + std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the live record's supervisor"), + ); + let supervisor_pid = supervisor.0.id(); + let make = |id: &str, host: &str, status: &str, pid: u32| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/reused-model", + "amd/reused-model", + host, + port, + "managed", + pid, + None, + None, + None, + ); + if pid != 0 { + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + } + record.status = status.to_owned(); + record.write().expect("write record"); + }; + make("svc-a", "::", "stopped", 0); + make("svc-b", "::1", "ready", supervisor_pid); + + let report = stop_managed_services_with(&paths, |_, _| { + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("the pass succeeds"); + + let ids: Vec<&str> = report + .failed + .iter() + .map(|f| f.service_id.as_str()) + .collect(); + assert_eq!(ids, vec!["svc-a"], "{report:?}"); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::StopWhatHoldsThePort + ); + assert_eq!(report.stopped, vec!["svc-b".to_owned()], "{report:?}"); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_real_stop_that_a_survivor_outlives_says_the_key_is_gone_and_it_is() { + // A keyed service is stopped for real (the production stop, which drops + // its endpoint key), then something still answers with its model, so the + // run aborts. The abort must say the key was dropped, and the key must + // actually be gone: the message and the state asserted together. + let endpoint = ServingEndpoint::serving("amd/keyed-model"); + let (root, paths) = test_paths("uninstall-keyed-survivor"); + let mut server = std::process::Command::new("sleep") + .arg("60") + .stdout(std::process::Stdio::null()) + .spawn() + .expect("spawn the managed server stand-in"); + let pid = server.id(); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-keyed", + "vllm", + "amd/keyed-model", + "amd/keyed-model", + "127.0.0.1", + endpoint.port, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + record.status = "ready".to_owned(); + record.write().expect("write record"); + endpoint_keys::store_endpoint_api_key(&paths, "svc-keyed", "live-secret") + .expect("store the key"); + assert!(endpoint_keys::endpoint_api_key(&paths, "svc-keyed").is_some()); + + let report = stop_managed_services_before_uninstall(&paths).expect("the pass runs"); + let message = format!( + "{:#}", + uninstall_removal_gate(&report).expect_err("the survivor aborts the run") + ); + assert!( + message.contains("the stop already dropped this service's endpoint key"), + "{message}" + ); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, "svc-keyed"), + None, + "the message claims the key was dropped: {message}" + ); + let _ = server.wait(); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn a_failed_live_partner_is_not_named_twice_through_its_stopped_twin() { + // Same port, one record stopped and one live. When the live one's stop + // fails, its own failure explains the listener there; the stopped twin + // must not add a second, wrong "the port holder" failure. + let endpoint = ServingEndpoint::serving("amd/reused-model"); + let (root, paths) = test_paths("uninstall-failed-partner"); + let make = |id: &str, status: &str| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/reused-model", + "amd/reused-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = status.to_owned(); + record.write().expect("write record"); + }; + make("svc-a", "stopped"); + make("svc-b", "ready"); + + let report = stop_managed_services_with(&paths, |_, _| anyhow::bail!("stop blew up")) + .expect("the pass succeeds"); + + let ids: Vec<&str> = report + .failed + .iter() + .map(|f| f.service_id.as_str()) + .collect(); + assert_eq!(ids, vec!["svc-b"], "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn the_daemon_stop_skips_pid_zero_and_its_own_pid() { + for pid in [0, std::process::id()] { + let (root, paths) = test_paths("uninstall-daemon-self-pid"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, _| { + panic!("pid {pid} must never be signalled") + }); + assert!( + report.failed.is_empty() && !report.helper_stopped, + "{report:?}" + ); + let _ = fs::remove_dir_all(root); + } + } + + #[test] + fn localhost_and_loopback_name_one_listener_but_other_hosts_do_not() { + assert!(hosts_name_one_listener("localhost", "127.0.0.1")); + assert!(hosts_name_one_listener("127.0.0.1", "127.0.0.1")); + // `localhost` resolves to `::1` on many hosts (typically Windows), so it + // can name a listener on either family. + assert!(hosts_name_one_listener("localhost", "::1")); + assert!(hosts_name_one_listener("::1", "localhost")); + assert!(!hosts_name_one_listener("127.0.0.1", "::1")); + assert!(!hosts_name_one_listener("10.0.0.5", "127.0.0.1")); + assert!(!hosts_name_one_listener("localhost", "10.0.0.5")); + } + + #[test] + fn a_helper_only_stop_prints_a_success_line() { + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: Vec::new(), + warnings: Vec::new(), + helper_stopped: true, + ..ManagedServiceStopReport::default() + }; + assert_eq!( + uninstall_removal_gate(&report).unwrap().as_deref(), + Some("stopped the background helper (rocmd)") + ); + } + + #[test] + fn the_stopped_service_advice_names_a_serve_command_that_restores_a_public_bind() { + let report = ManagedServiceStopReport { + stopped: vec!["svc-a".to_owned()], + relaunch: vec![format!( + "svc-a: `{}`", + rocm_engine_protocol::relaunch_command("0.0.0.0", true) + )], + failed: vec![FailedManagedServiceStop { + service_id: "svc-b".to_owned(), + reason: "r".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!( + message.contains(&format!( + "svc-a: `{}`", + rocm_engine_protocol::relaunch_command("0.0.0.0", true) + )) && message.contains("--allow-public-bind"), + "the stopped public service gets the command that restores its bind: {message}" + ); + // The same builder the two refusals use: a keyed service gets + // `--require-api-key`, and the key itself is optional. + assert!(message.contains("--require-api-key"), "{message}"); + assert!( + message.contains(rocm_engine_protocol::RELAUNCH_KEY_CHOICE), + "{message}" + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_survivor_behind_an_already_stopped_record_dooms_the_run_before_anything_is_stopped() { + // The retry the abort message asks for: last run's survivor reads as an + // already-stopped record whose endpoint still serves its own model. That + // verdict needs nothing this pass stops, so it must land before the live + // server is touched. + let endpoint = ServingEndpoint::serving("amd/survivor-model"); + let mut server = KillOnDrop( + std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn managed server"), + ); + let pid = server.0.id(); + let mut daemon = KillOnDrop( + std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a verified daemon stand-in"), + ); + let daemon_pid = daemon.0.id(); + let (root, paths) = test_paths("uninstall-doomed-by-stopped-record"); + let mut survivor = ManagedServiceRecord::new( + &paths, + "svc-survivor", + "vllm", + "amd/survivor-model", + "amd/survivor-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + survivor.status = "stopped".to_owned(); + survivor.write().expect("write survivor record"); + let mut state = runtime_state(true, daemon_pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(daemon_pid); + assert!(state.daemon_start_ticks.is_some(), "a verified daemon"); + state.write(&paths).expect("write runtime state"); + let mut live = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + live.status = "ready".to_owned(); + live.write().expect("write live record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.stopped.is_empty(), + "nothing may be stopped: {report:?}" + ); + assert!( + server.0.try_wait().expect("poll the server").is_none(), + "the live server must be left running by a run that aborts anyway" + ); + assert!( + daemon.0.try_wait().expect("poll the daemon").is_none() && !report.helper_stopped, + "a verified daemon must not be force-killed by a run that aborts anyway: {report:?}" + ); + assert!( + report + .failed + .iter() + .any(|failure| failure.service_id == "svc-survivor"), + "{report:?}" + ); + assert!(uninstall_removal_gate(&report).is_err()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stopped_helper_is_reported_apart_from_services_and_worded_as_a_helper() { + let failed_after = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "svc".to_owned(), + reason: "still ready".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: true, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&failed_after).unwrap_err()); + assert!( + message.contains("background helper (rocmd) was stopped"), + "{message}" + ); + assert!( + !message.contains("endpoint keys"), + "the helper has no endpoint key, and no services were stopped: {message}" + ); + + let ok = ManagedServiceStopReport { + stopped: vec!["a".to_owned()], + failed: Vec::new(), + warnings: Vec::new(), + helper_stopped: true, + ..ManagedServiceStopReport::default() + }; + assert_eq!( + uninstall_removal_gate(&ok).unwrap().as_deref(), + Some( + "stopped 1 managed service(s) before removal; stopped the background helper (rocmd)" + ) + ); + } + + #[test] + fn an_endpoint_that_answers_without_a_usable_listing_discloses_the_fail_open() { + // `ProceedUnidentified`: the port still accepts connections, the record is + // already stopped, and uninstall proceeds. The warning is the only thing + // that makes that a disclosed tradeoff. + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("addr").port(); + let (root, paths) = test_paths("uninstall-unidentified-listener"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-wedged", + "vllm", + "amd/m", + "amd/m", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!(report.failed.is_empty(), "{report:?}"); + assert!( + report.warnings.iter().any( + |w| w.contains("did not answer the identity probe") && w.contains("svc-wedged") + ), + "{report:?}" + ); + drop(listener); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unparseable_manifest_aborts_before_any_service_is_stopped() { + // Same shape as the daemon case and worse: a manifest that cannot be + // parsed may itself describe a live, GPU-holding server, so collecting + // it last meant every other service was already down — and its key + // already dropped — before the run turned out to be doomed. + // + // The daemon is staged live and fully verifiable here on purpose. It is + // the one thing the gate would otherwise still destroy on a doomed run: + // the manifest scan only reads, so putting the force-kill ahead of it + // terminates a healthy `rocmd` for an uninstall that removes nothing. + let server = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn managed server"); + let pid = server.id(); + let daemon = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn background helper"); + let daemon_pid = daemon.id(); + let (root, paths) = test_paths("uninstall-bad-manifest-ordering"); + paths.ensure().expect("create the app directories"); + let mut state = runtime_state(true, daemon_pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(daemon_pid); + state.write(&paths).expect("write runtime state"); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = managed_record_for_pid(&paths, pid, Some(real)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + fs::write(paths.services_dir().join("svc-corrupt.json"), b"{ not json") + .expect("seed an unparseable manifest"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.stopped.is_empty(), + "an unparseable manifest must abort before anything is stopped: {report:?}" + ); + let mut server = server; + let mut daemon = daemon; + assert!( + server.try_wait().expect("poll the server").is_none(), + "the healthy service must be untouched when the run aborts on another record" + ); + assert!( + daemon.try_wait().expect("poll the helper").is_none(), + "a live, verifiable rocmd must not be force-killed for a run that removes nothing" + ); + assert!( + report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::RepairTheRecord), + "the unparseable manifest must be the recorded failure: {report:?}" + ); + let _ = server.kill(); + let _ = server.wait(); + let _ = daemon.kill(); + let _ = daemon.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_record_without_a_start_time_is_a_legacy_record_where_start_times_are_readable() { + // The readable-start-time half of the predicate's contract (Linux, Windows), and no more than that. + // + // What actually prevents the conflation this predicate exists for — + // answering "can this platform report a start-time?" by reading the PID + // under inspection, which cannot tell "no `/proc` on this OS" from "that + // read just failed" — is the signature: `record_predates_start_ticks` + // takes no PID, so passing one is a compile error. It is prevented by + // construction, not caught by an assertion, and no assertion here should + // claim otherwise. + // + // The guard's end-to-end behaviour is pinned separately, by + // `uninstall_never_kills_a_daemon_pid_from_a_state_file_that_predates_\ + // start_ticks`, which stages a live unrelated process against a legacy + // record and fails if the guard is removed. + assert!( + record_predates_start_ticks(None), + "where start-times are readable, a record carrying none predates the field" + ); + assert!( + !record_predates_start_ticks(Some(1)), + "a record that carries a start-time is never a pre-upgrade record" + ); + } + + // `sleep` as a stand-in for the process that inherited the pid. + #[cfg(unix)] + #[test] + fn uninstall_never_signals_a_daemon_pid_recorded_as_already_stopped() { + // `running: false` is written by exactly one place — rocmd's clean + // shutdown, on its way out. (A daemon started without + // `--automations-enabled` returns before its first state write, so it + // never persists a false flag while alive.) The pid in such a record + // therefore belongs to a process that has already exited, and anything + // live under that number today inherited it. + // + // Windows is where this is the only defence: `process_start_ticks` is + // unset there for a record written before the upgrade, so the identity check falls back to its legacy + // `Matches` verdict and the force tree-kill lands on a stranger. That + // shape cannot be staged on Linux — identity genuinely works here — so + // this stages the same *decision*: a live pid that the identity check + // will confirm, which must still be left alone because the record says + // the daemon is stopped. + let mut stranger = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn unrelated process"); + let pid = stranger.id(); + let (root, paths) = test_paths("uninstall-daemon-recorded-stopped"); + let mut state = runtime_state(false, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + // `try_wait`, not `process_is_running`: a signalled child we have not + // reaped is a zombie, and a zombie still reads as running — so the + // liveness check would pass over the very kill this test exists to + // catch. An exit status here means it was signalled. + assert!( + stranger.try_wait().expect("poll the stranger").is_none(), + "a pid recorded as already stopped must never be signalled: uninstall killed an \ + unrelated process" + ); + assert!( + report.stopped.is_empty(), + "the daemon was already stopped, so nothing was stopped here: {report:?}" + ); + assert!( + report.failed.is_empty(), + "an already-stopped daemon is not an obstacle to uninstall: {report:?}" + ); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn every_wildcard_bind_spelling_is_probed_on_both_loopback_families() { + // A wildcard probed literally is not portable and would read as "nothing + // is serving" — the gate's failure-open direction, so it matters. + // + // Both families, not one: a wildcard record does not say which protocol + // the listener bound, and the two loopbacks are not interchangeable. A + // `::` bind with the usual `IPV6_V6ONLY=1` — the default on Windows — + // answers on `::1` and refuses `127.0.0.1`, so probing IPv4 alone reads + // a live engine as gone and waves the removal through. + for wildcard in [ + "0.0.0.0", + "::", + "[::]", + "0:0:0:0:0:0:0:0", + "*", + "", + " 0.0.0.0 ", + ] { + assert_eq!( + probe_hosts(wildcard), + vec!["127.0.0.1".to_owned(), "::1".to_owned()], + "wildcard {wildcard:?} must be probed on both loopback families" + ); + } + for literal in ["127.0.0.1", "192.168.1.10", "example.internal"] { + assert_eq!( + probe_hosts(literal), + vec![literal.to_owned()], + "a concrete host must be probed as recorded, and only there" + ); + } + // A concrete host still has to come back in a form that resolves. + // `(host, port).to_socket_addrs()` rejects a bracketed literal and + // anything padded, and the caller reads a resolution failure as "nothing + // is serving" — so handing the raw spelling through would delete the + // recovery tooling while the endpoint is live. + for (recorded, probed) in [ + ("[::1]", "::1"), + (" 127.0.0.1 ", "127.0.0.1"), + ("[FE80::1]", "fe80::1"), + ] { + assert_eq!( + probe_hosts(recorded), + vec![probed.to_owned()], + "a concrete host must be probed in a resolvable form" + ); + } + // Every probed form, wildcard expansions included, has to resolve: + // `(host, port).to_socket_addrs()` rejects a bracketed literal and + // anything padded, and the caller reads a resolution failure as "nothing + // is serving" — so handing a raw spelling through would delete the + // recovery tooling while the endpoint is live. + for recorded in ["[::1]", " 127.0.0.1 ", "[FE80::1]", "::", "0.0.0.0", "*"] { + for probed in probe_hosts(recorded) { + assert!( + std::net::ToSocketAddrs::to_socket_addrs(&(probed.as_str(), 1u16)).is_ok(), + "the probed form {probed:?} of {recorded:?} must resolve" + ); + } + } + } + + #[test] + fn only_an_uninstall_that_removes_the_recovery_tooling_stops_servers() { + // `--keep-binaries --keep-data` removes the cache alone: `rocm services + // stop` and every service record survive, so there is nothing to protect + // by force-stopping live servers. + let (root, paths) = test_paths("uninstall-recovery-tooling-scope"); + let cache_only = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "cache", + path: paths.cache_dir.clone(), + }], + ..UninstallPlan::default() + }; + assert!( + !plan_removes_recovery_tooling(&cache_only, &paths), + "a cache-only uninstall keeps the tooling that stops servers" + ); + + let with_binaries = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "binary", + path: PathBuf::from("/usr/local/bin/rocm"), + }], + ..UninstallPlan::default() + }; + assert!( + plan_removes_recovery_tooling(&with_binaries, &paths), + "removing the binaries takes away `rocm services stop`" + ); + + let with_data = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "data", + path: paths.data_dir.clone(), + }], + ..UninstallPlan::default() + }; + assert!( + plan_removes_recovery_tooling(&with_data, &paths), + "removing the data dir takes away the service records" + ); + + // The other direction: an entry INSIDE the services directory removes + // service records without being (or containing) the data dir itself. + let inside_services = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "data", + path: paths.services_dir().join("svc-a.json"), + }], + ..UninstallPlan::default() + }; + assert!( + plan_removes_recovery_tooling(&inside_services, &paths), + "removing a service record takes away what stops its server" + ); + let elsewhere = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "cache", + path: paths.data_dir.join("unrelated").join("file"), + }], + ..UninstallPlan::default() + }; + assert!( + !plan_removes_recovery_tooling(&elsewhere, &paths), + "a path beside the services directory leaves it alone" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_plan_does_not_promise_a_stop_for_a_live_record_the_gate_refuses() { + // A live record whose pid has no recorded start time is refused, not + // stopped. The plan is the last thing read before confirming, so it + // must say that — and the claim is only worth anything next to the + // outcome it describes, so both come from the one run. + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-plan-tickless"); + let mut record = managed_record_for_pid(&paths, pid, None); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("build the plan"); + let refusal = plan + .warnings + .iter() + .find(|warning| warning.contains("will refuse to remove anything")) + .expect("the plan says the run will refuse"); + assert!( + refusal.contains("no start time was recorded") + && refusal.contains("will stop nothing") + && refusal.contains("svc-managed-stop"), + "the plan must say the stop will be refused and for which record: {refusal}" + ); + let count = plan + .warnings + .iter() + .find(|warning| warning.contains("managed service record(s) under")) + .expect("the plan counts the managed services"); + assert!( + count.contains("none of them will be stopped") + && !count.contains("will be stopped before removal"), + "the plan must not promise the stop the gate refuses: {count}" + ); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("the record the plan said would be refused must not be stopped") + }) + .expect("stop pass"); + assert_eq!( + report.failed.len(), + 1, + "the gate refuses exactly what the plan said it would: {report:?}" + ); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::ConfirmTheServicePid + ); + assert!(child.try_wait().expect("poll").is_none(), "left alive"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + fn full_plan_options() -> UninstallOptions { + UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + } + } + + #[cfg(target_os = "linux")] + fn plan_warning_containing(plan: &UninstallPlan, needle: &str) -> Option { + plan.warnings + .iter() + .find(|warning| warning.contains(needle)) + .cloned() + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unparseable_record_beside_a_live_one_makes_the_plan_refuse_and_the_stop_pass_stop_nothing() + { + // A live, verifiable server plus a manifest that does not parse. The plan + // used to count the live record and say "will be stopped before removal" + // while the stop pass refused on the manifest before touching anything. + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-plan-bad-manifest"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + let bad = paths.services_dir().join("bad.json"); + fs::write(&bad, b"{").expect("plant an unparseable manifest"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let refusal = plan_warning_containing(&plan, "will refuse to remove anything") + .expect("the plan says the run will refuse"); + assert!( + refusal.contains(&bad.display().to_string()) && refusal.contains("will stop nothing"), + "{refusal}" + ); + let count = plan_warning_containing(&plan, "managed service record(s) under") + .expect("the plan counts the managed services"); + assert!( + count.contains("none of them will be stopped") + && !count.contains("will be stopped before removal"), + "{count}" + ); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("the run the plan said would refuse must stop nothing") + }) + .expect("stop pass"); + assert!(report.stopped.is_empty(), "{report:?}"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::RepairTheRecord); + assert!(child.try_wait().expect("poll").is_none(), "left alive"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_verifiable_record_beside_a_tickless_one_is_not_stopped_and_the_plan_says_so() { + // The refusal returns before any stop, so "the others will be stopped" + // was never true. Two live servers, one without a start time: neither is + // stopped, and the plan says so for both. + let mut ticked = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a verifiable server"); + let mut tickless = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a tickless server"); + let (root, paths) = test_paths("uninstall-plan-partial-refusal"); + let mut good = managed_record_named( + &paths, + "svc-good", + ticked.id(), + rocm_core::process_start_ticks(ticked.id()), + ); + good.status = "ready".to_owned(); + good.write().expect("write the verifiable record"); + let mut bad = managed_record_named(&paths, "svc-bad", tickless.id(), None); + bad.status = "ready".to_owned(); + bad.write().expect("write the tickless record"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let refusal = plan_warning_containing(&plan, "will refuse to remove anything") + .expect("the plan says the run will refuse"); + assert!( + refusal.contains("svc-bad") && !refusal.contains("svc-good"), + "{refusal}" + ); + let count = plan_warning_containing(&plan, "managed service record(s) under") + .expect("the plan counts the managed services"); + assert!( + count.contains("2 of 2") + && count.contains("none of them will be stopped") + && !count.contains("the others will be stopped"), + "{count}" + ); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("a refused run must not stop the verifiable record either") + }) + .expect("stop pass"); + assert!(report.stopped.is_empty(), "{report:?}"); + assert!(ticked.try_wait().expect("poll").is_none(), "left alive"); + for child in [&mut ticked, &mut tickless] { + let _ = child.kill(); + let _ = child.wait(); + } + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_dead_tickless_pid_beside_a_live_verifiable_one_does_not_make_the_run_refuse() { + // `unprovable_live_pids` filters on liveness: a recorded pid that is gone + // cannot be a reused one that gets killed. Dropping the filter would turn + // every record with a stale engine pid into a refusal. The plan says the + // server will be stopped, and the stop pass really does. + let child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the live supervisor"); + let pid = child.id(); + let mut gone = std::process::Command::new("true") + .spawn() + .expect("spawn a short-lived process"); + let dead_pid = gone.id(); + gone.wait().expect("reap it"); + let (root, paths) = test_paths("uninstall-dead-tickless-engine"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.engine_pid = Some(dead_pid); + record.engine_start_ticks = None; + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + assert!( + plan_warning_containing(&plan, "will refuse to remove anything").is_none(), + "{:?}", + plan.warnings + ); + let count = plan_warning_containing(&plan, "managed service record(s) under") + .expect("the plan counts the managed services"); + assert!(count.contains("will be stopped before removal"), "{count}"); + + let report = stop_managed_services_with(&paths, |_, _| { + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("stop pass"); + assert!(report.failed.is_empty(), "{report:?}"); + assert_eq!(report.stopped, vec!["svc-managed-stop".to_owned()]); + let mut child = child; + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_failed_record_whose_engine_is_still_alive_is_stopped_not_waved_through() { + // `refresh_from_engine_state` adopts `failed` while the engine process is + // still up, and the liveness refresh does not look at pids for a non-live + // status. The status alone read as "nothing to stop", and the tooling was + // removed under a process that may still hold the GPU. + let child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the engine"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-failed-record-live-engine"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "failed".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let count = plan_warning_containing(&plan, "managed service record(s) under") + .expect("the plan counts the managed services"); + assert!( + count.contains("1 of 1") && count.contains("will be stopped before removal"), + "{count}" + ); + + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert_eq!( + report.stopped, + vec!["svc-managed-stop".to_owned()], + "{report:?}" + ); + let mut child = child; + assert_stopped_within_grace( + &mut child, + "the failed record's live engine must be stopped", + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_stopped_record_with_a_reused_live_pid_is_left_to_the_port_probe() { + // The narrower half of the same predicate: a record that reads `stopped` + // is written only once its processes are confirmed gone, so a live pid + // under it is a reused number. Refusing on it would block every uninstall + // that finds an old, tickless record. + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let (root, paths) = test_paths("uninstall-stopped-record-reused-pid"); + let mut record = managed_record_for_pid(&paths, child.id(), None); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("a stopped record is not stopped again") + }) + .expect("stop pass"); + assert!(report.failed.is_empty(), "{report:?}"); + assert!(child.try_wait().expect("poll").is_none(), "left alone"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_stop_that_does_not_confirm_is_a_failure_whatever_the_status_says() { + // The decision is the confirmed flag. A supervisor's clean-exit write + // leaves the record reading `stopped` while a grandchild is still alive; + // the status string then says "stopped" for a stop that confirmed nothing. + let child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-unconfirmed-stop"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_confirmed_with(&paths, |_, _| { + Ok((serde_json::json!({ "status": "stopped" }), false)) + }) + .expect("stop pass"); + assert!(report.stopped.is_empty(), "{report:?}"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + assert!( + report.failed[0].reason.contains("could not confirm"), + "{}", + report.failed[0].reason + ); + + // And the converse: a confirmed stop counts whatever the report's status + // string says, because the flag is what the stop actually established. + let report = + stop_managed_services_confirmed_with(&paths, |_, _| Ok((serde_json::json!({}), true))) + .expect("stop pass"); + assert_eq!( + report.stopped, + vec!["svc-managed-stop".to_owned()], + "{report:?}" + ); + assert!(report.failed.is_empty(), "{report:?}"); + let mut child = child; + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_stop_that_finds_the_record_already_stopped_still_reports_it_unconfirmed() { + // The premise of the test above, on the real stop: a record that reads + // `stopped` keeps that status through a stop whose termination did not + // confirm. Only the flag distinguishes it from a confirmed stop. + let (root, paths) = test_paths("uninstall-stop-keeps-status"); + let mut record = managed_record_for_pid(&paths, std::process::id(), None); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let (report, confirmed) = + stop_internal_managed_service_confirmed_with(&paths, "svc-managed-stop", |_| { + (Vec::new(), false) + }) + .expect("stop"); + assert_eq!(report["status"], "stopped"); + assert!( + !confirmed, + "an unconfirmed termination is not a confirmed stop" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unreadable_helper_state_makes_the_plan_refuse_and_the_stop_pass_say_which_file() { + let (root, paths) = test_paths("uninstall-plan-helper-state-unreadable"); + let state_file = paths.automation_state_path(); + fs::create_dir_all(state_file.parent().expect("a parent")).expect("create the dir"); + fs::write(&state_file, b"{ not json").expect("plant an unparseable state file"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let refusal = plan_warning_containing(&plan, "will refuse to remove anything") + .expect("the plan says the run will refuse"); + assert!( + refusal.contains(&state_file.display().to_string()) + && refusal.contains("could not be read"), + "{refusal}" + ); + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::RepairTheDaemonState + ); + + // A run that keeps the tooling does not check, and says so. + let cache_only = UninstallOptions { + yes: true, + keep_binaries: true, + keep_config: true, + keep_data: true, + ..UninstallOptions::default() + }; + let plan = build_uninstall_plan(&paths, &cache_only).expect("build the cache plan"); + assert!( + plan_warning_containing(&plan, "will refuse to remove anything").is_none(), + "{:?}", + plan.warnings + ); + let note = plan_warning_containing(&plan, "background helper") + .expect("the cache-only plan names the unreadable state"); + assert!(note.contains("does not check it"), "{note}"); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_recycled_helper_pid_is_neither_planned_nor_stopped() { + // The recorded pid is alive but its start time does not match: that is a + // reused number, so there is nothing of ours to stop. The plan must not + // say "will be stopped", and the stop pass must leave the process alone. + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-helper-recycled"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid).map(|t| t.wrapping_add(1)); + state.write(&paths).expect("write runtime state"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + assert!( + plan_warning_containing(&plan, "background helper").is_none(), + "{:?}", + plan.warnings + ); + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert!( + !report.helper_stopped && report.failed.is_empty(), + "{report:?}" + ); + assert!(child.try_wait().expect("poll").is_none(), "left alone"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_helper_recorded_as_not_running_is_neither_planned_nor_stopped() { + // `running: false` is the clean-shutdown write: the pid in the file names + // a process that already exited, and whatever is alive under it now is + // someone else's — even when its start time happens to match. + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-helper-not-running"); + let mut state = runtime_state(false, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + assert!( + plan_warning_containing(&plan, "background helper").is_none(), + "{:?}", + plan.warnings + ); + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert!( + !report.helper_stopped && report.failed.is_empty(), + "{report:?}" + ); + assert!(child.try_wait().expect("poll").is_none(), "left alone"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_helper_is_stopped_with_the_whole_tree_scope() { + // The production stop hands the termination `HELPER_KILL_SCOPE`; a + // single-process kill would leave the helper's engine children holding + // the GPU. The injected termination sees the scope the real one would. + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-helper-scope"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + + let mut seen = None; + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, scope| { + seen = Some(scope); + rocm_core::TerminationOutcome::Graceful + }); + assert_eq!(seen, Some(rocm_core::KillScope::Tree)); + assert_eq!(seen, Some(HELPER_KILL_SCOPE)); + assert!(report.helper_stopped, "{report:?}"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + /// A live helper whose identity verifies: the one state in which the plan's + /// helper line can promise a stop. + #[cfg(target_os = "linux")] + fn plant_verified_helper(paths: &AppPaths) -> std::process::Child { + let helper = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let mut state = runtime_state(true, helper.id()); + state.daemon_start_ticks = rocm_core::process_start_ticks(helper.id()); + state.write(paths).expect("write runtime state"); + helper + } + + #[cfg(target_os = "linux")] + #[test] + fn a_verified_helper_is_left_running_by_a_run_that_refuses_on_a_tickless_service() { + // The plan is the last thing read before confirming. With a verified live + // helper and a service whose live pid has no recorded start time, the run + // refuses before stopping anything, so the helper must not be described as + // about to be stopped, and the stop pass must leave it alone: no helper + // termination, no per-service stop. + let mut helper = None; + let (root, paths) = test_paths("uninstall-helper-with-refusing-service"); + helper.replace(plant_verified_helper(&paths)); + let mut server = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let mut record = managed_record_for_pid(&paths, server.id(), None); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let note = plan_warning_containing(&plan, "background helper (rocmd") + .expect("the plan names the live helper"); + assert!( + note.contains("left running") && !note.contains("will be stopped"), + "the helper line must not promise a stop on a run that refuses: {note}" + ); + assert!( + plan_warning_containing(&plan, "will refuse to remove anything").is_some(), + "{:?}", + plan.warnings + ); + + let stops = std::sync::atomic::AtomicUsize::new(0); + let terminations = std::sync::atomic::AtomicUsize::new(0); + let report = stop_managed_services_with_helper_stop( + &paths, + |_, _| { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok((serde_json::json!({ "status": "stopped" }), true)) + }, + |_, _| { + terminations.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + rocm_core::TerminationOutcome::Graceful + }, + ) + .expect("stop pass"); + assert!( + report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::ConfirmTheServicePid), + "the run refuses exactly as the plan said: {report:?}" + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!(terminations.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(!report.helper_stopped, "{report:?}"); + let mut helper = helper.take().expect("helper"); + assert!(helper.try_wait().expect("poll").is_none(), "left running"); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = server.kill(); + let _ = server.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_verified_helper_is_left_running_by_a_run_whose_service_records_cannot_be_read() { + // A `*.json` entry that is a directory cannot be read, as root or not, so + // the plan's own read of the records fails and the stop pass errors. The + // helper line must say the same thing the abort does: left running. + let (root, paths) = test_paths("uninstall-helper-with-unreadable-records"); + let mut helper = plant_verified_helper(&paths); + fs::create_dir_all(paths.services_dir().join("boom.json")) + .expect("plant an unreadable entry"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + assert!( + plan_warning_containing(&plan, "could not read the managed service records") + .is_some_and(|line| line.contains("will refuse to remove anything")), + "{:?}", + plan.warnings + ); + let note = plan_warning_containing(&plan, "background helper (rocmd") + .expect("the plan names the live helper"); + assert!( + note.contains("left running") && !note.contains("will be stopped"), + "the helper line must not promise a stop on a run that aborts: {note}" + ); + + let terminations = std::sync::atomic::AtomicUsize::new(0); + let outcome = stop_managed_services_with_helper_stop( + &paths, + |_, _| panic!("no service may be stopped"), + |_, _| { + terminations.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + rocm_core::TerminationOutcome::Graceful + }, + ); + assert!(outcome.is_err(), "the unreadable records abort the pass"); + assert_eq!(terminations.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(helper.try_wait().expect("poll").is_none(), "left running"); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_verified_helper_is_stopped_by_a_run_that_proceeds_and_the_plan_says_so() { + let (root, paths) = test_paths("uninstall-helper-proceeds"); + let mut helper = plant_verified_helper(&paths); + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let note = plan_warning_containing(&plan, "background helper (rocmd") + .expect("the plan names the live helper"); + assert!(note.contains("will be stopped before removal"), "{note}"); + let terminations = std::sync::atomic::AtomicUsize::new(0); + let report = stop_managed_services_with_helper_stop( + &paths, + |_, _| panic!("no services"), + |_, _| { + terminations.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + rocm_core::TerminationOutcome::Graceful + }, + ) + .expect("stop pass"); + assert_eq!(terminations.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!( + report.helper_stopped && report.failed.is_empty(), + "{report:?}" + ); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unverifiable_helper_is_named_by_a_plan_that_also_cannot_read_the_records() { + // The records-read error aborts the run before the helper is looked at, so + // the pre-flight refusal line is absent; the helper line must still name the + // unverifiable helper and promise a refusal, never a stop. + let (root, paths) = test_paths("uninstall-unverifiable-helper-and-unreadable"); + let mut helper = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let mut state = runtime_state(true, helper.id()); + state.daemon_start_ticks = None; + state.write(&paths).expect("write runtime state"); + fs::create_dir_all(paths.services_dir().join("boom.json")) + .expect("plant an unreadable entry"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let line = plan + .warnings + .iter() + .find(|line| line.contains("rocmd") && line.contains("identity could not be verified")) + .unwrap_or_else(|| panic!("the helper is named: {:?}", plan.warnings)); + assert!( + line.contains("will refuse to remove anything") && !line.contains("will be stopped"), + "{line}" + ); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_helper_that_cannot_be_confirmed_stopped_stops_no_service_after_it() { + // The helper is stopped first so it cannot respawn what the services pass + // stops; if that stop does not confirm, no service may be stopped, since + // each stop drops that service's endpoint key for good. + let mut helper = plant_verified_helper(&test_paths("unused").1); + let _ = helper.kill(); + let _ = helper.wait(); + let (root, paths) = test_paths("uninstall-helper-timeout-stops-no-service"); + let mut helper = plant_verified_helper(&paths); + let mut server = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let ticks = rocm_core::process_start_ticks(server.id()); + let mut record = managed_record_for_pid(&paths, server.id(), ticks); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let stops = std::sync::atomic::AtomicUsize::new(0); + let report = stop_managed_services_with_helper_stop( + &paths, + |_, _| { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok((serde_json::json!({ "status": "stopped" }), true)) + }, + |_, _| rocm_core::TerminationOutcome::TimedOut, + ) + .expect("stop pass"); + assert!( + report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::StopTheDaemon), + "{report:?}" + ); + assert_eq!( + stops.load(std::sync::atomic::Ordering::SeqCst), + 0, + "no service may be stopped once the helper stop failed: {report:?}" + ); + assert!(report.stopped.is_empty(), "{report:?}"); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = server.kill(); + let _ = server.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_resolvable_host_with_nothing_listening_is_not_a_fail_open() { + // `host_port_resolves` separates "nothing is listening" (the normal, + // silent case) from "the name does not resolve" (evidence of nothing, + // disclosed). A host that resolves must not draw the disclosure. + let (root, paths) = test_paths("uninstall-resolvable-dead-endpoint"); + let mut record = managed_record_for_pid(&paths, std::process::id(), None); + record.status = "stopped".to_owned(); + record.host = "127.0.0.1".to_owned(); + record.port = 0; + record.write().expect("write service record"); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("a stopped record is not stopped again") + }) + .expect("stop pass"); + assert!(report.failed.is_empty(), "{report:?}"); + assert!( + report.warnings.is_empty(), + "a refused connect is evidence the port is free, not a fail-open: {:?}", + report.warnings + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn following_the_tickless_pid_advice_lets_the_retry_proceed() { + // The advice for a live pid with no start time: stop it yourself, or + // delete the record if it is not your server. Either has to clear the + // refusal. + let mut stopped_by_hand = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a tickless server"); + let (root, paths) = test_paths("uninstall-advice-tickless-stop"); + let mut record = managed_record_for_pid(&paths, stopped_by_hand.id(), None); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + let first = + stop_managed_services_with(&paths, |_, _| panic!("must not stop")).expect("first pass"); + assert_eq!(first.failed.len(), 1, "{first:?}"); + + stopped_by_hand.kill().expect("`kill `"); + stopped_by_hand.wait().expect("reap it"); + let second = stop_managed_services_with(&paths, |_, _| panic!("nothing left to stop")) + .expect("retry after killing the pid"); + assert!(second.failed.is_empty(), "{second:?}"); + let _ = fs::remove_dir_all(root); + + // Or: it was not our server, so delete the record file by hand. + let mut stranger = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let (root, paths) = test_paths("uninstall-advice-tickless-delete"); + let mut record = managed_record_for_pid(&paths, stranger.id(), None); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + let first = + stop_managed_services_with(&paths, |_, _| panic!("must not stop")).expect("first pass"); + assert_eq!(first.failed.len(), 1, "{first:?}"); + fs::remove_file(paths.services_dir().join("svc-managed-stop.json")) + .expect("delete the record file"); + let second = stop_managed_services_with(&paths, |_, _| panic!("no record left")) + .expect("retry after deleting the record"); + assert!(second.failed.is_empty(), "{second:?}"); + assert!(stranger.try_wait().expect("poll").is_none(), "left alone"); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn following_the_daemon_pid_advice_lets_the_retry_proceed() { + // The advice for a helper that cannot be proven to be rocmd: stop it if + // it is rocmd, or delete the runtime state file if it is something else. + let mut helper = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let (root, paths) = test_paths("uninstall-advice-daemon-stop"); + runtime_state(true, helper.id()) + .write(&paths) + .expect("write runtime state"); + let first = stop_managed_services_before_uninstall(&paths).expect("first pass"); + assert_eq!(first.failed.len(), 1, "{first:?}"); + assert_eq!( + first.failed[0].remedy, + StopFailureRemedy::ConfirmTheDaemonPid + ); + + helper.kill().expect("`kill `"); + helper.wait().expect("reap it"); + let second = stop_managed_services_before_uninstall(&paths).expect("retry after the kill"); + assert!(second.failed.is_empty(), "{second:?}"); + let _ = fs::remove_dir_all(root); + + let mut stranger = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let (root, paths) = test_paths("uninstall-advice-daemon-delete"); + runtime_state(true, stranger.id()) + .write(&paths) + .expect("write runtime state"); + let first = stop_managed_services_before_uninstall(&paths).expect("first pass"); + assert_eq!(first.failed.len(), 1, "{first:?}"); + fs::remove_file(paths.automation_state_path()).expect("delete the state file"); + let second = stop_managed_services_before_uninstall(&paths) + .expect("retry after deleting the state file"); + assert!( + second.failed.is_empty() && !second.helper_stopped, + "{second:?}" + ); + assert!(stranger.try_wait().expect("poll").is_none(), "left alone"); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn the_abort_names_what_the_run_already_stopped_and_the_helper_it_signalled() { + let report = ManagedServiceStopReport { + stopped: vec!["svc-down".to_owned()], + relaunch: vec![format!( + "svc-down: `{}`", + rocm_engine_protocol::relaunch_command("0.0.0.0", true) + )], + failed: vec![ + FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\"".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }, + FailedManagedServiceStop { + service_id: "rocmd (pid 7)".to_owned(), + reason: "not confirmed".to_owned(), + remedy: StopFailureRemedy::StopTheDaemon, + }, + ], + warnings: Vec::new(), + helper_stopped: false, + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!( + message.contains("stopped during this run and stay stopped: svc-down"), + "{message}" + ); + assert!(!message.contains("before the failure"), "{message}"); + assert!( + message.contains("signalled but not confirmed gone"), + "{message}" + ); + assert!(message.contains("--require-api-key"), "{message}"); + assert!(message.contains("--allow-public-bind"), "{message}"); + assert!(message.contains("ROCM_SERVE_API_KEY"), "{message}"); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_plan_reports_a_live_helper_even_when_no_service_record_exists() { + // A live `rocmd` is force-stopped whether or not any service record + // exists, so the plan cannot stay silent just because there are none. + let helper = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let pid = helper.id(); + let (root, paths) = test_paths("uninstall-plan-helper-only"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + let full = UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }; + let cache_only = UninstallOptions { + yes: true, + keep_binaries: true, + keep_config: true, + keep_data: true, + ..UninstallOptions::default() + }; + + let plan = build_uninstall_plan(&paths, &full).expect("build the full plan"); + let note = plan + .warnings + .iter() + .find(|warning| warning.contains("background helper")) + .expect("the plan names the live helper"); + assert!( + note.contains(&format!("pid {pid}")) && note.contains("will be stopped"), + "{note}" + ); + let cache_plan = build_uninstall_plan(&paths, &cache_only).expect("build the cache plan"); + let cache_note = cache_plan + .warnings + .iter() + .find(|warning| warning.contains("background helper")) + .expect("the cache-only plan names the live helper"); + assert!( + cache_note.contains("left running") && !cache_note.contains("will be stopped"), + "{cache_note}" + ); + + // The outcome the full plan promised. + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert!(report.helper_stopped, "{report:?}"); + let mut helper = helper; + assert_stopped_within_grace(&mut helper, "the helper the plan named must be stopped"); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_plan_says_an_unverifiable_helper_will_block_the_uninstall() { + // A helper whose state file carries no start time cannot be proven to be + // rocmd, so the gate refuses rather than kill it; the plan says so. + let helper = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let pid = helper.id(); + let (root, paths) = test_paths("uninstall-plan-helper-unverifiable"); + runtime_state(true, pid) + .write(&paths) + .expect("write runtime state"); + let plan = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("build the plan"); + let note = plan + .warnings + .iter() + .find(|warning| warning.contains("will refuse to remove anything")) + .expect("the plan names the unverifiable helper"); + assert!( + note.contains("could not be verified") && note.contains(&format!("pid {pid}")), + "{note}" + ); + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert!( + report + .failed + .iter() + .any(|f| f.remedy == StopFailureRemedy::ConfirmTheDaemonPid), + "the gate refuses exactly what the plan said it would: {report:?}" + ); + let mut helper = helper; + let _ = helper.kill(); + let _ = helper.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn removing_the_stopped_record_as_advised_lets_the_retry_proceed() { + // The advice for an authenticated stranger on a stopped record's old + // port is `rocm services remove --yes`. Following it has to clear + // the refusal; advice that left the next run aborting would be a dead + // end. The record is removed the way that command does, by deleting its + // manifest, and the same pass is run again. + let endpoint = ServingEndpoint::serving_with_authorization("amd/guarded-model"); + let (root, paths) = test_paths("uninstall-advice-then-proceed"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-stranger-port", + "vllm", + "amd/guarded-model", + "amd/guarded-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let first = stop_managed_services_before_uninstall(&paths).expect("first pass"); + assert!( + first + .failed + .iter() + .any(|f| f.remedy == StopFailureRemedy::RemoveTheStoppedRecord), + "{first:?}" + ); + + remove_managed_service_record(&paths, "svc-stranger-port", true) + .expect("the advised command must succeed"); + + let second = stop_managed_services_before_uninstall(&paths).expect("second pass"); + assert!( + second.failed.is_empty(), + "once the record is removed the retry must proceed: {second:?}" + ); + assert!(uninstall_removal_gate(&second).is_ok()); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_plan_warning_only_promises_a_stop_the_uninstall_will_perform() { + // The plan is printed and confirmed BEFORE the stop pass decides whether + // to run, so a warning that promises a stop on a run that keeps the + // tooling would have the operator confirm work that never happens. + // Asserting the predicate alone (above) cannot catch that drift; this + // pins the operator-visible text to the same condition. + let child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-warning-matches-behaviour"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let full = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("build the full plan"); + let full_warning = full + .warnings + .iter() + .find(|warning| warning.contains("managed service record")) + .expect("the plan warns about managed services"); + assert!( + full_warning.contains("will be stopped before removal"), + "a removal that takes the tooling away must promise the stop: {full_warning}" + ); + + let cache_only = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + keep_binaries: true, + keep_config: true, + keep_data: true, + ..UninstallOptions::default() + }, + ) + .expect("build the cache-only plan"); + let cache_warning = cache_only + .warnings + .iter() + .find(|warning| warning.contains("managed service record")) + .expect("the plan warns about managed services"); + assert!( + cache_warning.contains("left running"), + "a removal that keeps the tooling must not promise a stop: {cache_warning}" + ); + assert!( + !cache_warning.contains("will be stopped before removal"), + "a removal that keeps the tooling must not promise a stop: {cache_warning}" + ); + + let mut child = child; + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn the_plan_warning_for_only_stopped_records_promises_no_stop() { + let (root, paths) = test_paths("uninstall-warning-stopped-records"); + let mut record = managed_record_for_pid(&paths, u32::MAX - 1, None); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("build the plan"); + let warning = plan + .warnings + .iter() + .find(|warning| warning.contains("managed service record")) + .expect("the plan warns about managed services"); + assert!( + warning.contains("none is recorded as running"), + "records that are not live must be described as such: {warning}" + ); + assert!( + !warning.contains("will be stopped"), + "no stop is promised when nothing is live: {warning}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn uninstall_removal_gate_reports_count_when_all_stopped() { + let report = ManagedServiceStopReport { + stopped: vec!["a".to_owned(), "b".to_owned()], + failed: Vec::new(), + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let line = uninstall_removal_gate(&report).expect("all stopped must proceed"); + assert_eq!( + line.as_deref(), + Some("stopped 2 managed service(s) before removal") + ); + } + + #[test] + fn uninstall_removal_gate_is_silent_with_nothing_to_stop() { + let report = ManagedServiceStopReport::default(); + assert!( + uninstall_removal_gate(&report) + .expect("no services must proceed") + .is_none(), + "no managed services means no line to print" + ); + } + + pub(crate) fn test_paths(name: &str) -> (PathBuf, AppPaths) { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join(".rocm-work") + .join("tests") + .join("main") + .join(format!( + "rocm-cli-main-test-{name}-{}", + crate::test_support::unique_suffix() + )); + let _ = fs::remove_dir_all(&root); + ( + root.clone(), + AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }, + ) + } + + #[test] + fn test_paths_gives_each_call_its_own_root_even_for_one_label() { + crate::test_support::assert_each_call_gets_its_own_root(|| test_paths("same-label").0); + } + + // ---- Phase 9: reroute dispatch (bare `rocm` + interactive `rocm chat`) ---- + // + // The interactive branches require a real TTY (`interactive_terminal()`), + // which is unavailable in CI, and the dash visuals are trust-prior. These + // tests instead PROVE the dispatch TARGET changed: the two interactive + // handlers now call `dash::run_chat` and no longer call `tui::run`. We read + // this source file at test time and assert on the handler bodies. + fn main_rs_source() -> String { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src") + .join("main.rs"); + fs::read_to_string(path).expect("read apps/rocm/src/main.rs") + } + + /// Extract the body of `fn launch_default()` (brace-balanced). + fn launch_default_body(src: &str) -> String { + slice_braced_block(src, "fn launch_default() -> Result<()> {") + } + + /// Extract the whole `Some(Command::Chat { .. }) => { .. }` handler arm, + /// brace-balanced from the destructure pattern through the block body, so + /// both the destructured fields (e.g. `chat_mock`) and the dispatch code are + /// visible. Restricts to the production handler (the `=> {` form), not the + /// variant declaration which has no `=>`. + fn command_chat_handler_body(src: &str) -> String { + // Anchor on the handler arm specifically: the destructure that is + // immediately followed (after the closing `}` + `)`) by `=> {`. + let arm = "Some(Command::Chat {"; + let arm_at = src.find(arm).expect("Command::Chat handler present"); + // Balance from the `{` of the destructure to capture the full arm, + // including the `=> { ... }` block that follows. + slice_braced_to_arm_end(&src[arm_at..]) + } + + /// Given a slice beginning at `Some(Command::Chat {`, return the text from + /// that point through the end of the arm's `=> { .. }` block. + fn slice_braced_to_arm_end(src: &str) -> String { + // 1. balance the destructure `{ .. }`. + let pat_open = src.find('{').expect("destructure open brace"); + let bytes = src.as_bytes(); + let mut depth = 0usize; + let mut pat_end = pat_open; + for (i, &b) in bytes.iter().enumerate().skip(pat_open) { + match b { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + pat_end = i + 1; + break; + } + } + _ => {} + } + } + // 2. find the `=> {` block opener after the destructure and balance it. + let after = &src[pat_end..]; + let block_rel = after.find("=> {").expect("arm block opener `=> {`"); + let block_body = slice_braced_block(after, &after[block_rel..block_rel + 4]); + // Return destructure + block body together. + format!("{}{}", &src[pat_open..pat_end], block_body) + } + + /// Return the brace-balanced block that follows `opener` (which must end in + /// `{`), including the contents but excluding the trailing brace's tail. + fn slice_braced_block(src: &str, opener: &str) -> String { + let start = src.find(opener).unwrap_or_else(|| { + panic!("opener not found: {opener}"); + }); + let brace_start = start + opener.len() - 1; // index of the `{` + let bytes = src.as_bytes(); + let mut depth = 0usize; + let mut end = brace_start; + for (i, &b) in bytes.iter().enumerate().skip(brace_start) { + match b { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + end = i + 1; + break; + } + } + _ => {} + } + } + src[brace_start..end].to_string() + } + + /// Strip `//` line comments so assertions key on real code, not the prose + /// comments that still mention `tui::run` for documentation. + fn strip_line_comments(block: &str) -> String { + block + .lines() + .map(|line| match line.find("//") { + Some(idx) => &line[..idx], + None => line, + }) + .collect::>() + .join("\n") + } + + #[test] + fn dash_run_chat_is_the_bare_and_chat_dispatch_target() { + // Symbol exists with the expected signature (bare `rocm` / `rocm chat` + // interactive target). Coercing to the fn pointer fails to compile if + // the entrypoint is removed or its signature drifts; casting the pointer + // to an address gives the binding a real effect (no underscore-bind). + let target: fn(bool, ChatInferenceParams) -> Result<()> = dash::run_chat; + assert_ne!(target as usize, 0); + } + + #[test] + fn launch_default_routes_to_dash_run_chat_not_tui() { + let src = main_rs_source(); + let body = strip_line_comments(&launch_default_body(&src)); + assert!( + body.contains("dash::run_launcher(false)"), + "bare `rocm` interactive branch must route to the launcher; body:\n{body}" + ); + assert!( + !body.contains("tui::run"), + "launch_default must NOT call tui::run after the reroute; body:\n{body}" + ); + } + + #[test] + fn command_chat_interactive_routes_to_dash_run_chat_not_tui() { + let src = main_rs_source(); + let body = strip_line_comments(&command_chat_handler_body(&src)); + assert!( + body.contains("dash::run_chat("), + "interactive `rocm chat` must route to dash::run_chat(...); body:\n{body}" + ); + assert!( + !body.contains("tui::run"), + "Command::Chat handler must NOT call tui::run after the reroute; body:\n{body}" + ); + } + + #[test] + fn command_chat_interactive_notes_dropped_provider_flag() { + // Phase-9 polish: --provider on interactive `rocm chat` is no longer + // silently ignored — the handler emits a one-line note when provider is // set before rerouting to the dash. Proven by reading the handler body // (the interactive branch requires a TTY, unavailable in CI). let src = main_rs_source(); @@ -26988,6 +30659,412 @@ install therock"; } } + #[test] + fn every_daemon_identity_reading_maps_to_exactly_one_gate_outcome() { + use rocm_core::IdentityState::{Gone, Indeterminate, Matches, Recycled}; + // `Indeterminate` must abort: folding it into the proceed arm would + // delete the tooling while a live `rocmd` keeps respawning services. + assert_eq!( + daemon_identity_outcome(Gone, false), + DaemonIdentityOutcome::NothingOfOurs + ); + assert_eq!( + daemon_identity_outcome(Recycled, false), + DaemonIdentityOutcome::NothingOfOurs + ); + assert_eq!( + daemon_identity_outcome(Indeterminate, false), + DaemonIdentityOutcome::Unverifiable + ); + assert_eq!( + daemon_identity_outcome(Indeterminate, true), + DaemonIdentityOutcome::Unverifiable + ); + assert_eq!( + daemon_identity_outcome(Matches, true), + DaemonIdentityOutcome::Unverifiable + ); + assert_eq!( + daemon_identity_outcome(Matches, false), + DaemonIdentityOutcome::Stop + ); + } + + #[cfg(unix)] + #[test] + fn a_failed_per_service_stop_aborts_with_the_service_advice() { + let (root, paths) = test_paths("uninstall-stop-errs"); + let mut child = KillOnDrop( + std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"), + ); + let pid = child.0.id(); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_with(&paths, |_, _| anyhow::bail!("stop blew up")) + .expect("the pass itself succeeds"); + + assert_eq!( + report.failed.len(), + 1, + "the Err must be recorded as a failure" + ); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + assert!(report.failed[0].reason.contains("stop blew up")); + assert!(report.stopped.is_empty()); + let error = uninstall_removal_gate(&report).expect_err("the gate must abort"); + assert!(format!("{error:#}").contains("rocm services stop --yes")); + child.0.kill().ok(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn a_stop_result_without_a_status_is_not_a_confirmed_stop() { + // A missing `status` has to read as "unknown", never as "stopped": the + // stop result is the only evidence this gate has that the stop happened. + let (root, paths) = test_paths("uninstall-stop-no-status"); + let child = KillOnDrop( + std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"), + ); + let pid = child.0.id(); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_with(&paths, |_, _| Ok(serde_json::json!({}))) + .expect("the pass itself succeeds"); + + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + assert!(report.failed[0].reason.contains("unknown"), "{report:?}"); + assert!(report.stopped.is_empty(), "{report:?}"); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn a_stop_that_returns_ok_but_not_stopped_is_a_failure() { + let (root, paths) = test_paths("uninstall-stop-ok-not-stopped"); + let child = KillOnDrop( + std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"), + ); + let pid = child.0.id(); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_with(&paths, |_, _| Ok(serde_json::json!({ "status": "ready" }))) + .expect("the pass itself succeeds"); + + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + assert!(report.stopped.is_empty(), "{report:?}"); + assert!(uninstall_removal_gate(&report).is_err()); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_daemon_stop_that_is_not_confirmed_aborts_and_a_confirmed_one_is_recorded() { + let (root, paths) = test_paths("uninstall-daemon-stop-outcome"); + let child = KillOnDrop( + std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a live daemon stand-in"), + ); + let pid = child.0.id(); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + assert!(state.daemon_start_ticks.is_some()); + state.write(&paths).expect("write runtime state"); + + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, _| { + rocm_core::TerminationOutcome::TimedOut + }); + assert!(report.stopped.is_empty(), "{report:?}"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheDaemon); + assert!(uninstall_removal_gate(&report).is_err()); + + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, _| { + rocm_core::TerminationOutcome::Unverified + }); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::ConfirmTheDaemonPid + ); + + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, _| { + rocm_core::TerminationOutcome::Graceful + }); + assert!(report.failed.is_empty(), "{report:?}"); + assert!( + report.helper_stopped && report.stopped.is_empty(), + "{report:?}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn two_failures_of_one_class_get_that_classes_advice_once_and_name_both_ids() { + let failure = |id: &str| FailedManagedServiceStop { + service_id: id.to_owned(), + reason: "still ready".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }; + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![failure("svc-a"), failure("svc-b")], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert_eq!( + message.matches("rocm services stop --yes").count(), + 1, + "one class, one piece of advice: {message}" + ); + assert!( + message.contains("svc-a") && message.contains("svc-b"), + "{message}" + ); + assert!(message.ends_with("No files were removed."), "{message}"); + } + + #[test] + fn interleaved_failure_classes_print_each_advice_once() { + let failure = |id: &str, remedy| FailedManagedServiceStop { + service_id: id.to_owned(), + reason: "r".to_owned(), + remedy, + }; + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![ + failure("svc-a", StopFailureRemedy::StopWhatHoldsThePort), + failure("svc-b", StopFailureRemedy::RemoveTheStoppedRecord), + failure("svc-c", StopFailureRemedy::StopWhatHoldsThePort), + ], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert_eq!( + message.matches("Find what holds that port").count(), + 1, + "{message}" + ); + } + + #[test] + fn each_advice_names_only_the_ids_of_its_own_failure_class() { + // Advice that names every failed id sends the operator to hunt a port + // holder for a service whose real problem is a stale record, so the ids + // are asserted under the sentence that carries them, not just somewhere. + let failure = |id: &str, remedy| FailedManagedServiceStop { + service_id: id.to_owned(), + reason: "r".to_owned(), + remedy, + }; + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![ + failure("svc-port", StopFailureRemedy::StopWhatHoldsThePort), + failure("svc-old", StopFailureRemedy::RemoveTheStoppedRecord), + failure("svc-pid", StopFailureRemedy::ConfirmTheServicePid), + failure("svc-port-two", StopFailureRemedy::StopWhatHoldsThePort), + ], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!( + message.contains("Every process recorded for svc-port, svc-port-two is gone"), + "{message}" + ); + assert!( + message.contains("if it is, stop it first: svc-old."), + "{message}" + ); + assert!(message.contains("it was left alone: svc-pid."), "{message}"); + } + + #[cfg(unix)] + #[test] + fn a_live_record_without_start_ticks_is_refused_not_killed() { + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let child_pid = child.id(); + let (root, paths) = test_paths("tickless-live-record"); + let record = managed_record_for_pid(&paths, child_pid, None); + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write( + paths.services_dir().join("svc-managed-stop.json"), + serde_json::to_vec(&record).unwrap(), + ) + .unwrap(); + let report = stop_managed_services_with(&paths, |_, _| { + panic!("a tick-less live record must not be stopped") + }) + .unwrap(); + let alive = child.try_wait().unwrap().is_none(); + child.kill().ok(); + child.wait().ok(); + fs::remove_dir_all(root).ok(); + assert!(alive); + assert_eq!(report.failed.len(), 1); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::ConfirmTheServicePid + ); + // The advice tells the operator to "check what those pids are", so the + // refusal has to show them. + assert!( + report.failed[0].reason.contains(&child_pid.to_string()), + "the refusal must name the pid it left alone: {report:?}" + ); + } + + #[test] + fn the_tickless_pid_advice_text_names_the_id_and_both_platforms_kill_commands() { + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "svc-tickless".to_owned(), + reason: "r".to_owned(), + remedy: StopFailureRemedy::ConfirmTheServicePid, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!( + message.contains("it was left alone: svc-tickless."), + "{message}" + ); + assert!(message.contains("kill "), "{message}"); + assert!(message.contains("Stop-Process -Id "), "{message}"); + assert!(!message.contains("services remove"), "{message}"); + } + + #[cfg(unix)] + #[test] + fn a_vllm_shaped_record_with_a_distinct_engine_pid_carries_engine_ticks() { + let (root, paths) = test_paths("vllm-engine-ticks"); + let mut record = managed_record_for_pid(&paths, std::process::id(), Some(7)); + record.engine_pid = None; + fs::create_dir_all(&root).unwrap(); + record.status = "running".to_owned(); + record.engine_state_path = root.join("engine-state.json"); + fs::write( + &record.engine_state_path, + serde_json::json!({"status": "running", "pid": 4242, "server_pid": 4242, "start_ticks": 99}).to_string(), + ) + .unwrap(); + record.refresh_from_engine_state().unwrap(); + fs::remove_dir_all(root).ok(); + assert_eq!(record.engine_pid, Some(4242)); + assert_eq!(record.engine_start_ticks, Some(99)); + } + + #[test] + fn the_endpoint_sentence_follows_the_failure_class() { + let sentence = "may still be serving"; + for (remedy, expected) in [ + (StopFailureRemedy::StopTheService, true), + (StopFailureRemedy::StopWhatHoldsThePort, true), + (StopFailureRemedy::RemoveTheStoppedRecord, false), + (StopFailureRemedy::StopTheDaemon, false), + (StopFailureRemedy::ConfirmTheServicePid, false), + (StopFailureRemedy::ConfirmTheDaemonPid, false), + (StopFailureRemedy::RepairTheDaemonState, false), + (StopFailureRemedy::RepairTheRecord, false), + ] { + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "x".to_owned(), + reason: "r".to_owned(), + remedy, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert_eq!( + message.contains(sentence), + expected, + "{remedy:?}: {message}" + ); + } + } + + #[test] + fn the_abort_only_claims_endpoints_may_serve_when_a_service_failed() { + let repair_only = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "/x/broken.json".to_owned(), + reason: "service record could not be parsed".to_owned(), + remedy: StopFailureRemedy::RepairTheRecord, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&repair_only).unwrap_err()); + assert!(!message.contains("may still be serving"), "{message}"); + + let service = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "svc".to_owned(), + reason: "still ready".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&service).unwrap_err()); + assert!(message.contains("may still be serving"), "{message}"); + } + + #[cfg(unix)] + struct KillOnDrop(std::process::Child); + + #[cfg(unix)] + impl Drop for KillOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + /// Hosts whose engine choice differs, so the assertion below is exercised on /// more than one branch of `select_serve_engine`. fn engine_selection_hosts() -> Vec<(&'static str, rocm_core::HostGpuSummary)> { diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 3b05775eb..ba1a80c5d 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -199,18 +199,18 @@ pub(crate) fn ensure_public_service_has_endpoint_key( "managed service was launched with `--require-api-key` but has no endpoint API key, \ so restarting it would reopen it without authentication. Something outside this \ machine may still be publishing its port. The key is dropped when a service stops \ - and cannot be recovered. Launch it again with \ - `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ - to choose the key instead of generating one)." + and cannot be recovered. Launch it again with `{}` {}.", + rocm_engine_protocol::relaunch_command(host, true), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE ); } if rocm_engine_protocol::is_public_bind_host(host) && !key_present { bail!( "managed service is bound to the public host `{host}` but has no endpoint API key, \ so restarting it would reopen it without authentication. The key is dropped when a \ - service stops and cannot be recovered. Launch it again with \ - `rocm serve --host {host} --allow-public-bind` (add `--api-key `, or set \ - ROCM_SERVE_API_KEY, to choose the key instead of generating one)." + service stops and cannot be recovered. Launch it again with `{}` {}.", + rocm_engine_protocol::relaunch_command(host, false), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE ); } Ok(()) @@ -1149,7 +1149,7 @@ fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result --host 127.0.0.1 --port "), + "{rendered}" + ); + assert!( + !rendered.contains("--allow-public-bind"), + "a loopback service is not told to open a public bind: {rendered}" + ); + + // A public host that also demanded auth is told how to bind publicly. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, true) + .expect_err("a public service that demanded auth must not come back without it"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("--host 0.0.0.0 --port --allow-public-bind --require-api-key"), + "{rendered}" ); } @@ -1641,11 +1658,15 @@ mod tests { } /// Build an `AutomationRuntimeState` for the no-double-spawn guard tests. - fn runtime_state(running: bool, daemon_pid: u32) -> AutomationRuntimeState { + /// + /// `daemon_start_ticks` is left unrecorded; tests that care about the + /// daemon's identity set it explicitly. + pub(crate) fn runtime_state(running: bool, daemon_pid: u32) -> AutomationRuntimeState { AutomationRuntimeState { running, automations_enabled: true, daemon_pid, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 9a692c32c..18e42847d 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -4716,6 +4716,18 @@ fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { Ok(parent.join(file_name)) } +/// Stage-and-publish a file here, sharing only the publish step with `rocm-core`. +/// +/// Deliberately not [`rocm_core::write_file_atomically`], and not a copy of it +/// either: only the Windows-sensitive publish (`ReplaceFileW` and its fallbacks) +/// is single-sourced, via [`publish_temp_file`]. The staging half stays local +/// because it carries the `suffix_for_attempt` and `before_publish` seams the +/// tests below drive to force temp-name collisions, write failures and rename +/// races — injection points `rocm_core`'s caller-facing helper does not expose. +/// +/// Consequence worth knowing: this path does **not** `sync_all` before +/// publishing, so unlike the `rocm-core` helper it is atomic but carries no +/// crash-durability guarantee for the staged bytes. pub(crate) fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); write_file_atomically_with( @@ -4832,60 +4844,10 @@ where .with_context(|| format!("failed to publish {}", path.display())) } -#[cfg(not(windows))] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] +/// The publish step lives in `rocm-core` so there is one implementation of the +/// Windows `ReplaceFileW` handling for the whole workspace. fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } + rocm_core::publish_temp_file(tmp, path) } fn extract_tarball(archive_path: &Path, target_dir: &Path) -> Result<()> { diff --git a/apps/rocm/src/uninstall.rs b/apps/rocm/src/uninstall.rs index 7b8c7352d..af727bafb 100644 --- a/apps/rocm/src/uninstall.rs +++ b/apps/rocm/src/uninstall.rs @@ -9,7 +9,10 @@ //! only uninstall does on top of removing files: //! //! - undoing the PATH change the installer recorded making; -//! - deleting the running `rocm.exe` on Windows, which cannot unlink itself. +//! - deleting the running `rocm.exe` on Windows, which cannot unlink itself; +//! - stopping the servers this machine manages before any of that, and +//! refusing to remove anything when a stop cannot be confirmed (EAI-8014), +//! plus the report for a removal that stops partway. use std::ffi::OsString; use std::fmt::Write as _; @@ -20,9 +23,12 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, interactive_terminal}; +use crate::uninstall_gate::{ + ManagedServiceStopReport, stop_managed_services_before_uninstall, uninstall_removal_gate, +}; use crate::{ UninstallOptions, UninstallPlan, build_uninstall_plan, confirm_uninstall, - is_rocm_install_entry_name, remove_path, render_uninstall_plan, + is_rocm_install_entry_name, plan_removes_recovery_tooling, remove_path, render_uninstall_plan, }; /// The file, next to the installed binaries, in which `install.sh` and @@ -65,8 +71,17 @@ impl PathEdit { pub(crate) fn uninstall(options: UninstallOptions) -> Result<()> { let paths = AppPaths::discover()?; - let plan = build_uninstall_plan(&paths, &options)?; - print!("{}", render_uninstall_plan(&plan, &options)); + uninstall_with_paths(&paths, &options) +} + +/// The whole `uninstall` command against a given [`AppPaths`]. +/// +/// Split from [`uninstall`] only so a test can drive the real command — plan, +/// confirm gate, stop pass, removal — against an isolated root, instead of +/// exercising the pieces separately and taking the wiring between them on faith. +pub(crate) fn uninstall_with_paths(paths: &AppPaths, options: &UninstallOptions) -> Result<()> { + let plan = build_uninstall_plan(paths, options)?; + print!("{}", render_uninstall_plan(&plan, options)); if plan_is_empty(&plan) || options.dry_run { return Ok(()); @@ -82,7 +97,73 @@ pub(crate) fn uninstall(options: UninstallOptions) -> Result<()> { } } - execute_plan(&plan, &mut |line| println!("{line}"))?; + // Only an uninstall that takes away the means of stopping a server has to + // stop it first; a cache-only run leaves `rocm services stop` and every + // service record in place. + let removes_recovery_tooling = plan_removes_recovery_tooling(&plan, paths); + stop_managed_services_then_remove(&plan, || { + if removes_recovery_tooling { + stop_managed_services_before_uninstall(paths) + } else { + Ok(ManagedServiceStopReport::default()) + } + }) +} + +/// Stop the servers this machine manages, then remove the planned paths — in +/// that order, and only if every stop was confirmed. +/// +/// Uninstall used to report success while a publicly-bound, GPU-holding endpoint +/// kept serving, then delete the tooling needed to stop it (EAI-8014). The stop +/// runs first and the gate aborts on any unconfirmed stop, so the recovery +/// tooling stays in place. +/// +/// The stop pass is a parameter, not a direct call, so the ordering guarantee is +/// testable rather than merely inspectable: a test can hand in a stop that fails +/// — which no test can reliably provoke from a real process — and assert that +/// not one planned path was removed. Deleting the stop from this function makes +/// those tests fail. +fn stop_managed_services_then_remove( + plan: &UninstallPlan, + stop_managed_services: impl FnOnce() -> Result, +) -> Result<()> { + stop_then_remove_with(plan, stop_managed_services, remove_path) +} + +/// [`stop_managed_services_then_remove`] with the removal of one path supplied, +/// so a test can make it fail partway, which a real filesystem does not do on +/// demand. +fn stop_then_remove_with( + plan: &UninstallPlan, + stop_managed_services: impl FnOnce() -> Result, + remove: impl Fn(&Path) -> Result<()>, +) -> Result<()> { + stop_then_remove_writing_to(plan, stop_managed_services, remove, &mut std::io::stderr()) +} + +/// [`stop_then_remove_with`] with the stream the stop pass's warnings go to, so +/// a test can read what the operator would see on stderr and in which order +/// relative to the abort. +fn stop_then_remove_writing_to( + plan: &UninstallPlan, + stop_managed_services: impl FnOnce() -> Result, + remove: impl Fn(&Path) -> Result<()>, + warnings_out: &mut impl std::io::Write, +) -> Result<()> { + let stop_report = stop_managed_services()?; + // Every fail-open the stop pass took, on stderr rather than stdout: these + // are the places uninstall proceeded without proving the port was free, so + // they have to survive the operator piping its output somewhere. Printed + // before the gate, so they are on screen whether or not it aborts. + write_stop_warnings(warnings_out, &stop_report); + if let Some(line) = uninstall_removal_gate(&stop_report)? { + println!("{line}"); + } + + // PATH changes, the planned entries and the running executable go through + // `execute_plan_with`, which says exactly what is and is not removed when a + // step fails partway. + execute_plan_with(plan, &mut |line| println!("{line}"), &remove)?; println!("uninstall complete"); Ok(()) } @@ -95,7 +176,24 @@ pub(crate) fn uninstall(options: UninstallOptions) -> Result<()> { /// the record of the change are all still there to retry with. Then the /// config, data, and cache go, while `rocm` is still there to retry a failure /// with; then the binaries, and the record only once every change is undone. +#[cfg(all(test, unix))] fn execute_plan(plan: &UninstallPlan, report: &mut dyn FnMut(String)) -> Result<()> { + execute_plan_with(plan, report, &remove_path) +} + +/// [`execute_plan`] with the removal of one path supplied, so a test can make it +/// fail partway, which a real filesystem does not do on demand. +/// +/// The removal is not transactional: there is no rollback, so a failure partway +/// can leave the binaries and service records gone while a server the gate judged +/// stopped is not. The error therefore says what was removed, what failed (and +/// may be partly removed, since a directory is deleted child by child), and what +/// was not attempted, instead of a bare "failed to remove". +fn execute_plan_with( + plan: &UninstallPlan, + report: &mut dyn FnMut(String), + remove: &dyn Fn(&Path) -> Result<()>, +) -> Result<()> { for edit in &plan.path_edits { match apply_path_edit(edit) { Ok(line) => report(line), @@ -110,14 +208,40 @@ fn execute_plan(plan: &UninstallPlan, report: &mut dyn FnMut(String)) -> Result< let is_binary = |kind: &str| kind == "binary"; let non_binaries = plan.actions.iter().filter(|entry| !is_binary(entry.kind)); let binaries = plan.actions.iter().filter(|entry| is_binary(entry.kind)); - for entry in non_binaries.chain(binaries) { - remove_path(&entry.path) - .with_context(|| format!("failed to remove {}", entry.path.display()))?; - report(format!("removed {} {}", entry.kind, entry.path.display())); - } + // Every step in the order it runs, named as the operator will read it. + let mut steps: Vec<(String, PathBuf)> = non_binaries + .chain(binaries) + .map(|entry| { + ( + format!("{} {}", entry.kind, entry.path.display()), + entry.path.clone(), + ) + }) + .collect(); if let Some(record) = &plan.path_record { - remove_path(record).with_context(|| format!("failed to remove {}", record.display()))?; - report(format!("removed install record {}", record.display())); + steps.push(( + format!("install record {}", record.display()), + record.clone(), + )); + } + let mut removed: Vec = Vec::new(); + for (index, (name, path)) in steps.iter().enumerate() { + if let Err(error) = remove(path) { + let untouched: Vec = steps[index + 1..] + .iter() + .map(|(name, _)| name.clone()) + .chain( + plan.self_delete + .iter() + .map(|exe| format!("binary {} (the running program)", exe.display())), + ) + .collect(); + return Err(error + .context(format!("failed to remove {}", path.display())) + .context(partial_removal_note(&removed, name, &untouched))); + } + report(format!("removed {name}")); + removed.push(name.clone()); } if let Some(exe) = &plan.self_delete { @@ -126,6 +250,38 @@ fn execute_plan(plan: &UninstallPlan, report: &mut dyn FnMut(String)) -> Result< Ok(()) } +/// One `warning:` line per fail-open the stop pass took, on `out`. +fn write_stop_warnings(out: &mut impl std::io::Write, report: &ManagedServiceStopReport) { + for warning in &report.warnings { + let _ = writeln!(out, "warning: {warning}"); + } +} + +/// What a removal that stopped partway has to tell its operator. +/// +/// The entry that failed is reported apart from the untouched ones: removing a +/// directory deletes its children one by one, so it can fail after deleting +/// some of them, and "still present" would then overstate what is left. +fn partial_removal_note(removed: &[String], failed: &str, untouched: &[String]) -> String { + let removed_part = if removed.is_empty() { + "no entry was removed completely".to_owned() + } else { + format!("Already removed: {}", removed.join(", ")) + }; + let untouched_part = if untouched.is_empty() { + String::new() + } else { + format!(". Not attempted: {}", untouched.join(", ")) + }; + format!( + "uninstall stopped partway. {removed_part}. Failed, and may be partly removed: \ + {failed}{untouched_part}. What was already removed can include the `rocm`/`rocmd` \ + binaries and the service records, so `rocm services stop` may no longer work; if a \ + managed server may still be running, find and stop it by hand, then re-run \ + uninstall (or delete what is left) once the cause is fixed" + ) +} + pub(crate) const fn plan_is_empty(plan: &UninstallPlan) -> bool { plan.actions.is_empty() && plan.path_edits.is_empty() @@ -1327,3 +1483,753 @@ mod tests { } } } + +#[cfg(test)] +mod stop_gate_tests { + use std::fs; + use std::path::PathBuf; + + use anyhow::bail; + + use super::{ + stop_managed_services_then_remove, stop_then_remove_with, stop_then_remove_writing_to, + write_stop_warnings, + }; + use crate::uninstall_gate::{ + FailedManagedServiceStop, ManagedServiceStopReport, StopFailureRemedy, + }; + use crate::{UninstallPlan, UninstallPlanEntry}; + + /// A port nothing can ever be serving on. + /// + /// These tests need a record whose endpoint reads as dead, and the gate + /// really does probe it. Port 0 resolves — so the probe runs rather than + /// being skipped by a resolution failure — and the connect always fails, + /// on every platform. Binding an ephemeral port and dropping it would leave + /// a window in which something else on a busy runner grabs the port and + /// fails these tests through the gate's own probe. + /// + /// Both users are Linux-only, so the constant is too — `-D warnings` makes + /// dead code a build failure on the other platforms. + #[cfg(target_os = "linux")] + const UNSERVABLE_PORT: u16 = 0; + + /// An isolated root and a plan whose one action removes a real file in it. + /// + /// The file is what makes these tests assertions about *removal* rather than + /// about return values: it exists before the call, and its presence + /// afterwards is the evidence that the abort happened before the removal + /// loop, not after it. + fn plan_removing_one_file(name: &str) -> (PathBuf, UninstallPlan) { + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-{name}-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("failed to create the test root"); + let doomed = root.join("rocm"); + fs::write(&doomed, b"binary").expect("failed to seed the file the plan removes"); + ( + root, + UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "binary", + path: doomed, + }], + skipped: Vec::new(), + warnings: Vec::new(), + ..UninstallPlan::default() + }, + ) + } + + #[test] + fn a_removal_that_fails_partway_says_what_is_gone_and_what_is_left() { + // The removal is not transactional, so a failure after the first entry + // leaves that entry gone and the next one behind (config, data and cache + // go before the binaries). The error has to say so, and the report must + // match what is actually on disk. + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-partial-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("create the test root"); + let binary = root.join("config.toml"); + let data = root.join("data"); + fs::write(&binary, b"binary").expect("seed binary"); + fs::create_dir_all(&data).expect("seed data"); + let plan = UninstallPlan { + actions: vec![ + UninstallPlanEntry { + kind: "config", + path: binary.clone(), + }, + UninstallPlanEntry { + kind: "data", + path: data.clone(), + }, + ], + skipped: Vec::new(), + warnings: Vec::new(), + ..UninstallPlan::default() + }; + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |path| { + if path == data { + bail!("directory not empty") + } + fs::remove_file(path)?; + Ok(()) + }, + ) + .expect_err("a failing removal must fail the uninstall"); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!("Already removed: config {}", binary.display())), + "{message}" + ); + assert!( + message.contains(&format!( + "Failed, and may be partly removed: data {}", + data.display() + )), + "{message}" + ); + assert!(message.contains("directory not empty"), "{message}"); + assert!( + message.contains("`rocm services stop` may no longer work"), + "{message}" + ); + // The text above is only true if the disk agrees with it. + assert!(!binary.exists(), "the entry the note calls removed is gone"); + assert!( + data.exists(), + "the data the note calls present is still there" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_removal_failing_in_the_middle_names_the_later_entries_as_not_attempted() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-three-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("create the test root"); + let first = root.join("config.toml"); + let middle = root.join("data"); + let last = root.join("cache"); + fs::write(&first, b"config").expect("seed first"); + fs::create_dir_all(&middle).expect("seed middle"); + fs::create_dir_all(&last).expect("seed last"); + let entry = |kind: &'static str, path: &PathBuf| UninstallPlanEntry { + kind, + path: path.clone(), + }; + let plan = UninstallPlan { + actions: vec![ + entry("config", &first), + entry("data", &middle), + entry("cache", &last), + ], + skipped: Vec::new(), + warnings: Vec::new(), + ..UninstallPlan::default() + }; + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |path| { + if path == middle { + bail!("directory not empty") + } + if path.is_dir() { + fs::remove_dir_all(path)?; + } else { + fs::remove_file(path)?; + } + Ok(()) + }, + ) + .expect_err("a failing removal must fail the uninstall"); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!("Already removed: config {}", first.display())), + "{message}" + ); + assert!( + message.contains(&format!("Not attempted: cache {}", last.display())), + "{message}" + ); + assert!( + !message.contains("Not attempted: data"), + "the failed entry is reported apart from the untouched ones: {message}" + ); + assert!(!first.exists(), "removed, as the note says"); + assert!(middle.exists(), "the failed entry is still there"); + assert!(last.exists(), "the not-attempted entry is still on disk"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_removal_that_fails_first_reports_the_failed_entry_as_possibly_partial() { + let (root, plan) = plan_removing_one_file("fails-first"); + let doomed = plan.actions[0].path.clone(); + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |_| bail!("permission denied"), + ) + .expect_err("a failing removal must fail the uninstall"); + + let message = format!("{error:#}"); + assert!( + message.contains("no entry was removed completely"), + "{message}" + ); + assert!( + message.contains(&format!( + "Failed, and may be partly removed: {} {}", + plan.actions[0].kind, + doomed.display() + )), + "{message}" + ); + assert!(doomed.is_file(), "nothing was removed, as the note says"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_directory_removal_that_deletes_some_children_then_fails_is_reported_as_partial() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-partial-dir-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + let data = root.join("data"); + fs::create_dir_all(&data).expect("seed data"); + fs::write(data.join("a"), b"a").expect("seed a"); + fs::write(data.join("b"), b"b").expect("seed b"); + let plan = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "data", + path: data.clone(), + }], + skipped: Vec::new(), + warnings: Vec::new(), + ..UninstallPlan::default() + }; + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |path| { + // Like `remove_dir_all` failing midway: one child is gone, the + // directory is not. + fs::remove_file(path.join("a"))?; + bail!("directory not empty") + }, + ) + .expect_err("a failing removal must fail the uninstall"); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!( + "Failed, and may be partly removed: data {}", + data.display() + )), + "{message}" + ); + assert!(!data.join("a").exists() && data.join("b").exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn every_fail_open_the_stop_pass_took_is_written_as_a_warning_line() { + let report = ManagedServiceStopReport { + warnings: vec!["first fail-open".to_owned(), "second fail-open".to_owned()], + ..ManagedServiceStopReport::default() + }; + let mut out = Vec::new(); + write_stop_warnings(&mut out, &report); + assert_eq!( + String::from_utf8(out).expect("utf8"), + "warning: first fail-open\nwarning: second fail-open\n" + ); + } + + #[test] + fn the_stop_warnings_reach_the_operator_whether_or_not_the_run_aborts() { + // `write_stop_warnings` is tested on its own above; this pins the call + // site in the command, which a no-op there would otherwise leave green: + // the fail-opens are the only disclosure that uninstall proceeded without + // proving a port was free. + let (root, plan) = plan_removing_one_file("warnings-proceed"); + let mut out = Vec::new(); + stop_then_remove_writing_to( + &plan, + || { + Ok(ManagedServiceStopReport { + warnings: vec!["could not check port 9".to_owned()], + ..ManagedServiceStopReport::default() + }) + }, + |path| { + fs::remove_file(path)?; + Ok(()) + }, + &mut out, + ) + .expect("a run with only fail-open warnings proceeds"); + assert_eq!( + String::from_utf8(out).expect("utf8"), + "warning: could not check port 9\n" + ); + let _ = fs::remove_dir_all(root); + + // And on the abort path: printed before the gate refuses, so they are on + // screen even though nothing was removed. + let (root, plan) = plan_removing_one_file("warnings-abort"); + let doomed = plan.actions[0].path.clone(); + let mut out = Vec::new(); + let error = stop_then_remove_writing_to( + &plan, + || { + Ok(ManagedServiceStopReport { + failed: vec![FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\"".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: vec!["could not check port 9".to_owned()], + ..ManagedServiceStopReport::default() + }) + }, + |_| panic!("an aborted run removes nothing"), + &mut out, + ) + .expect_err("a failed stop aborts"); + assert!(format!("{error:#}").contains("svc-stuck")); + assert_eq!( + String::from_utf8(out).expect("utf8"), + "warning: could not check port 9\n" + ); + assert!(doomed.is_file(), "nothing was removed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_service_that_cannot_be_stopped_leaves_every_planned_path_in_place() { + // The EAI-8014 guarantee itself: while a managed server may still be + // serving, uninstall removes NOTHING — not the binaries, not the service + // records — so `rocm services stop` is still there to recover with. + let (root, plan) = plan_removing_one_file("stop-unconfirmed"); + let doomed = plan.actions[0].path.clone(); + + let error = stop_managed_services_then_remove(&plan, || { + Ok(ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\" after the stop attempt".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }) + }) + .expect_err("an unconfirmed stop must abort uninstall"); + + let message = format!("{error:#}"); + // Assert on text the GATE owns first. `svc-stuck` alone would be this + // test reading back its own input — the id is folded into the detail + // whatever the remedy is, so that assertion survives swapping the + // remedy to the daemon variant and would leave this branch of the + // remedy selection unexercised while reading as its coverage. + assert!( + message.contains("rocm services stop --yes"), + "the abort must carry the service remedy, not a generic one: {message}" + ); + assert!( + !message.contains("restarts managed services on its own"), + "a service is not the background helper, so its remedy must not be offered: {message}" + ); + assert!( + message.contains("svc-stuck"), + "the abort still names which service: {message}" + ); + assert!( + doomed.is_file(), + "the planned path must survive an aborted uninstall" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_background_helper_that_cannot_be_stopped_leaves_every_planned_path_in_place() { + // The daemon is the one failure class that is not a service: it restarts + // managed servers on its own, so removing the tooling while it is up + // recreates EAI-8014 from the other end. When its identity cannot be + // verified it is deliberately left running, which must abort the + // uninstall rather than quietly proceed. + let (root, plan) = plan_removing_one_file("daemon-unconfirmed"); + let doomed = plan.actions[0].path.clone(); + + let error = stop_managed_services_then_remove(&plan, || { + Ok(ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "rocmd (pid 4321)".to_owned(), + reason: "the background helper's identity could not be verified".to_owned(), + remedy: StopFailureRemedy::StopTheDaemon, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }) + }) + .expect_err("an unstopped background helper must abort uninstall"); + + let message = format!("{error:#}"); + // Assert on text the GATE owns, not on the strings this test handed it. + // The service id and reason above are folded into the detail whatever + // the remedy is, so asserting they appear only proves the input reached + // the output — it passes with the remedy swapped to the service variant, + // which would leave this remedy branch unexercised while reading as its + // coverage. The daemon remedy's own sentence is what distinguishes it. + assert!( + message.contains("restarts managed services on its own"), + "the abort must carry the daemon remedy, not a generic one: {message}" + ); + assert!( + !message.contains("rocm services stop --yes"), + "the daemon is not a service, so the service remedy must not be offered: {message}" + ); + assert!( + message.contains("rocmd (pid 4321)"), + "the abort still names which helper: {message}" + ); + assert!( + doomed.is_file(), + "the planned path must survive an aborted uninstall" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stop_pass_that_cannot_run_leaves_every_planned_path_in_place() { + // Discovery failing (an unreadable services directory) is the same class + // of danger as a stop failing: uninstall would be removing the tooling + // while blind to what it manages. + let (root, plan) = plan_removing_one_file("stop-undiscoverable"); + let doomed = plan.actions[0].path.clone(); + + let error = stop_managed_services_then_remove(&plan, || { + bail!("failed to read the services directory") + }) + .expect_err("a stop pass that cannot run must abort uninstall"); + + // Unlike the two remedy tests above, there is no gate-owned text to + // assert on here: a discovery failure never reaches the gate, it is + // propagated by the `?` on the stop pass. So this substring is + // deliberately the test's own input, and it pins exactly one thing — + // that the cause is carried through rather than swallowed or replaced + // by a generic "uninstall aborted", which is the `.ok()` defect an + // earlier round found on this path. The survival assertion below is + // what pins the abort itself. + assert!( + error.to_string().contains("services directory"), + "the abort carries the discovery failure verbatim: {error:#}" + ); + assert!( + doomed.is_file(), + "the planned path must survive an aborted uninstall" + ); + let _ = fs::remove_dir_all(root); + } + + // The bounded-exit assertion and its budget are shared with the crate root's + // test module rather than copied here. Two near-identical copies, carrying + // near-identical comments, is exactly what made it easy to believe both + // process-based tests were bounded when only one of them actually was. + #[cfg(target_os = "linux")] + use crate::tests::assert_stopped_within_grace; + + /// Gated to Linux because it spawns a real stand-in process and relies on the + /// zombie-state detection in `terminate_verified`, which is implemented only there. + #[cfg(target_os = "linux")] + #[test] + fn a_cache_only_uninstall_leaves_a_live_server_running() { + // The other direction of the recovery-tooling predicate, pinned through + // the command rather than on the predicate alone. `--keep-binaries + // --keep-data` leaves `rocm services stop` and every service record in + // place, so there is nothing to protect by force-stopping a live server + // — and doing it anyway would kill a user's running model for a run that + // only clears a cache. + // + // Unit tests already assert `plan_removes_recovery_tooling` is false for + // a cache-only plan, but nothing checked the wiring: forcing that + // predicate true left every other test green, because they all uninstall + // something that *does* remove the tooling. + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-cache-only-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + let paths = rocm_core::AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + paths.ensure().expect("create the app directories"); + fs::write(paths.cache_dir.join("blob"), b"cached").expect("seed a cache file"); + + let mut server = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("failed to spawn the managed server"); + let pid = server.id(); + let mut record = rocm_core::ManagedServiceRecord::new( + &paths, + "svc-cache-only", + "vllm", + "m", + "m", + "127.0.0.1", + UNSERVABLE_PORT, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + record.status = "ready".to_owned(); + record.write().expect("failed to write the service record"); + + super::uninstall_with_paths( + &paths, + &crate::UninstallOptions { + yes: true, + keep_binaries: true, + keep_data: true, + ..crate::UninstallOptions::default() + }, + ) + .expect("a cache-only uninstall must succeed"); + + assert!( + server.try_wait().expect("poll the server").is_none(), + "a cache-only uninstall must not stop a live managed server" + ); + assert!( + record.manifest_path.is_file(), + "a cache-only uninstall keeps the service records" + ); + let _ = server.kill(); + let _ = server.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn removal_proceeds_once_every_managed_server_is_confirmed_stopped() { + let (root, plan) = plan_removing_one_file("stop-confirmed"); + let doomed = plan.actions[0].path.clone(); + + stop_managed_services_then_remove(&plan, || { + Ok(ManagedServiceStopReport { + stopped: vec!["svc-stopped".to_owned()], + failed: Vec::new(), + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }) + }) + .expect("a confirmed stop must let uninstall proceed"); + + assert!( + !doomed.exists(), + "the planned path must be removed once nothing is left serving" + ); + let _ = fs::remove_dir_all(root); + } + + /// Gated to Linux: relies on a spawned stand-in and on + /// the zombie-state detection in `terminate_verified`. + #[cfg(target_os = "linux")] + #[test] + fn a_live_managed_server_is_stopped_before_the_planned_paths_are_removed() { + // The whole ordering, driven through the real stop pass rather than an + // injected report: a live server dies first, and only then do the files + // go. + // + // The mutation this is sensitive to is the ordering inversion — running + // the removal loop before the stop pass. That only bites because the + // plan below also removes the *services directory*: remove-first deletes + // the record, so the stop pass then finds nothing to stop, the stand-in + // survives its deadline and the assertion fails. With the plan removing + // only an unrelated file (as it did originally) the record survived the + // inversion, the stop still ran, and this test passed under exactly the + // defect it claims to guard. + let (root, mut plan) = plan_removing_one_file("live-server"); + let doomed = plan.actions[0].path.clone(); + let paths = rocm_core::AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + plan.actions.push(crate::UninstallPlanEntry { + kind: "data", + path: paths.data_dir.join("services"), + }); + let child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("failed to spawn the managed server"); + let pid = child.id(); + let free_port = UNSERVABLE_PORT; + let mut record = rocm_core::ManagedServiceRecord::new( + &paths, + "svc-live", + "vllm", + "m", + "m", + "127.0.0.1", + free_port, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + record.status = "ready".to_owned(); + record.write().expect("failed to write the service record"); + + stop_managed_services_then_remove(&plan, || { + crate::uninstall_gate::stop_managed_services_before_uninstall(&paths) + }) + .expect("a server that stops must not block uninstall"); + + // The stand-in is our own child, so its exit is observable directly — + // and bounded, so this pins that uninstall killed it rather than that it + // outlived the test. + let mut child = child; + assert_stopped_within_grace( + &mut child, + "the managed server must be stopped by uninstall", + ); + assert!( + !rocm_core::process_is_running(pid), + "the managed server must be stopped by uninstall" + ); + assert!( + !doomed.exists(), + "the planned path must be removed once the server is stopped" + ); + let _ = fs::remove_dir_all(root); + } + + /// Gated to Linux for the same reason as the test above: a spawned stand-in + /// and zombie-state detection. + #[cfg(target_os = "linux")] + #[test] + fn the_uninstall_command_itself_stops_a_managed_server_before_removing_anything() { + // Drives the real command end to end — plan, confirm gate, stop pass, + // removal — rather than the pieces separately. + // + // Two distinct claims, and it is worth being exact about which is which, + // because an earlier version of this comment claimed both and only + // delivered one. ORDERING: inlining the old remove-first loop back into + // `uninstall_with_paths` fails here even though every narrower test + // still passes. STOPPING: the assertion below is bounded, so skipping + // the stop pass fails it instead of merely making the suite take as long + // as the stand-in lives. + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-cmd-test-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + let paths = rocm_core::AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + for dir in [&paths.config_dir, &paths.data_dir, &paths.cache_dir] { + fs::create_dir_all(dir).expect("seed an isolated dir the plan will remove"); + } + let marker = paths.data_dir.join("state.json"); + fs::write(&marker, b"{}").expect("seed state the removal must delete"); + + let server = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn the managed server"); + let pid = server.id(); + let free_port = UNSERVABLE_PORT; + let mut record = rocm_core::ManagedServiceRecord::new( + &paths, + "svc-cmd-live", + "vllm", + "m", + "m", + "127.0.0.1", + free_port, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + record.status = "ready".to_owned(); + record.write().expect("write the service record"); + + // `--keep-binaries` keeps the test off the real executable-discovery + // path; removing the data dir still takes the service records away, so + // the stop pass is required to run. + super::uninstall_with_paths( + &paths, + &crate::UninstallOptions { + yes: true, + keep_binaries: true, + ..crate::UninstallOptions::default() + }, + ) + .expect("uninstall should succeed once the server is stopped"); + + let mut server = server; + assert_stopped_within_grace( + &mut server, + "`rocm uninstall` must stop the server it manages", + ); + assert!( + !rocm_core::process_is_running(pid), + "`rocm uninstall` must stop the server it manages" + ); + assert!( + !marker.exists(), + "`rocm uninstall` must remove the planned state once the server is stopped" + ); + let _ = fs::remove_dir_all(root); + } +} diff --git a/apps/rocm/src/uninstall_gate.rs b/apps/rocm/src/uninstall_gate.rs new file mode 100644 index 000000000..4ddc26b58 --- /dev/null +++ b/apps/rocm/src/uninstall_gate.rs @@ -0,0 +1,1376 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The stop-before-remove gate behind `rocm uninstall` (EAI-8014). +//! +//! Owns the types describing what a stop pass managed to do and the functions +//! that stop the background helper and every live managed service, classify +//! what is still answering afterwards, and decide whether removal may proceed. +//! The command handler in `uninstall.rs` calls in; the shared +//! `UninstallOptions`/`UninstallPlan` types stay in the crate root. + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, AutomationRuntimeState, ManagedServiceRecord}; +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use crate::chat::loopback_tcp_port_is_reachable; +use crate::{ + MANAGED_STOP_GRACE, endpoint_keys, host_port_resolves, load_managed_services, + managed_service_is_live, managed_service_record_is_in_use, + stop_internal_managed_service_confirmed, +}; + +/// A managed service uninstall could not confirm stopped, and why. +/// +/// The reason is carried rather than discarded so the abort message says what +/// went wrong — an operator facing "could not stop svc-x" with no cause has +/// nothing to act on. +#[derive(Debug, Clone)] +pub(crate) struct FailedManagedServiceStop { + pub(crate) service_id: String, + pub(crate) reason: String, + pub(crate) remedy: StopFailureRemedy, +} + +/// What will actually clear a failed stop. +/// +/// Not cosmetic. This gate refuses to remove anything while a stop is +/// unconfirmed, so the advice it prints is the operator's only way out, and +/// advice that cannot work turns the refusal into a dead end: `rocm services +/// stop` re-reads the same unparseable JSON and fails the same way, so a record +/// that will not parse would abort every retry identically. Each failure class +/// carries the remedy that can actually clear it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StopFailureRemedy { + /// The record parses and its processes are still there, so `rocm services + /// stop` can act on it. + StopTheService, + /// The recorded processes are gone but the endpoint still serves — an engine + /// grandchild outlived its supervisor. `rocm services stop` has nothing left + /// to kill, so the process holding the port has to be found and stopped. + StopWhatHoldsThePort, + /// A record already stopped before this run, whose old port now answers with + /// an authentication refusal. That is not evidence of a survivor of ours (a + /// keyed server of someone else's looks identical), and the record is what + /// keeps tripping the check, so removing it is the way out if it isn't ours. + RemoveTheStoppedRecord, + /// The record does not parse, so no `rocm` command can act on it — the file + /// itself has to be repaired or removed. + RepairTheRecord, + /// The background helper is still alive, so it can restart what was just + /// stopped. It has to go before anything is removed. + StopTheDaemon, + /// A live record carries no start time for a pid that is still alive, so the + /// pid cannot be proven to be the server and is left alone. + ConfirmTheServicePid, + /// A live pid is recorded as the helper but cannot be proven to be `rocmd` + /// (a pre-upgrade state file, or a reboot since). Killing it would risk an + /// unrelated process, so the operator has to decide, and the state file is the + /// thing to delete if it is not the helper. + ConfirmTheDaemonPid, + /// The helper's runtime-state file does not parse, so no pid was ever + /// recovered from it — "kill that pid" is advice nobody can act on. The file + /// itself has to be repaired or deleted, so the remedy names it. + RepairTheDaemonState, +} + +impl StopFailureRemedy { + /// The recovery advice for this class, naming `ids` where the advice is + /// useless without them. + /// + /// Exhaustive on purpose, like [`StoppedRecordVerdict::blocks`]: a new + /// variant must get advice, because the gate's whole contract is that its + /// advice is the operator's only way out. + pub(crate) fn advice(self, ids: &[String]) -> String { + match self { + Self::StopTheService => "Stop them with `rocm services stop --yes`, then re-run \ + uninstall. A server started by another user, or one wedged \ + in the kernel, needs elevated privileges or a manual kill \ + first." + .to_owned(), + Self::StopWhatHoldsThePort => format!( + "Every process recorded for {} is gone, yet the endpoint still answers — the \ + engine outlived its supervisor, so `rocm services stop` has nothing left to \ + kill. Find what holds that port (`ss -ltnp` on Linux, `Get-NetTCPConnection \ + -LocalPort ` on Windows), stop it, then re-run uninstall.", + ids.join(", ") + ), + Self::ConfirmTheServicePid => format!( + "No start time was recorded for a live pid of each service listed, so it cannot \ + be proven to still be the server (the pid may have been reused after a reboot) \ + and it was left alone: {}. Check what those pids are. If one is your server, \ + stop it yourself (`kill ` on Linux, `Stop-Process -Id ` on Windows). \ + If it is something else, delete the service's record file under the data \ + directory's `services` folder by hand. Then re-run uninstall.", + ids.join(", ") + ), + Self::StopTheDaemon => "The background helper restarts managed services on its own, \ + so it has to be stopped before uninstall can safely remove \ + anything: kill that pid (`kill ` on Linux, `Stop-Process -Id -Force` \ + on Windows), then re-run uninstall." + .to_owned(), + Self::RemoveTheStoppedRecord => format!( + "An authenticated server answers on the old port of a record that was already \ + stopped. If that server is not one of yours, delete the stale record with \ + `rocm services remove --yes` and re-run uninstall; if it is, stop it first: \ + {}.", + ids.join(", ") + ), + Self::ConfirmTheDaemonPid => format!( + "The recorded pid could not be proven to be the background helper, so it was left \ + alone. Check what that pid is ({}): if it is `rocmd`, stop it; if it is something \ + else (the state file may predate a reboot), delete the runtime state file named \ + there, then re-run uninstall.", + ids.join(", ") + ), + Self::RepairTheDaemonState => format!( + "The background helper's runtime state does not parse, so no pid could be read \ + from it and `rocm` cannot tell whether the helper is running. Check for a live \ + `rocmd` process and stop it, then repair or delete the file and re-run \ + uninstall: {}.", + ids.join(", ") + ), + Self::RepairTheRecord => format!( + "No `rocm` command can act on an unparseable record, so these have to be handled \ + on disk: check whether the server each one describes is still running (`rocm \ + services list` skips them), stop it, then repair or delete the file and re-run \ + uninstall: {}.", + ids.join(", ") + ), + } + } + + /// Where this class sits in the printed advice. + /// + /// Also exhaustive, and for a second reason beyond ordering: without it, a + /// new variant could compile an `advice()` arm and still never be printed, + /// because nothing would have added it to the list of classes to walk. + /// Ranking every variant means the set that gets advice is derived from the + /// failures themselves rather than from a list somebody has to remember to + /// extend. Most actionable first; the two "repair a file by hand" classes + /// last. + pub(crate) const fn advice_rank(self) -> u8 { + match self { + Self::StopTheService => 0, + Self::StopWhatHoldsThePort => 1, + Self::RemoveTheStoppedRecord => 2, + Self::ConfirmTheServicePid => 3, + Self::StopTheDaemon => 4, + Self::ConfirmTheDaemonPid => 5, + Self::RepairTheDaemonState => 6, + Self::RepairTheRecord => 7, + } + } +} + +/// The processes a stop of `record` would signal, each paired with the start +/// time recorded for it, de-duplicated on pid and preferring a known token. +/// +/// Shared by the stop itself and by this gate's pre-check, so the two cannot +/// disagree about which pid still needs a start time. +pub(crate) fn recorded_pid_identities(record: &ManagedServiceRecord) -> Vec<(u32, Option)> { + let mut entries: Vec<(u32, Option)> = Vec::new(); + for (pid, ticks) in [ + (Some(record.supervisor_pid), record.supervisor_start_ticks), + (record.engine_pid, record.engine_start_ticks), + ] { + let Some(pid) = pid.filter(|pid| *pid != 0 && *pid != std::process::id()) else { + continue; + }; + if let Some(existing) = entries.iter_mut().find(|(seen, _)| *seen == pid) { + if existing.1.is_none() { + existing.1 = ticks; + } + } else { + entries.push((pid, ticks)); + } + } + entries +} + +/// The live pids of `record` that carry no start time, and so cannot be told +/// apart from a reused pid. The stop refuses a record that has any, rather than +/// force-killing a tree it cannot prove is ours; the plan reads the same answer +/// so it never promises a stop this refuses. +pub(crate) fn unprovable_live_pids(record: &ManagedServiceRecord) -> Vec { + recorded_pid_identities(record) + .into_iter() + .filter(|(pid, ticks)| { + record_predates_start_ticks(*ticks) && rocm_core::process_is_running(*pid) + }) + .map(|(pid, _)| pid) + .collect() +} + +/// How long the gate waits for a still-listening endpoint to say what it serves. +/// +/// Only reached when something already answered a TCP connect, and only for +/// records that were already stopped, so it costs nothing on the normal path. +pub(crate) const ENDPOINT_IDENTITY_PROBE_TIMEOUT: Duration = Duration::from_secs(2); + +/// What [`stop_managed_services_before_uninstall`] managed to do, so the caller +/// can report the services it stopped and refuse to proceed while any is still +/// alive. +#[derive(Debug, Default)] +pub(crate) struct ManagedServiceStopReport { + /// Services confirmed stopped (every recorded process observed gone). + pub(crate) stopped: Vec, + /// Services that could not be confirmed stopped — a still-serving endpoint, + /// an unverifiable process, or a record too corrupt to locate one. Uninstall + /// must abort rather than remove the tooling that stops them. + pub(crate) failed: Vec, + /// Every place this pass decided to proceed without proving the port was + /// free, for the caller to put on stderr. + /// + /// These are values rather than `eprintln!`s so a test can assert on them. + /// While they were printed in place, the disclosure that keeps each + /// fail-open a tradeoff rather than a silent removal was pinned by nothing: + /// emptying a warning body left every test green while changing what a + /// destructive command tells its operator. Returning them makes the + /// disclosure part of this function's answer, and `stderr` the caller's + /// business. + pub(crate) warnings: Vec, + /// One relaunch command per entry of `stopped`, built from that service's own + /// record by the shared `rocm_engine_protocol::relaunch_command`, so the + /// abort message names the flags each one needs instead of a generic line. + pub(crate) relaunch: Vec, + /// The background helper (`rocmd`) was stopped by this pass. Kept apart from + /// `stopped` because it is not a service: it has no endpoint key, and the + /// consequence of its being down is that nothing supervises or recovers + /// services until it is restarted. + pub(crate) helper_stopped: bool, +} + +/// The kill breadth the stop applies to the background helper. +/// +/// A constant so a test can see the scope the production stop hands the +/// termination, rather than only that some termination was requested. +pub(crate) const HELPER_KILL_SCOPE: rocm_core::KillScope = rocm_core::KillScope::Tree; + +/// How much of a process tree the stop signals, in words, for the plan and the +/// docs: only Linux enumerates descendants (`process_tree_pids` returns just the +/// root on Windows), so the claim cannot be the same on both. +pub(crate) const fn tree_scope_words() -> &'static str { + if rocm_core::runtime_is_windows() { + "only the recorded process (descendants are not enumerated on Windows)" + } else { + "its whole process tree" + } +} + +/// What the gate decides about the background helper, read without touching it. +/// +/// The one decision behind the plan's note, the pre-flight and the stop itself, +/// so the plan cannot promise a stop that the stop then refuses. +pub(crate) enum HelperDecision { + /// No state file, `running` false, a free pid, or a pid that belongs to + /// someone else. + NothingToStop, + /// The state file exists but does not read or parse. + UnreadableState(String), + /// A live pid is recorded as the helper but cannot be proven to be it. + Unverifiable { pid: u32 }, + /// Verified as the helper: stop it. + Stop { + pid: u32, + identity: rocm_core::ProcessIdentity, + }, +} + +/// Decide [`HelperDecision`] from the state file and one reading of the pid. +pub(crate) fn helper_decision(paths: &AppPaths) -> HelperDecision { + // Three outcomes, not two. `load` returns `Ok(None)` only when there is no + // state file at all; a permission error or a half-written file returns + // `Err`, and discarding that would skip the daemon stop silently and let + // uninstall delete the tooling while a live `rocmd` respawns what the + // service pass just stopped. + let state = match AutomationRuntimeState::load(paths) { + Ok(Some(state)) => state, + Ok(None) => return HelperDecision::NothingToStop, + Err(error) => return HelperDecision::UnreadableState(format!("{error:#}")), + }; + // `running` is the same inactive contract `background_helper_already_running` + // spawns on. Only the clean-shutdown path writes `running: false`, so a false + // flag means the recorded pid belongs to a daemon that already exited, and + // anything alive under it now inherited the number. + if !state.running || state.daemon_pid == 0 || state.daemon_pid == std::process::id() { + return HelperDecision::NothingToStop; + } + if !rocm_core::process_is_running(state.daemon_pid) { + return HelperDecision::NothingToStop; + } + // Kill nothing this cannot identify. `terminate_verified` with `force` is a + // SIGKILL across the whole process tree, and `runtime-state.json` outlives a + // crash, OOM-kill or reboot with `running` still true, so the recorded pid + // can belong to an unrelated process by the time uninstall runs. An absent + // start-time is unverifiable on its own (see `record_predates_start_ticks`), + // and one reading of the daemon's start-time serves the whole decision, so + // a pid that flickers cannot clear one check and fail another. + let identity = rocm_core::ProcessIdentity::new(state.daemon_pid, state.daemon_start_ticks); + let observed_start_ticks = rocm_core::process_start_ticks(state.daemon_pid); + match daemon_identity_outcome( + rocm_core::identity_state_with_observed(&identity, observed_start_ticks), + record_predates_start_ticks(state.daemon_start_ticks), + ) { + DaemonIdentityOutcome::NothingOfOurs => HelperDecision::NothingToStop, + DaemonIdentityOutcome::Unverifiable => HelperDecision::Unverifiable { + pid: state.daemon_pid, + }, + DaemonIdentityOutcome::Stop => HelperDecision::Stop { + pid: state.daemon_pid, + identity, + }, + } +} + +/// The refusal a [`HelperDecision`] amounts to, if any. +fn helper_refusal(paths: &AppPaths, decision: &HelperDecision) -> Option { + match decision { + // Name the file, not a pid: nothing parsed, so no pid was ever + // recovered and "kill that pid" would be advice nobody can follow. + HelperDecision::UnreadableState(error) => Some(FailedManagedServiceStop { + service_id: format!("rocmd ({})", paths.automation_state_path().display()), + reason: format!( + "the background helper's runtime state could not be read, so it cannot be \ + confirmed stopped: {error}" + ), + remedy: StopFailureRemedy::RepairTheDaemonState, + }), + HelperDecision::Unverifiable { pid } => Some(daemon_identity_unverified( + *pid, + &paths.automation_state_path(), + )), + HelperDecision::NothingToStop | HelperDecision::Stop { .. } => None, + } +} + +/// What the plan should tell the operator about the background helper, derived +/// from the same decision the stop pass acts on, so the line cannot promise a +/// stop that pass then refuses. +/// +/// `None` when there is nothing to say: nothing to stop, or a refusal the plan's +/// consolidated refusal line already names. `run_refuses` is whether the run +/// aborts before stopping anything: a helper that is verified but sits in such a +/// run is left running, and saying "will be stopped" would be false. +/// `refusal_line_present` is whether the pre-flight ran and its refusal line is +/// in the plan; it is absent when the service records could not be read at all, +/// and then a helper that cannot be verified has to be named here. +pub(crate) fn background_helper_plan_note( + paths: &AppPaths, + removes_recovery_tooling: bool, + run_refuses: bool, + refusal_line_present: bool, +) -> Option { + let state_file = paths.automation_state_path(); + let decision = helper_decision(paths); + match (&decision, removes_recovery_tooling) { + (HelperDecision::NothingToStop, _) => None, + // Named by the refusal line, which comes from the same decision. + (HelperDecision::UnreadableState(_) | HelperDecision::Unverifiable { .. }, true) + if refusal_line_present => + { + None + } + (HelperDecision::UnreadableState(_) | HelperDecision::Unverifiable { .. }, true) => { + helper_refusal(paths, &decision).map(|failure| { + format!( + "{} ({}); uninstall will refuse to remove anything until this is resolved", + failure.service_id, failure.reason + ) + }) + } + (HelperDecision::UnreadableState(error), false) => Some(format!( + "could not read the background helper's runtime state {}: {error}. This run keeps \ + the tooling, so it does not check it", + state_file.display() + )), + (HelperDecision::Unverifiable { pid } | HelperDecision::Stop { pid, .. }, false) => { + Some(format!( + "the background helper (rocmd, pid {pid}) is running; this removal keeps the \ + tooling, so it is left running" + )) + } + (HelperDecision::Stop { pid, .. }, true) if run_refuses => Some(format!( + "the background helper (rocmd, pid {pid}) is running; uninstall will refuse (see \ + above) before stopping anything, so it is left running" + )), + (HelperDecision::Stop { pid, .. }, true) => Some(format!( + "the background helper (rocmd, pid {pid}) is running; it will be stopped before \ + removal, and the kill covers {}", + tree_scope_words() + )), + } +} + +/// The failure recorded when the background helper is live but cannot be proven +/// to be `rocmd`, so it was deliberately left alone. +pub(crate) fn daemon_identity_unverified( + daemon_pid: u32, + state_file: &std::path::Path, +) -> FailedManagedServiceStop { + FailedManagedServiceStop { + service_id: format!( + "rocmd (pid {daemon_pid}; state file {})", + state_file.display() + ), + reason: + "the background helper's identity could not be verified (no start time was recorded \ + for it, or its start time cannot be read now), so it was left running rather \ + than risk signalling an unrelated process that inherited its pid" + .to_owned(), + remedy: StopFailureRemedy::ConfirmTheDaemonPid, + } +} + +/// What the gate does with the daemon's recorded pid once its identity has been +/// read back. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DaemonIdentityOutcome { + /// The pid is free or belongs to someone else: nothing of ours to stop. + NothingOfOurs, + /// Could not be proven ours, so it is neither killed nor ignored: abort. + Unverifiable, + /// Verified as the daemon: stop it. + Stop, +} + +/// Map an identity reading to the gate's action. Pure so every input is pinned +/// by a test: folding `Indeterminate` into `NothingOfOurs` would remove the +/// tooling while a live `rocmd` keeps respawning services. +pub(crate) const fn daemon_identity_outcome( + state: rocm_core::IdentityState, + record_predates_start_ticks: bool, +) -> DaemonIdentityOutcome { + match state { + rocm_core::IdentityState::Gone | rocm_core::IdentityState::Recycled => { + DaemonIdentityOutcome::NothingOfOurs + } + rocm_core::IdentityState::Indeterminate => DaemonIdentityOutcome::Unverifiable, + rocm_core::IdentityState::Matches if record_predates_start_ticks => { + DaemonIdentityOutcome::Unverifiable + } + rocm_core::IdentityState::Matches => DaemonIdentityOutcome::Stop, + } +} + +/// Stop the background helper before uninstall stops the services it supervises. +/// +/// `rocmd` recovers managed services: a `ready`/`running` record whose endpoint +/// is unreachable is treated as recoverable and respawned +/// (`endpoint_status_unreachable`). That is precisely the state the stop pass +/// creates — the engine is killed, and the record still says `ready` until +/// `stop_internal_managed_service` writes it back. A daemon polling in that +/// window brings the server straight back, after which uninstall would delete +/// the binaries and every service record while a brand-new engine holds the GPU. +/// Stopping the supervisor first closes the window instead of racing it. +/// +/// A daemon that cannot be confirmed stopped is a blocking failure for the same +/// reason a service is: it can resurrect a server after the tooling is gone. +/// +/// It is never killed on the recorded pid alone. `runtime-state.json` outlives a +/// crash, OOM-kill or reboot with `running` still true, so that pid can name an +/// unrelated process — and this call site signals a whole tree with `force`. +/// Only a pid whose recorded start-time still matches is signalled; a recycled +/// one means the daemon is already gone (nothing to stop), and one that can be +/// neither confirmed nor refuted is left alone and reported as a failure, which +/// aborts the uninstall with the tooling intact. Being told to kill a pid is +/// recoverable; having an unrelated process tree killed is not. +/// +/// A record whose `running` is false is not signalled at all. Only rocmd's clean +/// shutdown writes that flag, on its way out, so the pid in such a record names +/// a process that has already exited and anything live under that number +/// inherited it. This is the same inactive contract `background_helper_already_running` +/// spawns on. +/// +/// Linux (`/proc`) and Windows (`GetProcessTimes` creation time) can record and +/// compare a start-time, so a recycled pid is told apart there. +/// +/// One residual gap, and the list above is otherwise the complete set of +/// inputs: a *missing* `runtime-state.json` is taken at face value as "no +/// daemon". Deleting that file by hand while `rocmd` is live therefore skips the +/// daemon stop silently. This is deliberate — a missing file is the ordinary +/// never-started case, and there is no pid to verify or signal without it — but +/// it does mean the gate is only as good as the state file. An unreadable one is +/// the case that aborts; an absent one is the case that proceeds. +/// +/// The verified termination is a parameter, so a test can make the stop come +/// back unconfirmed and see the [`HELPER_KILL_SCOPE`] the production stop asks +/// for; production passes `terminate_verified` with `force`. +pub(crate) fn stop_background_helper_with( + paths: &AppPaths, + report: &mut ManagedServiceStopReport, + terminate: impl FnOnce( + &rocm_core::ProcessIdentity, + rocm_core::KillScope, + ) -> rocm_core::TerminationOutcome, +) { + let decision = helper_decision(paths); + if let Some(failure) = helper_refusal(paths, &decision) { + report.failed.push(failure); + return; + } + let HelperDecision::Stop { pid, identity } = decision else { + return; + }; + println!("stopping the background helper (rocmd)"); + let outcome = terminate(&identity, HELPER_KILL_SCOPE); + if outcome.stopped() { + report.helper_stopped = true; + } else if matches!(outcome, rocm_core::TerminationOutcome::Unverified) { + report.failed.push(daemon_identity_unverified( + pid, + &paths.automation_state_path(), + )); + } else { + report.failed.push(FailedManagedServiceStop { + service_id: format!("rocmd (pid {pid})"), + reason: "the background helper was signalled but could not be confirmed stopped (the \ + servers it supervised may already be down), and it restarts managed \ + services whose endpoint stops answering" + .to_owned(), + remedy: StopFailureRemedy::StopTheDaemon, + }); + } +} + +/// Whether the stop pass has to deal with `record`: it reads as running, or its +/// status says something other than `stopped` while a recorded process is still +/// alive. +/// +/// The status string alone is not enough: `refresh_from_engine_state` adopts +/// `failed` from the engine's own state file, and the liveness refresh returns +/// early for a non-live status without consulting the pids, so a server whose +/// engine reported failure while its process is still up reads as removable. +/// `services remove` guards with [`managed_service_record_is_in_use`] for the +/// same reason. A record that reads `stopped` is left to the port probe: that +/// status is written only once the recorded processes are confirmed gone, so a +/// live pid under it is a reused number, and refusing on it would block every +/// uninstall that finds an old record. +pub(crate) fn record_needs_stopping(record: &ManagedServiceRecord) -> bool { + managed_service_is_live(record) + || (record.status != "stopped" && managed_service_record_is_in_use(record)) +} + +/// The records a stop pass partitions after the live ones are set aside: those +/// already stopped, and those stopped but sharing an endpoint with a live record. +struct RecordPartition<'a> { + already_stopped: Vec<&'a ManagedServiceRecord>, + deferred_partners: Vec<(&'a ManagedServiceRecord, Vec)>, +} + +fn partition_records(records: &[ManagedServiceRecord]) -> RecordPartition<'_> { + // A record already stopped before this run needs no stop to be judged, and a + // survivor from the previous run reads exactly like that: probe those first. + // + // Except a stopped record that shares its host:port with a live one: serving + // the same model again after a stop leaves exactly that pair, and the live + // record's own server answers there. That answer says nothing about the + // stopped record until the live one has been stopped, so it is probed after. + let live_endpoints: Vec<(String, Vec, u16)> = records + .iter() + .filter(|record| record_needs_stopping(record)) + .map(|record| { + ( + record.service_id.clone(), + probe_hosts(&record.host), + record.port, + ) + }) + .collect(); + // Overlap, not equality: `0.0.0.0`, `localhost` and `127.0.0.1` can all name + // the one listener, so any shared probe address on the same port counts. + let live_partners = |record: &ManagedServiceRecord| -> Vec { + let mine = probe_hosts(&record.host); + live_endpoints + .iter() + .filter(|(_, hosts, port)| { + *port == record.port + && mine + .iter() + .any(|a| hosts.iter().any(|b| hosts_name_one_listener(a, b))) + }) + .map(|(id, _, _)| id.clone()) + .collect() + }; + RecordPartition { + already_stopped: records + .iter() + .filter(|record| !record_needs_stopping(record) && live_partners(record).is_empty()) + .collect(), + deferred_partners: records + .iter() + .filter(|record| !record_needs_stopping(record)) + .map(|record| (record, live_partners(record))) + .filter(|(_, partners)| !partners.is_empty()) + .collect(), + } +} + +/// Everything the stop pass would refuse before it signals or removes anything, +/// decided without touching a process, and the records it read to decide. +/// +/// This is the one place those refusals live. The stop pass runs it first and +/// returns on any failure, so a run that is going to abort has stopped nothing +/// on the way (each confirmed stop drops that service's endpoint key, which +/// cannot be re-minted); the plan runs the same function, so the line the +/// operator confirms against describes the run that will happen. Order matters +/// and is the order of cost: an unparseable manifest, then a live pid that +/// cannot be proven ours, then a stopped record whose port still answers as its +/// own model, then a background helper that cannot be identified. Each group +/// returns before the next, as the stop pass always did. +/// +/// Fail-closed on discovery too: if the services directory exists but cannot be +/// enumerated, the error propagates so uninstall aborts rather than deleting the +/// tooling while blind to what it manages. +fn preflight_with_records( + paths: &AppPaths, +) -> Result<(ManagedServiceStopReport, Vec)> { + let mut report = ManagedServiceStopReport::default(); + // One reading of the helper serves the whole pre-flight, and its refusal is + // appended to whatever else refuses, so the plan can name it on a run that + // already refuses for another reason instead of the operator learning of it + // only on the retry. + let helper_failure = helper_refusal(paths, &helper_decision(paths)); + let with_helper = |mut report: ManagedServiceStopReport| { + report.failed.extend(helper_failure.clone()); + report + }; + // A manifest that exists but does not parse can describe a live, + // GPU-holding server; `load_managed_services` skips it silently. + for manifest in unreadable_service_manifests(paths)? { + report.failed.push(FailedManagedServiceStop { + // The full path, not the file name: the only remedy is to act on the + // file, so the message has to say which file. + service_id: manifest.display().to_string(), + reason: + "service record could not be parsed, so its server cannot be located or stopped" + .to_owned(), + remedy: StopFailureRemedy::RepairTheRecord, + }); + } + if !report.failed.is_empty() { + return Ok((with_helper(report), Vec::new())); + } + let records = load_managed_services(paths)?; + // A live record with no recorded start time cannot prove its pid is still the + // server (the pid may have been reused after a reboot), and the stop below + // force-kills a process tree. Refuse before anything is stopped, as the + // daemon path does for a tick-less state file. + for record in records + .iter() + .filter(|record| record_needs_stopping(record)) + { + let unprovable = unprovable_live_pids(record); + if !unprovable.is_empty() { + report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + reason: format!( + "no start time was recorded for live pid(s) {unprovable:?}, so they cannot be \ + told apart from a reused pid" + ), + remedy: StopFailureRemedy::ConfirmTheServicePid, + }); + } + } + if !report.failed.is_empty() { + return Ok((with_helper(report), records)); + } + let partition = partition_records(&records); + probe_records_for_survivors(paths, &partition.already_stopped, &[], &mut report); + Ok((with_helper(report), records)) +} + +/// [`preflight_with_records`] for the plan: the refusals a run would hit and the +/// fail-open disclosures it would make, with nothing stopped. +pub(crate) fn uninstall_preflight(paths: &AppPaths) -> Result { + preflight_with_records(paths).map(|(report, _)| report) +} + +/// The plan's one line for a run that will refuse, built from the failures +/// [`uninstall_preflight`] found. It says what the run does not do as well as +/// what blocks it: the pre-flight returns before anything is stopped. +pub(crate) fn preflight_refusal_line(report: &ManagedServiceStopReport) -> Option { + if report.failed.is_empty() { + return None; + } + let detail = report + .failed + .iter() + .map(|failure| format!("{} ({})", failure.service_id, failure.reason)) + .collect::>() + .join("; "); + Some(format!( + "uninstall will refuse to remove anything, and will stop nothing, until this is \ + resolved: {detail}" + )) +} + +/// Stop every managed service that is in use before uninstall removes the +/// binaries and service records needed to stop them. +/// +/// The defect this closes (EAI-8014): uninstall reported success while a +/// publicly-bound, GPU-holding managed server kept serving, and deleted the +/// `rocm`/`rocmd` binaries and service manifests — so the supported +/// `rocm services stop` path was gone and only a manual PID kill remained. +/// +/// A service is only counted stopped when the stop confirms every recorded +/// process gone (the boolean [`stop_internal_managed_service_confirmed`] +/// returns — the report's `status` string cannot say it, because a record that +/// already read `stopped` keeps that status through a stop that confirmed +/// nothing); anything else lands in `failed` so the caller aborts and keeps the +/// tooling. +pub(crate) fn stop_managed_services_before_uninstall( + paths: &AppPaths, +) -> Result { + stop_managed_services_confirmed_with(paths, stop_internal_managed_service_confirmed) +} + +/// [`stop_managed_services_before_uninstall`] with a stop that reports only a +/// `status`, which counts as confirmed when it reads `stopped`. Unix-test-only: the +/// production stop decides on the confirmed flag, and the tests that care drive +/// [`stop_managed_services_confirmed_with`] directly. +#[cfg(all(test, unix))] +pub(crate) fn stop_managed_services_with( + paths: &AppPaths, + stop_service: impl Fn(&AppPaths, &str) -> Result, +) -> Result { + stop_managed_services_confirmed_with(paths, |paths, service_id| { + let result = stop_service(paths, service_id)?; + let confirmed = result.get("status").and_then(serde_json::Value::as_str) == Some("stopped"); + Ok((result, confirmed)) + }) +} + +/// [`stop_managed_services_before_uninstall`] with the per-service stop +/// injectable, so a test can make it fail or fail to confirm — which no real +/// process reliably does. +pub(crate) fn stop_managed_services_confirmed_with( + paths: &AppPaths, + stop_service: impl Fn(&AppPaths, &str) -> Result<(serde_json::Value, bool)>, +) -> Result { + stop_managed_services_with_helper_stop(paths, stop_service, |identity, scope| { + rocm_core::terminate_verified(identity, scope, MANAGED_STOP_GRACE, true) + }) +} + +/// [`stop_managed_services_confirmed_with`] with the helper's termination +/// injectable too, so a test can make the helper stop come back unconfirmed +/// and see that no service is stopped after it. +pub(crate) fn stop_managed_services_with_helper_stop( + paths: &AppPaths, + stop_service: impl Fn(&AppPaths, &str) -> Result<(serde_json::Value, bool)>, + terminate_helper: impl FnOnce( + &rocm_core::ProcessIdentity, + rocm_core::KillScope, + ) -> rocm_core::TerminationOutcome, +) -> Result { + // The refusals are decided before anything is stopped, by the function the + // plan also calls. Past this point a stop can still fail, the helper can + // still not die, and a survivor can still answer after the stops, so the + // abort message names what the run already did. + let (mut report, records) = preflight_with_records(paths)?; + if !report.failed.is_empty() { + return Ok(report); + } + let partition = partition_records(&records); + stop_background_helper_with(paths, &mut report, terminate_helper); + if !report.failed.is_empty() { + return Ok(report); + } + let mut attempted: Vec<&ManagedServiceRecord> = Vec::new(); + for record in &records { + if !record_needs_stopping(record) { + continue; + } + attempted.push(record); + // Each stop waits out a bounded grace per recorded process (and the + // engine's own stop before that), so name the service first: without + // this, an uninstall with a live server reads as a hang. + println!("stopping managed service {}", record.service_id); + match stop_service(paths, &record.service_id) { + Ok((_, true)) => { + report.stopped.push(record.service_id.clone()); + report.relaunch.push(format!( + "{}: `{}`", + record.service_id, + rocm_engine_protocol::relaunch_command(&record.host, record.requires_api_key) + )); + } + Ok((result, false)) => { + let status = result + .get("status") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown"); + report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + reason: if status == "stopped" { + "its record reads \"stopped\", but the stop could not confirm every \ + recorded process gone" + .to_owned() + } else { + format!("still \"{status}\" after the stop attempt") + }, + remedy: StopFailureRemedy::StopTheService, + }); + } + Err(error) => report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + reason: format!("{error:#}"), + remedy: StopFailureRemedy::StopTheService, + }), + } + } + // Ground the PID bookkeeping in what is actually being served. Confirming a + // stop from recorded PIDs alone is thin: on Windows the kill scope is the + // recorded process only, so an engine grandchild can outlive it and keep the + // port and the GPU while the record reads "stopped". + // + // Every record is probed, not just the ones this pass stopped. A stop + // persists `status = "stopped"` before this loop runs, so the record that + // failed the gate reads as not-live on the very next run — probing only this + // pass's own work would let the retry the abort message asks for sail + // through and remove the tooling while the survivor keeps serving. That is + // the defect this gate exists to prevent, reached by following its own + // instructions. + // + // What differs is the evidence required, because a stopped record keeps its + // manifest and its old port until `rocm services remove` or `prune` deletes + // it, so an unrelated process that later binds that port must not brick + // uninstall with no way out: + // + // * stopped by this pass — any listener fails the gate. We killed the + // recorded processes seconds ago; something answering there is the + // survivor. + // * already stopped before this run — a listener alone proves nothing, so + // it has to identify itself as this record's own model before it counts. + // An unrelated service on a recycled port does not, and is not blocked. + let stopped_this_pass: Vec<&ManagedServiceRecord> = attempted.clone(); + probe_records_for_survivors(paths, &stopped_this_pass, &attempted, &mut report); + // A deferred record whose live partner failed to stop is already explained by + // the partner's own failure: the listener there IS the partner, so judging the + // stopped record too would name one survivor twice, under the wrong remedy. + let deferred: Vec<&ManagedServiceRecord> = partition + .deferred_partners + .iter() + .filter(|(_, partners)| { + !partners + .iter() + .any(|id| report.failed.iter().any(|f| &f.service_id == id)) + }) + .map(|(record, _)| *record) + .collect(); + probe_records_for_survivors(paths, &deferred, &attempted, &mut report); + Ok(report) +} + +/// Probe the endpoints of `to_probe` for a survivor, recording any that should +/// block the removal. `attempted` is the set this pass stopped; for those any +/// listener blocks, for the rest it has to identify itself as the record's own. +fn probe_records_for_survivors( + paths: &AppPaths, + to_probe: &[&ManagedServiceRecord], + attempted: &[&ManagedServiceRecord], + report: &mut ManagedServiceStopReport, +) { + for &record in to_probe { + if report + .failed + .iter() + .any(|failure| failure.service_id == record.service_id) + { + continue; + } + // A wildcard record yields both loopback families; whichever answers is + // the one the identity probe below has to talk to, so keep it rather + // than re-deriving a single host and guessing the family wrong. + let candidates = probe_hosts(&record.host); + let reachable_hosts: Vec = candidates + .iter() + .filter(|candidate| loopback_tcp_port_is_reachable(candidate, record.port)) + .cloned() + .collect(); + if reachable_hosts.is_empty() { + // Not reaching the port is the normal, silent case: nothing is + // listening, which is what a stopped service looks like. But + // `loopback_tcp_port_is_reachable` returns the same `false` when the + // address does not resolve at all, and those are opposite + // situations. A refused connect is evidence the port is free; a name + // that no longer resolves is evidence of nothing, and skipping on it + // means proceeding to delete the tooling without ever having asked + // whether a server is up. That is a fail-open, and it is disclosed. + if !candidates + .iter() + .any(|candidate| host_port_resolves(candidate, record.port)) + { + report.warnings.push(format!( + "{} is recorded for service {}, but that name does not resolve here, so \ + whether anything is still serving on port {} could not be checked at all, \ + and uninstall is proceeding without that check. A record written on \ + another machine, or under a hostname since removed, looks like this. If \ + that service may still be running, it has to be found and stopped by hand: \ + this uninstall does not wait for you to, and removes the tooling that \ + would have stopped it.", + record.host, record.service_id, record.port + )); + } + continue; + } + let stopped_by_this_pass = attempted + .iter() + .any(|candidate| candidate.service_id == record.service_id); + if !stopped_by_this_pass { + let endpoint_api_key = endpoint_keys::endpoint_api_key(paths, &record.service_id); + // A wildcard record is probed on every loopback family that accepts + // a connection, and ANY of them blocking blocks the removal: a + // stranger answering on `127.0.0.1` must not hide the record's own + // model still serving on `::1`. Each probe asks the address the + // reachability probe just succeeded against, because + // `record.endpoint_url` is built from the recorded host, so a + // `0.0.0.0` or `::` bind would be connected to literally — which + // does not resolve, fails the probe, and takes a fail-open branch, + // removing the tooling while a wildcard-bound engine is serving. + let mut blocking: Option = None; + let mut proceeding: Vec = Vec::new(); + for reachable_host in &reachable_hosts { + let mut probe_record = record.clone(); + probe_record.endpoint_url = + rocm_core::format_http_base_url(reachable_host, record.port); + let probe = rocm_core::managed_service_endpoint_identity( + &probe_record, + endpoint_api_key.as_deref(), + ENDPOINT_IDENTITY_PROBE_TIMEOUT, + ); + // Short-circuits: the extra round trip only happens when the + // probe produced no usable listing, which is the only case its + // answer can change. + let auth_refused = probe.is_err() + && endpoint_refused_authorization( + &probe_record.endpoint_url, + endpoint_api_key.as_deref(), + ); + // Every arm below is decided in `stopped_record_verdict` and + // pinned there by + // `every_identity_answer_maps_to_exactly_one_gate_outcome`, with + // `an_endpoint_listing_nothing_does_not_block_uninstall` and + // `a_listener_naming_another_model_does_not_block_uninstall` + // driving the two proceed-on-a-live-socket arms through this + // call site. The tests live in `main.rs`'s `mod tests`, far from + // here; change an arm and expect them, not this loop, to be what + // goes red. + let verdict = stopped_record_verdict(probe.ok(), auth_refused); + if verdict.blocks() { + blocking.get_or_insert(verdict); + } else if !proceeding.contains(&verdict) { + proceeding.push(verdict); + } + } + let Some(verdict) = blocking else { + // The two fail-open outcomes disclose themselves; only + // `ProceedUnrelated` is silent, because a listener that named + // its models and did not name ours is the one case that is + // actually evidence of a stranger. They go into the report + // rather than straight to stderr so the disclosure is a value a + // test can assert on — see the field's own comment. + for verdict in proceeding { + match verdict { + StoppedRecordVerdict::ProceedListingNothing => { + report.warnings.push(format!( + "{}:{} still accepts connections and answered the identity probe \ + with an empty model list, so it cannot be told from an unrelated \ + service; {} is already recorded stopped — proceeding. An engine \ + still loading or unloading looks like this. If that is a server \ + of yours, it has to be found and stopped by hand: this uninstall \ + does not wait for you to.", + record.host, record.port, record.service_id + )); + } + StoppedRecordVerdict::ProceedUnidentified => { + report.warnings.push(format!( + "{}:{} still accepts connections but did not answer the identity \ + probe with a usable model list, and service {} is already \ + recorded stopped — proceeding. That can be a wedged engine, an \ + unrelated server on the port, or a stale endpoint key. If it is \ + a server of yours, it has to be found and stopped by hand: this \ + uninstall does not wait for you to.", + record.host, record.port, record.service_id + )); + } + StoppedRecordVerdict::ProceedUnrelated => {} + StoppedRecordVerdict::BlockServingOurModel + | StoppedRecordVerdict::BlockAuthRefused => { + unreachable!("only non-blocking verdicts are collected") + } + } + } + continue; + }; + if verdict == StoppedRecordVerdict::BlockAuthRefused { + report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + reason: format!( + "an authenticated server answers on {}:{}, this record's old port", + record.host, record.port + ), + remedy: StopFailureRemedy::RemoveTheStoppedRecord, + }); + continue; + } + } + // A confirmed stop already dropped this service's endpoint key, and it + // cannot be re-minted: say so on the failure rather than let the abort + // message read as if the survivor were untouched. + let key_dropped = report.stopped.contains(&record.service_id); + report + .stopped + .retain(|stopped| stopped != &record.service_id); + report + .relaunch + .retain(|line| !line.starts_with(&format!("{}: ", record.service_id))); + report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + // Two different situations reach here and they need different + // sentences. Only one of them involved a stop: the other is a + // record that was already marked stopped before this run, which + // this pass never attempted to stop, and telling its operator the + // endpoint survived "the stop" describes something that did not + // happen — on the one output they have to reason from. + reason: if stopped_by_this_pass { + format!( + "{}:{} still accepts connections after the stop{}", + record.host, + record.port, + if key_dropped { + "; the stop already dropped this service's endpoint key" + } else { + "" + } + ) + } else { + format!( + "{}:{} is recorded stopped, but something there is still serving this \ + record's own model", + record.host, record.port + ) + }, + // Not `StopTheService`: the recorded processes are gone, so + // `rocm services stop` has nothing left to kill and every retry + // would abort identically. + remedy: StopFailureRemedy::StopWhatHoldsThePort, + }); + } +} + +/// Whether two probe addresses can reach the same listener on one port. +/// +/// `localhost` names whichever loopback family the resolver puts first — `::1` +/// on many Windows hosts — so it can reach a listener on either, and is never +/// told apart from one. +pub(crate) fn hosts_name_one_listener(a: &str, b: &str) -> bool { + let is_loopback = |h: &str| matches!(h, "127.0.0.1" | "::1"); + a == b || (a == "localhost" && is_loopback(b)) || (b == "localhost" && is_loopback(a)) +} + +/// The addresses to probe for a service recorded on `host`, in order. +/// +/// A service bound to a wildcard address is reachable on loopback; connecting to +/// the wildcard itself is not portable. Which loopback, though, depends on what +/// the listener actually bound: `::` with the usual `IPV6_V6ONLY=1` (the default +/// on Windows) answers on `::1` and *refuses* `127.0.0.1`, while an `0.0.0.0` +/// bind is the mirror image. Probing one family only would read a live, +/// port-holding engine as "nothing is serving" and wave the removal through — +/// the exact defect this gate exists to close — so a wildcard yields both and +/// the caller blocks if either answers. +/// +/// The normalized form is what gets probed, not just what gets classified. +/// `loopback_tcp_port_is_reachable` resolves with `(host, port)`, which rejects +/// a bracketed literal like `[::1]` and anything with stray whitespace — and a +/// resolution failure reads as "nothing is serving", which is the wrong +/// direction for a check whose whole job is to catch a surviving engine +/// grandchild. Records do carry bracketed spellings (`loopback_host_key` +/// normalizes them too) because `--host` is free-form. +pub(crate) fn probe_hosts(host: &str) -> Vec { + // Trimmed and case-folded so the spellings a record can carry — `0.0.0.0`, + // `::`, `[::]`, `0:0:0:0:0:0:0:0`, `*`, or empty — all resolve to loopback + // rather than being probed literally (a literal wildcard connect is not + // portable, and would silently read as "nothing is serving"). + let normalized = host.trim().trim_matches(['[', ']']).to_ascii_lowercase(); + match normalized.as_str() { + "0.0.0.0" | "::" | "0:0:0:0:0:0:0:0" | "*" | "" => { + vec!["127.0.0.1".to_owned(), "::1".to_owned()] + } + _ => vec![normalized], + } +} + +/// What the gate does about a listener answering on an already-stopped record's +/// recorded port. +/// +/// Lifted out of the stop pass so each outcome can be asserted directly. Two of +/// these arms once went untested because reaching them meant standing up a +/// server that answers in a particular way and then reading stderr: mutating +/// either "proceed" arm into a block, or deleting a warning, left every test +/// green while changing what a destructive command does. +/// +/// Both halves of that are closed now, so the next reader should not infer a +/// gap from the paragraph above. `every_identity_answer_maps_to_exactly_one_gate_outcome` +/// pins the mapping here; `an_endpoint_listing_nothing_does_not_block_uninstall` +/// and `a_listener_naming_another_model_does_not_block_uninstall` drive the two +/// proceed-on-a-live-socket arms through the real call site; and the warnings +/// are values on [`ManagedServiceStopReport`] rather than `eprintln!`s, so the +/// disclosure is asserted rather than merely emitted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StoppedRecordVerdict { + /// Serving this record's own model: the engine outlived the supervisor whose + /// death marked the record stopped. + BlockServingOurModel, + /// Refused our credentials — a live server stating it guards this path. + BlockAuthRefused, + /// Named its models and ours was not among them. The only "not ours" that is + /// evidence of anything, so the only one that proceeds silently. + ProceedUnrelated, + /// Answered, but listed nothing. Not evidence of a stranger: an engine still + /// loading or mid-unload looks exactly like this while holding the port. + ProceedListingNothing, + /// Listening but unidentifiable — not an OpenAI endpoint, an unparseable + /// reply, or a server erroring on a rotated key. + ProceedUnidentified, +} + +impl StoppedRecordVerdict { + /// Whether this outcome stops the uninstall. + /// + /// Exhaustive on purpose, rather than `matches!` over the blocking pair. + /// The wildcard that `matches!` implies would default a newly added variant + /// to "proceed" — the direction that removes the tooling — and it would + /// compile, leaving the mistake to be caught by a test that can only + /// enumerate the variants that existed when it was written. Spelled out, + /// the compiler stops the next variant until somebody decides which side of + /// a destructive command it belongs on. + pub(crate) const fn blocks(self) -> bool { + match self { + Self::BlockServingOurModel | Self::BlockAuthRefused => true, + Self::ProceedUnrelated | Self::ProceedListingNothing | Self::ProceedUnidentified => { + false + } + } + } +} + +/// Decide [`StoppedRecordVerdict`] from what the identity probe answered. +/// +/// `identity` is `None` when the probe produced no usable model list at all; +/// `auth_refused` then says whether that was a 401/403 from a live server, which +/// is stronger evidence the port is held than a listing would be. +pub(crate) const fn stopped_record_verdict( + identity: Option, + auth_refused: bool, +) -> StoppedRecordVerdict { + match identity { + Some(rocm_core::EndpointIdentity::ServesExpectedModel) => { + StoppedRecordVerdict::BlockServingOurModel + } + Some(rocm_core::EndpointIdentity::ServesOtherModels) => { + StoppedRecordVerdict::ProceedUnrelated + } + Some(rocm_core::EndpointIdentity::ListsNoModels) => { + StoppedRecordVerdict::ProceedListingNothing + } + None if auth_refused => StoppedRecordVerdict::BlockAuthRefused, + None => StoppedRecordVerdict::ProceedUnidentified, + } +} + +/// Whether a record carrying no start-time cannot be proven ours. Both supported +/// platforms (Linux, Windows) can record one, so an absent value means the +/// record predates the field or its writer never stamped it; either way the pid +/// is unproven. +pub(crate) const fn record_predates_start_ticks(recorded_start_ticks: Option) -> bool { + recorded_start_ticks.is_none() +} + +/// Whether `endpoint_url` answered the identity probe with an auth refusal. +/// +/// A 401/403 is not a failure to reach the endpoint — it is a live HTTP server +/// stating that it guards this path, which is stronger evidence that the port is +/// still held than a model listing would be. It is also the shape the retry run +/// takes for a public service: stopping the recorded processes clears the stored +/// key, so the next `rocm uninstall` probes an authenticated survivor with no +/// credentials and gets exactly this. +/// +/// Anything else — unreachable, a timeout, a non-HTTP listener, a 200 whose body +/// did not parse — is not an answer this can act on, and stays with the caller's +/// fail-open. +pub(crate) fn endpoint_refused_authorization( + endpoint_url: &str, + endpoint_api_key: Option<&str>, +) -> bool { + matches!( + rocm_core::http_get_with_auth( + endpoint_url, + "/v1/models", + endpoint_api_key, + ENDPOINT_IDENTITY_PROBE_TIMEOUT, + ), + Ok(parts) if parts.status == 401 || parts.status == 403 + ) +} + +/// Service manifests that exist but cannot be parsed back into a record. +/// +/// [`load_managed_services`] skips these silently, which is right for listing — +/// one bad file should not break `rocm services list` — but wrong for uninstall: +/// a corrupt manifest can describe a live, GPU-holding server, and removing the +/// tooling while blind to it is the exact defect this gate closes. Returns the +/// file names so the abort message can point at what to inspect. +/// +/// Only `*.json` directly under the services directory is a manifest; engine +/// state files live under their own engine directory +/// ([`AppPaths::service_engine_state_path`]), so they are not misread as corrupt +/// records here. +pub(crate) fn unreadable_service_manifests(paths: &AppPaths) -> Result> { + let services_dir = paths.services_dir(); + if !services_dir.is_dir() { + return Ok(Vec::new()); + } + let mut unreadable = Vec::new(); + for entry in fs::read_dir(&services_dir) + .with_context(|| format!("failed to read {}", services_dir.display()))? + { + let path = entry?.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let bytes = + fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + if serde_json::from_slice::(&bytes).is_err() { + unreadable.push(path); + } + } + unreadable.sort(); + Ok(unreadable) +} + +/// Decide whether uninstall may proceed to remove files, given the outcome of +/// stopping managed services. +/// +/// Returns an optional line to print before removal proceeds, or an error that +/// aborts uninstall with nothing removed when a service could not be stopped — +/// so the binaries and service records needed to recover stay in place. Kept +/// separate from the removal it guards so the abort branch (the core safety +/// guarantee) is unit-testable without an unkillable process. +pub(crate) fn uninstall_removal_gate(report: &ManagedServiceStopReport) -> Result> { + if !report.failed.is_empty() { + let detail = report + .failed + .iter() + .map(|failure| format!("{} ({})", failure.service_id, failure.reason)) + .collect::>() + .join("; "); + // Say what the aborted run already did. The stop pass is not atomic: the + // pre-flight refuses before anything is stopped, but a stop that fails, + // a helper that will not die, or a survivor found after the stops all + // abort a run that has already stopped some services. Each stopped one + // is down for good, and stopping it dropped its endpoint key, which + // cannot be re-minted — a publicly bound service, or one launched with + // `--require-api-key`, must be served again to come back. "No files + // were removed" alone would read as "nothing happened". + let services_note = if report.stopped.is_empty() { + String::new() + } else { + format!( + " These services were stopped during this run and stay stopped: {}. Stopping \ + them dropped their endpoint keys, so a service that was publicly bound, or \ + launched with `--require-api-key`, has to be served again; the command for \ + each, with its own model and port filled in: {} {}.", + report.stopped.join(", "), + report.relaunch.join("; "), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE + ) + }; + let helper_note = if report.helper_stopped { + " The background helper (rocmd) was stopped and stays stopped: nothing supervises or \ + recovers managed services until it is started again." + } else if report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::StopTheDaemon) + { + " The background helper (rocmd) was signalled but not confirmed gone, so the servers \ + it supervised may already be down." + } else { + "" + }; + let already_stopped = if services_note.is_empty() && helper_note.is_empty() { + String::new() + } else { + format!(" No files were removed, but:{services_note}{helper_note}") + }; + // Advice per failure class. A record that will not parse cannot be + // stopped by `rocm services stop` — that command loads the same file and + // fails identically — so pointing at it would make every retry abort the + // same way, which is exactly the dead end this gate must not create. + // + // The classes walked here are derived from the failures themselves and + // ordered by `advice_rank`, so no variant can be dropped by forgetting + // to extend a list; both that and `advice` are exhaustive matches. + let mut classes = report + .failed + .iter() + .map(|failure| failure.remedy) + .collect::>(); + classes.sort_unstable_by_key(|remedy| remedy.advice_rank()); + classes.dedup(); + let remedies = classes + .into_iter() + .map(|remedy| { + let ids = report + .failed + .iter() + .filter(|failure| failure.remedy == remedy) + .map(|failure| failure.service_id.clone()) + .collect::>(); + remedy.advice(&ids) + }) + .collect::>(); + let may_be_serving = report.failed.iter().any(|failure| { + matches!( + failure.remedy, + StopFailureRemedy::StopTheService | StopFailureRemedy::StopWhatHoldsThePort + ) + }); + bail!( + "uninstall aborted: could not stop managed service(s): {detail}.{} {}{}", + if may_be_serving { + " Their endpoints may still be serving and holding the GPU." + } else { + "" + }, + remedies.join(" "), + if already_stopped.is_empty() { + " No files were removed." + } else { + &already_stopped + } + ); + } + if report.stopped.is_empty() && !report.helper_stopped { + return Ok(None); + } + let mut line = String::new(); + if !report.stopped.is_empty() { + line = format!( + "stopped {} managed service(s) before removal", + report.stopped.len() + ); + } + if report.helper_stopped { + if !line.is_empty() { + line.push_str("; "); + } + line.push_str("stopped the background helper (rocmd)"); + } + Ok(Some(line)) +} diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18..945a47e90 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -24,6 +24,3 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } - -[target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 7f1df1746..2ef06a821 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -329,15 +329,19 @@ pub(crate) fn ensure_public_service_has_endpoint_key( bail!( "refusing to recover a service that was launched with an endpoint API key but no \ longer has one: it would come back up without authentication, and something \ - outside this machine may still be publishing its port. Relaunch it with \ - `rocm serve --require-api-key` to issue a new key." + outside this machine may still be publishing its port. Relaunch it with `{}` {} to \ + issue a new key.", + rocm_engine_protocol::relaunch_command(host, true), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE ); } if rocm_engine_protocol::is_public_bind_host(host) && !key_present { bail!( "refusing to respawn a service bound to the public host `{host}` without an endpoint \ - API key: it would come back up without authentication. Relaunch it with \ - `rocm serve --host {host} --allow-public-bind` to issue a new key." + API key: it would come back up without authentication. Relaunch it with `{}` {} to \ + issue a new key.", + rocm_engine_protocol::relaunch_command(host, false), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE ); } Ok(()) @@ -454,9 +458,11 @@ mod tests { for host in ["127.0.0.1", "localhost", "::1"] { let error = ensure_public_service_has_endpoint_key(host, false, true) .expect_err("a service launched with a key must not be recovered without one"); + let rendered = format!("{error:#}"); + assert!(rendered.contains("without authentication"), "{rendered}"); assert!( - format!("{error:#}").contains("without authentication"), - "{error:#}" + rendered.contains(&format!("rocm serve --host {host} --port ")), + "the relaunch command must be one that can be run: {rendered}" ); } // With the key still present, recovery proceeds. @@ -471,9 +477,18 @@ mod tests { // the key gone the child would listen on that public host anonymously, // so the spawn must be refused instead. let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + let rendered = format!("{error:#}"); + assert!(rendered.contains("without authentication"), "{rendered}"); + // The command it names must be the public-bind one: `--allow-public-bind` + // and no `--require-api-key`, which this branch never asked for. assert!( - error.to_string().contains("without authentication"), - "{error:#}" + rendered + .contains("rocm serve --host 0.0.0.0 --port --allow-public-bind`"), + "{rendered}" + ); + assert!( + rendered.contains(rocm_engine_protocol::RELAUNCH_KEY_CHOICE), + "{rendered}" ); ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); diff --git a/apps/rocmd/src/persistence.rs b/apps/rocmd/src/persistence.rs index 847c92b7a..a793b19d5 100644 --- a/apps/rocmd/src/persistence.rs +++ b/apps/rocmd/src/persistence.rs @@ -91,6 +91,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index d43e206e0..da7b2c8ad 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -661,6 +661,14 @@ fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { Ok(parent.join(file_name)) } +/// Stage-and-publish a file here, sharing only the publish step with `rocm-core`. +/// +/// Deliberately not [`rocm_core::write_file_atomically`]: only the +/// Windows-sensitive publish (`ReplaceFileW` and its fallbacks) is +/// single-sourced. The staging half stays local because it carries the +/// `suffix_for_attempt` and `before_publish` seams the tests drive. This path +/// does not `sync_all` before publishing, so it is atomic but carries no +/// crash-durability guarantee for the staged bytes. fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); write_file_atomically_with( @@ -746,60 +754,10 @@ where Ok(()) } -#[cfg(not(windows))] +/// The publish step lives in `rocm-core` so there is one implementation of the +/// Windows `ReplaceFileW` handling for the whole workspace. fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } + rocm_core::publish_temp_file(tmp, path) } pub(crate) fn sandbox_check_updates_value(output: CommandCapture) -> Value { diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 1a8c3335e..9efde14af 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -882,6 +882,9 @@ fn build_runtime_state( running: automations_enabled, automations_enabled, daemon_pid: std::process::id(), + // Captured while this process is alive so a later `rocm uninstall` can + // tell this daemon from an unrelated process that inherited the PID. + daemon_start_ticks: rocm_core::ProcessIdentity::capture(std::process::id()).start_ticks, started_at_unix_ms: now, last_tick_unix_ms: now, local_webhook_endpoint: None, @@ -1009,6 +1012,23 @@ fn wait_for_service_ready( #[cfg(test)] mod tests { use super::*; + + #[cfg(any(target_os = "linux", windows))] + #[test] + fn a_fresh_runtime_state_records_this_daemons_start_time() { + // `rocm uninstall` refuses to signal a daemon pid it cannot prove is + // rocmd, and the proof is this field: `None` here would make every + // record look pre-upgrade and abort every uninstall that finds a daemon. + let state = build_runtime_state(&RocmCliConfig::default(), true); + assert_eq!(state.daemon_pid, std::process::id()); + let recorded = state + .daemon_start_ticks + .expect("a live daemon must record its own start-time"); + assert_eq!( + Some(recorded), + rocm_core::process_start_ticks(std::process::id()) + ); + } use crate::test_support::identity_probe_record; use crate::test_support::{ UNCONFIRMED_STOP_PID, seed_keyed_service, temp_app_paths, unique_test_root, diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 79efcb269..06a331025 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1700,6 +1700,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -2882,6 +2883,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -2915,6 +2917,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -2981,6 +2984,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -3070,6 +3074,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -3125,6 +3130,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 790d4da4e..411937844 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -5,8 +5,9 @@ use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::ffi::OsString; +use std::ffi::{OsStr, OsString}; use std::fs; +use std::io::Write; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -103,21 +104,22 @@ pub use model_catalog::{ }; pub use net::{ Backoff, DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, - DownloadRequest, EndpointReadiness, EndpointReadinessOutcome, HttpResponseParts, - INFERENCE_PROBE_ATTEMPTED_STATE_KEY, INFERENCE_PROBE_RETRY_INTERVAL, INFERENCE_PROBE_TIMEOUT, - INFERENCE_VERIFIED_STATE_KEY, connect_tcp_stream, download_file_streaming, - download_file_streaming_with_progress, download_file_to_path, + DownloadRequest, EndpointIdentity, EndpointReadiness, EndpointReadinessOutcome, + HttpResponseParts, INFERENCE_PROBE_ATTEMPTED_STATE_KEY, INFERENCE_PROBE_RETRY_INTERVAL, + INFERENCE_PROBE_TIMEOUT, INFERENCE_VERIFIED_STATE_KEY, connect_tcp_stream, + download_file_streaming, download_file_streaming_with_progress, download_file_to_path, download_file_to_path_with_progress, engine_state_inference_verified, format_host_for_url, format_host_port, format_http_base_url, http_get_text, http_get_text_with_auth, - http_post_json_with_auth, managed_service_endpoint_model_ready, - managed_service_endpoint_readiness, merge_json_state_file, openai_chat_completion_probe, - openai_chat_completion_status, openai_models_endpoint_has_model, parse_http_endpoint, + http_get_with_auth, http_post_json_with_auth, managed_service_endpoint_identity, + managed_service_endpoint_model_ready, managed_service_endpoint_readiness, + merge_json_state_file, openai_chat_completion_probe, openai_chat_completion_status, + openai_models_endpoint_has_model, openai_models_endpoint_identity, parse_http_endpoint, read_http_response_bounded, write_all_tcp_stream, }; pub use paths::{AppPaths, engine_plugin_dirs}; pub use proc_lifecycle::{ IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, - process_start_ticks, terminate_verified, + identity_state_with_observed, process_start_ticks, terminate_verified, }; pub use process::{ FileLock, detach_command_session, process_is_running, terminate_process, terminate_process_tree, @@ -991,6 +993,28 @@ pub struct AutomationRuntimeState { pub running: bool, pub automations_enabled: bool, pub daemon_pid: u32, + /// The daemon's kernel start-time, captured at spawn, so `daemon_pid` can be + /// checked for PID recycling before anything signals it. + /// + /// `daemon_pid` alone is not safe to kill: this file survives a crash, + /// OOM-kill or reboot with `running` still true, after which the OS can + /// reissue that PID to an unrelated process. Pairing the PID with its + /// start-time makes a recycled PID detectable ([`identity_state`]). + /// + /// `None` on two very different occasions, which callers must not conflate: + /// a state file written before this field existed, and a platform where + /// [`process_start_ticks`] can never return anything. + /// + /// Conflating them is a live hazard, because [`identity_state`] maps a `None` + /// recorded value to [`IdentityState::Matches`] — the legacy best-effort arm + /// — so **no recycling check happens at all** and a caller that signals on + /// that verdict is killing on a bare pid. Both supported platforms (Linux + /// `/proc`, Windows `GetProcessTimes`) can record a start-time, so there an + /// absent value means the record predates the field or its writer never + /// stamped it, and the pid is unverifiable. Only an unsupported platform + /// (neither source, i.e. macOS) can never produce one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub daemon_start_ticks: Option, pub started_at_unix_ms: u128, pub last_tick_unix_ms: u128, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1012,15 +1036,21 @@ impl AutomationRuntimeState { Ok(Some(state)) } + /// Publish the runtime state atomically. + /// + /// Written on every daemon tick, and read by `rocm uninstall` to decide + /// whether a background helper is still up. A plain `fs::write` leaves a + /// truncated file if the daemon dies mid-write, and uninstall has to treat an + /// unreadable state file as "cannot confirm the helper is stopped" — so a + /// crash at the wrong moment would block uninstall until the operator + /// repaired the file by hand. pub fn write(&self, paths: &AppPaths) -> Result<()> { paths.ensure()?; let path = paths.automation_state_path(); - fs::write( - &path, - serde_json::to_vec_pretty(self) - .context("failed to serialize automation runtime state")?, - ) - .with_context(|| format!("failed to write {}", path.display()))?; + let bytes = serde_json::to_vec_pretty(self) + .context("failed to serialize automation runtime state")?; + write_file_atomically(&path, &bytes) + .with_context(|| format!("failed to write {}", path.display()))?; Ok(()) } @@ -1673,23 +1703,25 @@ impl ManagedServiceRecord { record } + /// Persist the record, atomically. + /// + /// Staged next to the manifest and published over it, so a concurrent reader + /// sees either the old record or the new one, never a half-written file. A + /// plain overwrite left a window in which a reader (`load_managed_services`, + /// or the uninstall gate that treats an unparseable manifest as a live server + /// it cannot account for) could observe a truncated record and act on it. + /// + /// Uses the workspace's one [`write_file_atomically`] rather than a local + /// rename: this runs on every status transition, including the ones the + /// uninstall stop pass performs, where a failed publish aborts the uninstall. pub fn write(&self) -> Result<()> { let mut host_record = self.clone(); host_record.normalize_paths_for_host(); - let parent = host_record - .manifest_path - .parent() - .context("service manifest path must have a parent directory")?; - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; let storage_record = host_record.with_storage_paths(); - fs::write( - &host_record.manifest_path, - serde_json::to_vec_pretty(&storage_record) - .context("failed to serialize service record")?, - ) - .with_context(|| format!("failed to write {}", host_record.manifest_path.display()))?; - Ok(()) + let bytes = serde_json::to_vec_pretty(&storage_record) + .context("failed to serialize service record")?; + write_file_atomically(&host_record.manifest_path, &bytes) + .with_context(|| format!("failed to write {}", host_record.manifest_path.display())) } } @@ -1982,6 +2014,186 @@ pub fn unix_time_millis() -> u128 { .as_millis() } +/// How many temp names to try before giving up on staging an atomic write. +const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; + +/// A unique temp path next to `path`, preserving the full file name so a +/// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes +/// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). +fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { + let parent = path.parent().context("file path has no parent directory")?; + let mut file_name = path + .file_name() + .context("file path has no file name")? + .to_os_string(); + file_name.push(".tmp-"); + file_name.push(suffix); + Ok(parent.join(file_name)) +} + +/// Move a staged temp file onto `path`, replacing whatever is there. +/// +/// The single implementation of the publish step for the whole workspace. It +/// lives here rather than in each caller because the Windows half is not +/// something to re-derive: a plain `rename` over an existing file is not +/// reliable while another process holds the destination open (an antivirus or +/// indexer scanning a file this CLI just wrote is enough), so an existing +/// destination goes through `ReplaceFileW`. Callers that got this wrong would +/// fail in the direction that matters — `ManagedServiceRecord::write` is on the +/// uninstall stop path, where a failed publish is reported as a service that +/// could not be stopped and aborts the whole uninstall. +#[cfg(not(windows))] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> std::io::Result<()> { + fs::rename(tmp, path) +} + +/// Move a staged temp file onto `path`, replacing whatever is there. +/// +/// See the non-Windows twin for why this is centralized. +#[cfg(windows)] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> std::io::Result<()> { + if path.try_exists()? { + return replace_file_windows(path, tmp); + } + + match fs::rename(tmp, path) { + Ok(()) => Ok(()), + Err(rename_error) => { + // The destination can appear between the check and the rename. + if path.try_exists()? { + replace_file_windows(path, tmp) + } else { + Err(rename_error) + } + } + } +} + +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn replace_file_windows(path: &Path, replacement: &Path) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; + + let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let replacement_wide: Vec = replacement + .as_os_str() + .encode_wide() + .chain(Some(0)) + .collect(); + + // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and + // remain alive for the duration of the synchronous Windows API call. The + // optional backup, exclude, and reserved pointers are intentionally null. + let replaced = unsafe { + ReplaceFileW( + path_wide.as_ptr(), + replacement_wide.as_ptr(), + std::ptr::null(), + 0, + std::ptr::null(), + std::ptr::null(), + ) + }; + if replaced == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +/// Write `bytes` to `path` so a concurrent reader sees either the old contents +/// or the new ones, never a half-written file. +/// +/// Staged under a unique sibling name reserved with `create_new` — so two +/// writers cannot pick the same scratch file — and published with +/// [`publish_temp_file`]. A failed publish takes the scratch file with it. +/// +/// Three limits worth knowing before reaching for this: +/// +/// * **Atomic, not durable.** The staged bytes are `sync_all`ed before the +/// publish, so a crash cannot leave a half-written or zero-length file. The +/// containing *directory* is never fsynced, so the rename itself can still be +/// lost by a power failure — the guarantee is that readers only ever see one +/// complete version, not that the newest one survives a crash. That flush is +/// held by construction, not by test: deleting it leaves every test green, +/// because reproducing what it prevents needs a real crash between the write +/// and the rename. Do not read a green suite as evidence it is still there. +/// * **Permissions are not carried over.** This replaces the target inode with +/// a freshly created file, so an existing file's mode does not survive (a +/// difference from the `fs::write` it usually replaces), and the new file gets +/// whatever the umask allows. What makes that acceptable today is only that +/// every current caller writes non-sensitive state — *not* the containing +/// directory, which `AppPaths::ensure` creates with plain `create_dir_all` and +/// no mode of its own. A caller with something sensitive to write must set the +/// mode explicitly here; it cannot lean on the directory. +/// * **A symlink at `path` is replaced, not written through.** The publish is a +/// rename onto `path` itself, so a symlink there is what gets replaced — where +/// `fs::write` would have followed it and overwritten the target. That is the +/// safer direction for state files (it cannot be aimed somewhere else by +/// planting a link) but it is a behaviour change, so a caller that means to +/// write through a link must resolve it first. +pub fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { + let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); + write_file_atomically_with_temp_id(path, bytes, &temp_id) +} + +/// [`write_file_atomically`] with the scratch-file id supplied, so a test can +/// plant files under the names the attempts will try and drive the +/// already-exists retry and the exhausted-attempts failure deterministically. +fn write_file_atomically_with_temp_id(path: &Path, bytes: &[u8], temp_id: &str) -> Result<()> { + let parent = path.parent().context("file path has no parent directory")?; + fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; + + let mut reserved = None; + for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { + let tmp = temp_sibling_path(path, &OsString::from(format!("{temp_id}-{attempt}")))?; + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + { + Ok(file) => { + reserved = Some((tmp, file)); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| format!("failed to create {}", tmp.display())); + } + } + } + let Some((tmp, mut file)) = reserved else { + bail!( + "failed to reserve a temporary file next to {} after {ATOMIC_WRITE_TEMP_ATTEMPTS} \ + attempts", + path.display() + ); + }; + + if let Err(error) = file.write_all(bytes) { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(disk_space::map_write_error(error, &tmp)); + } + // Flush before publishing. Without this the rename can reach the disk while + // the bytes have not, leaving a zero-length file after a crash — which for a + // service record is not a lost update but an unparseable manifest, and the + // uninstall gate refuses to remove anything while one of those exists. + if let Err(error) = file.sync_all() { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(disk_space::map_write_error(error, &tmp)); + } + drop(file); + + publish_temp_file(&tmp, path) + .inspect_err(|_| { + let _ = fs::remove_file(&tmp); + }) + .with_context(|| format!("failed to publish {}", path.display())) +} + #[cfg(test)] mod tests { use super::*; @@ -1991,6 +2203,385 @@ mod tests { use std::path::Path; use std::path::PathBuf; + /// A private root for one test, removed on the way out. + fn atomic_write_root(name: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!( + "rocm-core-atomic-write-{name}-{}-{}", + std::process::id(), + unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("failed to create the test root"); + root + } + + /// The scratch files `write_file_atomically` stages under, if any survived. + /// + /// `temp_sibling_path` names them `.tmp---`, so + /// a leftover is visible as a sibling containing `.tmp-`. + fn leftover_scratch_files(dir: &Path) -> Vec { + fs::read_dir(dir) + .expect("failed to read the test root") + .filter_map(|entry| { + let name = entry.ok()?.file_name().to_string_lossy().into_owned(); + name.contains(".tmp-").then_some(name) + }) + .collect() + } + + #[test] + fn a_taken_scratch_name_is_skipped_and_the_write_still_lands() { + // Another writer already holds attempt 0's name: the write must move on to + // the next attempt rather than fail or clobber the file it does not own. + let root = atomic_write_root("scratch-name-taken"); + let target = root.join("record.json"); + let taken = root.join("record.json.tmp-fixed-0"); + fs::write(&taken, b"someone else's scratch").expect("plant the taken name"); + + write_file_atomically_with_temp_id(&target, b"mine", "fixed").expect("write"); + + assert_eq!(fs::read(&target).expect("read"), b"mine"); + assert_eq!( + fs::read(&taken).expect("read"), + b"someone else's scratch", + "a scratch file that is not ours is left untouched" + ); + assert_eq!( + leftover_scratch_files(&root), + vec!["record.json.tmp-fixed-0".to_owned()], + "our own scratch file is gone once published" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn every_scratch_name_taken_is_an_error_not_a_clobber() { + let root = atomic_write_root("scratch-names-exhausted"); + let target = root.join("record.json"); + fs::write(&target, b"original").expect("seed the target"); + for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { + fs::write(root.join(format!("record.json.tmp-fixed-{attempt}")), b"x") + .expect("plant a taken name"); + } + + let error = write_file_atomically_with_temp_id(&target, b"new", "fixed") + .expect_err("no scratch name is free"); + + assert!( + format!("{error:#}").contains("failed to reserve a temporary file"), + "{error:#}" + ); + assert_eq!( + fs::read(&target).expect("read"), + b"original", + "a failed reservation leaves the target as it was" + ); + let _ = fs::remove_dir_all(root); + } + + /// Linux/unix: relies on a rename replacing the directory entry while an + /// already-open descriptor keeps reading the previous inode. Windows + /// publishes through `ReplaceFileW` and does not offer the same observation. + #[cfg(unix)] + #[test] + fn a_service_record_write_never_shows_a_reader_a_truncated_manifest() { + // `ManagedServiceRecord::write` runs on the uninstall stop path, where a + // torn manifest is not a lost update but an unparseable record that makes + // the gate refuse to remove anything. + // + // Asserting "the bytes landed" would not pin that: a plain `fs::write` + // also leaves correct final bytes and creates no scratch file, so such a + // test passes either way. The property only the atomic path has is what a + // *concurrent reader* sees — `fs::write` truncates the existing inode in + // place, so a reader holding it open watches the record disappear and + // come back, while the publish swaps in a new inode and leaves the old + // one whole. Hold the manifest open across a rewrite and require the old + // view to still be a complete, parseable record. + use std::io::{Read, Seek, SeekFrom}; + + let root = atomic_write_root("service-record-reader-safety"); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + paths.ensure().expect("create the app directories"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-reader-safety", + "vllm", + "amd/model", + "amd/model", + "127.0.0.1", + 8000, + "managed", + 0, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.write().expect("seed the manifest"); + + // A reader that opened the manifest before the rewrite began. + let mut reader = fs::File::open(&record.manifest_path).expect("open the manifest"); + + record.status = "stopped".to_owned(); + record.write().expect("rewrite the manifest"); + + reader.seek(SeekFrom::Start(0)).expect("rewind"); + let mut seen = Vec::new(); + reader + .read_to_end(&mut seen) + .expect("read the open manifest"); + let parsed: ManagedServiceRecord = serde_json::from_slice(&seen) + .expect("a reader holding the manifest open must never see a truncated document"); + assert_eq!( + parsed.status, "ready", + "the open descriptor must still see the pre-write record, not a rewritten inode" + ); + + // And the published file is the new one. + let published: ManagedServiceRecord = + serde_json::from_slice(&fs::read(&record.manifest_path).expect("read back")) + .expect("the published manifest must parse"); + assert_eq!(published.status, "stopped", "the rewrite must have landed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_runtime_state_write_never_shows_a_reader_a_truncated_file() { + // The uninstall gate reads this file to decide whether a daemon is live; a + // torn write makes it unparseable and aborts the run. Same property as + // the service record: a reader holding the old inode must keep seeing a + // whole document across a rewrite. + use std::io::{Read, Seek, SeekFrom}; + + let root = atomic_write_root("runtime-state-reader-safety"); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + paths.ensure().expect("create the app directories"); + let mut state = AutomationRuntimeState { + running: true, + automations_enabled: true, + daemon_pid: 1, + daemon_start_ticks: None, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers: Vec::new(), + }; + state.write(&paths).expect("seed the state file"); + let mut reader = + fs::File::open(paths.automation_state_path()).expect("open the state file"); + + state.running = false; + state.write(&paths).expect("rewrite the state file"); + + reader.seek(SeekFrom::Start(0)).expect("rewind"); + let mut seen = Vec::new(); + reader.read_to_end(&mut seen).expect("read the open file"); + let parsed: AutomationRuntimeState = serde_json::from_slice(&seen) + .expect("a reader holding the file open must never see a truncated document"); + assert!( + parsed.running, + "the open descriptor must see the pre-write state" + ); + let published = AutomationRuntimeState::load(&paths) + .expect("load") + .expect("present"); + assert!(!published.running, "the rewrite must have landed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_service_record_write_leaves_no_scratch_sibling() { + // This pins scratch-file hygiene, NOT atomic publishing: a plain + // `fs::write` creates no scratch file at all and would pass too. The + // atomicity property is pinned next door, by + // `a_service_record_write_never_shows_a_reader_a_truncated_manifest`, + // which fails against an in-place write. + // + // It still earns its place, but for a smaller reason than uninstall: + // scratch siblings are named `.json.tmp---`, + // whose extension is `tmp-…`, and both `unreadable_service_manifests` + // and the record loader filter on `extension == "json"` — so a leaked + // one is skipped, not mistaken for a corrupt record, and cannot abort + // uninstall. What this guards is directory hygiene: a publish that + // aborts between staging and rename leaves litter that accumulates + // silently, one file per failed write, in a directory an operator reads + // by hand. + let root = atomic_write_root("service-record-publish"); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + paths.ensure().expect("create the app directories"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-atomic-publish", + "vllm", + "amd/model", + "amd/model", + "127.0.0.1", + 8000, + "managed", + 0, + None, + None, + None, + ); + record.write().expect("first publish"); + record.status = "stopped".to_owned(); + record + .write() + .expect("republish over the existing manifest"); + + let services_dir = paths.services_dir(); + let stray = fs::read_dir(&services_dir) + .expect("read the services directory") + .filter_map(|entry| { + let name = entry.ok()?.file_name().to_string_lossy().into_owned(); + name.contains(".tmp-").then_some(name) + }) + .collect::>(); + assert!( + stray.is_empty(), + "publishing a record must leave no scratch manifest: {stray:?}" + ); + let written = fs::read(&record.manifest_path).expect("read the manifest back"); + let parsed: ManagedServiceRecord = + serde_json::from_slice(&written).expect("the published manifest must always parse"); + assert_eq!(parsed.status, "stopped", "the republish must have landed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn write_file_atomically_replaces_contents_and_leaves_no_scratch_file() { + // The publish must land the new bytes and take its staging file with it; + // a leaked scratch sibling in `services_dir` would be a file the uninstall + // gate has to reason about. + let root = atomic_write_root("replaces"); + let target = root.join("record.json"); + fs::write(&target, b"old contents").expect("seed the file"); + + write_file_atomically(&target, b"new contents").expect("the write must succeed"); + + assert_eq!( + fs::read(&target).expect("read back"), + b"new contents", + "the published file must hold the new bytes" + ); + assert!( + leftover_scratch_files(&root).is_empty(), + "a successful publish must leave no scratch file: {:?}", + leftover_scratch_files(&root) + ); + let _ = fs::remove_dir_all(root); + } + + /// Unix: needs a real symlink. + #[cfg(unix)] + #[test] + fn write_file_atomically_replaces_a_symlink_instead_of_writing_through_it() { + // Documented behaviour, and a change from the `fs::write` this replaced: + // that followed a symlink and overwrote whatever it pointed at, while + // the publish here renames onto the path itself. Security-relevant in + // the safe direction — planting a link next to a state file can no + // longer redirect the write somewhere else — but a change, so it is + // pinned rather than left as prose. + let root = atomic_write_root("symlink-target"); + let elsewhere = root.join("elsewhere"); + fs::write(&elsewhere, b"must not be touched").expect("seed the link target"); + let target = root.join("state.json"); + std::os::unix::fs::symlink(&elsewhere, &target).expect("plant a symlink at the target"); + + write_file_atomically(&target, b"published").expect("the write must succeed"); + + assert_eq!( + fs::read(&elsewhere).expect("read the link target"), + b"must not be touched", + "the write must not follow the symlink to its target" + ); + assert_eq!( + fs::read(&target).expect("read the published file"), + b"published", + "the published bytes must land at the path itself" + ); + assert!( + !fs::symlink_metadata(&target) + .expect("stat the published path") + .file_type() + .is_symlink(), + "the symlink must have been replaced by a regular file" + ); + assert!( + leftover_scratch_files(&root).is_empty(), + "no scratch file may survive a successful publish" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn write_file_atomically_creates_the_parent_directory() { + // `ManagedServiceRecord::write` targets a services dir that may not exist + // yet on a first write. + let root = atomic_write_root("creates-parent"); + let target = root.join("nested").join("deeper").join("record.json"); + + write_file_atomically(&target, b"contents").expect("the write must succeed"); + + assert_eq!(fs::read(&target).expect("read back"), b"contents"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn write_file_atomically_cleans_up_when_publishing_fails() { + // A failed publish must not strand its staging file. The destination is a + // non-empty directory, which no platform will let a file replace, so the + // failure happens at the publish step with the scratch file already + // written — exactly the path that has to clean up after itself. + let root = atomic_write_root("publish-fails"); + let target = root.join("record.json"); + fs::create_dir_all(&target).expect("seed a directory where the file goes"); + fs::write(target.join("occupant"), b"x").expect("make it non-empty"); + + let error = write_file_atomically(&target, b"contents") + .expect_err("replacing a non-empty directory must fail"); + + assert!( + leftover_scratch_files(&root).is_empty(), + "a failed publish must remove its scratch file, got {:?} ({error:#})", + leftover_scratch_files(&root) + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn write_file_atomically_refuses_a_path_with_no_parent_or_an_unusable_parent() { + let error = write_file_atomically(Path::new("/"), b"x") + .expect_err("a root path has no directory to stage in"); + assert!(format!("{error:#}").contains("no parent"), "{error:#}"); + + // A regular file where the parent directory should be: the directory + // cannot be created, and nothing may be left behind. + let root = atomic_write_root("parent-is-a-file"); + let blocker = root.join("blocker"); + fs::write(&blocker, b"x").expect("seed the blocking file"); + let error = write_file_atomically(&blocker.join("record.json"), b"x") + .expect_err("a file cannot be a parent directory"); + assert!( + format!("{error:#}").contains("failed to create"), + "{error:#}" + ); + assert!(leftover_scratch_files(&root).is_empty()); + let _ = fs::remove_dir_all(root); + } + #[test] fn service_id_accepts_generated_and_plain_ids() { // A freshly generated id must always validate. diff --git a/crates/rocm-core/src/net.rs b/crates/rocm-core/src/net.rs index 03b6616cb..27a3bbe89 100644 --- a/crates/rocm-core/src/net.rs +++ b/crates/rocm-core/src/net.rs @@ -521,6 +521,27 @@ pub fn http_get_text_with_auth( endpoint_api_key: Option<&str>, timeout: Duration, ) -> Result { + let parts = http_get_with_auth(endpoint_url, path, endpoint_api_key, timeout)?; + if parts.status != 200 { + bail!("HTTP endpoint returned HTTP {}", parts.status); + } + Ok(parts.body) +} + +/// GET `path` and return the response status alongside the body. +/// +/// The GET sibling of [`http_post_json_with_auth`], and it exists for the same +/// reason: a caller probing an endpoint has to tell "the server answered, with a +/// refusal" apart from "the server never answered", and only the former proves +/// something is alive on that port. [`http_get_text_with_auth`] collapses both +/// into `Err` — correct for callers that only want a 200's body, wrong for +/// anything deciding whether a port is still held. +pub fn http_get_with_auth( + endpoint_url: &str, + path: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { let deadline = Instant::now() + timeout; let (host, port) = parse_http_endpoint(endpoint_url) .with_context(|| format!("unsupported endpoint URL `{endpoint_url}`"))?; @@ -541,10 +562,12 @@ pub fn http_get_text_with_auth( .split_once("\r\n\r\n") .context("HTTP response was missing a body")?; let status_line = headers.lines().next().unwrap_or_default(); - if !status_line.contains(" 200 ") { - bail!("HTTP endpoint returned {status_line}"); - } - Ok(body.to_owned()) + let status = http_status_code(status_line) + .with_context(|| format!("unparsable HTTP status line `{status_line}`"))?; + Ok(HttpResponseParts { + status, + body: body.to_owned(), + }) } /// POST a JSON body and return the response status line plus body. @@ -777,19 +800,59 @@ pub fn openai_models_endpoint_has_model( endpoint_api_key: Option<&str>, timeout: Duration, ) -> Result { + Ok( + openai_models_endpoint_identity(endpoint_url, expected_model, endpoint_api_key, timeout)? + == EndpointIdentity::ServesExpectedModel, + ) +} + +/// What a listener said when asked which models it serves. +/// +/// The distinction [`openai_models_endpoint_has_model`] cannot draw: it answers +/// `false` both for "this is someone else's service" and for "this endpoint +/// answered, but listed nothing". For a readiness poll those are the same — not +/// ready either way — but for a caller deciding whether something still holds a +/// port they are opposites. An engine that is up but has not populated its model +/// list yet, or is mid-unload, lists nothing while very much holding the GPU. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum EndpointIdentity { + /// Listed a model matching the one asked about (or listed models when no + /// particular one was asked for). + ServesExpectedModel, + /// Listed models, none of which match — a different service on this port. + ServesOtherModels, + /// Answered `/v1/models` with an empty list. Says nothing either way about + /// whose service this is. + ListsNoModels, +} + +/// Ask a listener which models it serves, keeping "listed nothing" distinct. +pub fn openai_models_endpoint_identity( + endpoint_url: &str, + expected_model: Option<&str>, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { let body = http_get_text_with_auth(endpoint_url, "/v1/models", endpoint_api_key, timeout)?; let value = serde_json::from_str::(body.trim()) .context("failed to parse /v1/models JSON")?; let loaded_models = openai_loaded_model_ids(&value); if loaded_models.is_empty() { - return Ok(false); + return Ok(EndpointIdentity::ListsNoModels); } let Some(expected_model) = expected_model.filter(|value| !value.trim().is_empty()) else { - return Ok(true); + return Ok(EndpointIdentity::ServesExpectedModel); }; - Ok(loaded_models - .iter() - .any(|loaded| model_refs_match(loaded, expected_model))) + Ok( + if loaded_models + .iter() + .any(|loaded| model_refs_match(loaded, expected_model)) + { + EndpointIdentity::ServesExpectedModel + } else { + EndpointIdentity::ServesOtherModels + }, + ) } pub fn managed_service_endpoint_model_ready( @@ -810,6 +873,29 @@ pub fn managed_service_endpoint_model_ready( openai_models_endpoint_has_model(&record.endpoint_url, expected, endpoint_api_key, timeout) } +/// [`managed_service_endpoint_model_ready`] without collapsing "listed nothing" +/// into "not ours". +/// +/// An empty `endpoint_url` is the one case with no listener to ask about at all, +/// so it stays an error rather than being reported as an identity. +pub fn managed_service_endpoint_identity( + record: &ManagedServiceRecord, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + if record.endpoint_url.trim().is_empty() { + bail!("service record has no endpoint URL to identify"); + } + let expected = if !record.canonical_model_id.trim().is_empty() { + Some(record.canonical_model_id.as_str()) + } else if !record.model_ref.trim().is_empty() { + Some(record.model_ref.as_str()) + } else { + None + }; + openai_models_endpoint_identity(&record.endpoint_url, expected, endpoint_api_key, timeout) +} + /// How far along a managed service's endpoint is. /// /// The middle state is the one that matters: an engine lists a model within @@ -1185,6 +1271,145 @@ mod tests { server.join().expect("server thread should not panic")?; Ok(()) } + #[test] + fn an_empty_model_list_is_not_reported_as_somebody_elses_service() -> Result<()> { + // `openai_models_endpoint_has_model` answers `false` both for "these are + // someone else's models" and for "this endpoint listed nothing". For a + // readiness poll that is fine — not ready either way. For a caller + // deciding whether something still holds a port they are opposites: an + // engine that is up but has not populated its list yet, or is + // mid-unload, lists nothing while holding the port and the GPU, and + // reading that as "not ours" removes the tooling out from under it. + // + // Four connections: an empty list and a stranger's list, asked twice — + // once through the enum, once through the bool wrapper. The wrapper is + // where the three-way answer collapses back to the readiness contract + // every existing caller depends on, and that single comparison is not + // pinned by asserting on the enum alone. + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let server = std::thread::spawn(move || -> Result<()> { + let empty = r#"{"data":[]}"#; + let stranger = r#"{"data":[{"id":"someone/else"}]}"#; + for body in [empty, stranger, empty, stranger] { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 512]; + let _ = stream.read(&mut buffer)?; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + } + Ok(()) + }); + let endpoint = format!("http://127.0.0.1:{port}/v1"); + + assert_eq!( + openai_models_endpoint_identity( + &endpoint, + Some("amd/ours"), + None, + Duration::from_secs(2) + )?, + EndpointIdentity::ListsNoModels, + "an empty list says nothing about whose service this is" + ); + assert_eq!( + openai_models_endpoint_identity( + &endpoint, + Some("amd/ours"), + None, + Duration::from_secs(2) + )?, + EndpointIdentity::ServesOtherModels, + "a list that names other models is the only real evidence of a stranger" + ); + + // Across the wrapper. Only `ServesExpectedModel` may read as ready: + // mapping either of the other two arms to `true` would tell every + // readiness caller that an endpoint listing nothing, or listing a + // stranger's model, is serving theirs. + assert!( + !openai_models_endpoint_has_model( + &endpoint, + Some("amd/ours"), + None, + Duration::from_secs(2) + )?, + "an endpoint listing nothing is not serving our model" + ); + assert!( + !openai_models_endpoint_has_model( + &endpoint, + Some("amd/ours"), + None, + Duration::from_secs(2) + )?, + "an endpoint serving a stranger's model is not serving ours" + ); + + server.join().expect("server thread should not panic")?; + Ok(()) + } + + #[test] + fn a_get_reports_a_refusal_as_an_answer_not_as_a_failure() -> Result<()> { + // The distinction `http_get_text_with_auth` cannot make. Its `Err` means + // "no 200", which lumps a 401 from a live server together with a dead + // port — and a caller deciding whether something still holds a port has + // to tell those apart: only the refusal proves the port is held. + // + // Two connections: the same 401 read both ways. + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let server = std::thread::spawn(move || -> Result<()> { + for _ in 0..2 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 512]; + let _ = stream.read(&mut buffer)?; + write!( + stream, + "HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + } + Ok(()) + }); + let endpoint = format!("http://127.0.0.1:{port}/v1"); + + let parts = http_get_with_auth(&endpoint, "/models", None, Duration::from_secs(2))?; + assert_eq!( + parts.status, 401, + "a refusal is an answer, and its status has to survive" + ); + assert!( + http_get_text_with_auth(&endpoint, "/models", None, Duration::from_secs(2)).is_err(), + "the text helper still collapses every non-200 into an error" + ); + + server.join().expect("server thread should not panic")?; + + // And an unreachable port is an error either way — the two must not + // collapse in the other direction. + let dead = std::net::TcpListener::bind("127.0.0.1:0")?; + let dead_port = dead.local_addr()?.port(); + drop(dead); + assert!( + http_get_with_auth( + &format!("http://127.0.0.1:{dead_port}/v1"), + "/models", + None, + Duration::from_millis(500), + ) + .is_err(), + "nothing answered, so there is no status to report" + ); + Ok(()) + } + // Why readiness is gated on an inference probe (EAI-7333): a server that // lists the model on `/v1/models` but cannot yet serve // `/v1/chat/completions` still reports the model as present. This test pins diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 57da4bb1f..c5402fc29 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -12,6 +12,22 @@ //! truthfully: a stop is only "graceful" once the recorded process is observed to //! have actually exited within a bounded grace period, escalating to `SIGKILL` //! only when the caller opts into a forced stop. +//! +//! **The recycling defence needs a readable start-time.** [`process_start_ticks`] +//! reads it from `/proc` on Linux and from the process creation time +//! (`GetProcessTimes`) on Windows, and is a compile-time `None` everywhere +//! else, so on other platforms (macOS) every record *captured there* carries no +//! identity, and [`identity_state`] degrades to best-effort +//! [`IdentityState::Matches`] — the paragraph above describes what this module +//! enforces *where the platform can answer*. (A record reconstructed with a +//! start-time recorded elsewhere is the other case: unconfirmable rather than +//! absent, so it lands on [`IdentityState::Indeterminate`] and is not signalled +//! either.) The degradation is deliberate: a +//! host that cannot tell two processes apart must not therefore refuse to stop +//! anything, since that would make every service unstoppable rather than making +//! any of them safer. What holds on those hosts is the rest of the caller's +//! gate — the port reality-check, the endpoint identity probe, and aborting +//! with the recovery tooling intact on any unconfirmed stop. use std::time::{Duration, Instant}; @@ -31,8 +47,9 @@ pub enum KillScope { /// A spawned process identified in a way that is robust to PID recycling. /// -/// `start_ticks` is the kernel start-time of the process (clock ticks since -/// boot). It is `None` on platforms without `/proc`, where identity cannot be +/// `start_ticks` is the kernel start-time of the process (Linux: clock ticks since +/// boot; Windows: the creation time). It is `None` when the platform cannot read it (or the +/// process is gone), where identity cannot be /// verified and callers fall back to best-effort behaviour. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProcessIdentity { @@ -83,12 +100,73 @@ pub enum IdentityState { /// [`IdentityState::Indeterminate`], never a risky match. When no identity was /// recorded (legacy state files), it degrades to best-effort /// [`IdentityState::Matches`]. +/// +/// Reads the PID's current start-time itself. A caller that has already read it +/// — because it also needs the raw observation — should pass that one reading to +/// [`identity_state_with_observed`] rather than calling this and reading again, +/// so a process that exits between the two reads cannot produce two verdicts +/// derived from disagreeing observations. #[must_use] pub fn identity_state(id: &ProcessIdentity) -> IdentityState { - if !crate::process_is_running(id.pid) || process_has_exited(id.pid) { + identity_state_from_probes( + id, + || crate::process_is_running(id.pid) && !process_has_exited(id.pid), + || process_start_ticks(id.pid), + ) +} + +/// [`identity_state`] with its two observations taken lazily, so the order they +/// are taken in is a property of this body rather than of a call site. +/// +/// Liveness BEFORE the reading, and the early return is what enforces it. The +/// reading must not sit in argument position — Rust evaluates arguments before +/// entering the callee, which inverts the safe side of the race. A PID read +/// while it was still the recorded process, which then exits and is recycled +/// before the liveness check, would be compared as `expected == actual` and come +/// back [`IdentityState::Matches`]: the one verdict that authorises a kill, +/// handed out for a PID that is now somebody else. Reading only after liveness +/// means the reading always describes whatever holds the PID *now*, so a +/// recycled one disagrees and comes back [`IdentityState::Recycled`]. +/// +/// The two probes are parameters purely so that ordering is testable. Staging +/// the real race needs a process to exit and its PID to be reissued inside a +/// window of microseconds, which no test can do; two closures that record when +/// they were called pin it deterministically instead. They are `FnOnce` +/// generics, so this costs nothing at runtime — the real call above +/// monomorphizes back into the same two direct calls. +/// +/// What the tests pin is THIS function's ordering. That [`identity_state`] +/// hands it the reading as a closure, not as a value it read first, is the one +/// line `|| process_start_ticks(id.pid)` there, and no test can see it: a dead +/// PID reads `None` either way. Keep that line a closure. +fn identity_state_from_probes( + id: &ProcessIdentity, + is_live: impl FnOnce() -> bool, + read_start_ticks: impl FnOnce() -> Option, +) -> IdentityState { + if !is_live() { return IdentityState::Gone; } - match (id.start_ticks, process_start_ticks(id.pid)) { + // Compare only. Routing back through `identity_state_with_observed` would + // re-run the liveness check just performed, which on Linux is two more + // `/proc` reads per call — paid on every 25 ms tick of the bounded waits, + // which is where this is called from in a loop. + compare_start_ticks(id, read_start_ticks()) +} + +/// The identity comparison alone, for callers that have already established +/// liveness. +/// +/// Split out so the ordering guarantee above does not have to pay for a second +/// liveness check. Deliberately not public: on its own it cannot return +/// [`IdentityState::Gone`], so a caller that had not checked liveness would get +/// [`IdentityState::Matches`] for a dead PID — the one verdict that authorises a +/// kill. The two callers that establish liveness first are in this file. +const fn compare_start_ticks( + id: &ProcessIdentity, + observed_start_ticks: Option, +) -> IdentityState { + match (id.start_ticks, observed_start_ticks) { (Some(expected), Some(actual)) => { if expected == actual { IdentityState::Matches @@ -104,6 +182,31 @@ pub fn identity_state(id: &ProcessIdentity) -> IdentityState { } } +/// [`identity_state`] against a start-time the caller has already observed. +/// +/// `observed_start_ticks` is what [`process_start_ticks`] returned for `id.pid`: +/// `None` both where the platform has no `/proc` and where that one PID's +/// start-time could not be read. +/// +/// Liveness is checked here too, so a PID that simply exits after the caller's +/// reading still yields [`IdentityState::Gone`]. What that check cannot catch is +/// exit *and recycle* between the reading and this call: the PID is live again, +/// and a reading taken while it was still the recorded process matches the +/// record, so the verdict is [`IdentityState::Matches`] for a process that is no +/// longer ours. Callers holding a reading across anything slow should re-read +/// rather than pass a stale one; [`identity_state`] avoids the window entirely +/// by reading only after its own liveness check. +#[must_use] +pub fn identity_state_with_observed( + id: &ProcessIdentity, + observed_start_ticks: Option, +) -> IdentityState { + if !crate::process_is_running(id.pid) || process_has_exited(id.pid) { + return IdentityState::Gone; + } + compare_start_ticks(id, observed_start_ticks) +} + /// Whether `state` means the recorded process is definitively no longer running. /// /// [`IdentityState::Indeterminate`] is intentionally excluded: a process we can @@ -300,8 +403,8 @@ fn send_signal(pid: u32, signal: Signal, _tree: bool) -> bool { /// Read the kernel start-time (field 22 of `/proc//stat`) for `pid`. /// -/// Returns `None` when the value cannot be read, including on non-Linux -/// platforms, where identity verification degrades to best-effort. +/// Returns `None` when the value cannot be read, including on platforms other +/// than Linux and Windows, where identity verification degrades to best-effort. #[cfg(target_os = "linux")] #[must_use] pub fn process_start_ticks(pid: u32) -> Option { @@ -309,7 +412,47 @@ pub fn process_start_ticks(pid: u32) -> Option { parse_start_ticks(&stat) } -#[cfg(not(target_os = "linux"))] +/// On Windows the start-time is the process creation `FILETIME`. +/// +/// It counts 100 ns ticks since 1601. It is opaque to the identity comparison, which only checks +/// equality, and it is what lets a recycled PID be told from the recorded one. +#[cfg(windows)] +#[must_use] +#[allow(unsafe_code)] // Win32 FFI +pub fn process_start_ticks(pid: u32) -> Option { + use windows_sys::Win32::Foundation::{CloseHandle, FILETIME}; + use windows_sys::Win32::System::Threading::{ + GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + return None; + } + let zero = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let (mut created, mut exited, mut kernel, mut user) = (zero, zero, zero, zero); + // SAFETY: `handle` is a live process handle opened above and closed below; + // the four out-pointers reference initialized FILETIMEs on this stack. + let ok = GetProcessTimes( + handle, + &raw mut created, + &raw mut exited, + &raw mut kernel, + &raw mut user, + ); + CloseHandle(handle); + if ok == 0 { + return None; + } + Some((u64::from(created.dwHighDateTime) << 32) | u64::from(created.dwLowDateTime)) + } +} + +#[cfg(not(any(target_os = "linux", windows)))] #[must_use] // Platform stub. The `cfg` sibling calls non-const code, so making only this // arm `const` would give the two platforms different signatures and push @@ -366,9 +509,134 @@ fn parse_start_ticks(stat: &str) -> Option { #[cfg(test)] mod tests { use super::*; + + #[test] + fn an_exited_pid_reads_as_gone_whatever_start_time_is_observed() { + // The liveness check comes first: a pid that has exited must be `Gone` + // even when a start-time reading is supplied that would otherwise match. + let mut child = std::process::Command::new(if cfg!(windows) { "cmd" } else { "true" }) + .args(if cfg!(windows) { + vec!["/C", "exit"] + } else { + vec![] + }) + .spawn() + .expect("spawn a short-lived child"); + let pid = child.id(); + child.wait().expect("reap the child"); + let id = ProcessIdentity::new(pid, Some(42)); + assert_eq!( + identity_state_with_observed(&id, Some(42)), + IdentityState::Gone + ); + } + + #[cfg(windows)] + #[test] + fn windows_start_time_is_readable_and_stable_for_our_own_pid() { + let pid = std::process::id(); + let first = process_start_ticks(pid).expect("our own creation time is readable"); + std::thread::sleep(std::time::Duration::from_millis(50)); + assert_eq!(process_start_ticks(pid), Some(first)); + assert_ne!(first, 0); + } + + #[cfg(windows)] + #[test] + fn windows_start_time_is_none_for_an_unopenable_pid_and_differs_per_process() { + assert_eq!(process_start_ticks(u32::MAX - 1), None); + let mut child = std::process::Command::new("ping") + .args(["-n", "30", "127.0.0.1"]) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("spawn a child"); + std::thread::sleep(std::time::Duration::from_millis(50)); + let ours = process_start_ticks(std::process::id()); + let theirs = process_start_ticks(child.id()); + let _ = child.kill(); + let _ = child.wait(); + assert!(theirs.is_some()); + assert_ne!( + ours, theirs, + "a different process has a different creation time" + ); + } + #[cfg(unix)] use std::process::{Child, Command, Stdio}; + #[test] + fn a_dead_process_is_never_read_for_a_start_time() { + // The ordering inside `identity_state_from_probes`, pinned rather than + // inspected: the reading is taken only after the liveness check passes. + // If it were ever read first, a PID that exits and is recycled in between + // comes back `Matches`: the one verdict that authorises a kill, for a + // process that is no longer the recorded one. This covers the helper + // only; `identity_state` passing the reading lazily is not observable + // from a test (a dead PID reads `None` either way). + // + // The real race is microseconds wide and needs a PID-space wrap, so it + // cannot be staged. This stages the observable consequence instead: a + // reading taken for a process already known dead is a reading that + // could not have informed the verdict. + let id = ProcessIdentity { + pid: 4321, + start_ticks: Some(99), + }; + let mut read_start_ticks = false; + + let state = identity_state_from_probes( + &id, + || false, + || { + read_start_ticks = true; + Some(99) + }, + ); + + assert_eq!( + state, + IdentityState::Gone, + "a process that fails the liveness check is Gone whatever its start time reads as" + ); + assert!( + !read_start_ticks, + "`identity_state_from_probes` must not read the start time once liveness has \ + failed; this pins that helper's ordering only, not how `identity_state` calls it" + ); + } + + #[test] + fn a_live_process_is_judged_on_a_reading_taken_after_its_liveness_check() { + // The other half: when liveness passes, the reading is taken, and it is + // the reading — not the record — that decides. A recycled PID reads + // differently and must come back `Recycled`. + // + // The PID is arbitrary, and nothing here establishes whether it is + // running — it does not need to. Both probes are stubbed, so this path + // never consults the real process table at all, and that is the point: + // reaching the comparison at all proves the verdict came from the + // liveness this function was handed rather than from a second check of + // its own. While the comparison went back through + // `identity_state_with_observed`, this same call returned `Gone` from + // that function's own liveness check, whatever the stub said. + let id = ProcessIdentity { + pid: 4321, + start_ticks: Some(99), + }; + + assert_eq!( + identity_state_from_probes(&id, || true, || Some(99)), + IdentityState::Matches, + "a live PID still reading as its recorded start time is the recorded process" + ); + assert_eq!( + identity_state_from_probes(&id, || true, || Some(1234)), + IdentityState::Recycled, + "a live PID reading as a different start time is somebody else" + ); + } + /// Spawn a child that prints a line to stdout once it is ready, and block /// until that line arrives. This replaces sleep-based readiness guesses with /// a deterministic signal (e.g. that a shell has installed its SIGTERM trap), @@ -534,6 +802,51 @@ mod tests { reap(child); } + /// The observation the caller passes must be the one classified, not a fresh + /// read of the same PID. A caller that needs the raw start-time *and* the + /// verdict reads once and passes it down precisely so the two cannot + /// disagree; an implementation that quietly re-read `/proc` would restore + /// that hazard while every existing test still passed. Here the live child's + /// real start-time matches its recorded identity, so a re-reading + /// implementation returns `Matches` — only one that honours the argument + /// returns `Recycled`. + #[cfg(target_os = "linux")] + #[test] + fn identity_state_with_observed_classifies_the_reading_it_was_given() { + let (child, _) = spawn_ready("echo ready; while true; do sleep 1; done"); + let id = ProcessIdentity::capture(child.id()); + let real_ticks = id.start_ticks.expect("linux records a start-time"); + + // Collect every verdict first, then kill and reap, and only then assert. + // The child holds the harness's stdout pipe open, so an assertion that + // panics ahead of the kill does not merely fail this test — it leaks a + // looping process and hangs the whole suite on the pipe. Ask the + // questions, clean up unconditionally, then judge. + let agreeing = identity_state_with_observed(&id, Some(real_ticks)); + let disagreeing = identity_state_with_observed(&id, Some(real_ticks.wrapping_add(1))); + let unreadable = identity_state_with_observed(&id, None); + + let hard = terminate_verified(&id, KillScope::Single, Duration::from_secs(5), true); + reap(child); + + assert!(hard.stopped(), "the child must not outlive the test"); + assert_eq!( + agreeing, + IdentityState::Matches, + "the reading that agrees with the record is a match" + ); + assert_eq!( + disagreeing, + IdentityState::Recycled, + "a disagreeing reading must be classified, not discarded for a re-read" + ); + assert_eq!( + unreadable, + IdentityState::Indeterminate, + "an unreadable start-time against a recorded one is unconfirmable" + ); + } + /// A terminated child that has not been reaped yet is a zombie: it still has /// a PID, so `kill(pid, 0)` succeeds and `process_is_running` reports `true`. /// Termination checks must therefore go through `identity_state`, which diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index fbca3104f..bfba91f29 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -48,6 +48,34 @@ pub fn is_public_bind_host(host: &str) -> bool { !matches!(host, "127.0.0.1" | "localhost" | "::1") } +/// The `rocm serve` invocation that brings back a service whose endpoint key +/// was dropped, for the refusals and the uninstall abort that tell an operator +/// to launch it again. +/// +/// One builder for every place that prints it (the `rocm serve` and `rocmd` +/// refusals and the uninstall abort, one command per stopped service), so they +/// cannot drift: a public host needs `--allow-public-bind`, and a service that +/// asked for authentication needs `--require-api-key` even on loopback. The key +/// itself is optional ([`RELAUNCH_KEY_CHOICE`]). +#[must_use] +pub fn relaunch_command(host: &str, require_api_key: bool) -> String { + let public_flag = if is_public_bind_host(host) { + " --allow-public-bind" + } else { + "" + }; + let key_flag = if require_api_key { + " --require-api-key" + } else { + "" + }; + format!("rocm serve --host {host} --port {public_flag}{key_flag}") +} + +/// The parenthetical that follows a [`relaunch_command`]: supplying a key is +/// optional, and the environment variable keeps it out of shell history. +pub const RELAUNCH_KEY_CHOICE: &str = "(add `--api-key `, or set ROCM_SERVE_API_KEY, to choose the key instead of generating one)"; + /// Resolve the endpoint API key an engine adapter must enforce, if any. /// /// Reads the key file named by [`ENDPOINT_API_KEY_FILE_ENV`]. Returns `None` when @@ -771,6 +799,26 @@ mod tests { std::fs::remove_dir_all(&paths.data_dir).ok(); } + #[test] + fn relaunch_command_carries_exactly_the_flags_the_service_needs() { + assert_eq!( + relaunch_command("127.0.0.1", false), + "rocm serve --host 127.0.0.1 --port " + ); + assert_eq!( + relaunch_command("127.0.0.1", true), + "rocm serve --host 127.0.0.1 --port --require-api-key" + ); + assert_eq!( + relaunch_command("0.0.0.0", false), + "rocm serve --host 0.0.0.0 --port --allow-public-bind" + ); + assert_eq!( + relaunch_command("0.0.0.0", true), + "rocm serve --host 0.0.0.0 --port --allow-public-bind --require-api-key" + ); + } + #[test] fn is_public_bind_host_recognizes_only_the_accepted_loopback_spellings() { for host in ["127.0.0.1", "localhost", "::1"] { diff --git a/docs/architecture.md b/docs/architecture.md index 61c75b018..0c8f2fb31 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -53,7 +53,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns private types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — [`main.rs`](../apps/rocm/src/main.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and [`automations.rs`](../apps/rocm/src/automations.rs) all call into it. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. - [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install/update and the runtime-manifest model/loaders. The `runtimes` subcommand's own command enum, dispatch fn, and most of its domain logic — rendering, activation, and manifest import — stay in [`main.rs`](../apps/rocm/src/main.rs). -- [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns private types: `uninstall()`, plus the installer `PATH` undo and running-executable deletion. The shared plan type and its file-removal helpers stay in [`main.rs`](../apps/rocm/src/main.rs) because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them. +- [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns private types: `uninstall()`, the installer `PATH` undo, running-executable deletion, and the stop-before-remove ordering with its partial-removal report (the stop passes are in [`uninstall_gate.rs`](../apps/rocm/src/uninstall_gate.rs), whose pre-flight is shared by the plan and the stop pass, so the plan's helper and refusal lines come from the same decision as the run; the abort's relaunch advice comes from `rocm_engine_protocol::relaunch_command`, the builder `rocm serve` and `rocmd` refusals use). The shared plan type and its file-removal helpers stay in [`main.rs`](../apps/rocm/src/main.rs) because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them; so does the plan half (`build_uninstall_plan`). +- [`uninstall_gate.rs`](../apps/rocm/src/uninstall_gate.rs) — the stop-before-remove gate behind `rocm uninstall`: the shared pre-flight, the stop passes and their failure/report types. The plan's rendering of that pre-flight stays in `build_uninstall_plan` in [`main.rs`](../apps/rocm/src/main.rs). ### `apps/rocmd` — background daemon diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 3dcd76fd1..d0de473b2 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -645,3 +645,35 @@ Expected result: `PATH change left in place` and names it, and the `PATH` change stays. - With `rocm uninstall --keep-binaries`, the review says the `PATH` change stays too, and it does. + +### With a running managed server + +Start a managed server (`rocm serve `), then run: + +```powershell +rocm uninstall --yes +``` + +Expected result: + +- The plan warns the server will be stopped before removal. If the background + helper (`rocmd`) is running and the run proceeds, the plan says it will be stopped before removal (its whole process tree on Linux; only the recorded process on Windows, and the plan says so). On a run that will refuse, the plan says the helper is left running instead. +- The server (and helper) are stopped, then the files are removed and the + command exits 0. + +The refusal paths each need a fresh install and a fresh managed server, because +a successful `uninstall --yes` above removes the install. Reinstall, start a +managed server again, then: + +- **Unreadable record.** Write `{` into `/services/bad.json`, where + the data directory is the one `rocm examine` reports, and run + `rocm uninstall --yes`. It must exit non-zero, name that file's full path, + advise repairing or deleting it, and leave the install, config, and data + directories in place. +- **No recorded start time.** Edit the server's record under the same + `services` folder and delete its `supervisor_start_ticks` field (a record + written by an older version looks like this). The plan must say the stop + will be refused, and `rocm uninstall --yes` must exit non-zero naming the + pid, leave the server running, and remove nothing. +- **Keep binaries and data (removes only config and cache).** `rocm uninstall --yes --keep-binaries --keep-data` must leave + the server and the helper running and say so in the plan. diff --git a/docs/testing.md b/docs/testing.md index cc1f70b4a..ec21fe312 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1374,7 +1374,13 @@ generated public-key PEM installer verification; verify first-install PATH setup Linux writes the shell profile); reinstall stale-manifest purge and config preservation; a Windows loopback-HTTP install; isolated installed-binary directory smoke checks (`rocm examine` must read only the isolated -config/data/cache, never the real user `.rocm` state); and uninstall: a full +config/data/cache, never the real user `.rocm` state); and uninstall: stopping the local server it manages +(lifecycle-10 and lifecycle-24, Linux and Windows), the background helper +(lifecycle-26), an uninstall that keeps the binaries and data leaving the +server running and its record in place (lifecycle-27), refusing to remove +anything while a managed service record cannot be read (lifecycle-25), while a +live server it cannot prove is ours has no recorded start time (lifecycle-28), +and while the helper's runtime state cannot be read (lifecycle-29); and a full purge that also restores the shell profile, the Windows user `PATH` taken back off with its other entries and its registry type kept, the `PATH` change kept while another program shares the install folder or with `--keep-binaries`, a `PATH` change that cannot be undone removing nothing diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 3690e692c..41cabce3e 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -193,13 +193,12 @@ pub(crate) fn read_service_state(path: &Path) -> Result { } fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write( + // `rocmd` and `rocm uninstall` read this file while a launch rewrites it; + // the atomic publish means they never see a torn document. It creates the + // parent directory itself. + rocm_core::write_file_atomically( path, - serde_json::to_vec_pretty(value).context("failed to serialize vLLM state")?, + &serde_json::to_vec_pretty(value).context("failed to serialize vLLM state")?, ) .with_context(|| format!("failed to write {}", path.display())) } diff --git a/tests/e2e-cucumber/features/install_lifecycle.feature b/tests/e2e-cucumber/features/install_lifecycle.feature index 589db146e..d0f27b0a6 100644 --- a/tests/e2e-cucumber/features/install_lifecycle.feature +++ b/tests/e2e-cucumber/features/install_lifecycle.feature @@ -136,10 +136,28 @@ Feature: Release install lifecycle And the shell profile is back to the user's own settings And the record of the installer's PATH change is gone + # EAI-8014: uninstall used to report success while a server it manages kept + # serving and holding the GPU, then delete the very tooling (`rocm services + # stop`, the service records) needed to stop it. + @id:lifecycle-linux-uninstall-stops-managed-server @lifecycle @requires-os:linux + Scenario: lifecycle-10 - Linux - uninstall stops the local server it manages + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And a local server this machine manages is running + When the user uninstalls from the installed binary + Then the removal is reported as complete + And the uninstall plan says the managed servers will be stopped before removal + And the uninstall result says one managed service was stopped + And the local server this machine manages is no longer running + And the installed rocm and rocmd binaries are gone + And the isolated config, data, and cache folders are gone + # ── Windows user-PATH restoration, loopback HTTP install, isolated smoke ─ @id:lifecycle-windows-install-signed @lifecycle @requires-os:windows - Scenario: lifecycle-10 - Windows - a signed zip installs and verifies with native crypto + Scenario: lifecycle-11 - Windows - a signed zip installs and verifies with native crypto Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -149,7 +167,7 @@ Feature: Release install lifecycle And the install manifest is present @id:lifecycle-windows-key-rotation-fallback @lifecycle @requires-os:windows - Scenario: lifecycle-11 - Windows - a malformed current trust root falls through to the valid next key + Scenario: lifecycle-12 - Windows - a malformed current trust root falls through to the valid next key Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -159,7 +177,7 @@ Feature: Release install lifecycle And the installed rocm binary is present @id:lifecycle-windows-all-pinned-keys-malformed @lifecycle @requires-os:windows - Scenario: lifecycle-12 - Windows - all malformed pinned trust roots fail deterministically + Scenario: lifecycle-13 - Windows - all malformed pinned trust roots fail deterministically Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -169,7 +187,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-updates-user-path @lifecycle @requires-os:windows @serial - Scenario: lifecycle-13 - Windows - a default install updates the user PATH and restores it afterwards + Scenario: lifecycle-14 - Windows - a default install updates the user PATH and restores it afterwards Given a freshly built release tree And a generated signing keypair And the user PATH is captured for restoration @@ -180,7 +198,7 @@ Feature: Release install lifecycle And the install directory is on the user PATH @id:lifecycle-windows-verifies-without-openssl @lifecycle @requires-os:windows - Scenario: lifecycle-14 - Windows - the installer verifies with native crypto when openssl is absent from PATH + Scenario: lifecycle-15 - Windows - the installer verifies with native crypto when openssl is absent from PATH Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -190,7 +208,7 @@ Feature: Release install lifecycle And the install manifest is present @id:lifecycle-windows-rejects-bad-signature-without-openssl @lifecycle @requires-os:windows - Scenario: lifecycle-15 - Windows - a bad signature is rejected with native crypto when openssl is absent from PATH + Scenario: lifecycle-16 - Windows - a bad signature is rejected with native crypto when openssl is absent from PATH Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -200,7 +218,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-reinstall-purges-stale @lifecycle @requires-os:windows - Scenario: lifecycle-16 - Windows - reinstalling purges a stale prior entry + Scenario: lifecycle-17 - Windows - reinstalling purges a stale prior entry Given a freshly built release tree And a generated signing keypair And a signed bundle installed with the public key file @@ -211,7 +229,7 @@ Feature: Release install lifecycle And the install manifest is present @id:lifecycle-windows-reject-untrusted-key @lifecycle @requires-os:windows - Scenario: lifecycle-17 - Windows - an install with no public key rejects an untrusted signer + Scenario: lifecycle-18 - Windows - an install with no public key rejects an untrusted signer Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -220,7 +238,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-reject-bad-checksum @lifecycle @requires-os:windows - Scenario: lifecycle-18 - Windows - a tampered checksum is rejected before activation + Scenario: lifecycle-19 - Windows - a tampered checksum is rejected before activation Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -230,7 +248,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-reject-bad-signature @lifecycle @requires-os:windows - Scenario: lifecycle-19 - Windows - a tampered signature is rejected before activation + Scenario: lifecycle-20 - Windows - a tampered signature is rejected before activation Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -240,7 +258,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-reject-missing-signature @lifecycle @requires-os:windows - Scenario: lifecycle-20 - Windows - a missing signature is rejected when a signature is required + Scenario: lifecycle-21 - Windows - a missing signature is rejected when a signature is required Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -250,7 +268,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-http-install @lifecycle @requires-os:windows - Scenario: lifecycle-21 - Windows - a signed bundle installs over a loopback HTTP download + Scenario: lifecycle-22 - Windows - a signed bundle installs over a loopback HTTP download Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -260,7 +278,7 @@ Feature: Release install lifecycle And the install manifest is present @id:lifecycle-windows-installed-binary-smoke @lifecycle @requires-os:windows - Scenario: lifecycle-22 - Windows - the installed binary honors isolated directories and removes itself on uninstall + Scenario: lifecycle-23 - Windows - the installed binary honors isolated directories and removes itself on uninstall Given a freshly built release tree And a generated signing keypair And a signed bundle installed with the public key file @@ -278,10 +296,114 @@ Feature: Release install lifecycle And the non-running installed rocmd binary is gone And the install manifest is gone + # The Windows half of EAI-8014. Worth its own scenario rather than trusting the + # Linux one because the termination path differs: Windows terminates only the + # recorded process, where Linux signals the whole tree. + # + # What this pins is that PID-based termination, not the gate's port + # reality-check. The planted record carries port 0, which nothing can ever be + # serving on, so the port branch is satisfied trivially on every run. Staging + # the case the port check exists for — an engine grandchild outliving the + # recorded process and still serving while the record reads "stopped" — needs + # a real listener the scenario does not stand up. That branch is covered by + # the unit tests around `stopped_record_verdict` instead. + @id:lifecycle-windows-uninstall-stops-managed-server @lifecycle @requires-os:windows + Scenario: lifecycle-24 - Windows - uninstall stops the local server it manages + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated directories with state + And a local server this machine manages is running + When the user uninstalls from the installed binary + Then the removal is reported as complete + And the uninstall plan says the managed servers will be stopped before removal + And the uninstall result says one managed service was stopped + And the local server this machine manages is no longer running + And the non-running installed rocmd binary is gone + And the isolated config, data, and cache folders are gone + + # EAI-8014: when a managed service record cannot be read, uninstall cannot tell + # whether a server is still running, so it must refuse rather than delete the + # tooling that would be needed to stop it. + @id:lifecycle-linux-uninstall-refuses-unreadable-record @lifecycle @requires-os:linux + Scenario: lifecycle-25 - Linux - uninstall refuses and keeps everything when a service record cannot be read + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And a managed service record that cannot be parsed + When the user uninstalls from the installed binary + Then the removal is refused with advice to repair or delete that record + And the installed rocm binary and manifest are still present + And the isolated XDG state is still present + + # EAI-8014: the background helper restarts managed services on its own, so + # uninstall stops it first. Pinned alone, with no service record, so it is the + # helper stop being proven and not the service stop. + @id:lifecycle-linux-uninstall-stops-background-helper @lifecycle @requires-os:linux + Scenario: lifecycle-26 - Linux - uninstall stops the background helper + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And the background helper is running + When the user uninstalls from the installed binary + Then the removal is reported as complete + And the uninstall plan says the background helper will be stopped + And the uninstall result says the background helper was stopped + And the background helper is no longer running + And the isolated config, data, and cache folders are gone + + # EAI-8014: a run that keeps the binaries and data (and so removes only config and cache) keeps `rocm services stop` and the + # service records, so it must not force-stop a server it leaves stoppable. + @id:lifecycle-linux-uninstall-keep-binaries-data-leaves-server @lifecycle @requires-os:linux + Scenario: lifecycle-27 - Linux - uninstall keeping binaries and data leaves the managed server running and stoppable + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And a local server this machine manages is running + When the user uninstalls keeping binaries and data + Then the removal is reported as complete + And the keep-binaries-and-data run says the server is left running + And the local server this machine manages is still running + And the managed service record is still present + And only the isolated config and cache folders are gone + And the installed rocm and rocmd binaries are still present + + # EAI-8014: a live server whose record has no start time cannot be proven ours, + # so uninstall refuses and kills nothing instead of signalling an unverified pid. + @id:lifecycle-linux-uninstall-refuses-tickless-server @lifecycle @requires-os:linux + Scenario: lifecycle-28 - Linux - uninstall refuses a live server it cannot prove is ours + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And a local server this machine manages is running with no recorded start time + When the user uninstalls from the installed binary + Then the removal is refused because the server cannot be proven ours + And the installed rocm binary and manifest are still present + And the managed service record is still present + And the local server this machine manages is still running + + # EAI-8014: an unreadable helper runtime state means the helper cannot be + # confirmed stopped, so uninstall refuses and keeps everything. + @id:lifecycle-linux-uninstall-refuses-unreadable-helper-state @lifecycle @requires-os:linux + Scenario: lifecycle-29 - Linux - uninstall refuses when the helper runtime state cannot be read + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And the background helper runtime state cannot be parsed + When the user uninstalls from the installed binary + Then the removal is refused because the helper state cannot be read + And the installed rocm binary and manifest are still present + And the isolated XDG state is still present + # ── Uninstall: the installer's PATH change ────────────────────────────── @id:lifecycle-linux-uninstall-keeps-shared-path @lifecycle @requires-os:linux - Scenario: lifecycle-23 - Linux - uninstall keeps the shell PATH change while another program shares the install directory + Scenario: lifecycle-30 - Linux - uninstall keeps the shell PATH change while another program shares the install directory Given a freshly built release tree And a generated signing keypair And a signed bundle installed with the shell profile updated @@ -295,7 +417,7 @@ Feature: Release install lifecycle And the record of the installer's PATH change is still in the install directory @id:lifecycle-windows-uninstall-restores-user-path @lifecycle @requires-os:windows @serial - Scenario: lifecycle-24 - Windows - uninstall takes the install directory back off the user PATH and keeps the rest of it + Scenario: lifecycle-31 - Windows - uninstall takes the install directory back off the user PATH and keeps the rest of it Given a freshly built release tree And a generated signing keypair And the user PATH is captured for restoration @@ -309,7 +431,7 @@ Feature: Release install lifecycle And the record of the installer's PATH change is gone @id:lifecycle-linux-uninstall-keep-binaries-keeps-path @lifecycle @requires-os:linux - Scenario: lifecycle-25 - Linux - uninstall that keeps the binaries keeps the shell PATH change + Scenario: lifecycle-32 - Linux - uninstall that keeps the binaries keeps the shell PATH change Given a freshly built release tree And a generated signing keypair And a signed bundle installed with the shell profile updated @@ -322,7 +444,7 @@ Feature: Release install lifecycle And the record of the installer's PATH change is still in the install directory @id:lifecycle-linux-uninstall-path-failure-keeps-binaries @lifecycle @requires-os:linux - Scenario: lifecycle-26 - Linux - uninstall that cannot undo the PATH change removes nothing, so it can be retried + Scenario: lifecycle-33 - Linux - uninstall that cannot undo the PATH change removes nothing, so it can be retried Given a freshly built release tree And a generated signing keypair And the shell profile already holds the user's own settings @@ -342,7 +464,7 @@ Feature: Release install lifecycle And the record of the installer's PATH change is gone @id:lifecycle-linux-uninstall-follows-the-installer-record @lifecycle @requires-os:linux - Scenario: lifecycle-27 - Linux - uninstall undoes the PATH change in a profile it finds only through the installer's record + Scenario: lifecycle-34 - Linux - uninstall undoes the PATH change in a profile it finds only through the installer's record Given a freshly built release tree And a generated signing keypair And the install directory is reached through a link diff --git a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs index 5563f3a0e..ff0e4ed61 100644 --- a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs @@ -18,9 +18,10 @@ use std::fmt::Write as _; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use cucumber::{given, then, when}; +use e2e_cucumber::mock_server::{ServiceRecordOptions, write_service_record_with}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; @@ -54,6 +55,14 @@ pub struct LifecycleState { smoke_config: Option, smoke_data: Option, smoke_cache: Option, + /// A stand-in for a local server this machine manages, planted with a + /// service record so uninstall must stop it. Killed on `Drop` if a scenario + /// fails before uninstall reaches it, so no scenario leaks a process. + managed_server: Option, + /// A stand-in for the background `rocmd` helper, planted via its runtime-state + /// file. Killed on `Drop` like `managed_server`. + background_helper: Option, + broken_record: Option, /// The shell profile contents seeded before install, which uninstall must /// restore exactly. profile_before_install: Option, @@ -81,6 +90,15 @@ fn shell_profile(st: &LifecycleState) -> PathBuf { impl Drop for LifecycleState { fn drop(&mut self) { + // Never leak the managed-server stand-in, even when a scenario fails + // before uninstall would have stopped it. + for mut child in [self.managed_server.take(), self.background_helper.take()] + .into_iter() + .flatten() + { + let _ = child.kill(); + let _ = child.wait(); + } // Restore the machine user PATH if a scenario mutated it, even on panic. #[cfg(windows)] if let Some(previous) = self.captured_user_path.take() { @@ -532,6 +550,9 @@ async fn given_release_tree(world: &mut E2eWorld) { smoke_config: None, smoke_data: None, smoke_cache: None, + managed_server: None, + background_helper: None, + broken_record: None, profile_before_install: None, custom_profile: None, }); @@ -662,6 +683,155 @@ fn seed_isolated_dirs(world: &mut E2eWorld) { st.smoke_cache = Some(cache); } +/// A single long-lived process to stand in for a managed server, on either OS. +/// +/// One process, not a shell that spawns one: the CLI terminates the recorded +/// PID, and a grandchild would leave the scenario asserting against the wrong +/// process. +fn spawn_long_lived_child() -> std::process::Child { + #[cfg(windows)] + let mut command = { + let mut command = Command::new("powershell"); + command.args(["-NoProfile", "-Command", "Start-Sleep -Seconds 600"]); + command + }; + #[cfg(not(windows))] + let mut command = { + let mut command = Command::new("sleep"); + command.arg("600"); + command + }; + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the managed server stand-in") +} + +/// Start a local server this machine manages: a real long-lived child process +/// plus the on-disk service record `rocm serve --managed` would leave behind, +/// planted in the isolated data dir the installed binary reads. +/// +/// Black-box on purpose — a process the CLI can find only through its record, +/// exactly as a real managed vLLM server appears to `rocm uninstall`. The record +/// carries the supervisor start time, so the CLI's verified +/// termination verifies it against the recorded start time and stops it. +#[given("a local server this machine manages is running")] +async fn given_managed_server_running(world: &mut E2eWorld) { + plant_managed_server(world, true); +} + +/// A live server whose record carries no start time, as an older version wrote +/// it: uninstall cannot prove the pid is ours, so it must refuse. +#[given("a local server this machine manages is running with no recorded start time")] +async fn given_managed_server_running_tickless(world: &mut E2eWorld) { + plant_managed_server(world, false); +} + +fn plant_managed_server(world: &mut E2eWorld, with_start_ticks: bool) { + let services_dir = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir must be seeded before planting a service record") + .join("services"); + let server = spawn_long_lived_child(); + // Port 0 is the one port nothing can ever be serving on: it resolves, so the + // CLI really does run its probe, and the connect always fails. Liveness then + // falls through to the recorded process, which is alive. Binding an + // ephemeral port and dropping it would leave a window in which something + // else on a busy runner grabs it and fails the scenario. + let free_port = 0; + write_service_record_with( + &services_dir, + "amd/test-model", + free_port, + ServiceRecordOptions { + status: "ready", + startup_phase: None, + supervisor_pid: server.id(), + engine_pid: Some(server.id()), + ..ServiceRecordOptions::default() + }, + ); + // A record written by a current `rocmd` carries the supervisor's start time; + // a tick-less live record is refused by uninstall rather than killed. + if !with_start_ticks { + state_mut(world).managed_server = Some(server); + return; + } + let record_path = services_dir.join("e2e-mock.json"); + let mut record: serde_json::Value = + serde_json::from_slice(&std::fs::read(&record_path).expect("failed to read record")) + .expect("failed to parse record"); + record["supervisor_start_ticks"] = + serde_json::json!(rocm_core::process_start_ticks(server.id())); + std::fs::write( + &record_path, + serde_json::to_vec_pretty(&record).expect("failed to serialize record"), + ) + .expect("failed to rewrite record"); + state_mut(world).managed_server = Some(server); +} + +/// The background helper `rocmd` as `rocm uninstall` finds it: only through the +/// runtime-state file a live daemon leaves, pid plus start time. +#[given("the background helper is running")] +async fn given_background_helper_running(world: &mut E2eWorld) { + let automations_dir = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir must be seeded before planting helper state") + .join("automations"); + std::fs::create_dir_all(&automations_dir).expect("failed to create automations dir"); + let helper = spawn_long_lived_child(); + let runtime_state = serde_json::json!({ + "running": true, + "automations_enabled": true, + "daemon_pid": helper.id(), + "daemon_start_ticks": rocm_core::process_start_ticks(helper.id()), + "started_at_unix_ms": 0, + "last_tick_unix_ms": 0, + "active_watchers": [], + }); + std::fs::write( + automations_dir.join("runtime-state.json"), + serde_json::to_vec_pretty(&runtime_state).expect("failed to serialize helper state"), + ) + .expect("failed to write helper state"); + state_mut(world).background_helper = Some(helper); +} + +/// A runtime-state file the CLI cannot parse: it cannot tell whether a helper +/// is running, so uninstall must refuse rather than delete the tooling. +#[given("the background helper runtime state cannot be parsed")] +async fn given_unparseable_helper_state(world: &mut E2eWorld) { + let automations_dir = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir must be seeded before planting helper state") + .join("automations"); + std::fs::create_dir_all(&automations_dir).expect("failed to create automations dir"); + let path = automations_dir.join("runtime-state.json"); + std::fs::write(&path, b"{ not json").expect("failed to write unparseable helper state"); + state_mut(world).broken_record = Some(path); +} + +/// An unparseable `*.json` under the isolated `services/` dir: the CLI cannot +/// tell whether it describes a running server. +#[given("a managed service record that cannot be parsed")] +async fn given_unparseable_service_record(world: &mut E2eWorld) { + let services_dir = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir must be seeded before planting a service record") + .join("services"); + std::fs::create_dir_all(&services_dir).expect("failed to create services dir"); + std::fs::write(services_dir.join("broken.json"), b"{ not json") + .expect("failed to write unparseable record"); + state_mut(world).broken_record = Some(services_dir.join("broken.json")); +} + // ── When: package ────────────────────────────────────────────────────── async fn package_with_key_file(world: &mut E2eWorld) { @@ -1152,6 +1322,11 @@ fn run_uninstall(world: &mut E2eWorld, extra_args: &[&str]) { st.last_rc = output.status.code().unwrap_or(-1); } +#[when("the user uninstalls keeping binaries and data")] +async fn when_uninstall_cache_only(world: &mut E2eWorld) { + run_uninstall(world, &["--keep-binaries", "--keep-data"]); +} + /// Apply the isolated config/data/cache env for installed-binary invocations. fn apply_smoke_env(cmd: &mut Command, st: &LifecycleState) { if let Some(dir) = &st.smoke_config { @@ -1406,6 +1581,260 @@ async fn then_examine_isolated(world: &mut E2eWorld) { } } +#[then("the removal is reported as complete")] +async fn then_removal_complete(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert_eq!( + state(world).last_rc, + 0, + "uninstall did not exit zero:\n{out}" + ); + assert!( + out.contains("uninstall complete"), + "uninstall did not report completion:\n{out}" + ); +} + +#[then("the removal is refused with advice to repair or delete that record")] +async fn then_removal_refused(world: &mut E2eWorld) { + let st = state(world); + let out = &st.last_output; + let record = st.broken_record.as_ref().expect("no broken record planted"); + assert_ne!( + st.last_rc, 0, + "uninstall succeeded despite an unreadable record:\n{out}" + ); + assert!( + out.contains("repair or delete"), + "refusal did not advise repairing or deleting the record:\n{out}" + ); + assert!( + out.contains(&record.display().to_string()), + "refusal did not name the unreadable record {}:\n{out}", + record.display() + ); + assert!( + !out.contains("uninstall complete"), + "refused uninstall still reported completion:\n{out}" + ); +} + +#[then("the removal is refused because the server cannot be proven ours")] +async fn then_refused_tickless(world: &mut E2eWorld) { + let out = state(world).last_output.clone(); + assert_ne!(state(world).last_rc, 0, "uninstall succeeded:\n{out}"); + assert!( + out.contains("uninstall will refuse to remove anything") + && out.contains("no start time was recorded"), + "the plan did not say the removal will be refused for want of a start time:\n{out}" + ); + assert!( + !out.contains("uninstall complete"), + "refused uninstall still reported completion:\n{out}" + ); + let server = state_mut(world) + .managed_server + .as_mut() + .expect("no managed server was started"); + assert!( + server.try_wait().expect("failed to poll").is_none(), + "uninstall killed a pid it could not prove was ours" + ); +} + +#[then("the removal is refused because the helper state cannot be read")] +async fn then_refused_helper_state(world: &mut E2eWorld) { + let st = state(world); + let out = &st.last_output; + let path = st.broken_record.as_ref().expect("no broken state planted"); + assert_ne!(st.last_rc, 0, "uninstall succeeded:\n{out}"); + assert!( + out.contains("uninstall will refuse to remove anything") + && out.contains(&path.display().to_string()), + "the plan did not name the unreadable helper state {}:\n{out}", + path.display() + ); + assert!( + !out.contains("uninstall complete"), + "refused uninstall still reported completion:\n{out}" + ); +} + +#[then("the installed rocm binary and manifest are still present")] +async fn then_install_kept(world: &mut E2eWorld) { + let st = state(world); + for path in [ + installed_binary(&st.install_dir, "rocm"), + st.install_dir.join(".rocm-cli-manifest"), + ] { + assert!( + path.exists(), + "refused uninstall removed {}", + path.display() + ); + } +} + +#[then("the isolated XDG state is still present")] +async fn then_refused_xdg_kept(world: &mut E2eWorld) { + let st = state(world); + let record = st.broken_record.as_ref().expect("no broken record planted"); + assert!( + record.exists(), + "refused uninstall removed {}", + record.display() + ); + for dir in [&st.smoke_config, &st.smoke_data, &st.smoke_cache] + .into_iter() + .flatten() + { + assert!(dir.exists(), "refused uninstall removed {}", dir.display()); + } +} + +/// Wait for our own child to exit, bounded by uninstall's stop grace. +fn child_exits_within_grace(child: &mut std::process::Child) -> bool { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + match child.try_wait().expect("failed to poll the child") { + Some(_) => return true, + None if std::time::Instant::now() >= deadline => return false, + None => std::thread::sleep(std::time::Duration::from_millis(100)), + } + } +} + +#[then("the background helper is no longer running")] +async fn then_background_helper_stopped(world: &mut E2eWorld) { + let mut helper = state_mut(world) + .background_helper + .take() + .expect("no background helper was started"); + let exited = child_exits_within_grace(&mut helper); + if !exited { + let _ = helper.kill(); + let _ = helper.wait(); + } + let out = &state(world).last_output; + assert!( + exited, + "uninstall reported success while the background helper kept running:\n{out}" + ); +} + +#[then("the uninstall plan says the background helper will be stopped")] +async fn then_plan_names_helper(world: &mut E2eWorld) { + // Asserted next to the outcome the `lifecycle-linux-uninstall-stops-background-helper` scenario checks in its following steps: + // the plan is only worth reading if the stop it promises happens. + let out = &state(world).last_output; + assert!( + out.contains("the background helper (rocmd, pid") && out.contains("will be stopped"), + "the plan did not say the helper will be stopped:\n{out}" + ); +} + +#[then("the uninstall plan says the managed servers will be stopped before removal")] +async fn then_plan_names_servers(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert!( + out.contains("those servers will be stopped before removal"), + "the plan did not say the managed servers will be stopped before removal:\n{out}" + ); +} + +#[then("the uninstall result says one managed service was stopped")] +async fn then_result_names_stopped_service(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert!( + out.contains("stopped 1 managed service(s) before removal"), + "the result did not report the stopped service:\n{out}" + ); +} + +#[then("the uninstall result says the background helper was stopped")] +async fn then_result_names_stopped_helper(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert!( + out.contains("stopped the background helper (rocmd)"), + "the result did not report the stopped helper:\n{out}" + ); +} + +#[then("the local server this machine manages is still running")] +async fn then_managed_server_still_running(world: &mut E2eWorld) { + let server = state_mut(world) + .managed_server + .as_mut() + .expect("no managed server was started"); + // A run keeping binaries and data keeps the tooling that stops servers, so it must leave + // the server alone; give a wrongly-issued stop time to land first. + std::thread::sleep(std::time::Duration::from_secs(2)); + assert!( + server.try_wait().expect("failed to poll").is_none(), + "an uninstall keeping binaries and data stopped the managed server" + ); +} + +#[then("the keep-binaries-and-data run says the server is left running")] +async fn then_cache_only_says_left_running(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert!( + out.contains("so those servers are left running"), + "the keep-binaries-and-data run did not say the server is left running:\n{out}" + ); + assert_eq!( + state(world).last_rc, + 0, + "the keep-binaries-and-data run exited non-zero:\n{out}" + ); + assert!( + !out.contains("will refuse to remove anything"), + "the keep-binaries-and-data run printed a refusal:\n{out}" + ); +} + +#[then("the managed service record is still present")] +async fn then_service_record_present(world: &mut E2eWorld) { + let record = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir not seeded") + .join("services") + .join("e2e-mock.json"); + assert!(record.exists(), "the service record was removed"); +} + +#[then("the local server this machine manages is no longer running")] +async fn then_managed_server_stopped(world: &mut E2eWorld) { + let mut server = state_mut(world) + .managed_server + .take() + .expect("no managed server was started"); + // The stand-in is our own child, so its exit is observable directly rather + // than through a PID probe that a zombie would answer wrongly. Uninstall + // waits out its own bounded stop grace, so allow for that before failing. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let exited = loop { + match server + .try_wait() + .expect("failed to poll the managed server") + { + Some(_) => break true, + None if std::time::Instant::now() >= deadline => break false, + None => std::thread::sleep(std::time::Duration::from_millis(100)), + } + }; + if !exited { + let _ = server.kill(); + let _ = server.wait(); + } + let out = &state(world).last_output; + assert!( + exited, + "uninstall reported success while the server it manages kept running:\n{out}" + ); +} + #[then("the installed rocm binary that ran uninstall is gone")] async fn then_running_rocm_gone(world: &mut E2eWorld) { let st = state(world); @@ -1638,6 +2067,39 @@ async fn then_rocmd_gone(world: &mut E2eWorld) { assert_missing(&installed_binary(&st.install_dir, "rocmd")); } +/// The isolated folders are the CLI's own roots (the `ROCM_CLI_*_DIR` values), +/// so a purge removes the folder itself, not a `rocm-cli` subfolder of it. +#[then("the isolated config, data, and cache folders are gone")] +async fn then_isolated_folders_gone(world: &mut E2eWorld) { + let st = state(world); + for dir in [&st.smoke_config, &st.smoke_data, &st.smoke_cache] + .into_iter() + .flatten() + { + assert_missing(dir); + } +} + +#[then("only the isolated config and cache folders are gone")] +async fn then_only_cache_gone(world: &mut E2eWorld) { + let st = state(world); + assert_missing(st.smoke_cache.as_ref().expect("no isolated cache dir")); + assert_missing(st.smoke_config.as_ref().expect("no isolated config dir")); + let data = st.smoke_data.as_ref().expect("no isolated data dir"); + assert!( + data.exists(), + "an uninstall keeping the data removed {}", + data.display() + ); +} + +#[then("the installed rocm and rocmd binaries are still present")] +async fn then_binaries_still_present(world: &mut E2eWorld) { + let st = state(world); + assert_exists(&installed_binary(&st.install_dir, "rocm")); + assert_exists(&installed_binary(&st.install_dir, "rocmd")); +} + #[then("the isolated XDG config, data, and cache state is gone")] async fn then_xdg_gone(world: &mut E2eWorld) { let st = state(world);