From 2ea3982d676deba6e3433b3ee0c52805c0aaca82 Mon Sep 17 00:00:00 2001 From: politerealism Date: Thu, 17 Sep 2026 19:17:59 -0400 Subject: [PATCH] fix(podman): resolve local callbacks to loopback on Linux Since RFC 0012 (#2942), the callback-capable supervisor always runs in the host network namespace, for both rootful and rootless Podman. The driver still requested a callback listener via bridge-network gateway inspection (rootful) or default-route negotiation (rootless pasta), and aliased host.containers.internal/host.openshell.internal through Podman's host-gateway resolver. On multi-homed hosts, Podman's host-gateway resolution and the driver's own negotiation can each pick a different interface than the one the supervisor actually shares with the gateway (loopback), leaving sandbox creation unreachable. Request loopback directly for the automatic local-callback path on Linux, and resolve the host aliases to 127.0.0.1 to match, regardless of rootful/rootless mode or which rootless network helper Podman reports. An explicit host_gateway_ip override is unaffected. macOS Podman Machine behavior (gvproxy host-loopback forwarding) is unchanged. Fixes #3412 Signed-off-by: politerealism --- architecture/gateway.md | 13 +- crates/openshell-driver-podman/NETWORKING.md | 11 +- crates/openshell-driver-podman/src/client.rs | 20 -- crates/openshell-driver-podman/src/config.rs | 13 +- .../openshell-driver-podman/src/container.rs | 63 ++++- crates/openshell-driver-podman/src/driver.rs | 240 ++++++------------ docs/reference/gateway-config.mdx | 8 +- docs/reference/sandbox-compute-drivers.mdx | 2 +- skills/debug-openshell-cluster/SKILL.md | 28 +- 9 files changed, 183 insertions(+), 215 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index f991e591bc..320e3efe7a 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -1026,13 +1026,12 @@ system entry instead of pretending to delete package-manager owned state. the host can bind that bridge IP. - Podman-backed macOS gateways use gvproxy's host-loopback IP for sandbox host aliases by default so stale Podman machine images do not need Podman's - `host-gateway` resolver. Linux Podman keeps the resolver unless - `host_gateway_ip` is configured. Rootful Podman can request its exact bridge - gateway listener. Rootless Podman explicitly reporting pasta requests the - private IPv4 source selected by the host default route rather than an - arbitrary private interface. Slirp4netns, other helpers, and missing helper - metadata fail closed for local callbacks until a rootless-network namespace - relay is available. + `host-gateway` resolver. Linux Podman resolves sandbox host aliases to + loopback directly by default, for both rootful and rootless operation: the + callback-capable supervisor always shares the host network namespace with + the gateway (RFC 0012), so loopback is reachable independent of host + topology and rootless network helper. An explicit `host_gateway_ip` + overrides either platform default with a concrete address. - Gateway restarts recover persisted objects from storage, but live relay streams must be re-established by supervisors. - User-facing behavior changes must update published docs in `docs/`; this file diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 4ad9eedae6..3f18d805c5 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -31,10 +31,13 @@ the upstream corporate proxy apply only to the supervisor. The gateway's SSH tunnel uses the supervisor relay over its private Unix socket, so the driver does not publish a supervisor port. -Rootful Podman uses the configured bridge and its gateway address. Rootless -local callbacks require the existing pasta path; slirp4netns or unknown helpers -require an explicitly remote `grpc_endpoint`. On macOS, Podman Machine provides -the runtime and host-loopback forwarding. +On Linux, the supervisor's host network namespace is the same as the +gateway's, for both rootful and rootless Podman, so the automatic callback +route resolves the sandbox host aliases to loopback directly rather than +Podman's `host-gateway` resolver, which can select a different, unreachable +interface on multi-homed hosts. Set `host_gateway_ip` explicitly to override +with a different address (for example, a containerized or remote gateway). +On macOS, Podman Machine provides the runtime and host-loopback forwarding. These runtime-managed network helpers are outside the workload trust boundary. Sharing the workload's user namespace preserves volume UID/GID mapping; it diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index be5dcea0b9..4455551018 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -767,26 +767,6 @@ impl PodmanClient { .await } - /// Inspect a network and return the gateway IP of its first subnet. - /// - /// The gateway IP is the host's address on the bridge network, used by - /// sandbox containers to call back to the gateway server. - pub async fn network_gateway_ip(&self, name: &str) -> Result, PodmanApiError> { - validate_name(name)?; - let encoded = url_encode(name); - let path = format!("/libpod/networks/{encoded}/json"); - let resp: Value = self.request_json(hyper::Method::GET, &path, None).await?; - // The response has "subnets": [{"gateway": "10.89.1.1", "subnet": "..."}] - let gateway = resp - .get("subnets") - .and_then(|s| s.as_array()) - .and_then(|arr| arr.first()) - .and_then(|sub| sub.get("gateway")) - .and_then(|g| g.as_str()) - .map(String::from); - Ok(gateway) - } - // ── Image operations ──────────────────────────────────────────────── /// Pull an image if it is not already present locally. diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index d0ca11669e..3d75ddd1dc 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -53,10 +53,15 @@ pub struct PodmanComputeConfig { pub network_name: String, /// Host gateway IP used for sandbox host aliases. /// - /// Empty uses Podman's `host-gateway` resolver. macOS defaults to - /// gvproxy's host-loopback IP because stale Podman machines may fail to - /// resolve `host-gateway` while still serving `host.containers.internal` - /// through gvproxy. + /// Empty on Linux resolves the alias to loopback directly: since RFC + /// 0012, the callback-capable supervisor always shares the host network + /// namespace with the gateway, so loopback is always reachable and + /// independent of host topology (see #3412 for the multi-homed-host + /// failure this replaced). Empty on macOS uses gvproxy's host-loopback + /// IP because stale Podman machines may fail to resolve `host-gateway` + /// while still serving `host.containers.internal` through gvproxy. Set + /// explicitly to override either default, e.g. for a containerized or + /// remote gateway. pub host_gateway_ip: String, /// Container stop timeout in seconds (SIGTERM → SIGKILL). pub stop_timeout_secs: u32, diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 4f502da8bc..4b18347996 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1612,16 +1612,32 @@ fn provider_spiffe_workload_api_socket_mount_source(config: &PodmanComputeConfig fn hostadd_entries(config: &PodmanComputeConfig) -> Vec { let host_gateway_ip = config.host_gateway_ip.trim(); - if host_gateway_ip.is_empty() { + if !host_gateway_ip.is_empty() { return vec![ - "host.containers.internal:host-gateway".into(), - "host.openshell.internal:host-gateway".into(), + format!("host.containers.internal:{host_gateway_ip}"), + format!("host.openshell.internal:{host_gateway_ip}"), ]; } + if cfg!(target_os = "linux") { + // Since RFC 0012 (#2942), the callback-capable supervisor always + // shares the host network namespace with the gateway. Resolve the + // alias to loopback directly instead of Podman's `host-gateway` + // magic value, which independently picks a host interface and can + // select a different, unreachable one on multi-homed hosts + // (#3412). `gateway_listener_requirements` requests a matching + // loopback listener for the same reason. + return vec![ + "host.containers.internal:127.0.0.1".into(), + "host.openshell.internal:127.0.0.1".into(), + ]; + } + + // Non-Linux (Podman Machine): gvproxy already forwards `host-gateway` + // to the VM host's loopback, so the magic value resolves correctly. vec![ - format!("host.containers.internal:{host_gateway_ip}"), - format!("host.openshell.internal:{host_gateway_ip}"), + "host.containers.internal:host-gateway".into(), + "host.openshell.internal:host-gateway".into(), ] } @@ -2600,7 +2616,44 @@ mod tests { } #[test] + #[cfg(target_os = "linux")] + fn container_spec_injects_host_aliases() { + // Since RFC 0012 (#2942), the supervisor shares the host network + // namespace with the gateway, so the automatic (no explicit + // host_gateway_ip) alias resolves directly to loopback rather than + // Podman's `host-gateway` magic value, which can select a + // different, unreachable interface on multi-homed hosts (#3412). + let sandbox = test_sandbox("test-id", "test-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let hostadd: Vec<&str> = spec["hostadd"] + .as_array() + .expect("hostadd should be an array") + .iter() + .filter_map(|v| v.as_str()) + .collect(); + + assert!( + hostadd.contains(&"host.containers.internal:127.0.0.1"), + "missing Podman host alias" + ); + assert!( + hostadd.contains(&"host.openshell.internal:127.0.0.1"), + "missing OpenShell stable host alias" + ); + assert!( + !hostadd.contains(&"host.docker.internal:127.0.0.1"), + "Podman should not inject Docker's host alias" + ); + } + + #[test] + #[cfg(not(target_os = "linux"))] fn container_spec_injects_host_aliases() { + // Podman Machine's gvproxy forwards the `host-gateway` magic value + // to the VM host's loopback correctly, so the automatic path keeps + // using it unchanged on non-Linux hosts. let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); let spec = build_container_spec(&sandbox, &config); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 62142a3157..494a39047c 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -21,9 +21,7 @@ use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, }; -#[cfg(target_os = "linux")] -use openshell_core::proto::compute::v1::GatewayDefaultRouteInterfaceRequirement; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "linux", target_os = "macos"))] use openshell_core::proto::compute::v1::GatewayLoopbackInterfaceRequirement; use openshell_core::proto::compute::v1::{ CpuResourceCapabilities, DriverSandbox, GatewayListenerRequirement, GetCapabilitiesResponse, @@ -75,9 +73,6 @@ impl From for ComputeDriverError { pub struct PodmanComputeDriver { client: PodmanClient, config: PodmanComputeConfig, - /// The host's IP on the bridge network, when that bridge exists in the - /// gateway's network namespace (notably rootful Podman). - network_gateway_ip: Option, /// Whether Podman's service is running without root privileges. rootless: bool, /// Rootless network helper reported by Podman, such as `pasta`. @@ -470,27 +465,8 @@ impl PodmanComputeDriver { ); } - // Ensure the bridge network exists. Inspect its gateway only when the - // selected Linux callback route will bind that exact address. client.ensure_network(&config.network_name).await?; - let uses_local_callback_alias = Url::parse(&config.grpc_endpoint) - .ok() - .as_ref() - .is_some_and(callback_endpoint_uses_local_alias); - let needs_network_gateway_ip = cfg!(target_os = "linux") - && uses_local_callback_alias - && !rootless - && config.host_gateway_ip.trim().is_empty(); - let network_gateway_ip = if needs_network_gateway_ip { - client.network_gateway_ip(&config.network_name).await? - } else { - None - }; - info!( - network = %config.network_name, - gateway_ip = ?network_gateway_ip, - "Bridge network ready" - ); + info!(network = %config.network_name, "Bridge network ready"); let (gpu_inventory, allow_all_default_gpu) = local_podman_gpu_selector_state(); if !gpu_inventory.is_empty() { @@ -503,7 +479,6 @@ impl PodmanComputeDriver { Ok(Self { client, config, - network_gateway_ip, rootless, rootless_network_cmd, gpu_selector: Arc::new(CdiGpuDefaultSelector::new( @@ -515,15 +490,6 @@ impl PodmanComputeDriver { }) } - /// The host's IP on the bridge network, if available. - /// - /// Used to request the exact rootful gateway callback listener when no - /// explicit host-gateway override is configured. - #[must_use] - pub fn network_gateway_ip(&self) -> Option<&str> { - self.network_gateway_ip.as_deref() - } - /// Report driver capabilities. pub fn capabilities(&self) -> Result { Ok(GetCapabilitiesResponse { @@ -584,39 +550,31 @@ impl PodmanComputeDriver { #[cfg(target_os = "linux")] { - if self.rootless { - validate_rootless_local_callback_helper(&self.rootless_network_cmd)?; - - if self.config.host_gateway_ip.trim().is_empty() { - return Ok(vec![GatewayListenerRequirement { - reason: - "Podman rootless pasta callback uses the host default-route interface" - .to_string(), - selector: Some(Selector::DefaultRouteInterface( - GatewayDefaultRouteInterfaceRequirement {}, - )), - }]); - } + if self.config.host_gateway_ip.trim().is_empty() { + // Since RFC 0012 (#2942), the callback-capable supervisor + // always runs in the host network namespace, for both + // rootful and rootless Podman. It therefore always shares + // the host's loopback with the gateway; request that + // listener directly rather than inspecting the (no longer + // callback-relevant) bridge network gateway or negotiating + // the host's default-route interface, either of which can + // resolve to a different, unreachable interface on + // multi-homed hosts (see #3412). + return Ok(vec![GatewayListenerRequirement { + reason: "Podman supervisor callback shares the host network namespace with the gateway".to_string(), + selector: Some(Selector::LoopbackInterface( + GatewayLoopbackInterfaceRequirement {}, + )), + }]); } - - let gateway_ip = if self.config.host_gateway_ip.trim().is_empty() { - self.network_gateway_ip.as_deref().ok_or_else(|| { - ComputeDriverError::Precondition(format!( - "Podman network '{}' did not report a host bridge gateway address for local callback alias '{}'", - self.config.network_name, - endpoint.host_str().unwrap_or_default() - )) - })? - } else { - self.config.host_gateway_ip.trim() - }; + let gateway_ip = self.config.host_gateway_ip.trim(); let gateway_ip = gateway_ip.parse::().map_err(|err| { ComputeDriverError::Precondition(format!( "Podman callback gateway address '{gateway_ip}' is invalid: {err}" )) })?; Ok(vec![GatewayListenerRequirement { - reason: format!("Podman network '{}' host gateway", self.config.network_name), + reason: "Podman explicit host gateway override".to_string(), selector: Some(Selector::ExactBindAddress( SocketAddr::new(gateway_ip, callback_port).to_string(), )), @@ -1646,7 +1604,6 @@ impl PodmanComputeDriver { Self { client, config, - network_gateway_ip: None, rootless: false, rootless_network_cmd: String::new(), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( @@ -1734,25 +1691,6 @@ fn callback_endpoint_uses_local_alias(endpoint: &Url) -> bool { .is_some_and(|host| matches!(host, "host.containers.internal" | "host.openshell.internal")) } -#[cfg(any(target_os = "linux", test))] -fn validate_rootless_local_callback_helper( - rootless_network_cmd: &str, -) -> Result<(), ComputeDriverError> { - let rootless_network_cmd = rootless_network_cmd.trim(); - if rootless_network_cmd == "pasta" { - return Ok(()); - } - - let reported = if rootless_network_cmd.is_empty() { - "" - } else { - rootless_network_cmd - }; - Err(ComputeDriverError::Precondition(format!( - "Podman rootless network helper '{reported}' does not support direct local gateway callbacks; configure pasta or use an explicitly remote grpc_endpoint" - ))) -} - // ── Sandbox binary extraction (userns fallback) ──────────────────────── async fn extract_sandbox_bin( @@ -2598,31 +2536,33 @@ mod tests { #[test] #[cfg(target_os = "linux")] - fn rootful_local_callback_alias_requests_discovered_network_gateway() { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + fn rootful_local_callback_alias_requests_loopback() { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { grpc_endpoint: "http://host.openshell.internal:17670".to_string(), ..PodmanComputeConfig::default() }); - driver.network_gateway_ip = Some("10.89.1.1".to_string()); let requirements = driver.gateway_listener_requirements().unwrap(); + // Since #2942, the supervisor always shares the host network + // namespace with the gateway, so the automatic path requests + // loopback directly instead of a bridge-network gateway address + // that may not be reachable on multi-homed hosts (#3412). assert_eq!(requirements.len(), 1); - assert_eq!( + assert!(matches!( requirements[0].selector, - Some(Selector::ExactBindAddress("10.89.1.1:17670".to_string())) - ); + Some(Selector::LoopbackInterface(_)) + )); } #[test] #[cfg(target_os = "linux")] - fn configured_host_gateway_overrides_discovered_network_gateway() { + fn configured_host_gateway_overrides_automatic_loopback() { let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { grpc_endpoint: "http://host.containers.internal:17670".to_string(), host_gateway_ip: "10.90.1.1".to_string(), ..PodmanComputeConfig::default() }); - driver.network_gateway_ip = Some("10.89.1.1".to_string()); driver.rootless = true; driver.rootless_network_cmd = "pasta".to_string(); @@ -2656,7 +2596,7 @@ mod tests { #[test] #[cfg(target_os = "linux")] - fn rootless_pasta_requests_default_route_interface() { + fn rootless_pasta_local_callback_alias_requests_loopback() { let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { grpc_endpoint: "http://host.openshell.internal:17670".to_string(), ..PodmanComputeConfig::default() @@ -2668,34 +2608,17 @@ mod tests { assert!(matches!( requirements[0].selector, - Some(Selector::DefaultRouteInterface(_)) + Some(Selector::LoopbackInterface(_)) )); } - #[test] - fn rootless_non_pasta_helpers_are_rejected() { - for (rootless_network_cmd, reported) in [ - ("slirp4netns", "slirp4netns"), - ("", ""), - ("unknown-helper", "unknown-helper"), - ] { - let err = validate_rootless_local_callback_helper(rootless_network_cmd).unwrap_err(); - - assert!(matches!(err, ComputeDriverError::Precondition(_))); - assert!(err.to_string().contains(reported)); - assert!(err.to_string().contains("configure pasta")); - assert!(err.to_string().contains("remote grpc_endpoint")); - } - } - - #[test] - fn rootless_pasta_is_accepted_for_local_callbacks() { - validate_rootless_local_callback_helper("pasta").unwrap(); - } - #[test] #[cfg(target_os = "linux")] - fn rootless_slirp_rejects_explicit_host_gateway_override() { + fn rootless_slirp_explicit_host_gateway_override_still_works() { + // The automatic path no longer depends on the rootless network + // helper at all, so an explicit override works regardless of which + // helper Podman reports (previously slirp4netns was rejected even + // with an explicit override). let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { grpc_endpoint: "http://host.openshell.internal:17670".to_string(), host_gateway_ip: "10.90.1.1".to_string(), @@ -2704,17 +2627,18 @@ mod tests { driver.rootless = true; driver.rootless_network_cmd = "slirp4netns".to_string(); - let err = driver.gateway_listener_requirements().unwrap_err(); + let requirements = driver.gateway_listener_requirements().unwrap(); - assert!(matches!(err, ComputeDriverError::Precondition(_))); - assert!(err.to_string().contains("slirp4netns")); + assert_eq!( + requirements[0].selector, + Some(Selector::ExactBindAddress("10.90.1.1:17670".to_string())) + ); } - #[cfg(target_os = "linux")] #[tokio::test] - async fn constructor_preserves_required_network_gateway_discovery_error() { - let (socket_path, _request_log, handle) = spawn_podman_stub( - "network-gateway-error", + async fn constructor_skips_network_gateway_discovery_for_remote_callback() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "remote-callback-no-network-gateway", vec![ StubResponse::new(StatusCode::OK, ""), StubResponse::new( @@ -2723,41 +2647,48 @@ mod tests { "host": { "cgroupVersion": "v2", "networkBackend": "netavark", - "security": {"rootless": false}, - "remoteSocket": {"path": "/run/podman/podman.sock"} - }, - "version": {"Version": "5.0.0"} + "security": {"rootless": false} + } }"#, ), StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new( - StatusCode::INTERNAL_SERVER_ERROR, - r#"{"message":"network gateway inspection failed"}"#, - ), ], ); let config = PodmanComputeConfig { socket_path: Some(socket_path.clone()), - grpc_endpoint: "http://host.containers.internal:8080".to_string(), + grpc_endpoint: "https://gateway.example.test:9443".to_string(), ..PodmanComputeConfig::default() }; - let Err(err) = PodmanComputeDriver::new(config).await else { - panic!("required network gateway discovery failure should prevent startup"); - }; + let driver = PodmanComputeDriver::new(config) + .await + .expect("remote callbacks must not require bridge gateway inspection"); - assert!( - err.to_string() - .contains("network gateway inspection failed"), - "unexpected startup error: {err}" - ); + assert!(driver.gateway_listener_requirements().unwrap().is_empty()); handle.await.expect("stub task should finish"); + assert_eq!( + request_log + .lock() + .expect("request log lock should not be poisoned") + .as_slice(), + [ + "GET /_ping".to_string(), + format!("GET {}", api_path("/libpod/info")), + format!("POST {}", api_path("/libpod/networks/create")), + ] + ); } + #[cfg(target_os = "linux")] #[tokio::test] - async fn constructor_skips_network_gateway_discovery_for_remote_callback() { + async fn constructor_skips_network_gateway_discovery_for_local_callback() { + // Regression coverage for #3412: the constructor must never inspect + // the bridge network's gateway address for the local callback alias + // anymore, on rootful or rootless, since the supervisor no longer + // reaches the gateway through that bridge (RFC 0012 host-netns + // supervisor). Only /_ping, /info, and network create should fire. let (socket_path, request_log, handle) = spawn_podman_stub( - "remote-callback-no-network-gateway", + "local-callback-no-network-gateway", vec![ StubResponse::new(StatusCode::OK, ""), StubResponse::new( @@ -2775,16 +2706,21 @@ mod tests { ); let config = PodmanComputeConfig { socket_path: Some(socket_path.clone()), - grpc_endpoint: "https://gateway.example.test:9443".to_string(), + grpc_endpoint: "http://host.containers.internal:8080".to_string(), + gateway_port: 8080, ..PodmanComputeConfig::default() }; let driver = PodmanComputeDriver::new(config) .await - .expect("remote callbacks must not require bridge gateway inspection"); + .expect("local callbacks must not require bridge gateway inspection"); - assert!(driver.network_gateway_ip().is_none()); - assert!(driver.gateway_listener_requirements().unwrap().is_empty()); + let requirements = driver.gateway_listener_requirements().unwrap(); + assert_eq!(requirements.len(), 1); + assert!(matches!( + requirements[0].selector, + Some(Selector::LoopbackInterface(_)) + )); handle.await.expect("stub task should finish"); assert_eq!( request_log @@ -2799,22 +2735,6 @@ mod tests { ); } - #[test] - #[cfg(target_os = "linux")] - fn rootful_local_callback_alias_requires_concrete_gateway_address() { - let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), - ..PodmanComputeConfig::default() - }); - - let err = driver.gateway_listener_requirements().unwrap_err(); - - assert!( - err.to_string() - .contains("did not report a host bridge gateway address") - ); - } - #[test] #[cfg(target_os = "macos")] fn podman_machine_callback_alias_requests_loopback_listener() { diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 04890f8843..de6d7bb3b1 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -766,8 +766,12 @@ grpc_endpoint = "https://host.containers.internal:17670" # The gateway overwrites gateway_port from bind_address at runtime. gateway_port = 17670 network_name = "openshell" -# Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. -# Set "" to force Podman's host-gateway resolver. +# Omit for the platform default. On Linux the supervisor always shares the +# gateway's host network namespace, so the default resolves to loopback +# regardless of host topology. On macOS Podman machine, the default is +# gvproxy's host-loopback IP (192.168.127.254). Set explicitly only to +# override with a different reachable address (e.g. a containerized or +# remote gateway). # host_gateway_ip = "192.168.127.254" ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index d24ff67ef4..f73efd4d33 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -274,7 +274,7 @@ stopped sandboxes alone. For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. -On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. +On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` resolves sandbox host aliases to loopback directly, for both rootful and rootless Podman: the callback-capable supervisor always shares the host network namespace with the gateway, so loopback is reachable regardless of host topology (this replaced Podman's `host-gateway` resolver, which could select an unreachable interface on multi-homed hosts). Set `host_gateway_ip` explicitly to override with a different address, for example a containerized or remote gateway. ### Podman Driver Config Mounts diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index c0a30f14a7..f62c8b7422 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -281,17 +281,21 @@ Common findings: probe and the companion supervisor's private health check. Do not add capabilities, attach a workload network, or disable the runtime seccomp profile. There is no sandbox nftables or nested-network setup to repair. -- Gateway exits before becoming healthy with a callback-listener discovery - error: inspect `podman info --debug`, the configured Podman network, and the - host's IPv4 default route. Rootless pasta uses the private source address - selected by that route; rootful Podman uses the bridge gateway address. -- Current gateways reuse the primary listener when it covers Podman's callback - address. If the primary does not cover that address, inspect the gateway - startup logs for the additional callback-only listener and its provenance. -- Rootless slirp4netns, another named helper, or missing helper metadata - requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip` - cannot bypass slirp4netns host-loopback isolation. Do not work around - discovery failures by broadening the primary gateway listener to `0.0.0.0`. +- The supervisor exits with a policy-fetch failure and the sandbox container + follows into `Error` with a misleading `ContainerExited: code 0` (the + workload's own exit state, not the supervisor's): check the + `openshell-supervisor-` container's own logs first, not the sandbox + container's. On a multi-homed host (multiple interfaces, VPN/mesh + adapters), this usually means the default local callback resolved to an + unreachable interface — see below. +- On Linux, the automatic (no explicit `host_gateway_ip`) callback route + resolves to loopback directly, for both rootful and rootless Podman: since + RFC 0012, the supervisor always shares the host network namespace with the + gateway, so loopback is reachable independent of host topology or rootless + network helper. Do not work around a mismatch by broadening the primary + gateway listener to `0.0.0.0`; set `host_gateway_ip` explicitly only when + the gateway itself is not on the same host (containerized or remote + gateway). When `userns` is configured (e.g. `userns = "auto"` or `userns = "keep-id"`): @@ -760,7 +764,7 @@ credential failures. | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | -| Gateway exits while resolving compute-driver listener requirements | The callback hostname is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | +| Gateway exits while resolving compute-driver listener requirements | The callback hostname is unsupported, or the selected address is not private/authorized | Gateway startup error, driver logs | | Admin, health, reflection, or HTTP request is denied on an additional Docker/Podman callback-only listener | Additional callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected | | Docker or Podman sandbox never registers | Wrong callback endpoint or supervisor startup failure | Gateway logs and sandbox container logs | | Docker GPU sandbox fails before startup | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` |