diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 66310ed4c2..7f279778c4 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -112,6 +112,7 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --namespace openshell --create-namespace \ --set server.disableTls=true \ --set server.telemetryEnabled=false \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --wait --timeout 5m kubectl wait --namespace openshell \ diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index d7ba5d9e38..7a458e7ef7 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -35,15 +35,28 @@ jobs: launchctl setenv OPENSHELL_COMPUTE_DRIVER vm launchctl setenv OPENSHELL_TELEMETRY_ENABLED "$OPENSHELL_TELEMETRY_ENABLED" - - name: Install and check status + - name: Install and check gateway status run: | set -euo pipefail curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh + openshell --version openshell status - sandbox="release-canary-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - openshell sandbox create --name "$sandbox" --detach - openshell sandbox exec --name "$sandbox" --no-tty -- true - openshell sandbox delete "$sandbox" + + # GitHub-hosted macOS runners do not expose the Hypervisor.framework + # support libkrun needs. Sandbox launch is covered by the VM E2E lane. + - name: Collect Homebrew diagnostics + if: failure() + run: | + set +e + brew services info openshell + for log in \ + "$(brew --prefix)/var/log/openshell/openshell-gateway.out.log" \ + "$(brew --prefix)/var/log/openshell/openshell-gateway.err.log"; do + if [ -f "$log" ]; then + echo "--- $log ---" + tail -n 300 "$log" + fi + done ubuntu: name: Ubuntu Docker @@ -68,7 +81,7 @@ jobs: set -euo pipefail curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh openshell status - sandbox="release-canary-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + sandbox="rc-${GITHUB_RUN_ID}" openshell sandbox create --name "$sandbox" --detach openshell sandbox exec --name "$sandbox" --no-tty -- true openshell sandbox delete "$sandbox" @@ -153,7 +166,7 @@ jobs: DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \ OPENSHELL_VERSION="$OPENSHELL_VERSION" \ OPENSHELL_TELEMETRY_ENABLED="$OPENSHELL_TELEMETRY_ENABLED" \ - CANARY_SANDBOX="release-canary-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + CANARY_SANDBOX="rc-${GITHUB_RUN_ID}" \ INSTALL_SH_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh" \ bash -s <<'EOF' set -euo pipefail @@ -235,7 +248,7 @@ jobs: - name: Create and exercise a sandbox run: | set -euo pipefail - sandbox="release-canary-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + sandbox="rc-${GITHUB_RUN_ID}" openshell sandbox create --name "$sandbox" --detach openshell sandbox exec --name "$sandbox" --no-tty -- true openshell sandbox delete "$sandbox" @@ -292,7 +305,9 @@ jobs: --version 0.0.0-dev \ --namespace "$RELEASE_NAMESPACE" --create-namespace \ --set server.disableTls=true \ + --set server.auth.allowUnauthenticatedUsers=true \ --set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}" \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --wait --timeout 5m - name: Verify gateway pod is Ready @@ -333,7 +348,7 @@ jobs: set -euo pipefail openshell gateway add http://127.0.0.1:8080 --local --name "$KIND_GATEWAY_NAME" openshell status - sandbox="release-canary-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + sandbox="rc-${GITHUB_RUN_ID}" openshell sandbox create --name "$sandbox" --detach openshell sandbox exec --name "$sandbox" --no-tty -- true openshell sandbox delete "$sandbox" diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 0077c86689..4a7ea2194c 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -51,8 +51,9 @@ creation. Drivers that enforce policy outside the standard supervisor fetch later revisions through `GetSandboxConfig` and acknowledge them through `ReportPolicyStatus`. Process-identity omissions are preserved across this boundary so every driver -can apply its native image or runtime defaults. Driver-requested listeners are -structurally validated and remain restricted to sandbox callback RPCs. +can apply its native image or runtime defaults. Drivers connect supervisors to +the operator-configured gateway endpoint; they do not request additional +gateway listeners. Canonical main-process support is part of the `ComputeDriver` contract. Every in-tree and extension driver must forward the exact specification; it is not an diff --git a/architecture/gateway.md b/architecture/gateway.md index b277c32daa..45e934d610 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -6,7 +6,7 @@ attachments; and asks compute runtimes to create or delete sandbox workloads. ## Responsibilities -- Authenticate clients and sandbox callbacks. +- Authenticate clients and sandbox supervisor sessions. - Serve gRPC APIs for sandbox lifecycle, provider management, policy updates, settings, logs, watch streams, and relay forwarding. - Serve HTTP endpoints for health, WebSocket tunnels, and edge-auth flows. @@ -49,7 +49,7 @@ versions fail before runtime construction, and driver settings belong only to Package lifecycle code may replace an exact package-generated v1 default, but it preserves edited configurations for explicit operator migration. -Gateway listener TLS and sandbox callback TLS are separate inputs. A selected +Gateway listener TLS and sandbox supervisor TLS are separate inputs. A selected local Docker, Podman, or VM driver requires a complete guest bundle whenever the gateway listener uses TLS; package-managed local TLS can supply that bundle. Kubernetes instead projects guest credentials through its configured Secret. @@ -141,24 +141,16 @@ template, provider, and service collection list RPCs use the same field with an all-workspaces marker. Platform-global policy operations omit both `sandbox` and `workspace_scope`, while sandbox policy operations require both. -Docker and Podman report the local address through which their sandboxes can -reach the gateway. When the primary listener covers that address, the gateway -reuses it; sandbox JWT authentication and its RPC allowlist remain the callback -authorization boundary. When the primary listener does not cover the address, -the gateway adds a callback-only listener. Additional callback listeners accept -only gRPC methods classified as sandbox-callable by the gateway's generated -authorization metadata. They reject user and administrator APIs, health, -reflection, and HTTP routes before normal request -authentication. The operator-configured primary listener retains the full -multiplexed API surface. - -The `rpc_auth` classification is also the source of truth for negotiated -listener exposure: marking an RPC as `sandbox` or `dual` makes it callable on -these listeners. Review such changes as both authorization and network-surface -changes. Listener requirements are currently authorized only for the built-in -Docker and Podman drivers. Operator-granted listener capabilities for external -drivers are tracked in -[#2539](https://github.com/NVIDIA/OpenShell/issues/2539). +Docker and Podman supervisors use host networking and connect through the +gateway's primary listener. On Linux, local supervisors use the primary +loopback endpoint. Sandbox JWT authentication and the generated sandbox RPC +allowlist remain the authorization boundary; the gateway does not negotiate or +bind compute-driver-specific listeners. + +The `rpc_auth` classification is the source of truth for supervisor access. +Marking an RPC as `sandbox` or `dual` makes it callable by an authenticated +sandbox principal on the primary listener. Review such changes as +authorization-surface changes. Operators can configure a gateway-wide gRPC request rate limit. The limit is applied only to gRPC API traffic after protocol multiplexing; health, metrics, @@ -1059,25 +1051,16 @@ system entry instead of pretending to delete package-manager owned state. - Gateway TLS and client certificate distribution are deployment concerns owned by the operator or packaging layer. -- Compute runtimes own the mechanics of starting workloads and injecting - callback configuration. Local Docker, Podman, and VM callback endpoints can - be derived from their fixed host aliases. Kubernetes requires an explicit - endpoint from driver placement; Helm renders it from the gateway Service - name and namespace rather than inferring it from sandbox placement. -- Docker-backed local gateways use Docker's `host-gateway` callback alias on - macOS and Docker Desktop-style runtimes. They request IPv4 loopback callback - reachability and add a listener only when the primary does not cover it. - Native Linux Docker may expose an additional bridge-gateway listener because - the host can bind that bridge IP. -- Podman-backed macOS gateways use gvproxy's host-loopback IP for sandbox host - aliases by default so stale Podman machine images do not need Podman's - `host-gateway` resolver. Linux Podman keeps the resolver unless - `host_gateway_ip` is configured. Rootful Podman can request its exact bridge - gateway listener. Rootless Podman explicitly reporting pasta requests the - private IPv4 source selected by the host default route rather than an - arbitrary private interface. Slirp4netns, other helpers, and missing helper - metadata fail closed for local callbacks until a rootless-network namespace - relay is available. +- Compute runtimes own the mechanics of starting workloads and injecting the + gateway endpoint. Docker and Podman supervisors use host networking; local + Linux supervisors use the gateway's primary loopback endpoint. Kubernetes + uses the gateway Service rendered by Helm. VM supervisors use their + runtime-specific host route. +- Docker Desktop requires host networking to be enabled and cannot combine it + with Enhanced Container Isolation. Set an explicit remote `grpc_endpoint` + when the gateway is not reachable on the Docker daemon host. +- Podman Machine uses gvproxy's host-loopback route on macOS. Native Linux + Podman uses the primary loopback endpoint. - Gateway restarts recover persisted objects from storage, but live relay streams must be re-established by supervisors. - User-facing behavior changes must update published docs in `docs/`; this file diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 74bbfb4aea..cf661d32c5 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -400,7 +400,7 @@ The canonical main process receives the declared workload environment before supervisor-only values are stripped and provider placeholders are injected. Template environment is treated like user-provided sandbox environment. It can shape the workload child, but it cannot override driver-controlled identity, -gateway callback, TLS, relay socket, proxy, provider, or supervisor coordination +gateway endpoint, TLS, relay socket, proxy, provider, or supervisor coordination variables. Drivers and the supervisor rewrite those reserved values after image and template environment are considered. diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 0ab1049533..33bff374e5 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -7,58 +7,6 @@ use std::path::{Path, PathBuf}; use crate::proto::compute::v1::DriverSandbox; -/// Built-in sandbox network callback routes used to derive a callback endpoint -/// when an operator does not configure a per-driver `grpc_endpoint` override. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GatewayCallbackRoute { - /// A Docker container reaches the host through Docker's gateway alias. - Docker, - /// A Podman container reaches the host through Podman's gateway alias. - Podman, - /// A libkrun guest reaches the host through gvproxy's gateway alias. - Vm, -} - -/// Build the endpoint a sandbox uses to call its gateway for a known route. -/// -/// The result is deliberately derived by the gateway rather than baked into -/// individual driver defaults. A configured `grpc_endpoint` remains an -/// operator override for remote or non-standard deployments. -#[must_use] -pub fn gateway_callback_endpoint( - route: GatewayCallbackRoute, - gateway_port: u16, - gateway_tls_enabled: bool, -) -> String { - let scheme = if gateway_tls_enabled { "https" } else { "http" }; - let host = match route { - GatewayCallbackRoute::Docker | GatewayCallbackRoute::Vm => "host.openshell.internal", - GatewayCallbackRoute::Podman => "host.containers.internal", - }; - format!("{scheme}://{host}:{gateway_port}") -} - -#[cfg(test)] -mod callback_endpoint_tests { - use super::{GatewayCallbackRoute, gateway_callback_endpoint}; - - #[test] - fn derives_endpoint_for_each_builtin_route() { - assert_eq!( - gateway_callback_endpoint(GatewayCallbackRoute::Docker, 17670, false), - "http://host.openshell.internal:17670" - ); - assert_eq!( - gateway_callback_endpoint(GatewayCallbackRoute::Podman, 17670, true), - "https://host.containers.internal:17670" - ); - assert_eq!( - gateway_callback_endpoint(GatewayCallbackRoute::Vm, 17670, true), - "https://host.openshell.internal:17670" - ); - } -} - // --------------------------------------------------------------------------- // Sandbox container/pod label keys (openshell.ai/ namespace) // --------------------------------------------------------------------------- diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index b82eb3f3d7..aa45b14745 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -35,9 +35,9 @@ mediates every supported TCP and DNS operation, attributes it to the calling binary, and sends the request across the private channel. The supervisor authorizes the request before it opens an upstream connection. Docker's absent workload network is the mandatory outer fence if mediation fails or is -bypassed. Only the trusted supervisor companion joins the driver-owned bridge, -where it originates approved egress and can resolve other services on that -network. +bypassed. The trusted supervisor companion uses Docker host networking, where +it reaches the gateway's primary loopback listener and originates approved +egress. The driver copies trusted runtime bytes from the configured supervisor image through the Docker archive API. No workload launch depends on a host bind @@ -73,7 +73,7 @@ LSM decisions remain authoritative. | `cap_drop = ALL`, no `cap_add`, no-new-privileges | Prevents either container from acquiring Linux capabilities. | | Docker default seccomp and AppArmor profiles | Retains runtime hardening; startup confirmation fails closed if nested seccomp notification is unavailable. | | `network_mode = none` on the workload | Removes direct external routes. | -| Driver-owned bridge on the supervisor | Lets the trusted supervisor originate approved gateway and upstream connections and use Docker service discovery. | +| `network_mode = host` on the supervisor | Lets the trusted supervisor reach the gateway's primary loopback listener and originate approved upstream connections. | | `restart_policy = no` | Keeps canonical main-process exit terminal. | | `PidsLimit` | Applies the configured sandbox PID budget. Omit `sandbox_pids_limit` to use OpenShell's default. Explicit zero is invalid. | | Private named volumes | One carries the authenticated sandbox/supervisor channel. The other is mounted only into the supervisor and contains its JWT and private gateway credentials. | @@ -128,22 +128,25 @@ dynamically linked glibc `/openshell-supervisor` binary that runs in the host-networked supervisor container. Release and gateway image builds bake matching image tags into the binary. -## Callback and TLS +## Gateway session and TLS `OPENSHELL_ENDPOINT` and gateway authentication material are injected only into the supervisor companion. The workload never receives the sandbox JWT, gateway client TLS key, policy authority, or interception CA private key. -When no endpoint is configured, the driver derives -`host.openshell.internal:`. Native Linux uses the managed bridge -gateway. Docker Desktop and compatible VM-backed daemons use Docker's -`host-gateway` route. A configured HTTPS server certificate must include the +When no endpoint is configured, the supervisor connects to +`127.0.0.1:`. Set `grpc_endpoint` when the gateway is not on the +Docker daemon host. A configured HTTPS server certificate must include the endpoint host in its subject alternative names. -The driver pins a concrete managed-bridge address in the sandbox descriptor. -For Docker's special `host-gateway` route, it leaves the address unpinned so -the supervisor resolves the driver-injected alias in its own container. This -avoids treating the Docker VM's loopback as the desktop host. +The driver publishes host loopback as the backend address for +`host.openshell.internal`. Policy DNS resolves that reserved name through the +mediated path, so policies can reach host services without a Docker bridge, +container DNS alias, or another gateway listener. + +Docker Engine on Linux supports host networking directly. Docker Desktop +requires host networking to be enabled in Settings and does not support it +when Enhanced Container Isolation is enabled. The supervisor owns these security-critical variables: diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 4230f75136..bdc1b27a64 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -13,8 +13,8 @@ use bollard::errors::Error as BollardError; use bollard::models::{ ContainerCreateBody, ContainerState, ContainerStateStatusEnum, ContainerSummary, ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, HealthConfig, HealthStatusEnum, - HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, - ProgressDetail, SystemInfo, VolumeCreateRequest, + HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, ProgressDetail, + SystemInfo, VolumeCreateRequest, }; use bollard::query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, @@ -26,11 +26,10 @@ use futures::{Stream, StreamExt}; use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - CONDITION_EXITED, CONDITION_RUNTIME_RESTART, GatewayCallbackRoute, LABEL_MANAGED_BY, - LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, - LABEL_SANDBOX_WORKSPACE, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, - gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, - validate_linux_elf_binary, + CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, + LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, + SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, + temp_extract_container_name, validate_linux_elf_binary, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -44,16 +43,14 @@ use openshell_core::proto::compute::v1::{ CpuResourceCapabilities, CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, DriverSandboxTemplate, - EnsureWorkspaceRequest, EnsureWorkspaceResponse, GatewayListenerRequirement, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - GpuResourceCapabilities, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, + EnsureWorkspaceRequest, EnsureWorkspaceResponse, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, GpuResourceCapabilities, + GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, MemoryResourceCapabilities, ResourceCapabilities, StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, - watch_sandboxes_event, + compute_driver_server::ComputeDriver, watch_sandboxes_event, }; use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, @@ -128,9 +125,8 @@ const BOUNDARY_CERTIFICATE_FILE: &str = "boundary-server.crt"; const BOUNDARY_PRIVATE_KEY_FILE: &str = "boundary-server.key"; const SUPERVISOR_AUTH_BUNDLE_FILE: &str = "supervisor-auth.json"; const START_GENERATION_FILE: &str = "start-generation"; -const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; +const HOST_OPEN_SHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; -const DOCKER_NETWORK_DRIVER: &str = "bridge"; fn provisioning_span( parent: &opentelemetry::Context, @@ -196,12 +192,6 @@ pub struct DockerComputeConfig { /// Host-side private key for Docker sandbox mTLS. pub guest_tls_key: Option, - /// Docker bridge network that sandbox containers join. - pub network_name: String, - - /// Host gateway IP used for sandbox host aliases. - pub host_gateway_ip: String, - /// Unix socket path used for interactive sandbox access. pub ssh_socket_path: String, @@ -251,7 +241,6 @@ impl DockerComputeConfig { openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) .map_err(Error::config)?; } - parse_optional_host_gateway_ip(&self.host_gateway_ip)?; if gateway_bind_address.port() == 0 { return Err(Error::config( "docker compute driver requires a fixed non-zero gateway bind port", @@ -275,8 +264,6 @@ impl Default for DockerComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), - host_gateway_ip: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, @@ -299,15 +286,11 @@ struct DockerDriverRuntimeConfig { default_image: String, image_pull_policy: ImagePullPolicy, sandbox_namespace: String, - network_name: String, - gateway_route: DockerGatewayRoute, - gateway_callback_bind_address: Option, stop_timeout_secs: u32, log_level: String, sandbox_binary: Arc>, supervisor_image_id: String, supervisor_grpc_endpoint: String, - gateway_tls_server_name: Option, ssh_socket_path: String, guest_tls: Option, daemon_version: String, @@ -325,12 +308,6 @@ struct DockerGpuRuntimeCapabilities { wsl_all_gpu_fallback_enabled: bool, } -#[derive(Debug, Clone, PartialEq, Eq)] -enum DockerGatewayRoute { - Bridge { bind_address: SocketAddr }, - HostGateway, -} - #[derive(Clone)] pub struct DockerComputeDriver { docker: Arc, @@ -883,46 +860,20 @@ impl DockerComputeDriver { "docker compute driver requires a fixed non-zero gateway bind port", )); } - let network_name = docker_network_name(docker_config); - let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; - let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; - let gateway_route = - docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); - let gateway_callback_bind_address = - docker_gateway_callback_bind_address(&gateway_route, gateway_bind_address); let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { - docker_config.grpc_endpoint = gateway_callback_endpoint( - GatewayCallbackRoute::Docker, + docker_config.grpc_endpoint = default_docker_supervisor_grpc_endpoint( gateway_port, docker_guest_tls_configured(&docker_config), ); } - let host_grpc_endpoint = - docker_host_openshell_endpoint(&docker_config.grpc_endpoint, &gateway_route)?; - let original_gateway_url = Url::parse(&docker_config.grpc_endpoint).map_err(|error| { + Url::parse(&docker_config.grpc_endpoint).map_err(|error| { Error::config(format!( "invalid docker grpc_endpoint '{}': {error}", docker_config.grpc_endpoint )) })?; - let host_gateway_url = Url::parse(&host_grpc_endpoint).map_err(|error| { - Error::config(format!( - "invalid normalized Docker host grpc_endpoint '{host_grpc_endpoint}': {error}" - )) - })?; - let gateway_tls_server_name = (original_gateway_url.scheme() == "https" - && original_gateway_url.host_str() != host_gateway_url.host_str()) - .then(|| { - original_gateway_url - .host_str() - .unwrap_or_default() - .to_string() - }); - let supervisor_grpc_endpoint = match &gateway_route { - DockerGatewayRoute::Bridge { .. } => host_grpc_endpoint, - DockerGatewayRoute::HostGateway => docker_config.grpc_endpoint.clone(), - }; + let supervisor_grpc_endpoint = docker_config.grpc_endpoint.clone(); let supervisor_image = docker_config .supervisor_image .clone() @@ -961,15 +912,11 @@ impl DockerComputeDriver { default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy, sandbox_namespace: docker_config.sandbox_label.clone(), - network_name, - gateway_route, - gateway_callback_bind_address, stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: gateway_log_level.to_string(), sandbox_binary, supervisor_image_id, supervisor_grpc_endpoint, - gateway_tls_server_name, ssh_socket_path: docker_config.ssh_socket_path.clone(), guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), @@ -2915,18 +2862,6 @@ impl ComputeDriver for ComputeDriverService { .await } - async fn get_gateway_listener_requirements( - &self, - request: Request, - ) -> Result, Status> { - self.rpc_tracer - .trace( - openshell_otel::rpc::GET_GATEWAY_LISTENER_REQUIREMENTS, - ComputeDriver::get_gateway_listener_requirements(&self.driver, request), - ) - .await - } - async fn validate_sandbox_create( &self, request: Request, @@ -3072,28 +3007,6 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(self.capabilities())) } - async fn get_gateway_listener_requirements( - &self, - _request: Request, - ) -> Result, Status> { - let requirements = - self.config - .gateway_callback_bind_address - .map_or_else(Vec::new, |bind_address| { - vec![GatewayListenerRequirement { - reason: match self.config.gateway_route { - DockerGatewayRoute::Bridge { .. } => "docker managed bridge gateway", - DockerGatewayRoute::HostGateway => "docker host-gateway IPv4 loopback", - } - .to_string(), - selector: Some(Selector::ExactBindAddress(bind_address.to_string())), - }] - }); - Ok(Response::new(GetGatewayListenerRequirementsResponse { - requirements, - })) - } - async fn validate_sandbox_create( &self, request: Request, @@ -4400,7 +4313,6 @@ async fn prepare_docker_boundary_files( Status::failed_precondition("Docker sandbox launch authentication is required") }) .and_then(|spec| decode_docker_launch_authentication(&spec.launch_authentication))?; - let host_gateway_ip = docker_boundary_host_gateway_ip(&config.gateway_route); let session_id = launch_authentication.supervisor.session_id; let tls = generate_sandbox_tls_material(session_id) .map_err(|error| Status::internal(format!("generate Docker boundary TLS: {error}")))?; @@ -4429,7 +4341,11 @@ async fn prepare_docker_boundary_files( server_name: tls.server_name.clone(), trust_anchor_pem: tls.trust_anchor_pem.clone(), }, - host_gateway_ip, + // Pin the reserved host alias to the same address used by the + // host-networked supervisor. This is normally loopback, but container + // CI reaches the gateway and host fixtures through the job + // container's bridge address. + host_gateway_ip: docker_supervisor_host_address(&config.supervisor_grpc_endpoint), workload_identity: workload_identity.clone(), child_env: docker_child_environment(sandbox), } @@ -4814,6 +4730,63 @@ fn docker_auxiliary_container_labels( ]) } +fn docker_supervisor_host_config(mounts: Vec, grpc_endpoint: &str) -> HostConfig { + HostConfig { + // The supervisor is trusted infrastructure and originates every + // approved upstream connection. Host networking lets it reach the + // configured gateway and host-side services directly; the workload + // remains fenced by network=none. + network_mode: Some("host".to_string()), + extra_hosts: docker_supervisor_host_aliases(grpc_endpoint), + mounts: Some(mounts), + cap_drop: Some(vec!["ALL".to_string()]), + cap_add: None, + security_opt: Some(vec!["no-new-privileges:true".to_string()]), + readonly_rootfs: Some(true), + tmpfs: Some(HashMap::from([ + ( + "/run".to_string(), + format!( + "rw,noexec,nosuid,size=64m,uid={SUPERVISOR_UID},gid={SUPERVISOR_GID},mode=0700" + ), + ), + ( + "/tmp".to_string(), + "rw,noexec,nosuid,size=64m,mode=1777".to_string(), + ), + ( + "/var/log".to_string(), + format!( + "rw,noexec,nosuid,size=64m,uid={SUPERVISOR_UID},gid={SUPERVISOR_GID},mode=0700" + ), + ), + ])), + restart_policy: None, + ..Default::default() + } +} + +fn docker_supervisor_host_aliases(grpc_endpoint: &str) -> Option> { + let address = docker_supervisor_host_address(grpc_endpoint)?; + Some(vec![ + format!("{HOST_OPEN_SHELL_INTERNAL}:{address}"), + format!("{HOST_DOCKER_INTERNAL}:{address}"), + ]) +} + +fn docker_supervisor_host_address(grpc_endpoint: &str) -> Option { + let endpoint = Url::parse(grpc_endpoint).ok()?; + match endpoint.host()? { + url::Host::Ipv4(address) => Some(IpAddr::V4(address)), + url::Host::Domain(domain) if domain.eq_ignore_ascii_case("localhost") => { + Some(IpAddr::V4(Ipv4Addr::LOCALHOST)) + } + // Docker's extra-host syntax for IPv6 differs across daemon versions. + // Leave IPv6 and named remote endpoints to normal name resolution. + url::Host::Ipv6(_) | url::Host::Domain(_) => None, + } +} + async fn spawn_docker_control_process( docker: &Docker, sandbox: &DriverSandbox, @@ -4877,12 +4850,6 @@ async fn spawn_docker_control_process( openshell_core::telemetry::enabled_env_value() ), ]; - if let Some(server_name) = config.gateway_tls_server_name.as_deref() { - environment.push(format!( - "{}={server_name}", - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME - )); - } if config.guest_tls.is_some() { environment.extend([ format!( @@ -4989,48 +4956,10 @@ async fn spawn_docker_control_process( start_period: Some(SUPERVISOR_HEALTH_START_PERIOD_NS), start_interval: Some(SUPERVISOR_HEALTH_INTERVAL_NS), }), - host_config: Some(HostConfig { - // The supervisor is trusted infrastructure and originates every - // approved upstream connection. The driver-owned bridge provides - // Docker DNS and service discovery while the workload remains - // fenced by network=none. - network_mode: Some(config.network_name.clone()), - mounts: Some(supervisor_mounts), - cap_drop: Some(vec!["ALL".to_string()]), - cap_add: None, - security_opt: Some(vec!["no-new-privileges:true".to_string()]), - readonly_rootfs: Some(true), - tmpfs: Some(HashMap::from([ - ( - "/run".to_string(), - format!( - "rw,noexec,nosuid,size=64m,uid={SUPERVISOR_UID},gid={SUPERVISOR_GID},mode=0700" - ), - ), - ( - "/tmp".to_string(), - "rw,noexec,nosuid,size=64m,mode=1777".to_string(), - ), - ( - "/var/log".to_string(), - format!( - "rw,noexec,nosuid,size=64m,uid={SUPERVISOR_UID},gid={SUPERVISOR_GID},mode=0700" - ), - ), - ])), - extra_hosts: Some(vec![ - format!( - "{HOST_OPENSHELL_INTERNAL}:{}", - docker_supervisor_host_alias(&config.gateway_route) - ), - format!( - "{HOST_DOCKER_INTERNAL}:{}", - docker_supervisor_host_alias(&config.gateway_route) - ), - ]), - restart_policy: None, - ..Default::default() - }), + host_config: Some(docker_supervisor_host_config( + supervisor_mounts, + &config.supervisor_grpc_endpoint, + )), ..Default::default() }; let created = docker @@ -5665,252 +5594,6 @@ fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<() Ok(()) } -fn docker_host_openshell_endpoint( - endpoint: &str, - route: &DockerGatewayRoute, -) -> CoreResult { - let mut url = Url::parse(endpoint) - .map_err(|error| Error::config(format!("invalid docker grpc_endpoint: {error}")))?; - if !matches!( - url.host_str(), - Some(HOST_OPENSHELL_INTERNAL | HOST_DOCKER_INTERNAL) - ) { - return Ok(url.to_string()); - } - let host = match route { - DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), - DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), - }; - url.set_host(Some(&host.to_string())).map_err(|error| { - Error::config(format!( - "failed to map Docker gateway alias to its host listener: {error}" - )) - })?; - Ok(url.to_string()) -} - -fn docker_supervisor_host_alias(route: &DockerGatewayRoute) -> String { - match route { - DockerGatewayRoute::Bridge { bind_address } => bind_address.ip().to_string(), - DockerGatewayRoute::HostGateway => "host-gateway".to_string(), - } -} - -fn docker_boundary_host_gateway_ip(route: &DockerGatewayRoute) -> Option { - match route { - DockerGatewayRoute::Bridge { bind_address } => Some(bind_address.ip()), - // Docker resolves this special alias inside the supervisor container. - // Pinning it to loopback would target the daemon VM rather than the - // desktop host on Docker Desktop and compatible runtimes. - DockerGatewayRoute::HostGateway => None, - } -} - -fn docker_network_name(config: &DockerComputeConfig) -> String { - let name = config.network_name.trim(); - if name.is_empty() { - return DEFAULT_DOCKER_NETWORK_NAME.to_string(); - } - name.to_string() -} - -fn parse_optional_host_gateway_ip(value: &str) -> CoreResult> { - let trimmed = value.trim(); - if trimmed.is_empty() { - return Ok(None); - } - - trimmed - .parse() - .map(Some) - .map_err(|err| Error::config(format!("invalid host_gateway_ip value '{trimmed}': {err}"))) -} - -fn docker_gateway_route( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, -) -> DockerGatewayRoute { - docker_gateway_route_for_host( - info, - bridge_gateway_ip, - port, - host_gateway_ip, - host_runtime_requires_host_gateway_alias(), - ) -} - -fn docker_gateway_route_for_host( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, - host_requires_host_gateway_alias: bool, -) -> DockerGatewayRoute { - if let Some(host_alias_ip) = host_gateway_ip { - return DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(host_alias_ip, port), - }; - } - - if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { - DockerGatewayRoute::HostGateway - } else { - DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(bridge_gateway_ip, port), - } - } -} - -fn docker_gateway_callback_bind_address( - route: &DockerGatewayRoute, - primary_bind_address: SocketAddr, -) -> Option { - match route { - DockerGatewayRoute::Bridge { bind_address, .. } => Some(*bind_address), - DockerGatewayRoute::HostGateway => match primary_bind_address.ip() { - IpAddr::V4(ip) if ip.is_unspecified() || ip == Ipv4Addr::LOCALHOST => None, - _ => Some(SocketAddr::new( - IpAddr::V4(Ipv4Addr::LOCALHOST), - primary_bind_address.port(), - )), - }, - } -} - -fn host_runtime_requires_host_gateway_alias() -> bool { - cfg!(target_os = "macos") -} - -/// Detect Docker Desktop and behaviourally compatible runtimes - Colima, -/// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing -/// constraint: the bridge gateway IP is reachable from inside containers but -/// not from the `OpenShell` server process running on the host, so callbacks -/// must traverse `host-gateway`. -/// -/// Each runtime is detected via the daemon's reported OS string or hostname, -/// supplemented by labels where the runtime publishes them. -fn uses_host_gateway_alias(info: &SystemInfo) -> bool { - let operating_system = info - .operating_system - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if operating_system.contains("docker desktop") { - return true; - } - - let name = info - .name - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if name.starts_with("colima") - || name.starts_with("lima-") - || name.starts_with("rancher-desktop") - || name.starts_with("orbstack") - { - return true; - } - - info.labels.as_ref().is_some_and(|labels| { - labels.iter().any(|label| { - label.starts_with("com.docker.desktop.") - || label.starts_with("dev.rancherdesktop.") - || label.starts_with("dev.orbstack.") - }) - }) -} - -async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { - match docker.inspect_network(network_name, None).await { - Ok(network) => return validate_bridge_network(network_name, &network), - Err(err) if !is_not_found_error(&err) => { - return Err(Error::execution(format!( - "failed to inspect Docker network '{network_name}': {err}" - ))); - } - Err(_) => {} - } - - docker - .create_network(NetworkCreateRequest { - name: network_name.to_string(), - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - attachable: Some(true), - labels: Some(HashMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }) - .await - .map(|_| ()) - .or_else(|err| { - if is_conflict_error(&err) { - Ok(()) - } else { - Err(Error::execution(format!( - "failed to create Docker network '{network_name}': {err}" - ))) - } - })?; - - let network = docker - .inspect_network(network_name, None) - .await - .map_err(|err| { - Error::execution(format!( - "failed to inspect Docker network '{network_name}' after create: {err}" - )) - })?; - validate_bridge_network(network_name, &network) -} - -fn validate_bridge_network( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - if network.driver.as_deref() != Some(DOCKER_NETWORK_DRIVER) { - return Err(Error::config(format!( - "Docker network '{network_name}' must use the '{DOCKER_NETWORK_DRIVER}' driver, found '{}'", - network.driver.as_deref().unwrap_or("unknown") - ))); - } - - docker_bridge_gateway_ip(network_name, network) -} - -fn docker_bridge_gateway_ip( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - let Some(configs) = network.ipam.as_ref().and_then(|ipam| ipam.config.as_ref()) else { - return Err(Error::config(format!( - "Docker bridge network '{network_name}' does not expose IPAM gateway configuration" - ))); - }; - - for config in configs { - let Some(gateway) = config.gateway.as_deref() else { - continue; - }; - let ip = gateway.parse::().map_err(|err| { - Error::config(format!( - "Docker bridge network '{network_name}' has invalid gateway '{gateway}': {err}" - )) - })?; - if matches!(ip, IpAddr::V4(_)) { - return Ok(ip); - } - } - - Err(Error::config(format!( - "Docker bridge network '{network_name}' does not have an IPv4 IPAM gateway" - ))) -} - fn docker_resource_limits( template: &DriverSandboxTemplate, ) -> Result { @@ -6531,6 +6214,11 @@ fn docker_guest_tls_configured(docker_config: &DockerComputeConfig) -> bool { && docker_config.guest_tls_key.is_some() } +fn default_docker_supervisor_grpc_endpoint(gateway_port: u16, tls: bool) -> String { + let scheme = if tls { "https" } else { "http" }; + format!("{scheme}://127.0.0.1:{gateway_port}") +} + pub(crate) fn docker_guest_tls_paths( docker_config: &DockerComputeConfig, ) -> CoreResult> { @@ -6592,16 +6280,6 @@ fn is_not_found_error(err: &BollardError) -> bool { ) } -fn is_conflict_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) -} - fn is_removal_in_progress_error(err: &BollardError) -> bool { matches!( err, @@ -6642,4 +6320,3 @@ fn internal_status(operation: &str, err: BollardError) -> Status { #[cfg(test)] mod tests; -pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index a5ca29887a..8630d37c11 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -22,7 +22,7 @@ struct Args { #[arg(long, env = "OPENSHELL_DOCKER_DRIVER_CONFIG")] config: PathBuf, - /// Gateway listener address used to derive sandbox callback routing. + /// Gateway listener address used to derive the supervisor endpoint. #[arg( long, env = "OPENSHELL_GATEWAY_BIND", diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 21412a4d25..5b767979c7 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 use super::*; -use openshell_core::config::DEFAULT_SERVER_PORT; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, @@ -17,12 +16,10 @@ use openshell_core::progress::{ PROGRESS_STEP_STARTING_SANDBOX, }; use openshell_core::proto::compute::v1::{ - DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, - GetGatewayListenerRequirementsRequest, GpuResourceRequirements, ResourceRequirements, - WorkloadIdentityRequest, gateway_listener_requirement::Selector, + DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, GpuResourceRequirements, + ResourceRequirements, WorkloadIdentityRequest, }; use std::fs; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; use tempfile::TempDir; @@ -124,23 +121,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), image_pull_policy: ImagePullPolicy::IfNotPresent, sandbox_namespace: "default".to_string(), - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), - gateway_route: DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new( - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - ), - }, - gateway_callback_bind_address: Some(SocketAddr::new( - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - )), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: "info".to_string(), sandbox_binary: Arc::new(b"\x7fELFtest".to_vec()), supervisor_image_id: "sha256:supervisor-test".to_string(), supervisor_grpc_endpoint: "https://host.openshell.internal:8443".to_string(), - gateway_tls_server_name: None, ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), @@ -941,359 +926,6 @@ async fn tracing_in_process_stream_leaves_status_unset_when_dropped() { provider.shutdown().unwrap(); } -#[tokio::test] -async fn gateway_listener_requirements_report_managed_bridge_address() { - let config = runtime_config(); - let expected_address = match config.gateway_route { - DockerGatewayRoute::Bridge { bind_address, .. } => bind_address, - DockerGatewayRoute::HostGateway => panic!("test config must use a managed bridge"), - }; - let driver = test_driver_with_config(config); - - let response = driver - .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) - .await - .unwrap() - .into_inner(); - - assert_eq!(response.requirements.len(), 1); - assert_eq!( - response.requirements[0].selector, - Some(Selector::ExactBindAddress(expected_address.to_string())) - ); -} - -#[tokio::test] -async fn gateway_listener_requirements_are_empty_for_host_gateway_route() { - let mut config = runtime_config(); - config.gateway_route = DockerGatewayRoute::HostGateway; - config.gateway_callback_bind_address = None; - let driver = test_driver_with_config(config); - - let response = driver - .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) - .await - .unwrap() - .into_inner(); - - assert!(response.requirements.is_empty()); -} - -#[tokio::test] -async fn host_gateway_route_reports_ipv4_loopback_callback_listener() { - let mut config = runtime_config(); - config.gateway_route = DockerGatewayRoute::HostGateway; - config.gateway_callback_bind_address = Some("127.0.0.1:17670".parse().unwrap()); - let driver = test_driver_with_config(config); - - let response = driver - .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) - .await - .unwrap() - .into_inner(); - - assert_eq!(response.requirements.len(), 1); - assert_eq!( - response.requirements[0].selector, - Some(Selector::ExactBindAddress("127.0.0.1:17670".to_string())) - ); -} - -#[test] -fn docker_bridge_gateway_ip_requires_ipv4_gateway() { - let network = bollard::models::NetworkInspect { - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - ipam: Some(bollard::models::Ipam { - config: Some(vec![ - bollard::models::IpamConfig { - gateway: Some("fd00::1".to_string()), - ..Default::default() - }, - bollard::models::IpamConfig { - gateway: Some("172.18.0.1".to_string()), - ..Default::default() - }, - ]), - ..Default::default() - }), - ..Default::default() - }; - - assert_eq!( - docker_bridge_gateway_ip(DEFAULT_DOCKER_NETWORK_NAME, &network).unwrap(), - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)) - ); - - let ipv6_only_network = bollard::models::NetworkInspect { - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - ipam: Some(bollard::models::Ipam { - config: Some(vec![bollard::models::IpamConfig { - gateway: Some("fd00::1".to_string()), - ..Default::default() - }]), - ..Default::default() - }), - ..Default::default() - }; - - assert!( - docker_bridge_gateway_ip(DEFAULT_DOCKER_NETWORK_NAME, &ipv6_only_network) - .unwrap_err() - .to_string() - .contains("IPv4 IPAM gateway") - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_docker_desktop() { - let info = SystemInfo { - operating_system: Some("Docker Desktop".to_string()), - labels: Some(vec![ - "com.docker.desktop.address=unix:///tmp/docker.sock".to_string(), - ]), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn vm_backed_docker_daemon_uses_daemon_local_companion_transport() { - let desktop = SystemInfo { - operating_system: Some("Docker Desktop".to_string()), - ..Default::default() - }; - let native = SystemInfo { - operating_system: Some("Ubuntu 24.04".to_string()), - ..Default::default() - }; - - assert!(uses_host_gateway_alias(&desktop)); - assert!(!uses_host_gateway_alias(&native)); -} - -#[test] -fn host_gateway_route_requests_ipv4_loopback_for_ipv6_primary() { - assert_eq!( - docker_gateway_callback_bind_address( - &DockerGatewayRoute::HostGateway, - "[::1]:17670".parse().unwrap(), - ), - Some("127.0.0.1:17670".parse().unwrap()) - ); -} - -#[test] -fn host_gateway_route_reuses_ipv4_primary_when_it_covers_loopback() { - for primary in ["127.0.0.1:17670", "0.0.0.0:17670"] { - assert_eq!( - docker_gateway_callback_bind_address( - &DockerGatewayRoute::HostGateway, - primary.parse().unwrap(), - ), - None, - "{primary} already covers the IPv4 loopback callback" - ); - } -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_colima() { - let info = SystemInfo { - name: Some("colima".to_string()), - operating_system: Some("Ubuntu 24.04.4 LTS".to_string()), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 20, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_colima_named_profile() { - let info = SystemInfo { - operating_system: Some("Ubuntu 24.04 LTS".to_string()), - // `colima start --profile ` sets the daemon hostname to - // `colima-`; the prefix match still catches it. - name: Some("colima-default".to_string()), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_rancher_desktop() { - let info = SystemInfo { - operating_system: Some("Alpine Linux v3.20".to_string()), - name: Some("lima-rancher-desktop".to_string()), - labels: Some(vec![ - "dev.rancherdesktop.profile=Rancher Desktop".to_string(), - ]), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_orbstack() { - let info = SystemInfo { - operating_system: Some("OrbStack".to_string()), - name: Some("orbstack".to_string()), - labels: Some(vec!["dev.orbstack.machine_type=docker".to_string()]), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_uses_bridge_gateway_for_linux_docker() { - let info = SystemInfo { - operating_system: Some("Ubuntu 24.04 LTS".to_string()), - ..Default::default() - }; - - let route = docker_gateway_route_for_host( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - false, - ); - - assert_eq!( - route, - DockerGatewayRoute::Bridge { - bind_address: "172.18.0.1:17670".parse().unwrap(), - } - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_when_host_runtime_requires_it() { - let info = SystemInfo { - operating_system: Some("Ubuntu 24.04 LTS".to_string()), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route_for_host( - &info, - IpAddr::V4(Ipv4Addr::new(10, 89, 10, 1)), - DEFAULT_SERVER_PORT, - None, - true, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_prefers_configured_host_gateway_ip() { - let info = SystemInfo { - operating_system: Some("Ubuntu 24.04 LTS".to_string()), - ..Default::default() - }; - - let route = docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - Some(IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4))), - ); - - assert_eq!( - route, - DockerGatewayRoute::Bridge { - bind_address: "172.20.0.4:17670".parse().unwrap(), - } - ); -} - -#[test] -fn docker_supervisor_alias_matches_the_trusted_gateway_route() { - assert_eq!( - docker_supervisor_host_alias(&DockerGatewayRoute::Bridge { - bind_address: "172.20.0.4:17670".parse().unwrap(), - }), - "172.20.0.4" - ); - assert_eq!( - docker_supervisor_host_alias(&DockerGatewayRoute::HostGateway), - "host-gateway" - ); -} - -#[test] -fn docker_boundary_pins_only_concrete_host_gateway_addresses() { - assert_eq!( - docker_boundary_host_gateway_ip(&DockerGatewayRoute::Bridge { - bind_address: "172.20.0.4:17670".parse().unwrap(), - }), - Some(IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4))) - ); - assert_eq!( - docker_boundary_host_gateway_ip(&DockerGatewayRoute::HostGateway), - None - ); -} - -#[test] -fn parse_optional_host_gateway_ip_rejects_invalid_values() { - assert_eq!(parse_optional_host_gateway_ip("").unwrap(), None); - assert_eq!( - parse_optional_host_gateway_ip("172.20.0.4").unwrap(), - Some(IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4))) - ); - assert!( - parse_optional_host_gateway_ip("not-an-ip") - .unwrap_err() - .to_string() - .contains("host_gateway_ip") - ); -} - #[test] fn parse_cpu_limit_supports_cores_and_millicores() { assert_eq!(parse_cpu_limit("250m").unwrap(), Some(250_000_000)); @@ -2902,6 +2534,62 @@ fn build_container_create_body_disables_docker_networking() { assert_eq!(host_config.dns, Some(vec!["127.0.0.53".to_string()])); } +#[test] +fn docker_supervisor_uses_host_network() { + let host = docker_supervisor_host_config(Vec::new(), "https://127.0.0.1:17670"); + + assert_eq!(host.network_mode.as_deref(), Some("host")); + assert_eq!( + host.extra_hosts, + Some(vec![ + "host.openshell.internal:127.0.0.1".to_string(), + "host.docker.internal:127.0.0.1".to_string(), + ]) + ); + assert_eq!(host.cap_drop, Some(vec!["ALL".to_string()])); + assert_eq!(host.cap_add, None); +} + +#[test] +fn docker_supervisor_maps_host_aliases_to_the_gateway_address() { + let host = docker_supervisor_host_config(Vec::new(), "https://172.20.0.4:17670"); + + assert_eq!( + host.extra_hosts, + Some(vec![ + "host.openshell.internal:172.20.0.4".to_string(), + "host.docker.internal:172.20.0.4".to_string(), + ]) + ); + assert_eq!( + docker_supervisor_host_address("https://172.20.0.4:17670"), + Some("172.20.0.4".parse().unwrap()) + ); +} + +#[test] +fn docker_supervisor_leaves_named_gateway_hosts_to_dns() { + let host = docker_supervisor_host_config(Vec::new(), "https://gateway.example.com:17670"); + + assert_eq!(host.extra_hosts, None); + assert_eq!( + docker_supervisor_host_address("https://gateway.example.com:17670"), + None + ); +} + +#[test] +fn docker_supervisor_defaults_to_the_primary_loopback_endpoint() { + assert_eq!( + default_docker_supervisor_grpc_endpoint(17_670, false), + "http://127.0.0.1:17670" + ); + assert_eq!( + default_docker_supervisor_grpc_endpoint(17_670, true), + "https://127.0.0.1:17670" + ); +} + #[test] fn build_container_create_body_limits_writable_runtime_storage_to_supervisor_ca() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 82d54ecea0..65c6a8b85b 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -8,8 +8,7 @@ use openshell_core::proto::compute::v1::{ AuthenticateSandboxRequest, AuthenticateSandboxResponse, CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, EnsureWorkspaceResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, @@ -83,22 +82,6 @@ impl ComputeDriver for ComputeDriverService { .await } - async fn get_gateway_listener_requirements( - &self, - _request: Request, - ) -> Result, Status> { - self.rpc_tracer - .trace( - openshell_otel::rpc::GET_GATEWAY_LISTENER_REQUIREMENTS, - async { - Ok(Response::new(GetGatewayListenerRequirementsResponse { - requirements: Vec::new(), - })) - }, - ) - .await - } - async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-mxc/src/grpc.rs b/crates/openshell-driver-mxc/src/grpc.rs index 9a166057ef..004fca381f 100644 --- a/crates/openshell-driver-mxc/src/grpc.rs +++ b/crates/openshell-driver-mxc/src/grpc.rs @@ -12,8 +12,7 @@ use openshell_core::proto::compute::v1::{ AuthenticateSandboxRequest, AuthenticateSandboxResponse, CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, EnsureWorkspaceResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, @@ -51,17 +50,6 @@ impl ComputeDriver for ComputeDriverService { )) } - async fn get_gateway_listener_requirements( - &self, - _request: Request, - ) -> Result, Status> { - // MXC is an in-process, single-host driver: it needs no extra gateway - // listeners (no relay/surrogate/remote endpoint), so it reports none. - Ok(Response::new(GetGatewayListenerRequirementsResponse { - requirements: Vec::new(), - })) - } - async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 4ad9eedae6..3e52cccd3c 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -9,7 +9,7 @@ environment variable. workload container supervisor container agent -> sandbox -- private UDS / gRPC -> policy proxy -> host network -> destination | - +-- authenticated gateway callback + +-- authenticated gateway session ``` ## Outer network fence @@ -24,17 +24,13 @@ TCP opens, TCP byte streams, DNS requests/replies, and lifecycle operations share the authenticated gRPC channel. DNS is resolved and authorized by the supervisor. General UDP is unsupported. -## Supervisor callback network +## Supervisor network -The supervisor companion uses Podman's host network. Host-gateway aliases and -the upstream corporate proxy apply only to the supervisor. The gateway's SSH -tunnel uses the supervisor relay over its private Unix socket, so the driver -does not publish a supervisor port. - -Rootful Podman uses the configured bridge and its gateway address. Rootless -local callbacks require the existing pasta path; slirp4netns or unknown helpers -require an explicitly remote `grpc_endpoint`. On macOS, Podman Machine provides -the runtime and host-loopback forwarding. +The supervisor companion uses Podman's host network. On Linux it connects to +the gateway's primary loopback endpoint. On macOS, Podman Machine provides the +host-loopback route. The upstream corporate proxy applies only to the +supervisor. The gateway's SSH tunnel uses the supervisor relay over its private +Unix socket, so the driver does not publish a supervisor port. These runtime-managed network helpers are outside the workload trust boundary. Sharing the workload's user namespace preserves volume UID/GID mapping; it @@ -50,8 +46,8 @@ Inspect both containers with the same sandbox-ID label, distinguishing kernel/runtime primitive. Do not add capabilities or disable runtime seccomp. - Sandbox cannot authenticate to supervisor: check the private channel volume, matching user namespace mappings, and shared SELinux label. -- Supervisor cannot call back: inspect its configured gateway endpoint, - credentials, host network, and gateway callback listener. +- Supervisor cannot connect: inspect its configured gateway endpoint, + credentials, host network, and the gateway's primary listener. - DNS or egress denied: inspect supervisor policy decisions. Do not add a workload network, resolver bypass, or direct gateway route. - Pair is not Ready: check the supervisor health socket and gateway session. diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 912a79ac7a..85cdb33259 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -60,10 +60,9 @@ The channel contains the sandbox bootstrap and sandbox-side TLS identity only. Supervisor private keys and the runtime descriptor stay in the supervisor's private filesystem. Landlock denies agent access to the top-level `/.openshell` control hierarchy. The driver verifies Podman's reported `network=none` fence before launch and -restart. `host.containers.internal` and callback networking apply to the -supervisor, not the agent. +restart. Host networking applies to the supervisor, not the agent. -Gateway callbacks use the existing sandbox JWT and optional configured mTLS +Gateway sessions use the existing sandbox JWT and optional configured mTLS bundle. The sandbox/supervisor channel always uses its separate, per-sandbox mutual TLS material. These are distinct authentication relationships. @@ -112,7 +111,7 @@ operator's `enable_bind_mounts` opt-in. Reserved control paths and the workspace root cannot be replaced. User-owned volumes are never created or deleted. See [gateway configuration](../../docs/reference/gateway-config.mdx) for -operator settings and [NETWORKING.md](NETWORKING.md) for callback networking. +operator settings and [NETWORKING.md](NETWORKING.md) for supervisor networking. The supervisor uses Podman's host network and owns the upstream proxy settings. Omit `health_check_interval_secs` to disable Podman's periodic health command. Explicit zero is invalid. OpenShell still gates readiness on the supervisor's diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index be5dcea0b9..4455551018 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -767,26 +767,6 @@ impl PodmanClient { .await } - /// Inspect a network and return the gateway IP of its first subnet. - /// - /// The gateway IP is the host's address on the bridge network, used by - /// sandbox containers to call back to the gateway server. - pub async fn network_gateway_ip(&self, name: &str) -> Result, PodmanApiError> { - validate_name(name)?; - let encoded = url_encode(name); - let path = format!("/libpod/networks/{encoded}/json"); - let resp: Value = self.request_json(hyper::Method::GET, &path, None).await?; - // The response has "subnets": [{"gateway": "10.89.1.1", "subnet": "..."}] - let gateway = resp - .get("subnets") - .and_then(|s| s.as_array()) - .and_then(|arr| arr.first()) - .and_then(|sub| sub.get("gateway")) - .and_then(|g| g.as_str()) - .map(String::from); - Ok(gateway) - } - // ── Image operations ──────────────────────────────────────────────── /// Pull an image if it is not already present locally. diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index d0ca11669e..3812367234 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -36,8 +36,8 @@ pub struct PodmanComputeConfig { pub image_pull_policy: ImagePullPolicy, /// Gateway gRPC endpoint the sandbox connects back to. /// - /// When empty, the driver auto-detects the endpoint using - /// `gateway_port` and `host.containers.internal`. + /// When empty, the driver selects loopback on Linux or + /// `host.containers.internal` with Podman Machine, using `gateway_port`. pub grpc_endpoint: String, /// Port the gateway server is actually listening on. /// @@ -48,8 +48,7 @@ pub struct PodmanComputeConfig { pub gateway_port: u16, /// Unix socket path the in-container supervisor bridges relay traffic to. pub ssh_socket_path: String, - /// Name of the Podman bridge network. - /// Created automatically if it does not exist. + /// Name of the Podman bridge network used for driver-managed resources. pub network_name: String, /// Host gateway IP used for sandbox host aliases. /// diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 4f502da8bc..572f741ac2 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1086,9 +1086,7 @@ fn build_base_spec( .collect() }); - // Network configuration -- always bridge mode. - // Matches libpod's network spec format `{name: {opts}}`; the unit-struct - // value mirrors empty opts in the JSON. + // The isolation roles override this base network configuration below. #[allow(clippy::zero_sized_map_values)] let mut networks = BTreeMap::new(); networks.insert(config.network_name.clone(), NetworkAttachment {}); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 32419fdf3e..bacc58769c 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -13,35 +13,69 @@ use crate::watcher::{ use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; use openshell_core::driver_utils::{ - GatewayCallbackRoute, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, - gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, - validate_linux_elf_binary, write_cache_binary_atomic, + SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, + temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, }; -#[cfg(target_os = "linux")] -use openshell_core::proto::compute::v1::GatewayDefaultRouteInterfaceRequirement; -#[cfg(target_os = "macos")] -use openshell_core::proto::compute::v1::GatewayLoopbackInterfaceRequirement; use openshell_core::proto::compute::v1::{ - CpuResourceCapabilities, DriverSandbox, GatewayListenerRequirement, GetCapabilitiesResponse, - GpuResourceCapabilities, GpuResourceRequirements, MemoryResourceCapabilities, - ResourceCapabilities, gateway_listener_requirement::Selector, + CpuResourceCapabilities, DriverSandbox, GetCapabilitiesResponse, GpuResourceCapabilities, + GpuResourceRequirements, MemoryResourceCapabilities, ResourceCapabilities, }; use std::collections::HashMap; -#[cfg(target_os = "linux")] -use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use tracing::{Instrument as _, debug, info, warn}; -use url::Url; const STOP_COMPLETION_POLL_INTERVAL: Duration = Duration::from_millis(50); const STOP_COMPLETION_TIMEOUT_HEADROOM: Duration = Duration::from_secs(5); +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PodmanEndpointEnvironment { + LinuxHost, + PodmanMachine, +} + +impl PodmanEndpointEnvironment { + const fn current() -> Self { + if cfg!(target_os = "linux") { + Self::LinuxHost + } else { + Self::PodmanMachine + } + } + + const fn gateway_host(self) -> &'static str { + match self { + Self::LinuxHost => "127.0.0.1", + Self::PodmanMachine => "host.containers.internal", + } + } +} + +fn select_grpc_endpoint( + config: &PodmanComputeConfig, + environment: PodmanEndpointEnvironment, +) -> String { + if !config.grpc_endpoint.is_empty() { + return config.grpc_endpoint.clone(); + } + + let scheme = if config.tls_enabled() { + "https" + } else { + "http" + }; + format!( + "{scheme}://{}:{}", + environment.gateway_host(), + config.gateway_port + ) +} + fn decode_launch_authentication( encoded: &[u8], ) -> Result { @@ -75,13 +109,8 @@ impl From for ComputeDriverError { pub struct PodmanComputeDriver { client: PodmanClient, config: PodmanComputeConfig, - /// The host's IP on the bridge network, when that bridge exists in the - /// gateway's network namespace (notably rootful Podman). - network_gateway_ip: Option, /// Whether Podman's service is running without root privileges. rootless: bool, - /// Rootless network helper reported by Podman, such as `pasta`. - rootless_network_cmd: String, gpu_selector: Arc, gpu_inventory_refresh: Arc (CdiGpuInventory, bool) + Send + Sync>, lifecycle_event_fences: LifecycleEventFences, @@ -94,7 +123,6 @@ impl std::fmt::Debug for PodmanComputeDriver { .field("default_image", &self.config.default_image) .field("network_name", &self.config.network_name) .field("rootless", &self.rootless) - .field("rootless_network_cmd", &self.rootless_network_cmd) .field("gpu_inventory", &self.gpu_selector.device_ids()) .finish() } @@ -417,7 +445,7 @@ impl PodmanComputeDriver { } // Verify cgroups v2, detect rootless mode, and log system info. - let (rootless, rootless_network_cmd) = match client.system_info().await { + let rootless = match client.system_info().await { Ok(info) => { if info.host.cgroup_version != "v2" { return Err(PodmanApiError::Connection(format!( @@ -439,7 +467,7 @@ impl PodmanComputeDriver { apparmor_enabled = info.host.security.apparmor_enabled, "Connected to Podman" ); - (info.host.security.rootless, info.host.rootless_network_cmd) + info.host.security.rootless } Err(e) => { return Err(PodmanApiError::Connection(format!( @@ -455,14 +483,12 @@ impl PodmanComputeDriver { check_subuid_range(); } - // Auto-detect the gRPC callback endpoint before deciding whether this - // callback route needs the Podman bridge gateway address. - if config.grpc_endpoint.is_empty() { - config.grpc_endpoint = gateway_callback_endpoint( - GatewayCallbackRoute::Podman, - config.gateway_port, - config.tls_enabled(), - ); + // The supervisor shares the Podman host network. Linux supervisors can + // therefore use gateway loopback directly; Podman Machine retains its + // standard desktop-host alias. + let endpoint_was_selected = config.grpc_endpoint.is_empty(); + config.grpc_endpoint = select_grpc_endpoint(&config, PodmanEndpointEnvironment::current()); + if endpoint_was_selected { info!( grpc_endpoint = %config.grpc_endpoint, tls = config.tls_enabled(), @@ -470,27 +496,8 @@ impl PodmanComputeDriver { ); } - // Ensure the bridge network exists. Inspect its gateway only when the - // selected Linux callback route will bind that exact address. client.ensure_network(&config.network_name).await?; - let uses_local_callback_alias = Url::parse(&config.grpc_endpoint) - .ok() - .as_ref() - .is_some_and(callback_endpoint_uses_local_alias); - let needs_network_gateway_ip = cfg!(target_os = "linux") - && uses_local_callback_alias - && !rootless - && config.host_gateway_ip.trim().is_empty(); - let network_gateway_ip = if needs_network_gateway_ip { - client.network_gateway_ip(&config.network_name).await? - } else { - None - }; - info!( - network = %config.network_name, - gateway_ip = ?network_gateway_ip, - "Bridge network ready" - ); + info!(network = %config.network_name, "Podman network ready"); let (gpu_inventory, allow_all_default_gpu) = local_podman_gpu_selector_state(); if !gpu_inventory.is_empty() { @@ -503,9 +510,7 @@ impl PodmanComputeDriver { Ok(Self { client, config, - network_gateway_ip, rootless, - rootless_network_cmd, gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -515,15 +520,6 @@ impl PodmanComputeDriver { }) } - /// The host's IP on the bridge network, if available. - /// - /// Used to request the exact rootful gateway callback listener when no - /// explicit host-gateway override is configured. - #[must_use] - pub fn network_gateway_ip(&self) -> Option<&str> { - self.network_gateway_ip.as_deref() - } - /// Report driver capabilities. pub fn capabilities(&self) -> Result { Ok(GetCapabilitiesResponse { @@ -550,94 +546,6 @@ impl PodmanComputeDriver { }) } - /// Report the gateway exposure needed by Podman's standard local callback aliases. - /// - /// Rootful Podman binds the exact bridge address behind the sandbox alias. - /// Rootless pasta follows the host's default-route interface, while Podman - /// Machine forwards the alias to gateway loopback. Other rootless helpers - /// cannot use a direct host listener. - pub fn gateway_listener_requirements( - &self, - ) -> Result, ComputeDriverError> { - let endpoint = Url::parse(&self.config.grpc_endpoint).map_err(|err| { - ComputeDriverError::Precondition(format!( - "invalid Podman gateway callback endpoint '{}': {err}", - self.config.grpc_endpoint - )) - })?; - let uses_local_callback_alias = callback_endpoint_uses_local_alias(&endpoint); - if !uses_local_callback_alias { - return Ok(Vec::new()); - } - let callback_port = endpoint.port_or_known_default().ok_or_else(|| { - ComputeDriverError::Precondition(format!( - "Podman gateway callback endpoint '{}' has no port", - self.config.grpc_endpoint - )) - })?; - if callback_port != self.config.gateway_port { - return Err(ComputeDriverError::Precondition(format!( - "Podman local callback endpoint '{}' uses port {callback_port}, but the gateway primary listener uses port {}; configure grpc_endpoint with the gateway primary listener port", - self.config.grpc_endpoint, self.config.gateway_port - ))); - } - - #[cfg(target_os = "linux")] - { - if self.rootless { - validate_rootless_local_callback_helper(&self.rootless_network_cmd)?; - - if self.config.host_gateway_ip.trim().is_empty() { - return Ok(vec![GatewayListenerRequirement { - reason: - "Podman rootless pasta callback uses the host default-route interface" - .to_string(), - selector: Some(Selector::DefaultRouteInterface( - GatewayDefaultRouteInterfaceRequirement {}, - )), - }]); - } - } - - let gateway_ip = if self.config.host_gateway_ip.trim().is_empty() { - self.network_gateway_ip.as_deref().ok_or_else(|| { - ComputeDriverError::Precondition(format!( - "Podman network '{}' did not report a host bridge gateway address for local callback alias '{}'", - self.config.network_name, - endpoint.host_str().unwrap_or_default() - )) - })? - } else { - self.config.host_gateway_ip.trim() - }; - let gateway_ip = gateway_ip.parse::().map_err(|err| { - ComputeDriverError::Precondition(format!( - "Podman callback gateway address '{gateway_ip}' is invalid: {err}" - )) - })?; - Ok(vec![GatewayListenerRequirement { - reason: format!("Podman network '{}' host gateway", self.config.network_name), - selector: Some(Selector::ExactBindAddress( - SocketAddr::new(gateway_ip, callback_port).to_string(), - )), - }]) - } - #[cfg(target_os = "macos")] - { - Ok(vec![GatewayListenerRequirement { - reason: "Podman machine callback forwarding terminates on gateway loopback" - .to_string(), - selector: Some(Selector::LoopbackInterface( - GatewayLoopbackInterfaceRequirement {}, - )), - }]) - } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - { - Ok(Vec::new()) - } - } - #[must_use] pub fn default_image(&self) -> &str { &self.config.default_image @@ -1644,9 +1552,7 @@ impl PodmanComputeDriver { Self { client, config, - network_gateway_ip: None, rootless: false, - rootless_network_cmd: String::new(), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -1726,31 +1632,6 @@ fn check_subuid_range() { } } -fn callback_endpoint_uses_local_alias(endpoint: &Url) -> bool { - endpoint - .host_str() - .is_some_and(|host| matches!(host, "host.containers.internal" | "host.openshell.internal")) -} - -#[cfg(any(target_os = "linux", test))] -fn validate_rootless_local_callback_helper( - rootless_network_cmd: &str, -) -> Result<(), ComputeDriverError> { - let rootless_network_cmd = rootless_network_cmd.trim(); - if rootless_network_cmd == "pasta" { - return Ok(()); - } - - let reported = if rootless_network_cmd.is_empty() { - "" - } else { - rootless_network_cmd - }; - Err(ComputeDriverError::Precondition(format!( - "Podman rootless network helper '{reported}' does not support direct local gateway callbacks; configure pasta or use an explicitly remote grpc_endpoint" - ))) -} - // ── Sandbox binary extraction (userns fallback) ──────────────────────── async fn extract_sandbox_bin( @@ -2528,37 +2409,34 @@ mod tests { // ── grpc_endpoint auto-detection ─────────────────────────────────── // - // PodmanComputeDriver::new() fills grpc_endpoint when it is empty. - // The scheme (http vs https) depends on whether TLS client certs are - // configured. These tests simulate the auto-detection logic. + // PodmanComputeDriver::new() fills grpc_endpoint through + // select_grpc_endpoint() when it is empty. #[test] - fn grpc_endpoint_http_without_tls() { - let mut cfg = PodmanComputeConfig { + fn grpc_endpoint_uses_loopback_on_linux() { + let cfg = PodmanComputeConfig { gateway_port: 8081, ..PodmanComputeConfig::default() }; - if cfg.grpc_endpoint.is_empty() { - let scheme = if cfg.tls_enabled() { "https" } else { "http" }; - cfg.grpc_endpoint = format!("{scheme}://host.containers.internal:{}", cfg.gateway_port); - } - assert_eq!(cfg.grpc_endpoint, "http://host.containers.internal:8081"); + assert_eq!( + select_grpc_endpoint(&cfg, PodmanEndpointEnvironment::LinuxHost), + "http://127.0.0.1:8081" + ); } #[test] - fn grpc_endpoint_https_with_tls() { - let mut cfg = PodmanComputeConfig { + fn grpc_endpoint_uses_host_alias_on_podman_machine() { + let cfg = PodmanComputeConfig { gateway_port: 8080, guest_tls_ca: Some(PathBuf::from("/tls/ca.crt")), guest_tls_cert: Some(PathBuf::from("/tls/tls.crt")), guest_tls_key: Some(PathBuf::from("/tls/tls.key")), ..PodmanComputeConfig::default() }; - if cfg.grpc_endpoint.is_empty() { - let scheme = if cfg.tls_enabled() { "https" } else { "http" }; - cfg.grpc_endpoint = format!("{scheme}://host.containers.internal:{}", cfg.gateway_port); - } - assert_eq!(cfg.grpc_endpoint, "https://host.containers.internal:8080"); + assert_eq!( + select_grpc_endpoint(&cfg, PodmanEndpointEnvironment::PodmanMachine), + "https://host.containers.internal:8080" + ); } #[test] @@ -2586,67 +2464,14 @@ mod tests { #[test] fn explicit_grpc_endpoint_takes_precedence() { - let mut cfg = PodmanComputeConfig { + let cfg = PodmanComputeConfig { grpc_endpoint: "https://gateway.internal:9000".to_string(), gateway_port: 8081, ..PodmanComputeConfig::default() }; - if cfg.grpc_endpoint.is_empty() { - let scheme = if cfg.tls_enabled() { "https" } else { "http" }; - cfg.grpc_endpoint = format!("{scheme}://host.containers.internal:{}", cfg.gateway_port); - } - assert_eq!(cfg.grpc_endpoint, "https://gateway.internal:9000"); - } - - #[test] - fn rootless_slirp_allows_remote_callback_endpoint() { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "https://gateway.internal:9000".to_string(), - ..PodmanComputeConfig::default() - }); - driver.rootless = true; - driver.rootless_network_cmd = "slirp4netns".to_string(); - - let requirements = driver.gateway_listener_requirements().unwrap(); - - assert!(requirements.is_empty()); - } - - #[test] - #[cfg(target_os = "linux")] - fn rootful_local_callback_alias_requests_discovered_network_gateway() { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), - ..PodmanComputeConfig::default() - }); - driver.network_gateway_ip = Some("10.89.1.1".to_string()); - - let requirements = driver.gateway_listener_requirements().unwrap(); - - assert_eq!(requirements.len(), 1); - assert_eq!( - requirements[0].selector, - Some(Selector::ExactBindAddress("10.89.1.1:17670".to_string())) - ); - } - - #[test] - #[cfg(target_os = "linux")] - fn configured_host_gateway_overrides_discovered_network_gateway() { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.containers.internal:17670".to_string(), - host_gateway_ip: "10.90.1.1".to_string(), - ..PodmanComputeConfig::default() - }); - driver.network_gateway_ip = Some("10.89.1.1".to_string()); - driver.rootless = true; - driver.rootless_network_cmd = "pasta".to_string(); - - let requirements = driver.gateway_listener_requirements().unwrap(); - assert_eq!( - requirements[0].selector, - Some(Selector::ExactBindAddress("10.90.1.1:17670".to_string())) + select_grpc_endpoint(&cfg, PodmanEndpointEnvironment::LinuxHost), + "https://gateway.internal:9000" ); } @@ -2670,221 +2495,6 @@ mod tests { .expect("an omitted profile preserves Podman's runtime behavior"); } - #[test] - #[cfg(target_os = "linux")] - fn rootless_pasta_requests_default_route_interface() { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), - ..PodmanComputeConfig::default() - }); - driver.rootless = true; - driver.rootless_network_cmd = "pasta".to_string(); - - let requirements = driver.gateway_listener_requirements().unwrap(); - - assert!(matches!( - requirements[0].selector, - Some(Selector::DefaultRouteInterface(_)) - )); - } - - #[test] - fn rootless_non_pasta_helpers_are_rejected() { - for (rootless_network_cmd, reported) in [ - ("slirp4netns", "slirp4netns"), - ("", ""), - ("unknown-helper", "unknown-helper"), - ] { - let err = validate_rootless_local_callback_helper(rootless_network_cmd).unwrap_err(); - - assert!(matches!(err, ComputeDriverError::Precondition(_))); - assert!(err.to_string().contains(reported)); - assert!(err.to_string().contains("configure pasta")); - assert!(err.to_string().contains("remote grpc_endpoint")); - } - } - - #[test] - fn rootless_pasta_is_accepted_for_local_callbacks() { - validate_rootless_local_callback_helper("pasta").unwrap(); - } - - #[test] - #[cfg(target_os = "linux")] - fn rootless_slirp_rejects_explicit_host_gateway_override() { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), - host_gateway_ip: "10.90.1.1".to_string(), - ..PodmanComputeConfig::default() - }); - driver.rootless = true; - driver.rootless_network_cmd = "slirp4netns".to_string(); - - let err = driver.gateway_listener_requirements().unwrap_err(); - - assert!(matches!(err, ComputeDriverError::Precondition(_))); - assert!(err.to_string().contains("slirp4netns")); - } - - #[cfg(target_os = "linux")] - #[tokio::test] - async fn constructor_preserves_required_network_gateway_discovery_error() { - let (socket_path, _request_log, handle) = spawn_podman_stub( - "network-gateway-error", - vec![ - StubResponse::new(StatusCode::OK, ""), - StubResponse::new( - StatusCode::OK, - r#"{ - "host": { - "cgroupVersion": "v2", - "networkBackend": "netavark", - "security": {"rootless": false}, - "remoteSocket": {"path": "/run/podman/podman.sock"} - }, - "version": {"Version": "5.0.0"} - }"#, - ), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new( - StatusCode::INTERNAL_SERVER_ERROR, - r#"{"message":"network gateway inspection failed"}"#, - ), - ], - ); - let config = PodmanComputeConfig { - socket_path: Some(socket_path.clone()), - grpc_endpoint: "http://host.containers.internal:8080".to_string(), - ..PodmanComputeConfig::default() - }; - - let Err(err) = PodmanComputeDriver::new(config).await else { - panic!("required network gateway discovery failure should prevent startup"); - }; - - assert!( - err.to_string() - .contains("network gateway inspection failed"), - "unexpected startup error: {err}" - ); - handle.await.expect("stub task should finish"); - } - - #[tokio::test] - async fn constructor_skips_network_gateway_discovery_for_remote_callback() { - let (socket_path, request_log, handle) = spawn_podman_stub( - "remote-callback-no-network-gateway", - vec![ - StubResponse::new(StatusCode::OK, ""), - StubResponse::new( - StatusCode::OK, - r#"{ - "host": { - "cgroupVersion": "v2", - "networkBackend": "netavark", - "security": {"rootless": false} - } - }"#, - ), - StubResponse::new(StatusCode::CREATED, "{}"), - ], - ); - let config = PodmanComputeConfig { - socket_path: Some(socket_path.clone()), - grpc_endpoint: "https://gateway.example.test:9443".to_string(), - ..PodmanComputeConfig::default() - }; - - let driver = PodmanComputeDriver::new(config) - .await - .expect("remote callbacks must not require bridge gateway inspection"); - - assert!(driver.network_gateway_ip().is_none()); - assert!(driver.gateway_listener_requirements().unwrap().is_empty()); - handle.await.expect("stub task should finish"); - assert_eq!( - request_log - .lock() - .expect("request log lock should not be poisoned") - .as_slice(), - [ - "GET /_ping".to_string(), - format!("GET {}", api_path("/libpod/info")), - format!("POST {}", api_path("/libpod/networks/create")), - ] - ); - } - - #[test] - #[cfg(target_os = "linux")] - fn rootful_local_callback_alias_requires_concrete_gateway_address() { - let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), - ..PodmanComputeConfig::default() - }); - - let err = driver.gateway_listener_requirements().unwrap_err(); - - assert!( - err.to_string() - .contains("did not report a host bridge gateway address") - ); - } - - #[test] - #[cfg(target_os = "macos")] - fn podman_machine_callback_alias_requests_loopback_listener() { - let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), - ..PodmanComputeConfig::default() - }); - - let requirements = driver.gateway_listener_requirements().unwrap(); - - assert_eq!(requirements.len(), 1); - assert!(matches!( - requirements[0].selector, - Some(Selector::LoopbackInterface(_)) - )); - } - - #[test] - fn explicit_remote_callback_does_not_request_gateway_listener() { - let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "https://gateway.example.test:9443".to_string(), - gateway_port: 17670, - ..PodmanComputeConfig::default() - }); - - assert!(driver.gateway_listener_requirements().unwrap().is_empty()); - } - - #[test] - fn local_callback_alias_requires_primary_listener_port() { - for grpc_endpoint in [ - "http://host.openshell.internal:17671", - "http://host.containers.internal", - ] { - let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: grpc_endpoint.to_string(), - gateway_port: 17670, - ..PodmanComputeConfig::default() - }); - - let err = driver.gateway_listener_requirements().unwrap_err(); - - assert!( - matches!(err, ComputeDriverError::Precondition(_)), - "mismatched local callback port should fail precondition: {err}" - ); - assert!( - err.to_string() - .contains("gateway primary listener uses port 17670"), - "unexpected error for {grpc_endpoint}: {err}" - ); - } - } - #[test] fn local_podman_cdi_gpu_inventory_maps_nvidia_device_nodes() { let root = std::env::temp_dir().join(format!( diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 4e06183805..ef336f4dce 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -7,12 +7,11 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DeleteWorkspaceRequest, DeleteWorkspaceResponse, EnsureWorkspaceRequest, - EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, - GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, - GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, - StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, - ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, - WatchSandboxesRequest, compute_driver_server::ComputeDriver, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, + GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, + StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, + compute_driver_server::ComputeDriver, }; use std::pin::Pin; use tonic::{Request, Response, Status}; @@ -79,25 +78,6 @@ impl ComputeDriver for ComputeDriverService { .await } - async fn get_gateway_listener_requirements( - &self, - _request: Request, - ) -> Result, Status> { - self.rpc_tracer - .trace( - openshell_otel::rpc::GET_GATEWAY_LISTENER_REQUIREMENTS, - async { - Ok(Response::new(GetGatewayListenerRequirementsResponse { - requirements: self - .driver - .gateway_listener_requirements() - .map_err(Status::from)?, - })) - }, - ) - .await - } - async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 62a2dde5a1..93a3f92966 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -79,7 +79,7 @@ struct Args { )] sandbox_ssh_socket_path: String, - /// Podman bridge network name. + /// Podman network name retained for driver-managed resources. #[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)] network_name: String, diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index c019543152..14f1f9e45b 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -49,8 +49,7 @@ use openshell_core::proto::compute::v1::{ DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, EnsureWorkspaceRequest, EnsureWorkspaceResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, GpuResourceCapabilities, ListSandboxesRequest, ListSandboxesResponse, MemoryResourceCapabilities, ResourceCapabilities, StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, @@ -4242,15 +4241,6 @@ impl ComputeDriver for VmDriver { Ok(Response::new(self.capabilities())) } - async fn get_gateway_listener_requirements( - &self, - _request: Request, - ) -> Result, Status> { - Ok(Response::new(GetGatewayListenerRequirementsResponse { - requirements: Vec::new(), - })) - } - async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-otel/src/grpc.rs b/crates/openshell-otel/src/grpc.rs index 65d498eccb..e22af83968 100644 --- a/crates/openshell-otel/src/grpc.rs +++ b/crates/openshell-otel/src/grpc.rs @@ -94,10 +94,6 @@ pub mod rpc { "GetCapabilities", "openshell.compute.v1.ComputeDriver/GetCapabilities", ); - pub const GET_GATEWAY_LISTENER_REQUIREMENTS: ComputeDriverRpc = ComputeDriverRpc::new( - "GetGatewayListenerRequirements", - "openshell.compute.v1.ComputeDriver/GetGatewayListenerRequirements", - ); pub const VALIDATE_SANDBOX_CREATE: ComputeDriverRpc = ComputeDriverRpc::new( "ValidateSandboxCreate", "openshell.compute.v1.ComputeDriver/ValidateSandboxCreate", @@ -192,7 +188,6 @@ pub fn compute_driver_rpc_operation(path: &str) -> Option { match path.rsplit('/').next() { Some("AuthenticateSandbox") => Some(rpc::AUTHENTICATE_SANDBOX), Some("GetCapabilities") => Some(rpc::GET_CAPABILITIES), - Some("GetGatewayListenerRequirements") => Some(rpc::GET_GATEWAY_LISTENER_REQUIREMENTS), Some("ValidateSandboxCreate") => Some(rpc::VALIDATE_SANDBOX_CREATE), Some("CreateSandbox") => Some(rpc::CREATE_SANDBOX), Some("GetSandbox") => Some(rpc::GET_SANDBOX), @@ -315,7 +310,6 @@ mod tests { for rpc in [ rpc::AUTHENTICATE_SANDBOX, rpc::GET_CAPABILITIES, - rpc::GET_GATEWAY_LISTENER_REQUIREMENTS, rpc::VALIDATE_SANDBOX_CREATE, rpc::CREATE_SANDBOX, rpc::GET_SANDBOX, diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 9a057519b7..54c44a7bed 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -33,7 +33,7 @@ k8s-openapi = { workspace = true } # Async runtime tokio = { workspace = true } -socket2 = { workspace = true } +socket2 = { workspace = true, features = ["all"] } nix = { workspace = true } libc = "0.2" diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 6b7e7c43e7..8e2b5cb462 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -25,16 +25,13 @@ use openshell_core::proto::compute::v1::{ AuthenticateSandboxRequest, CreateSandboxRequest, DeleteSandboxRequest, DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent, DriverResourceRequirements, DriverSandbox, DriverSandboxSpec, DriverSandboxStatus, DriverSandboxTemplate, - EnsureWorkspaceRequest, EnsureWorkspaceResponse, - GatewayListenerRequirement as ProtoGatewayListenerRequirement, GetCapabilitiesRequest, - GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, - GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, - ListSandboxesRequest, ResourceCapabilities as DriverResourceCapabilities, + EnsureWorkspaceRequest, EnsureWorkspaceResponse, GetCapabilitiesRequest, GetSandboxRequest, + GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, + ResourceCapabilities as DriverResourceCapabilities, ResourceRequirements as DriverSandboxResourceRequirements, StartSandboxRequest, StopSandboxRequest, ValidateSandboxCreateRequest, WatchSandboxesEvent, WatchSandboxesRequest, WorkloadIdentityRequest, compute_driver_client::ComputeDriverClient, - compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, - watch_sandboxes_event, + compute_driver_server::ComputeDriver, watch_sandboxes_event, }; use openshell_core::proto::{ PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxSpec, SandboxStatus, @@ -46,7 +43,6 @@ use prost::Message; use std::collections::HashMap; use std::fmt; use std::future::Future; -use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::{Arc, Mutex as StdMutex, Weak}; @@ -177,41 +173,6 @@ mod traced_driver { const DELETE_PHASE_CAS_RETRY_LIMIT: usize = 3; const SUPERVISOR_SESSION_CAS_RETRY_LIMIT: usize = 3; -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum GatewayListenerRequirement { - Exact { - address: SocketAddr, - driver_name: String, - reason: String, - }, - DefaultRouteInterface { - driver_name: String, - reason: String, - }, - LoopbackInterface { - driver_name: String, - reason: String, - }, -} - -impl GatewayListenerRequirement { - pub fn driver_name(&self) -> &str { - match self { - Self::Exact { driver_name, .. } - | Self::DefaultRouteInterface { driver_name, .. } - | Self::LoopbackInterface { driver_name, .. } => driver_name, - } - } - - pub fn reason(&self) -> &str { - match self { - Self::Exact { reason, .. } - | Self::DefaultRouteInterface { reason, .. } - | Self::LoopbackInterface { reason, .. } => reason, - } - } -} - /// Serializes request-side lifecycle mutations for the same stable sandbox ID. /// /// Watch events deliberately do not use these gates, so a slow driver delete @@ -525,14 +486,6 @@ impl ComputeDriver for RemoteComputeDriver { client.authenticate_sandbox(request).await } - async fn get_gateway_listener_requirements( - &self, - request: Request, - ) -> Result, Status> { - let mut client = self.client(); - client.get_gateway_listener_requirements(request).await - } - async fn validate_sandbox_create( &self, request: Request, @@ -639,7 +592,6 @@ pub struct ComputeRuntime { supervisor_sessions: Arc, sync_lock: Arc>, lifecycle_gates: Arc, - gateway_listener_requirements: Vec, replica_id: String, /// Gateway-issued staging slots for rootfs tar archives. Shared across /// clones: `ServerState` holds `ComputeRuntime` by value, so a per-clone @@ -699,59 +651,6 @@ impl ComputeRuntime { rootfs_tar_max_bytes: capabilities.rootfs_tar_max_bytes, }; let default_image = capabilities.default_image; - let gateway_listener_requirements = match driver - .get_gateway_listener_requirements(Request::new( - GetGatewayListenerRequirementsRequest {}, - )) - .await - { - Ok(response) => response - .into_inner() - .requirements - .into_iter() - .map(|requirement: ProtoGatewayListenerRequirement| { - let Some(selector) = requirement.selector else { - return Err(ComputeError::Message(format!( - "compute driver '{driver_name}' returned a gateway listener requirement without a selector" - ))); - }; - match selector { - Selector::ExactBindAddress(bind_address) => { - let address = bind_address.parse::().map_err(|err| { - ComputeError::Message(format!( - "compute driver '{driver_name}' returned invalid gateway listener address '{bind_address}': {err}" - )) - })?; - Ok(GatewayListenerRequirement::Exact { - address, - driver_name: driver_name.clone(), - reason: requirement.reason, - }) - } - Selector::DefaultRouteInterface(_) => { - Ok(GatewayListenerRequirement::DefaultRouteInterface { - driver_name: driver_name.clone(), - reason: requirement.reason, - }) - } - Selector::LoopbackInterface(_) => { - Ok(GatewayListenerRequirement::LoopbackInterface { - driver_name: driver_name.clone(), - reason: requirement.reason, - }) - } - } - }) - .collect::, ComputeError>>()?, - Err(status) if status.code() == Code::Unimplemented => { - debug!( - driver = %driver_name, - "Compute driver does not implement gateway listener requirements" - ); - Vec::new() - } - Err(status) => return Err(compute_error_from_status(status)), - }; let rootfs_tar_staging = Arc::new(rootfs_tar::RootfsTarStagingRegistry::new( (!driver_info.rootfs_tar_staging_dir.is_empty()) .then(|| PathBuf::from(&driver_info.rootfs_tar_staging_dir)), @@ -771,7 +670,6 @@ impl ComputeRuntime { supervisor_sessions, sync_lock: Arc::new(Mutex::new(())), lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), - gateway_listener_requirements, replica_id: lease::replica_id(), rootfs_tar_staging, }) @@ -886,11 +784,6 @@ impl ComputeRuntime { self } - #[must_use] - pub(crate) fn gateway_listener_requirements(&self) -> &[GatewayListenerRequirement] { - &self.gateway_listener_requirements - } - pub(crate) async fn ensure_workspace(&self, workspace: &str) -> Result<(), Status> { let workspace = workspace.to_string(); match self @@ -5300,15 +5193,6 @@ impl ComputeDriver for NoopTestDriver { )) } - async fn get_gateway_listener_requirements( - &self, - _request: Request, - ) -> Result, Status> { - Ok(tonic::Response::new( - GetGatewayListenerRequirementsResponse::default(), - )) - } - async fn validate_sandbox_create( &self, _request: Request, @@ -5454,7 +5338,6 @@ pub fn new_test_runtime_with_driver( supervisor_sessions: Arc::new(SupervisorSessionRegistry::new()), sync_lock: Arc::new(Mutex::new(())), lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), - gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), rootfs_tar_staging: Arc::new(rootfs_tar::RootfsTarStagingRegistry::disabled()), } @@ -5859,15 +5742,6 @@ mod tests { })) } - async fn get_gateway_listener_requirements( - &self, - _request: Request, - ) -> Result, Status> { - Ok(tonic::Response::new( - GetGatewayListenerRequirementsResponse::default(), - )) - } - async fn validate_sandbox_create( &self, _request: Request, @@ -6212,15 +6086,6 @@ mod tests { })) } - async fn get_gateway_listener_requirements( - &self, - _request: Request, - ) -> Result, Status> { - Ok(tonic::Response::new( - GetGatewayListenerRequirementsResponse::default(), - )) - } - async fn validate_sandbox_create( &self, _request: Request, @@ -6437,7 +6302,6 @@ mod tests { supervisor_sessions: Arc::new(SupervisorSessionRegistry::new()), sync_lock: Arc::new(Mutex::new(())), lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), - gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), rootfs_tar_staging: Arc::new(rootfs_tar::RootfsTarStagingRegistry::disabled()), } @@ -7636,14 +7500,6 @@ mod tests { self.0.get_capabilities(request).await } - async fn get_gateway_listener_requirements( - &self, - request: Request, - ) -> Result, Status> - { - self.0.get_gateway_listener_requirements(request).await - } - async fn validate_sandbox_create( &self, request: Request, @@ -11902,12 +11758,6 @@ mod tests { .get_capabilities(Request::new(GetCapabilitiesRequest {})) .await .unwrap(); - remote - .get_gateway_listener_requirements(Request::new( - GetGatewayListenerRequirementsRequest {}, - )) - .await - .unwrap(); remote .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { sandbox: Some(sandbox.clone()), @@ -11959,7 +11809,7 @@ mod tests { let traceparents = driver.traceparents(); assert_eq!( traceparents.len(), - 9, + 8, "the client interceptor should cover every RPC" ); assert!( @@ -12002,14 +11852,14 @@ mod tests { let traceparents = driver.traceparents(); assert_eq!( traceparents.len(), - 2, - "the capability and listener-requirements probes should carry initialization trace context" + 1, + "the capability probe should carry initialization trace context" ); assert!( traceparents .iter() .all(|traceparent| traceparent.contains(&trace_id)), - "both initialization probes should be part of the initialization trace" + "the initialization probe should be part of the initialization trace" ); } @@ -12023,11 +11873,7 @@ mod tests { let driver = FakeComputeDriver::new() .with_driver_name("fake-remote-driver") .with_default_image("openshell/sandbox:remote") - .with_gateway_manages_lifecycle() - .with_gateway_listener_requirement( - "172.19.0.1:17670", - "external driver managed bridge", - ); + .with_gateway_manages_lifecycle(); let _server = driver.serve_uds(&socket_path).unwrap(); let endpoint = connect_remote_compute_driver("docker", &socket_path) @@ -12044,15 +11890,6 @@ mod tests { ) .await .unwrap(); - assert_eq!( - runtime.gateway_listener_requirements(), - &[GatewayListenerRequirement::Exact { - address: "172.19.0.1:17670".parse().unwrap(), - driver_name: "docker".to_string(), - reason: "external driver managed bridge".to_string(), - }] - ); - let mut sandbox = sandbox_record("sb-uds", "uds-sandbox", SandboxPhase::Provisioning); sandbox.spec = Some(SandboxSpec { log_level: "debug".to_string(), @@ -12084,8 +11921,8 @@ mod tests { runtime.validate_sandbox_create(&sandbox).await.unwrap(); runtime.create_sandbox(sandbox, None, false).await.unwrap(); let calls = driver.calls(); - assert_eq!(calls.len(), 4, "unexpected calls: {calls:?}"); - let validated = match &calls[2] { + assert_eq!(calls.len(), 3, "unexpected calls: {calls:?}"); + let validated = match &calls[1] { FakeComputeDriverCall::ValidateSandboxCreate { sandbox: Some(sandbox), } => sandbox, @@ -12108,7 +11945,7 @@ mod tests { Some(42) ); assert!(matches!( - &calls[3], + &calls[2], FakeComputeDriverCall::CreateSandbox { sandbox: Some(sandbox) } if sandbox.spec.as_ref().and_then(|spec| spec.policy.as_ref()) .is_some_and(|policy| policy.version == 42) @@ -12155,43 +11992,6 @@ mod tests { } } - #[tokio::test] - #[cfg(unix)] - async fn remote_compute_driver_accepts_unimplemented_listener_requirements_api() { - use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; - - let dir = tempfile::tempdir().unwrap(); - let socket_path = dir.path().join("compute-driver.sock"); - let driver = FakeComputeDriver::new() - .with_driver_name("legacy-remote-driver") - .without_gateway_listener_requirements_api(); - let _server = driver.serve_uds(&socket_path).unwrap(); - - let endpoint = connect_remote_compute_driver("external-test", &socket_path) - .await - .unwrap(); - let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); - let runtime = ComputeRuntime::new_remote_driver( - endpoint, - store, - SandboxIndex::new(), - SandboxWatchBus::new(), - TracingLogBus::new(), - Arc::new(SupervisorSessionRegistry::new()), - ) - .await - .unwrap(); - - assert!(runtime.gateway_listener_requirements().is_empty()); - assert_eq!( - driver.calls(), - vec![ - FakeComputeDriverCall::GetCapabilities, - FakeComputeDriverCall::GetGatewayListenerRequirements, - ] - ); - } - #[tokio::test] async fn create_sandbox_returns_resource_version_one() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index b638fc33e4..afc2f4a763 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -1,728 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use crate::compute::GatewayListenerRequirement; use openshell_core::{Error, Result}; -use socket2::{Domain, Protocol, Socket, Type}; -use std::net::{IpAddr, SocketAddr}; +use std::net::SocketAddr; use tokio::net::TcpListener; use tracing::info; -/// Authorization scope associated with a gateway listener. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum GatewayListenerScope { - Primary, - ComputeDriverCallback, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct CoveredGatewayAddress { - pub address: SocketAddr, - pub scope: GatewayListenerScope, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct GatewayListenerSpec { - pub address: SocketAddr, - pub scope: GatewayListenerScope, - covered_addresses: Vec, - provenance: Option, -} - -/// Diagnostic source of a driver-requested listener. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct GatewayListenerProvenance { - pub driver_name: String, - pub reason: String, -} - -/// A gateway listener together with the context needed to serve it. pub struct BoundGatewayListener { pub listener: TcpListener, - pub spec: GatewayListenerSpec, -} - -impl GatewayListenerSpec { - pub fn new(address: SocketAddr, scope: GatewayListenerScope) -> Self { - Self { - address, - scope, - covered_addresses: Vec::new(), - provenance: None, - } - } - - pub fn scope_for_local_addr(&self, local_addr: SocketAddr) -> GatewayListenerScope { - self.covered_addresses - .iter() - .find(|covered| covered.address == local_addr) - .map_or(self.scope, |covered| covered.scope) - } - - fn bind_to(mut self, local_addr: SocketAddr) -> Self { - let requested_addr = self.address; - self.address = local_addr; - self.covered_addresses = - resolve_bound_covered_addresses(&self.covered_addresses, requested_addr, local_addr); - self - } -} - -fn gateway_listener_specs( - bind_address: SocketAddr, - requirements: &[GatewayListenerRequirement], -) -> Result> { - let needs_default_route_resolution = requirements.iter().any(|requirement| { - matches!( - requirement, - GatewayListenerRequirement::DefaultRouteInterface { .. } - ) - }); - let default_route_ip = if needs_default_route_resolution { - Some(gateway_default_route_ip()?) - } else { - None - }; - gateway_listener_specs_with_default_route_ip(bind_address, requirements, default_route_ip) -} - -fn gateway_listener_specs_with_default_route_ip( - bind_address: SocketAddr, - requirements: &[GatewayListenerRequirement], - default_route_ip: Option, -) -> Result> { - let mut specs = vec![GatewayListenerSpec::new( - bind_address, - GatewayListenerScope::Primary, - )]; - - // Resolve exact requirements first so they can satisfy a later semantic - // requirement regardless of driver response ordering. - for requirement in requirements { - let GatewayListenerRequirement::Exact { address, .. } = requirement else { - continue; - }; - validate_gateway_listener_requirement(bind_address, requirement)?; - add_callback_listener_spec(&mut specs, *address, requirement); - } - - for requirement in requirements { - let GatewayListenerRequirement::DefaultRouteInterface { .. } = requirement else { - continue; - }; - validate_gateway_listener_requirement(bind_address, requirement)?; - let Some(ip) = default_route_ip else { - return Err(Error::config(format!( - "compute driver '{}' requested the gateway default-route interface, but no IPv4 source address was resolved (reason: {})", - requirement.driver_name(), - requirement.reason() - ))); - }; - if !gateway_default_route_ip_is_usable(ip) { - return Err(Error::config(format!( - "compute driver '{}' requested the gateway default-route interface, but its resolved address {ip} is not a private IPv4 address (reason: {})", - requirement.driver_name(), - requirement.reason() - ))); - } - let address = SocketAddr::new(ip, bind_address.port()); - validate_resolved_gateway_listener(bind_address, address)?; - add_callback_listener_spec(&mut specs, address, requirement); - } - - for requirement in requirements { - let GatewayListenerRequirement::LoopbackInterface { .. } = requirement else { - continue; - }; - validate_gateway_listener_requirement(bind_address, requirement)?; - let address = SocketAddr::from(([127, 0, 0, 1], bind_address.port())); - validate_resolved_gateway_listener(bind_address, address)?; - add_callback_listener_spec(&mut specs, address, requirement); - } - - Ok(specs) -} - -fn add_callback_listener_spec( - specs: &mut Vec, - address: SocketAddr, - requirement: &GatewayListenerRequirement, -) { - let scope = GatewayListenerScope::ComputeDriverCallback; - if let Some(existing) = specs - .iter_mut() - .find(|existing| listener_covers(existing.address, address)) - { - if existing.scope == GatewayListenerScope::Primary { - return; - } - if existing.address == address { - return; - } - if !existing - .covered_addresses - .iter() - .any(|covered| covered.address == address) - { - existing - .covered_addresses - .push(CoveredGatewayAddress { address, scope }); - } - return; - } - specs.push(callback_listener_spec(address, requirement)); -} - -fn callback_listener_spec( - address: SocketAddr, - requirement: &GatewayListenerRequirement, -) -> GatewayListenerSpec { - GatewayListenerSpec { - address, - scope: GatewayListenerScope::ComputeDriverCallback, - covered_addresses: Vec::new(), - provenance: Some(GatewayListenerProvenance { - driver_name: requirement.driver_name().to_string(), - reason: requirement.reason().to_string(), - }), - } -} - -fn validate_gateway_listener_requirement( - primary_listener: SocketAddr, - requirement: &GatewayListenerRequirement, -) -> Result<()> { - match requirement { - GatewayListenerRequirement::Exact { address, .. } => { - validate_resolved_gateway_listener(primary_listener, *address) - } - GatewayListenerRequirement::DefaultRouteInterface { .. } - | GatewayListenerRequirement::LoopbackInterface { .. } => Ok(()), - } -} - -fn validate_resolved_gateway_listener( - primary_listener: SocketAddr, - requested_listener: SocketAddr, -) -> Result<()> { - if requested_listener.ip().is_unspecified() { - return Err(Error::config(format!( - "compute driver requested wildcard gateway listener {requested_listener}" - ))); - } - if requested_listener.ip().is_multicast() { - return Err(Error::config(format!( - "compute driver requested multicast gateway listener {requested_listener}" - ))); - } - if requested_listener.port() == 0 { - return Err(Error::config(format!( - "compute driver requested zero-port gateway listener {requested_listener}" - ))); - } - if requested_listener.port() != primary_listener.port() { - return Err(Error::config(format!( - "compute driver requested gateway listener {requested_listener} with port {}, but the primary listener uses port {}", - requested_listener.port(), - primary_listener.port() - ))); - } - Ok(()) -} - -fn gateway_default_route_ip_is_usable(address: IpAddr) -> bool { - matches!(address, IpAddr::V4(address) if address.is_private()) -} - -#[cfg(target_os = "linux")] -fn gateway_default_route_ip() -> Result { - // UDP connect performs a local route lookup without sending a packet. The - // selected source address follows the IPv4 default route, matching pasta's - // default upstream-interface selection. - let socket = - std::net::UdpSocket::bind((std::net::Ipv4Addr::UNSPECIFIED, 0)).map_err(|err| { - Error::config(format!("failed to open default-route probe socket: {err}")) - })?; - socket - .connect((std::net::Ipv4Addr::new(192, 0, 2, 1), 9)) - .map_err(|err| Error::config(format!("failed to resolve IPv4 default route: {err}")))?; - socket - .local_addr() - .map(|address| address.ip()) - .map_err(|err| Error::config(format!("failed to read IPv4 default-route address: {err}"))) -} - -#[cfg(not(target_os = "linux"))] -fn gateway_default_route_ip() -> Result { - Err(Error::config( - "default-route gateway listener requirements are supported only on Linux", - )) -} - -pub async fn bind_gateway_listeners( - bind_address: SocketAddr, - requirements: &[GatewayListenerRequirement], -) -> Result> { - let specs = gateway_listener_specs(bind_address, requirements)?; - let mut listeners = Vec::with_capacity(specs.len()); - for spec in &specs { - let ipv6_only = matches!( - spec.address.ip(), - IpAddr::V6(address) if address.is_unspecified() - ) && specs.iter().any(|candidate| { - candidate.address.port() == spec.address.port() && candidate.address.is_ipv4() - }); - let listener = bind_gateway_listener(spec.address, ipv6_only) - .await - .map_err(|e| Error::transport(format!("failed to bind to {}: {e}", spec.address)))?; - let local_addr = listener.local_addr().unwrap_or(spec.address); - match spec.scope { - GatewayListenerScope::Primary => { - info!( - address = %local_addr, - listener_purpose = "primary", - authorization_scope = "full-multiplexed-api", - "Gateway listener bound" - ); - } - GatewayListenerScope::ComputeDriverCallback => { - let provenance = spec - .provenance - .as_ref() - .expect("callback listener spec must include provenance"); - info!( - address = %local_addr, - listener_purpose = "compute-driver-callback", - driver = %provenance.driver_name, - reason = %provenance.reason, - authorization_scope = "sandbox-callable-grpc-only", - "Gateway listener bound" - ); - } - } - listeners.push(BoundGatewayListener { - listener, - spec: spec.clone().bind_to(local_addr), - }); - } - Ok(listeners) -} - -fn resolve_bound_covered_addresses( - covered_addresses: &[CoveredGatewayAddress], - requested_listener_addr: SocketAddr, - bound_listener_addr: SocketAddr, -) -> Vec { - covered_addresses - .iter() - .map(|covered| CoveredGatewayAddress { - address: resolve_ephemeral_port( - covered.address, - requested_listener_addr, - bound_listener_addr, - ), - scope: covered.scope, - }) - .collect() -} - -fn resolve_ephemeral_port( - address: SocketAddr, - requested_listener_addr: SocketAddr, - bound_listener_addr: SocketAddr, -) -> SocketAddr { - if requested_listener_addr.port() == 0 && address.port() == 0 { - SocketAddr::new(address.ip(), bound_listener_addr.port()) - } else { - address - } -} - -async fn bind_gateway_listener( - address: SocketAddr, - ipv6_only: bool, -) -> std::io::Result { - if ipv6_only { - let socket = Socket::new(Domain::IPV6, Type::STREAM, Some(Protocol::TCP))?; - socket.set_reuse_address(true)?; - socket.set_only_v6(true)?; - socket.set_nonblocking(true)?; - socket.bind(&address.into())?; - socket.listen(1024)?; - let listener: std::net::TcpListener = socket.into(); - return TcpListener::from_std(listener); - } - - TcpListener::bind(address).await -} - -fn listener_covers(existing: SocketAddr, requested: SocketAddr) -> bool { - if existing == requested { - return true; - } - if existing.port() != requested.port() { - return false; - } - - match (existing.ip(), requested.ip()) { - (IpAddr::V4(existing), IpAddr::V4(_)) => existing.is_unspecified(), - (IpAddr::V6(existing), IpAddr::V6(_)) => existing.is_unspecified(), - _ => false, - } + pub address: SocketAddr, } -#[cfg(test)] -mod tests { - use super::{ - GatewayListenerProvenance, GatewayListenerScope, GatewayListenerSpec, - bind_gateway_listeners, gateway_listener_specs, - gateway_listener_specs_with_default_route_ip, - }; - use crate::compute::GatewayListenerRequirement; - use std::net::SocketAddr; - use std::sync::atomic::{AtomicBool, Ordering}; - use tokio::net::TcpListener; - - #[test] - fn gateway_listener_specs_reuse_primary_when_wildcard_covers_driver_address() { - let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); - let requirements = [ - exact_listener_requirement(callback), - exact_listener_requirement(callback), - ]; - - assert_eq!( - gateway_listener_specs(primary, &requirements).unwrap(), - vec![primary_listener_spec(primary)] - ); - } - - #[test] - fn gateway_listener_scope_for_reused_primary_remains_primary() { - let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); - let loopback: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let [spec] = gateway_listener_specs(primary, &[exact_listener_requirement(callback)]) - .unwrap() - .try_into() - .unwrap(); - - assert_eq!( - spec.scope_for_local_addr(callback), - GatewayListenerScope::Primary, - ); - assert_eq!( - spec.scope_for_local_addr(loopback), - GatewayListenerScope::Primary, - ); - } - - #[test] - fn gateway_listener_specs_preserve_driver_callback_scope() { - let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); - let requirements = [ - exact_listener_requirement(callback), - exact_listener_requirement(callback), - ]; - - assert_eq!( - gateway_listener_specs(primary, &requirements).unwrap(), - vec![ - GatewayListenerSpec { - address: primary, - scope: GatewayListenerScope::Primary, - covered_addresses: Vec::new(), - provenance: None, - }, - GatewayListenerSpec { - address: callback, - scope: GatewayListenerScope::ComputeDriverCallback, - covered_addresses: Vec::new(), - provenance: Some(GatewayListenerProvenance { - driver_name: "alpha".to_string(), - reason: "managed bridge".to_string(), - }), - }, - ] - ); - } - - #[test] - fn gateway_listener_specs_accept_safe_external_driver_requirement() { - let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let requirement = GatewayListenerRequirement::Exact { - address: "172.18.0.1:8080".parse().unwrap(), - driver_name: "external-test".to_string(), - reason: "external bridge".to_string(), - }; - - let specs = gateway_listener_specs(primary, &[requirement]).unwrap(); - assert_eq!(specs.len(), 2); - assert_eq!(specs[1].address, "172.18.0.1:8080".parse().unwrap()); - assert_eq!(specs[1].scope, GatewayListenerScope::ComputeDriverCallback); - } - - #[test] - fn gateway_listener_specs_reject_invalid_exact_addresses() { - let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - for address in [ - "0.0.0.0:8080", - "224.0.0.1:8080", - "172.18.0.1:0", - "172.18.0.1:9090", - ] { - let requirement = exact_listener_requirement(address.parse().unwrap()); - assert!( - gateway_listener_specs(primary, &[requirement]).is_err(), - "{address} should be rejected" - ); - } - } - - #[test] - fn gateway_listener_specs_use_exact_network_gateway() { - let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let network_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); - - assert_eq!( - gateway_listener_specs(primary, &[network_listener_requirement(network_gateway)]) - .unwrap(), - vec![ - primary_listener_spec(primary), - callback_listener_spec(network_gateway, "beta", "managed bridge",), - ] - ); - } - - #[test] - fn gateway_listener_specs_reuse_primary_when_it_covers_exact_requirement() { - let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let network_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); - - assert_eq!( - gateway_listener_specs(primary, &[network_listener_requirement(network_gateway)],) - .unwrap(), - vec![primary_listener_spec(primary)] - ); - } - - #[test] - fn gateway_listener_specs_resolve_default_route_source() { - let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let default_route_ip = "192.168.20.20".parse().unwrap(); - - assert_eq!( - gateway_listener_specs_with_default_route_ip( - primary, - &[default_route_listener_requirement()], - Some(default_route_ip), - ) - .unwrap(), - vec![ - primary_listener_spec(primary), - callback_listener_spec( - "192.168.20.20:8080".parse().unwrap(), - "beta", - "default route interface", - ), - ] - ); - } - - #[test] - fn gateway_listener_specs_reject_public_default_route_source() { - let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - - let err = gateway_listener_specs_with_default_route_ip( - primary, - &[default_route_listener_requirement()], - Some("203.0.113.20".parse().unwrap()), - ) - .unwrap_err(); - - assert!(err.to_string().contains("not a private IPv4 address")); - } - - #[test] - fn gateway_listener_specs_reuse_ipv4_wildcard_for_default_route() { - let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let default_route_ip = "192.168.20.20".parse().unwrap(); - assert_eq!( - gateway_listener_specs_with_default_route_ip( - primary, - &[default_route_listener_requirement()], - Some(default_route_ip), - ) - .unwrap(), - vec![primary_listener_spec(primary)] - ); - } - - #[test] - fn gateway_listener_specs_resolve_loopback_separately() { - let primary: SocketAddr = "192.168.20.20:8080".parse().unwrap(); - - assert_eq!( - gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), - vec![ - primary_listener_spec(primary), - callback_listener_spec("127.0.0.1:8080".parse().unwrap(), "beta", "host forwarder",), - ] - ); - } - - #[test] - fn gateway_listener_specs_reuse_wildcard_primary_for_loopback() { - let primary = "0.0.0.0:8080".parse().unwrap(); - - assert_eq!( - gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), - vec![primary_listener_spec(primary)] - ); - } - - #[test] - fn gateway_listener_specs_reuse_matching_primary_address() { - let primary = "127.0.0.1:8080".parse().unwrap(); - - assert_eq!( - gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), - vec![primary_listener_spec(primary)] - ); - } - - #[test] - fn gateway_listener_specs_do_not_use_ipv6_listener_for_ipv4_loopback_requirement() { - for primary in ["[::1]:8080", "[::]:8080"] { - let primary = primary.parse().unwrap(); - let specs = - gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(); - - assert_eq!(specs.len(), 2); - assert_eq!(specs[1].address, SocketAddr::from(([127, 0, 0, 1], 8080))); - } - } - - #[test] - fn gateway_listener_specs_validate_selector_independently_of_driver_name() { - let primary: SocketAddr = "192.168.20.20:8080".parse().unwrap(); - let requirement = GatewayListenerRequirement::LoopbackInterface { - driver_name: "alpha".to_string(), - reason: "wrong selector".to_string(), - }; - - let specs = gateway_listener_specs(primary, &[requirement]).unwrap(); - assert_eq!(specs.len(), 2); - assert_eq!(specs[0].address, primary); - assert_eq!(specs[1].address, "127.0.0.1:8080".parse().unwrap()); - assert_eq!(specs[1].scope, GatewayListenerScope::ComputeDriverCallback); - } - - #[tokio::test] - async fn failed_bind_does_not_return_partially_bound_listeners() { - let occupied_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let occupied_address = occupied_listener.local_addr().unwrap(); - let continuation_reached = AtomicBool::new(false); - let primary_address: SocketAddr = "127.0.0.1:0".parse().unwrap(); - - let result: openshell_core::Result<()> = async { - let _listeners = bind_gateway_listeners( - primary_address, - &[exact_listener_requirement(occupied_address)], - ) - .await?; - continuation_reached.store(true, Ordering::SeqCst); - Ok(()) - } - .await; - - assert!( - result.is_err(), - "binding the occupied extra gateway address should fail" - ); - assert!( - !continuation_reached.load(Ordering::SeqCst), - "binding must fail before returning a partial listener set" - ); - } - - #[tokio::test] - #[cfg(target_os = "linux")] - #[ignore = "flaky under concurrent test execution"] - async fn gateway_listeners_bind_ipv6_wildcard_and_ipv4_callback_on_same_port() { - let probe = TcpListener::bind("[::1]:0") - .await - .expect("IPv6 loopback probe should bind"); - let port = probe.local_addr().unwrap().port(); - drop(probe); - - let primary = format!("[::]:{port}").parse().unwrap(); - let listeners = bind_gateway_listeners(primary, &[loopback_listener_requirement()]) - .await - .expect("IPv6 wildcard and IPv4 callback listeners should both bind"); - - assert_eq!(listeners.len(), 2); - assert_eq!(listeners[0].spec.address, primary); - assert_eq!( - listeners[1].spec.address, - SocketAddr::from(([127, 0, 0, 1], port)) - ); - } - - fn exact_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { - GatewayListenerRequirement::Exact { - address, - driver_name: "alpha".to_string(), - reason: "managed bridge".to_string(), - } - } - - fn network_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { - GatewayListenerRequirement::Exact { - address, - driver_name: "beta".to_string(), - reason: "managed bridge".to_string(), - } - } - - fn default_route_listener_requirement() -> GatewayListenerRequirement { - GatewayListenerRequirement::DefaultRouteInterface { - driver_name: "beta".to_string(), - reason: "default route interface".to_string(), - } - } - - fn loopback_listener_requirement() -> GatewayListenerRequirement { - GatewayListenerRequirement::LoopbackInterface { - driver_name: "beta".to_string(), - reason: "host forwarder".to_string(), - } - } - - fn primary_listener_spec(address: SocketAddr) -> GatewayListenerSpec { - GatewayListenerSpec { - address, - scope: GatewayListenerScope::Primary, - covered_addresses: Vec::new(), - provenance: None, - } - } - - fn callback_listener_spec( - address: SocketAddr, - driver_name: &str, - reason: &str, - ) -> GatewayListenerSpec { - GatewayListenerSpec { - address, - scope: GatewayListenerScope::ComputeDriverCallback, - covered_addresses: Vec::new(), - provenance: Some(GatewayListenerProvenance { - driver_name: driver_name.to_string(), - reason: reason.to_string(), - }), - } - } +/// Bind the operator-configured gateway endpoint. +pub async fn bind_gateway_listener(address: SocketAddr) -> Result { + let listener = TcpListener::bind(address) + .await + .map_err(|error| Error::transport(format!("failed to bind to {address}: {error}")))?; + let local_addr = listener.local_addr().unwrap_or(address); + info!(address = %local_addr, "Gateway listener bound"); + Ok(BoundGatewayListener { + listener, + address: local_addr, + }) } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 80cc2acd57..a8cd09acca 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -84,7 +84,7 @@ pub(crate) fn install_jsonwebtoken_crypto_provider() { } use compute::ComputeRuntime; -use gateway_listener::{BoundGatewayListener, GatewayListenerScope, bind_gateway_listeners}; +use gateway_listener::{BoundGatewayListener, bind_gateway_listener}; pub use grpc::OpenShellService; pub use http::{health_router, http_router, metrics_router, service_http_router}; @@ -723,11 +723,7 @@ pub(crate) async fn run_server( )) })?; - let gateway_listeners = bind_gateway_listeners( - config.bind_address, - state.compute.gateway_listener_requirements(), - ) - .await?; + let gateway_listener = bind_gateway_listener(config.bind_address).await?; // Create the multiplexed service let service = MultiplexService::new(state.clone()); @@ -801,17 +797,14 @@ pub(crate) async fn run_server( None }; - let mut listener_tasks = Vec::with_capacity(gateway_listeners.len()); let enable_loopback_service_http = config.service_routing.enable_loopback_service_http; - for listener in gateway_listeners { - listener_tasks.push(tokio::spawn(serve_gateway_listener( - listener, - service.clone(), - tls_acceptor.clone(), - enable_loopback_service_http, - shutdown_rx.clone(), - ))); - } + let listener_task = tokio::spawn(serve_gateway_listener( + gateway_listener, + service.clone(), + tls_acceptor.clone(), + enable_loopback_service_http, + shutdown_rx.clone(), + )); // Deadlines must run while restored supervisors wait for policy repair. let (startup_tx, startup_rx) = watch::channel(false); @@ -819,10 +812,8 @@ pub(crate) async fn run_server( .compute .spawn_watchers(shutdown_rx.clone(), startup_rx); - // Restored supervisors need the callback listeners while the compute - // driver reconciles persisted sandboxes. Serve them before starting that - // reconciliation so policy fetch and supervisor-session registration - // cannot deadlock gateway startup. + // Serve the gateway before reconciling persisted sandboxes so restored + // supervisors can fetch policy and register their sessions. if let Err(err) = state .compute .start_persisted_sandboxes_with_authentication( @@ -857,10 +848,8 @@ pub(crate) async fn run_server( state.gateway_shutting_down.store(true, Ordering::Release); let _ = shutdown_tx.send(true); - for task in listener_tasks { - if let Err(err) = task.await { - warn!(error = %err, "Gateway listener task failed during shutdown"); - } + if let Err(err) = listener_task.await { + warn!(error = %err, "Gateway listener task failed during shutdown"); } state @@ -879,8 +868,10 @@ async fn serve_gateway_listener( enable_loopback_service_http: bool, mut shutdown: watch::Receiver, ) { - let BoundGatewayListener { listener, spec } = bound_listener; - let listen_addr = spec.address; + let BoundGatewayListener { + listener, + address: listen_addr, + } = bound_listener; loop { let accepted = tokio::select! { @@ -900,21 +891,12 @@ async fn serve_gateway_listener( continue; } }; - let listener_scope = match stream.local_addr() { - Ok(local_addr) => spec.scope_for_local_addr(local_addr), - Err(e) => { - debug!(error = %e, client = %addr, listen = %listen_addr, "Failed to inspect accepted local address"); - spec.scope - } - }; - set_tcp_nodelay_best_effort(&stream); spawn_gateway_connection( stream, addr, listen_addr, - listener_scope, service.clone(), tls_acceptor.clone(), enable_loopback_service_http, @@ -975,19 +957,14 @@ fn allow_plaintext_service_http( enabled: bool, listen_addr: SocketAddr, peer_addr: SocketAddr, - listener_scope: GatewayListenerScope, ) -> bool { - enabled - && matches!(listener_scope, GatewayListenerScope::Primary) - && listen_addr.ip().is_loopback() - && peer_addr.ip().is_loopback() + enabled && listen_addr.ip().is_loopback() && peer_addr.ip().is_loopback() } fn spawn_gateway_connection( stream: TcpStream, addr: SocketAddr, listen_addr: SocketAddr, - listener_scope: GatewayListenerScope, service: MultiplexService, tls_acceptor: Option, enable_loopback_service_http: bool, @@ -1000,13 +977,9 @@ fn spawn_gateway_connection( enable_loopback_service_http, listen_addr, addr, - listener_scope, ) => { - if let Err(e) = service - .serve_service_http_on_listener(stream, listener_scope) - .await - { + if let Err(e) = service.serve_service_http(stream).await { if is_benign_connection_close(e.as_ref()) { debug!(error = %e, client = %addr, listen = %listen_addr, "Plaintext service HTTP connection closed"); } else { @@ -1018,7 +991,6 @@ fn spawn_gateway_connection( warn!( client = %addr, listen = %listen_addr, - scope = ?listener_scope, "Rejected plaintext HTTP on gateway listener" ); } @@ -1030,11 +1002,7 @@ fn spawn_gateway_connection( Ok(tls_stream) => { let peer_identity = multiplex::extract_peer_identity(&tls_stream); if let Err(e) = service - .serve_with_peer_identity_on_listener( - tls_stream, - peer_identity, - listener_scope, - ) + .serve_with_peer_identity(tls_stream, peer_identity) .await { if is_benign_connection_close(e.as_ref()) { @@ -1060,7 +1028,7 @@ fn spawn_gateway_connection( }); } else { tokio::spawn(async move { - if let Err(e) = service.serve_on_listener(stream, listener_scope).await { + if let Err(e) = service.serve(stream).await { if is_benign_connection_close(e.as_ref()) { debug!(error = %e, client = %addr, "Connection closed"); } else { @@ -1778,10 +1746,10 @@ pub(crate) async fn ensure_default_workspace(store: &Store) -> Result<()> { mod tests { use super::{ BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, ExtensionKind, - GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor, - allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - configured_compute_driver, extension_token_ttl, is_benign_tls_handshake_failure, - mint_gateway_extension_credential, serve_gateway_listener, + MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, + bind_gateway_listener, classify_initial_bytes, configured_compute_driver, + extension_token_ttl, is_benign_tls_handshake_failure, mint_gateway_extension_credential, + serve_gateway_listener, }; use openshell_core::{ Config, @@ -1799,10 +1767,7 @@ mod tests { use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; - use crate::{ - compute::GatewayListenerRequirement, gateway_listener::GatewayListenerSpec, - tls_test_utils::generate_test_certs_with_ca, - }; + use crate::tls_test_utils::generate_test_certs_with_ca; static DETECTION_PROBE_ORDER: LazyLock>> = LazyLock::new(|| Mutex::new(Vec::new())); @@ -2049,7 +2014,7 @@ mod tests { let handle = tokio::spawn(serve_gateway_listener( BoundGatewayListener { listener, - spec: GatewayListenerSpec::new(listen_addr, GatewayListenerScope::Primary), + address: listen_addr, }, service, Some(tls_acceptor), @@ -2147,23 +2112,10 @@ mod tests { let peer: SocketAddr = "127.0.0.1:54000".parse().unwrap(); let wildcard: SocketAddr = "0.0.0.0:8080".parse().unwrap(); let remote_peer: SocketAddr = "192.0.2.10:54000".parse().unwrap(); - let primary = GatewayListenerScope::Primary; - let callback = GatewayListenerScope::ComputeDriverCallback; - - assert!(allow_plaintext_service_http(true, loopback, peer, primary)); - assert!(!allow_plaintext_service_http( - false, loopback, peer, primary - )); - assert!(!allow_plaintext_service_http(true, wildcard, peer, primary)); - assert!(!allow_plaintext_service_http( - true, - loopback, - remote_peer, - primary - )); - assert!(!allow_plaintext_service_http( - true, loopback, peer, callback - )); + assert!(allow_plaintext_service_http(true, loopback, peer)); + assert!(!allow_plaintext_service_http(false, loopback, peer)); + assert!(!allow_plaintext_service_http(true, wildcard, peer)); + assert!(!allow_plaintext_service_http(true, loopback, remote_peer)); } #[tokio::test] @@ -2462,14 +2414,8 @@ mod tests { let occupied_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let occupied_address = occupied_listener.local_addr().unwrap(); let start_attempted = AtomicBool::new(false); - let primary_address: SocketAddr = "127.0.0.1:0".parse().unwrap(); - let result: openshell_core::Result<()> = async { - let _listeners = bind_gateway_listeners( - primary_address, - &[docker_listener_requirement(occupied_address)], - ) - .await?; + let _listener = bind_gateway_listener(occupied_address).await?; start_attempted.store(true, Ordering::SeqCst); Ok(()) } @@ -2477,19 +2423,11 @@ mod tests { assert!( result.is_err(), - "binding the occupied extra gateway address should fail" + "binding the occupied gateway address should fail" ); assert!( !start_attempted.load(Ordering::SeqCst), - "persisted sandbox start must not run before every gateway listener is bound" + "persisted sandbox start must not run before the gateway listener is bound" ); } - - fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { - GatewayListenerRequirement::Exact { - address, - driver_name: "docker".to_string(), - reason: "managed bridge".to_string(), - } - } } diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 87f0a9fbc6..2df0ea5403 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -7,7 +7,7 @@ //! to either the gRPC service or HTTP endpoints based on the request headers. use bytes::{Bytes, BytesMut}; -use http::{Extensions, HeaderValue, Request, Response, StatusCode}; +use http::{Extensions, HeaderValue, Request, Response}; use http_body::Body; use http_body_util::{BodyExt, Full, LengthLimitError, Limited, StreamBody}; use hyper::body::Incoming; @@ -49,7 +49,6 @@ use crate::{ auth::oidc::{self, OidcAuthenticator}, auth::principal::{Principal, UserPrincipal}, auth::workspace_authz::{MinWorkspaceRole, authorize_workspace}, - gateway_listener::GatewayListenerScope, http_router, service_http_router, }; @@ -220,22 +219,7 @@ impl MultiplexService { where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - self.serve_on_listener(stream, GatewayListenerScope::Primary) - .await - } - - /// Serve a connection and preserve its listener scope in request - /// extensions for downstream routing and policy decisions. - pub(crate) async fn serve_on_listener( - &self, - stream: S, - listener_scope: GatewayListenerScope, - ) -> Result<(), Box> - where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, - { - self.serve_with_peer_identity_on_listener(stream, None, listener_scope) - .await + self.serve_with_peer_identity(stream, None).await } /// Serve a TLS connection with an optional mTLS peer identity. @@ -244,25 +228,6 @@ impl MultiplexService { stream: S, peer_identity: Option, ) -> Result<(), Box> - where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, - { - self.serve_with_peer_identity_on_listener( - stream, - peer_identity, - GatewayListenerScope::Primary, - ) - .await - } - - /// Serve a TLS connection and preserve its listener scope in request - /// extensions for downstream routing and policy decisions. - pub(crate) async fn serve_with_peer_identity_on_listener( - &self, - stream: S, - peer_identity: Option, - listener_scope: GatewayListenerScope, - ) -> Result<(), Box> where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { @@ -299,10 +264,7 @@ impl MultiplexService { let grpc_service = request_id_middleware!(grpc_service); let http_service = request_id_middleware!(http_service); - let service = GatewayListenerContextService::new( - MultiplexedService::new(grpc_service, http_service), - listener_scope, - ); + let service = MultiplexedService::new(grpc_service, http_service); let mut builder = Builder::new(TokioExecutor::new()); // Server-side HTTP/2 keepalive: supervisors hold long-lived sessions, and without @@ -331,26 +293,9 @@ impl MultiplexService { where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - self.serve_service_http_on_listener(stream, GatewayListenerScope::Primary) - .await - } - - /// Serve a plaintext service HTTP connection and preserve its listener - /// scope in request extensions. - pub(crate) async fn serve_service_http_on_listener( - &self, - stream: S, - listener_scope: GatewayListenerScope, - ) -> Result<(), Box> - where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, - { - let http_service = GatewayListenerContextService::new( - TowerToHyperService::new(request_id_middleware!(service_http_router( - self.state.clone() - ))), - listener_scope, - ); + let http_service = TowerToHyperService::new(request_id_middleware!(service_http_router( + self.state.clone() + ))); Builder::new(TokioExecutor::new()) .serve_connection_with_upgrades(TokioIo::new(stream), http_service) @@ -360,36 +305,6 @@ impl MultiplexService { } } -/// Adds the immutable listener authorization scope to every served request. -#[derive(Clone)] -struct GatewayListenerContextService { - inner: S, - listener_scope: GatewayListenerScope, -} - -impl GatewayListenerContextService { - fn new(inner: S, listener_scope: GatewayListenerScope) -> Self { - Self { - inner, - listener_scope, - } - } -} - -impl hyper::service::Service> for GatewayListenerContextService -where - S: hyper::service::Service>, -{ - type Response = S::Response; - type Error = S::Error; - type Future = S::Future; - - fn call(&self, mut request: Request) -> Self::Future { - request.extensions_mut().insert(self.listener_scope); - self.inner.call(request) - } -} - /// `OpenShell` gRPC wrapper that applies configured gateway interceptors before /// tonic dispatches to a specific RPC handler. #[derive(Clone)] @@ -1153,38 +1068,6 @@ impl MultiplexedService { } } -fn listener_allows_request( - listener_scope: Option<&GatewayListenerScope>, - is_grpc: bool, - path: &str, -) -> bool { - match listener_scope { - Some(GatewayListenerScope::ComputeDriverCallback) => { - is_grpc && crate::auth::sandbox_methods::is_sandbox_callable(path) - } - Some(GatewayListenerScope::Primary) | None => true, - } -} - -fn callback_listener_rejection(is_grpc: bool) -> Response { - if is_grpc { - let response: Response = tonic::Status::permission_denied( - "compute-driver callback listeners accept sandbox callback RPCs only", - ) - .into_http(); - let (parts, body) = response.into_parts(); - let body = body.map_err(Into::into).boxed_unsync(); - Response::from_parts(parts, BoxBody(body)) - } else { - Response::builder() - .status(StatusCode::FORBIDDEN) - .body(boxed_body_from_bytes(Bytes::from_static( - b"compute-driver callback listeners accept gRPC callbacks only", - ))) - .expect("static callback listener rejection response must be valid") - } -} - impl hyper::service::Service> for MultiplexedService where G: tower::Service, Response = Response> + Clone + Send + 'static, @@ -1208,15 +1091,6 @@ where .get("content-type") .is_some_and(|v| v.as_bytes().starts_with(b"application/grpc")); - if !listener_allows_request( - req.extensions().get::(), - is_grpc, - req.uri().path(), - ) { - let response = callback_listener_rejection(is_grpc); - return Box::pin(async move { Ok(response) }); - } - if is_grpc { let method = grpc_method_from_path(req.uri().path()); let start = Instant::now(); @@ -1400,6 +1274,7 @@ impl Body for BoxBody { mod tests { use super::*; use bytes::Bytes; + use http::StatusCode; use http_body_util::Empty; use openshell_core::GatewayInterceptorConfig; use openshell_core::proto::CreateSandboxRequest; @@ -1415,113 +1290,6 @@ mod tests { use tokio_stream::wrappers::TcpListenerStream; use tower::Service; - #[tokio::test] - async fn listener_context_service_preserves_listener_scope() { - let observed = Arc::new(Mutex::new(None)); - let captured = observed.clone(); - let inner = hyper::service::service_fn(move |request: Request>| { - *captured.lock().unwrap() = request.extensions().get::().copied(); - async move { Ok::<_, Infallible>(Response::new(Empty::::new())) } - }); - let service = GatewayListenerContextService::new(inner, GatewayListenerScope::Primary); - hyper::service::Service::call(&service, Request::new(Empty::::new())) - .await - .unwrap(); - - assert_eq!( - *observed.lock().unwrap(), - Some(GatewayListenerScope::Primary) - ); - } - - fn callback_listener_scope() -> GatewayListenerScope { - GatewayListenerScope::ComputeDriverCallback - } - - #[test] - fn callback_listener_allows_sandbox_callback_rpcs() { - let scope = callback_listener_scope(); - let callback_paths = [ - "/openshell.v1.OpenShell/ConnectSupervisor", - "/openshell.v1.OpenShell/RelayStream", - "/openshell.v1.OpenShell/GetSandboxConfig", - "/openshell.v1.OpenShell/ReportPolicyStatus", - "/openshell.v1.OpenShell/PushSandboxLogs", - "/openshell.v1.OpenShell/GetSandboxProviderEnvironment", - "/openshell.v1.OpenShell/SubmitPolicyAnalysis", - "/openshell.v1.OpenShell/RefreshSandboxToken", - ]; - - for path in callback_paths { - assert!( - listener_allows_request(Some(&scope), true, path), - "callback listener should allow {path}" - ); - } - } - - #[test] - fn callback_listener_surface_matches_rpc_auth_metadata() { - let scope = callback_listener_scope(); - - for path in crate::auth::method_authz::all_paths() { - assert_eq!( - listener_allows_request(Some(&scope), true, path), - crate::auth::method_authz::is_sandbox_callable(path), - "callback listener exposure must follow rpc_auth metadata for {path}" - ); - } - } - - #[test] - fn callback_listener_rejects_non_callback_routes() { - let scope = callback_listener_scope(); - let rejected_grpc_paths = [ - "/grpc.health.v1.Health/Check", - "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo", - "/openshell.v1.OpenShell/ListSandboxes", - "/openshell.v1.OpenShell/DeleteSandbox", - "/openshell.v1.OpenShell/CreateProvider", - ]; - - for path in rejected_grpc_paths { - assert!( - !listener_allows_request(Some(&scope), true, path), - "callback listener should reject {path}" - ); - } - assert!(!listener_allows_request(Some(&scope), false, "/health")); - assert!(!listener_allows_request(Some(&scope), false, "/service")); - } - - #[test] - fn primary_listener_routing_is_unchanged() { - let primary = GatewayListenerScope::Primary; - let paths = [ - "/grpc.health.v1.Health/Check", - "/openshell.v1.OpenShell/ListSandboxes", - "/health", - "/service", - ]; - - for path in paths { - assert!(listener_allows_request(Some(&primary), true, path)); - assert!(listener_allows_request(Some(&primary), false, path)); - assert!(listener_allows_request(None, true, path)); - assert!(listener_allows_request(None, false, path)); - } - } - - #[test] - fn callback_listener_rejections_use_protocol_appropriate_statuses() { - let grpc = callback_listener_rejection(true); - assert_eq!(grpc.status(), StatusCode::OK); - assert_eq!(grpc.headers().get("grpc-status").unwrap(), "7"); - - let http = callback_listener_rejection(false); - assert_eq!(http.status(), StatusCode::FORBIDDEN); - } - #[derive(Clone)] struct PostCommitTestInterceptor; @@ -1624,12 +1392,6 @@ mod tests { } async fn start_http_server_with_middleware() -> std::net::SocketAddr { - start_http_server_with_middleware_on_listener(GatewayListenerScope::Primary).await - } - - async fn start_http_server_with_middleware_on_listener( - listener_scope: GatewayListenerScope, - ) -> std::net::SocketAddr { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -1637,7 +1399,6 @@ mod tests { let http_service = request_id_middleware!(http_service); let service = MultiplexedService::new(http_service.clone(), http_service); - let service = GatewayListenerContextService::new(service, listener_scope); tokio::spawn(async move { loop { @@ -1689,39 +1450,6 @@ mod tests { http1_request(addr, "GET", path, headers).await } - #[tokio::test] - async fn callback_listener_filter_is_applied_before_route_dispatch() { - let addr = start_http_server_with_middleware_on_listener(callback_listener_scope()).await; - - let health = http1_get(addr, "/healthz", &[]).await; - assert_eq!(health.status(), StatusCode::FORBIDDEN); - - let admin = http1_request( - addr, - "POST", - "/openshell.v1.OpenShell/ListSandboxes", - &[("content-type", "application/grpc")], - ) - .await; - assert_eq!(admin.status(), StatusCode::OK); - assert_eq!(admin.headers().get("grpc-status").unwrap(), "7"); - - let callback = http1_request( - addr, - "POST", - "/openshell.v1.OpenShell/ConnectSupervisor", - &[("content-type", "application/grpc")], - ) - .await; - assert_ne!( - callback - .headers() - .get("grpc-status") - .and_then(|value| value.to_str().ok()), - Some("7") - ); - } - #[tokio::test] async fn intercepted_grpc_body_collection_rejects_oversized_body() { let oversized = Bytes::from(vec![0_u8; MAX_INTERCEPTED_GRPC_BODY_SIZE + 1]); diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index b0beda43c6..bd3f4790ac 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -518,7 +518,7 @@ mod tests { } assert_eq!( compiled_method_count, - 102 + PROVIDER_READINESS_RPC_SIGNATURES.len(), + 101 + PROVIDER_READINESS_RPC_SIGNATURES.len(), "classify every compiled RPC" ); assert_eq!( diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 5f024de692..8ff4060e83 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -19,13 +19,11 @@ use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServ use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverSandbox, EnsureWorkspaceRequest, - EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest, - GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, + GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, + StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, - compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, + compute_driver_server::ComputeDriver, }; use std::collections::HashMap; #[cfg(unix)] @@ -96,7 +94,6 @@ pub fn authenticate_as_dev_user(mut request: Request<()>) -> Result, #[derive(Debug, Clone, PartialEq)] pub enum FakeComputeDriverCall { GetCapabilities, - GetGatewayListenerRequirements, ValidateSandboxCreate { sandbox: Option, }, @@ -131,8 +128,6 @@ pub struct FakeComputeDriver { #[derive(Debug)] struct FakeComputeDriverState { capabilities: GetCapabilitiesResponse, - gateway_listener_requirements: Vec, - gateway_listener_requirements_supported: bool, sandboxes: HashMap, calls: Vec, traceparents: Vec, @@ -160,8 +155,6 @@ impl FakeComputeDriver { rootfs_tar_staging_dir: String::new(), rootfs_tar_max_bytes: 0, }, - gateway_listener_requirements: Vec::new(), - gateway_listener_requirements_supported: true, sandboxes: HashMap::new(), calls: Vec::new(), traceparents: Vec::new(), @@ -193,29 +186,6 @@ impl FakeComputeDriver { self } - #[must_use] - pub fn with_gateway_listener_requirement( - self, - bind_address: impl Into, - reason: impl Into, - ) -> Self { - self.with_state(|state| { - state - .gateway_listener_requirements - .push(GatewayListenerRequirement { - reason: reason.into(), - selector: Some(Selector::ExactBindAddress(bind_address.into())), - }); - }); - self - } - - #[must_use] - pub fn without_gateway_listener_requirements_api(self) -> Self { - self.with_state(|state| state.gateway_listener_requirements_supported = false); - self - } - #[must_use] pub fn calls(&self) -> Vec { self.with_state(|state| state.calls.clone()) @@ -323,25 +293,6 @@ impl ComputeDriver for FakeComputeDriver { Ok(Response::new(response)) } - async fn get_gateway_listener_requirements( - &self, - request: Request, - ) -> Result, Status> { - self.record_traceparent(request.metadata()); - self.with_state(|state| { - state - .calls - .push(FakeComputeDriverCall::GetGatewayListenerRequirements); - state - .gateway_listener_requirements_supported - .then(|| GetGatewayListenerRequirementsResponse { - requirements: state.gateway_listener_requirements.clone(), - }) - .map(Response::new) - .ok_or_else(|| Status::unimplemented("listener requirements unsupported")) - }) - } - async fn validate_sandbox_create( &self, request: Request, diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 4d82964189..e491b825c3 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -79,13 +79,8 @@ services: user: "0" ports: - # gRPC / control-plane API (used by the openshell CLI and sandbox callbacks) - # The Docker driver injects host.openshell.internal: into sandbox - # containers as the callback endpoint. The gateway's internal port is 8080, so - # host port 8080 must be published at the same number so that - # host.openshell.internal:8080 routes to the gateway container. - # gateway.toml binds to 127.0.0.1 — the Docker driver adds the bridge listener - # automatically so sandbox containers can reach the gateway without 0.0.0.0. + # gRPC / control-plane API. The Docker supervisor uses host networking + # and reaches this primary loopback endpoint through the published port. - "127.0.0.1:${OPENSHELL_PORT:-8080}:8080" # Health endpoint (GET /healthz, GET /readyz) - "127.0.0.1:${OPENSHELL_HEALTH_PORT:-8081}:8081" diff --git a/deploy/helm/openshell/tests/grpc_endpoint_test.yaml b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml index f84442f605..82b3ca5cb0 100644 --- a/deploy/helm/openshell/tests/grpc_endpoint_test.yaml +++ b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -suite: Kubernetes gateway callback endpoint +suite: Kubernetes supervisor gateway endpoint templates: - templates/gateway-config.yaml diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index af2e97a94f..7ae1f6f15e 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -24,9 +24,9 @@ compute_driver = "podman" ``` The RPM does not override `bind_address`. The primary listener uses the -built-in `127.0.0.1:17670` default. The Podman driver reports the callback -interface it needs, and the gateway adds a separate listener scoped to that -interface. This keeps the general API off unrelated host interfaces. +built-in `127.0.0.1:17670` default. Host-networked Podman supervisors connect +to this same loopback listener, so the gateway does not expose another host +interface. `compute_driver = "podman"` pins the compute driver to Podman. Without this, the gateway auto-detects in order: Kubernetes, Podman, Docker. Pinning @@ -68,8 +68,7 @@ systemctl --user edit openshell-gateway The RPM enables mutual TLS by default. The gateway requires a valid client certificate for all API connections. Its primary listener uses -`127.0.0.1:17670`; Podman callback traffic uses the additional listener -described in "Default configuration" above. +`127.0.0.1:17670`; Podman supervisor sessions use that same listener. ### Auto-generated certificates @@ -244,10 +243,10 @@ version = 2 compute_driver = "podman" [openshell.drivers.podman] +network_name = "openshell" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "if_not_present" health_check_interval_secs = 10 -network_name = "openshell" stop_timeout_secs = 10 ``` diff --git a/deploy/rpm/QUICKSTART.md b/deploy/rpm/QUICKSTART.md index f37f611351..107f0a4421 100644 --- a/deploy/rpm/QUICKSTART.md +++ b/deploy/rpm/QUICKSTART.md @@ -66,8 +66,8 @@ On first start, the gateway automatically generates: - A self-signed PKI bundle (CA, server cert, client cert) for mTLS > **Note:** The primary gateway listener uses the loopback default, -> `127.0.0.1:17670`. The Podman driver requests a separate callback listener -> scoped to the interface its sandboxes can reach. Mutual TLS (mTLS) is +> `127.0.0.1:17670`. Host-networked Podman supervisors use this same listener. +> Mutual TLS (mTLS) is > enabled automatically on first start, requiring a valid client certificate > for every connection. See CONFIGURATION.md for details. diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index a0a6e296f0..6270ace419 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -19,8 +19,7 @@ version = 2 [openshell.gateway] # Keep the primary listener on the built-in 127.0.0.1:17670 default. The -# Podman driver reports the callback interface it needs, and the gateway -# adds a separate listener scoped to that interface. +# host-networked Podman supervisor uses this same loopback endpoint. # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver diff --git a/deploy/rpm/gateway.toml.default.v1 b/deploy/rpm/gateway.toml.default.v1 index cd7e0d99c3..2dcae9344d 100644 --- a/deploy/rpm/gateway.toml.default.v1 +++ b/deploy/rpm/gateway.toml.default.v1 @@ -19,8 +19,7 @@ version = 1 [openshell.gateway] # Keep the primary listener on the built-in 127.0.0.1:17670 default. The -# Podman driver reports the callback interface it needs, and the gateway -# adds a separate listener scoped to that interface. +# host-networked Podman supervisor uses this same loopback endpoint. # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 57d8ca6b39..e27d378381 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -56,7 +56,7 @@ For detailed driver behavior, refer to [Sandbox Compute Drivers](/reference/sand On macOS, the install script uses Homebrew. The Homebrew package installs the `openshell` CLI, `openshell-prover`, the gateway binary, and a Homebrew-managed gateway service. -The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew upgrades migrate exact package-generated schema-v1 prefix configs, including the affected IPv6 variant. They preserve edited prefix configs and all user configs. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) for an edited v1 file. +The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Host-networked Docker Desktop and Podman Machine supervisors reuse the primary listener when they can reach host loopback. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew upgrades migrate exact package-generated schema-v1 prefix configs, including the affected IPv6 variant. They preserve edited prefix configs and all user configs. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) for an edited v1 file. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. diff --git a/docs/get-started/tutorials/docker-compose.mdx b/docs/get-started/tutorials/docker-compose.mdx index 77ff79f638..1efc2abfeb 100644 --- a/docs/get-started/tutorials/docker-compose.mdx +++ b/docs/get-started/tutorials/docker-compose.mdx @@ -34,7 +34,12 @@ The Compose configuration lives at [`deploy/docker/`](https://github.com/NVIDIA/ ## Port note -The Docker compute driver injects `host.openshell.internal:` into every sandbox container as its callback address. The gateway listens on port 8080 inside the container, so **port 8080 must be published at the same number on the Docker host**. Publishing it as a different host port (for example `18080:8080`) causes sandbox containers to call back to the wrong port and remain stuck in the `Provisioning` phase. +The Docker supervisor uses host networking and connects to the gateway through +the host's loopback port. The gateway listens on port 8080 inside the container, +so **port 8080 must be published at the same number on the Docker host**. +Publishing it as a different host port (for example `18080:8080`) leaves the +supervisor unable to establish its gateway session and the sandbox remains in +the `Provisioning` phase. If port 8080 is taken, change `OPENSHELL_SERVER_PORT` and update the port mapping to `:8080`, then set `OPENSHELL_PORT=` in an `.env` file. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 75599ae19f..ead6f89f11 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -31,9 +31,9 @@ Package-managed gateways use either built-in defaults or a package-seeded TOML f | Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml`; the systemd user service seeds this file from the packaged template on first start. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | -The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. +The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. Host-networked Podman supervisors use that same loopback endpoint, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. -The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. +The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Host-networked Docker and Podman supervisors reuse that listener when the desktop runtime exposes host loopback. A user config takes precedence. Homebrew and RPM upgrades migrate only exact package-generated schema-v1 defaults. Homebrew recognizes both its empty v1 prefix config and the affected IPv6-loopback variant. RPM recognizes the v1 file seeded by its systemd user service. Package upgrades never rewrite an edited file; migrate an edited v1 file manually with the steps below. @@ -704,15 +704,13 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "if_not_present" # Value assigned to the openshell.sandbox_namespace label on sandbox containers. sandbox_label = "docker-dev" -# Optional override. When omitted, the gateway derives -# https://host.openshell.internal: for this driver. -grpc_endpoint = "https://host.openshell.internal:17670" +# Optional override. When omitted, the host-networked supervisor uses the +# gateway's primary loopback endpoint. +grpc_endpoint = "https://127.0.0.1:17670" # Workload-side runtime. Defaults to the gateway version. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" # Supervisor runtime. Defaults to the gateway version. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -network_name = "openshell-docker" -host_gateway_ip = "172.17.0.1" # Unsafe operator override. Host bind mounts, including Docker local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. @@ -755,6 +753,7 @@ guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] +network_name = "openshell" # Rootless socket path. For root Podman use /run/podman/podman.sock. # Omit to auto-detect: the driver probes for a responsive Podman socket, then # asks the podman CLI where its socket is, and fails to start if neither finds @@ -762,11 +761,11 @@ guest_tls_key = "/etc/openshell/certs/client-key.pem" socket_path = "/run/user/1000/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "if_not_present" # always | if_not_present | never | newer -# Optional override. When omitted, the gateway derives this endpoint. -grpc_endpoint = "https://host.containers.internal:17670" +# Optional override. When omitted, Linux uses gateway loopback and Podman +# Machine uses host.containers.internal. +grpc_endpoint = "https://127.0.0.1:17670" # The gateway overwrites gateway_port from bind_address at runtime. gateway_port = 17670 -network_name = "openshell" # Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. # Set "" to force Podman's host-gateway resolver. # host_gateway_ip = "192.168.127.254" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index d24ff67ef4..fab785e91f 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -62,7 +62,7 @@ Common gateway options: |---|---| | `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. +Set driver-specific values such as sandbox images, gateway endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -152,15 +152,10 @@ It overrides the gateway's configured default runtime class for that sandbox, while a typed `SandboxTemplate.runtime_class_name` value from the API still takes precedence. -Docker and Podman report the address through which their sandboxes can reach -the gateway. If the primary listener covers that address, the gateway reuses -it and sandbox JWT authentication restricts the supervisor to its callback RPC -allowlist. If the primary listener is not reachable through that address, the -gateway creates an additional callback-only listener. Use the primary endpoint -for CLI, administrator, health, reflection, inference-route management, and -HTTP requests. A `PermissionDenied` response from an additional callback-only -listener is expected for those requests. Do not broaden the primary listener -to `0.0.0.0` solely to make sandbox callbacks reachable. +Docker and Podman supervisors use host networking. On Linux they connect to the +gateway's primary loopback listener. Sandbox JWT authentication restricts each +supervisor to the sandbox-callable RPC allowlist; no additional gateway +listener is created. The published supervisor container uses a shell-free distroless Debian 13 image. Inspect its container logs and health status with your runtime tools; it does not @@ -173,14 +168,16 @@ change the tools available inside your workload image. The gateway talks to the Docker daemon to create sandbox containers. Docker is also required for local image builds from directories or Dockerfiles. -Docker Desktop and compatible macOS runtimes route `host.openshell.internal` -through an IPv4 host-gateway alias. The gateway reuses an IPv4 primary listener -that already covers loopback. Otherwise, the Docker driver requests a separate -`127.0.0.1:` callback-only listener. +The trusted supervisor companion uses Docker host networking; the agent +workload retains `network=none`. On Linux the supervisor reaches the gateway at +its primary loopback endpoint. Docker Desktop requires host networking to be +enabled and does not support this mode together with Enhanced Container +Isolation. Set `grpc_endpoint` when the gateway is not reachable on the Docker +daemon host. For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -256,7 +253,7 @@ namespace roots. These checks do not make host bind mounts safe. The gateway talks to the Podman API socket. The Podman driver requires Podman 5.x, cgroups v2, rootless networking, and an active Podman user socket. When `socket_path` is not set, the driver probes known socket paths, then uses the `podman` CLI to resolve the active native or machine-backed connection. It fails to start if neither method finds a socket. -The agent workload uses `network=none`. Its trusted supervisor companion uses Podman's host network for gateway callbacks and policy-approved upstream connections. +The agent workload uses `network=none`. Its trusted supervisor companion uses Podman's host network for its gateway session and policy-approved upstream connections. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). @@ -274,7 +271,11 @@ stopped sandboxes alone. For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. -On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. +On Linux, the host-networked supervisor uses the gateway's primary loopback +endpoint. On macOS with `podman machine`, the driver uses gvproxy's +host-loopback IP, `192.168.127.254`, by default. Set `host_gateway_ip` only when +your Podman machine uses a non-standard host-loopback address, or set +`grpc_endpoint` explicitly when the gateway is remote. ### Podman Driver Config Mounts @@ -426,7 +427,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | -| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | +| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `sandbox_runtime_image` | `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` | Override the image that provides `openshell-sandbox`. The default repository with an empty tag uses the version pinned into the gateway. | | `sandbox_runtime_image_pull_policy` | `sandboxRuntime.image.pullPolicy` | Set the Kubernetes image pull policy for the sandbox runtime image. | diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index ca6236f214..e982ccefac 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -165,7 +165,7 @@ supervisor_runtime = f"localhost/openshell/supervisor@{image_digest}" base_runtime = f"docker.io/library/debian@{image_digest}" pull_policy = "missing" if schema == 1 else "if_not_present" gateway_port = 18181 -callback = f"https://host.containers.internal:{gateway_port}" +grpc_endpoint = f"https://127.0.0.1:{gateway_port}" driver_socket = f"/tmp/{variant}-driver.sock" podman_socket = f"/tmp/{variant}-podman.sock" network = f"{variant}-network" @@ -243,7 +243,7 @@ launch = { if external: launch.update( { - "external_driver_grpc_endpoint": callback, + "external_driver_grpc_endpoint": grpc_endpoint, "external_driver_host_gateway_ip": "host-gateway", "external_driver_userns": None, "external_driver_spiffe": False, @@ -257,7 +257,7 @@ if external: "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": pull_policy, "OPENSHELL_SANDBOX_RUNTIME_IMAGE": sandbox_boundary, "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10, - "OPENSHELL_GRPC_ENDPOINT": callback, + "OPENSHELL_GRPC_ENDPOINT": grpc_endpoint, "OPENSHELL_GATEWAY_PORT": gateway_port, "OPENSHELL_NETWORK_NAME": network, "OPENSHELL_STOP_TIMEOUT": 15, diff --git a/e2e/parity/verify-results.py b/e2e/parity/verify-results.py index 4e8a4619bb..d785c1f3ef 100644 --- a/e2e/parity/verify-results.py +++ b/e2e/parity/verify-results.py @@ -357,12 +357,12 @@ def verify_variant( f"{launch_path}: external driver pre-execution hash mismatch", ) gateway_port = launch.get("gateway_port") - callback_endpoint = f"https://host.containers.internal:{gateway_port}" + grpc_endpoint = f"https://127.0.0.1:{gateway_port}" require( isinstance(gateway_port, int) and 0 < gateway_port <= 65535 - and launch.get("external_driver_grpc_endpoint") == callback_endpoint, - f"{launch_path}: external driver callback endpoint is not isolated", + and launch.get("external_driver_grpc_endpoint") == grpc_endpoint, + f"{launch_path}: external driver gRPC endpoint is not isolated", ) require( launch.get("external_driver_host_gateway_ip") == "host-gateway" @@ -417,7 +417,7 @@ def verify_variant( and driver_environment["OPENSHELL_SANDBOX_IMAGE_PULL_POLICY"] == expected_policy and driver_environment["OPENSHELL_HEALTH_CHECK_INTERVAL_SECS"] == 10 - and driver_environment["OPENSHELL_GRPC_ENDPOINT"] == callback_endpoint + and driver_environment["OPENSHELL_GRPC_ENDPOINT"] == grpc_endpoint and driver_environment["OPENSHELL_GATEWAY_PORT"] == gateway_port and isinstance(driver_environment["OPENSHELL_NETWORK_NAME"], str) and driver_environment["OPENSHELL_NETWORK_NAME"] @@ -443,12 +443,12 @@ def verify_variant( and Path(tls_input["path"]).is_absolute() and isinstance(tls_input["sha256"], str) and SHA256_RE.fullmatch(tls_input["sha256"]) is not None, - f"{launch_path}: invalid external driver callback TLS input {field}", + f"{launch_path}: invalid external driver TLS input {field}", ) tls_paths.add(tls_input["path"]) require( len(tls_paths) == 3, - f"{launch_path}: external driver callback TLS paths are not distinct", + f"{launch_path}: external driver TLS paths are not distinct", ) else: require( diff --git a/e2e/rust/src/harness/container.rs b/e2e/rust/src/harness/container.rs index 542b9db8e6..5e23a70ff9 100644 --- a/e2e/rust/src/harness/container.rs +++ b/e2e/rust/src/harness/container.rs @@ -162,11 +162,18 @@ impl ContainerHttpServer { let engine = ContainerEngine::from_env()?; let host_port = find_free_port(); let network = e2e_network_name(); - let host = network.as_ref().map_or_else( - || "host.openshell.internal".to_string(), - |_| alias.to_string(), - ); - let port = if network.is_some() { 8000 } else { host_port }; + // A host-networked Docker supervisor cannot use a Docker network's DNS + // aliases, but it can route directly to containers on the bridge. Use + // the fixture's bridge address instead of overloading the reserved + // host alias, which may point at the CI job container. Podman keeps the + // shared-network alias path. + let use_host_port = network.is_none(); + let mut host = if use_host_port { + "host.openshell.internal".to_string() + } else { + alias.to_string() + }; + let port = if use_host_port { host_port } else { 8000 }; let mut args = vec![ "run".to_string(), @@ -175,15 +182,19 @@ impl ContainerHttpServer { "--entrypoint".to_string(), "python3".to_string(), ]; - if let Some(network) = network.as_deref() { + if use_host_port { + args.extend(["-p".to_string(), format!("{host_port}:8000")]); + } else { + let network = network.as_deref().ok_or_else(|| { + "container fixture network was not configured despite network mode selection" + .to_string() + })?; args.extend([ "--network".to_string(), network.to_string(), "--network-alias".to_string(), alias.to_string(), ]); - } else { - args.extend(["-p".to_string(), format!("{host_port}:8000")]); } args.extend([ DEFAULT_TEST_SERVER_IMAGE.to_string(), @@ -208,6 +219,18 @@ impl ContainerHttpServer { )); } + if is_e2e_driver("docker") + && let Some(network) = network.as_deref() + { + match container_network_ip(&engine, &stdout, network, alias) { + Ok(ip) => host = ip, + Err(err) => { + let _ = engine.command().args(["rm", "-f", &stdout]).output(); + return Err(err); + } + } + } + let server = Self { host, port, @@ -325,6 +348,64 @@ impl HostSupportContainer { Ok(fixture) } + /// Start a Python fixture with several ports published on the test host. + /// + /// This is useful for a host-networked supervisor when one fixture must + /// exercise several destination ports without relying on container DNS. + pub async fn start_python_with_host_bindings( + script: &str, + bindings: &[(u16, u16)], + ready_port: u16, + capabilities: &[&str], + ) -> Result { + let published_ready_port = bindings + .iter() + .find_map(|(host_port, container_port)| { + (*container_port == ready_port).then_some(*host_port) + }) + .ok_or_else(|| "host fixture bindings must include the readiness port".to_string())?; + let engine = ContainerEngine::from_env()?; + let mut args = vec![ + "run".to_string(), + "--detach".to_string(), + "--entrypoint".to_string(), + "python3".to_string(), + ]; + args.extend(bindings.iter().flat_map(|(host_port, container_port)| { + ["-p".to_string(), format!("{host_port}:{container_port}")] + })); + args.extend( + capabilities + .iter() + .map(|capability| format!("--cap-add={capability}")), + ); + args.extend([ + DEFAULT_TEST_SERVER_IMAGE.to_string(), + "-c".to_string(), + script.to_string(), + ]); + let output = engine + .command() + .args(&args) + .output() + .map_err(|err| format!("start {} host fixture: {err}", engine.name()))?; + if !output.status.success() { + return Err(format!( + "{} run failed (exit {:?}):\n{}", + engine.name(), + output.status.code(), + String::from_utf8_lossy(&output.stderr) + )); + } + let fixture = Self { + port: published_ready_port, + container_id: String::from_utf8_lossy(&output.stdout).trim().to_string(), + engine, + }; + fixture.wait_until_listening(ready_port).await?; + Ok(fixture) + } + async fn wait_until_listening(&self, container_port: u16) -> Result<(), String> { let deadline = timeout(Duration::from_secs(60), async { let mut tick = interval(Duration::from_millis(500)); @@ -489,27 +570,32 @@ impl SupportContainer { /// The container's IP address on the shared e2e network. pub fn ip(&self) -> Result { - let format = format!( - "{{{{with index .NetworkSettings.Networks \"{}\"}}}}{{{{.IPAddress}}}}{{{{end}}}}", - self.network - ); - let output = self - .engine - .command() - .args(["inspect", "--format", &format, &self.container_id]) - .output() - .map_err(|e| format!("inspect {} container: {e}", self.engine.name()))?; - let ip = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if !output.status.success() || ip.is_empty() { - return Err(format!( - "could not resolve IP of support container '{}' on network '{}':\n{}", - self.alias, - self.network, - String::from_utf8_lossy(&output.stderr) - )); - } - Ok(ip) + container_network_ip(&self.engine, &self.container_id, &self.network, &self.alias) + } +} + +fn container_network_ip( + engine: &ContainerEngine, + container_id: &str, + network: &str, + label: &str, +) -> Result { + let format = format!( + "{{{{with index .NetworkSettings.Networks \"{network}\"}}}}{{{{.IPAddress}}}}{{{{end}}}}" + ); + let output = engine + .command() + .args(["inspect", "--format", &format, container_id]) + .output() + .map_err(|e| format!("inspect {} container: {e}", engine.name()))?; + let ip = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !output.status.success() || ip.is_empty() { + return Err(format!( + "could not resolve IP of support container '{label}' on network '{network}':\n{}", + String::from_utf8_lossy(&output.stderr) + )); } + Ok(ip) } impl Drop for SupportContainer { diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs index e82a269812..3b1883f11f 100644 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ b/e2e/rust/tests/sandbox_lifecycle.rs @@ -10,6 +10,7 @@ use openshell_e2e::harness::binary::{openshell_cmd, openshell_tty_cmd}; use openshell_e2e::harness::cli::{run_cli, wait_for_sandbox_phase}; use openshell_e2e::harness::output::{extract_field, strip_ansi}; use openshell_e2e::harness::sandbox::SandboxGuard; +use serial_test::serial; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::time::{Instant, sleep}; @@ -211,6 +212,7 @@ async fn reconnect_with_input_ownership( } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn sandbox_stop_start_preserves_workspace() { const SENTINEL: &str = "openshell-stop-start-sentinel"; const SENTINEL_PATH: &str = "/sandbox/.openshell-stop-start-e2e"; @@ -287,6 +289,7 @@ async fn sandbox_stop_start_preserves_workspace() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn sandbox_can_be_deleted_while_stopped() { let mut sandbox = SandboxGuard::create_keep( &["sh", "-c", "echo stop-ready; exec sleep infinity"], @@ -328,6 +331,7 @@ async fn sandbox_can_be_deleted_while_stopped() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn canonical_main_exit_zero_completes_persistent_sandbox() { let mut cmd = openshell_tty_cmd(&["sandbox", "create", "--", "echo", "OK"]); cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); @@ -377,6 +381,7 @@ async fn canonical_main_exit_zero_completes_persistent_sandbox() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn canonical_main_nonzero_exit_preserves_status() { let mut cmd = openshell_tty_cmd(&[ "sandbox", @@ -429,6 +434,7 @@ async fn canonical_main_nonzero_exit_preserves_status() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn detached_canonical_main_exit_zero_reaches_completed() { const RELEASE_PATH: &str = "/sandbox/.openshell-detached-success-release"; let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 0"); @@ -453,6 +459,7 @@ async fn detached_canonical_main_exit_zero_reaches_completed() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn detached_canonical_main_nonzero_exit_reaches_error() { const RELEASE_PATH: &str = "/sandbox/.openshell-detached-failure-release"; let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 11"); @@ -477,6 +484,7 @@ async fn detached_canonical_main_nonzero_exit_reaches_error() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn canonical_main_and_exec_receive_declared_environment() { for mode in ["--tty", "--no-tty"] { let script = r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}"; while true; do sleep 1; done"#; @@ -515,6 +523,7 @@ async fn canonical_main_and_exec_receive_declared_environment() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn detached_main_exit_during_provisioning_is_classified_as_workload_result() { let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", "exit 11"]) .await @@ -548,6 +557,7 @@ async fn detached_main_exit_during_provisioning_is_classified_as_workload_result } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn canonical_tty_main_uses_sandbox_environment() { let script = r#"printf 'canonical_env home=%s user=%s term=%s\n' "$HOME" "$USER" "$TERM"; while true; do sleep 1; done"#; let mut sandbox = @@ -584,6 +594,7 @@ async fn canonical_tty_main_uses_sandbox_environment() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn canonical_main_disconnect_reconnect_replays_history_for_same_process() { const FIRST_MARKER: &str = "sequence=0001"; let script = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'main_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'main_pid=%s input=%s\n' "$$" "$line"; done"#; @@ -679,6 +690,7 @@ async fn canonical_main_disconnect_reconnect_replays_history_for_same_process() } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn sandbox_create_with_no_keep_cleans_up_after_tty_command() { let name = format!("tty-{:015x}", rand::random::() & 0x0fff_ffff_ffff_ffff); // Capture startup diagnostics before --no-keep removes a failed container. @@ -763,6 +775,7 @@ async fn sandbox_create_with_no_keep_cleans_up_after_tty_command() { } #[tokio::test] +#[serial(sandbox_lifecycle)] async fn sandbox_create_with_no_keep_preserves_failure_then_cleans_up() { let mut cmd = openshell_tty_cmd(&[ "sandbox", diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 6f3f751193..780f37df0f 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -9,6 +9,8 @@ use std::process::{Command, Stdio}; use openshell_e2e::harness::binary::openshell_cmd; use openshell_e2e::harness::container::{SupportContainer, is_e2e_driver}; +use openshell_e2e::harness::host_process::HostPythonFixture; +use openshell_e2e::harness::port::find_free_port; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; @@ -78,13 +80,13 @@ impl MuslDnsProbe { } } -fn write_policy() -> Result { +fn write_policy(host: &str, fixture_port: u16, tcp_dns_port: u16) -> Result { write_policy_for_identity( - FIXTURE_ALIAS, + host, "sandbox", "sandbox", &[], - &[FIXTURE_PORT, TCP_DNS_PORT], + &[fixture_port, tcp_dns_port], ) } @@ -253,9 +255,8 @@ async fn local_container_native_tcp_uses_policy_dns_and_fails_closed() { return; } - let fixture = SupportContainer::start_python_with_capabilities( - FIXTURE_ALIAS, - &format!( + let fixture_script = |fixture_port: u16, tcp_dns_port: u16, transparent_port: u16| { + format!( r#"import socket, threading def listen(port): s = socket.socket() @@ -271,21 +272,63 @@ def serve(s): c.sendall(b'native-tcp-ok:' + data) c.close() -transparent_listener = listen({TRANSPARENT_LISTENER_PORT}) -tcp_dns_listener = listen({TCP_DNS_PORT}) -fixture_listener = listen({FIXTURE_PORT}) +transparent_listener = listen({transparent_port}) +tcp_dns_listener = listen({tcp_dns_port}) +fixture_listener = listen({fixture_port}) threading.Thread(target=serve, args=(transparent_listener,), daemon=True).start() threading.Thread(target=serve, args=(tcp_dns_listener,), daemon=True).start() serve(fixture_listener) "# - ), - FIXTURE_PORT, - &["NET_BIND_SERVICE"], - ) - .await - .expect("start TCP fixture"); - let real_ip = fixture.ip().expect("fixture IP"); - let policy = write_policy().expect("write policy"); + ) + }; + enum Fixture { + Host(HostPythonFixture), + Network(SupportContainer), + } + let (fixture, policy_host, real_ip, fixture_port, tcp_dns_port, transparent_port) = + if is_e2e_driver("docker") { + let fixture_host_port = find_free_port(); + let tcp_dns_host_port = find_free_port(); + let transparent_host_port = find_free_port(); + let fixture = HostPythonFixture::start( + &fixture_script(fixture_host_port, tcp_dns_host_port, transparent_host_port), + fixture_host_port, + ) + .await + .expect("start host TCP fixture"); + ( + Fixture::Host(fixture), + "host.openshell.internal".to_string(), + "127.0.0.1".to_string(), + fixture_host_port, + tcp_dns_host_port, + transparent_host_port, + ) + } else { + let fixture = SupportContainer::start_python_with_capabilities( + FIXTURE_ALIAS, + &fixture_script(FIXTURE_PORT, TCP_DNS_PORT, TRANSPARENT_LISTENER_PORT), + FIXTURE_PORT, + &["NET_BIND_SERVICE"], + ) + .await + .expect("start shared-network TCP fixture"); + let real_ip = fixture.ip().expect("fixture IP"); + ( + Fixture::Network(fixture), + FIXTURE_ALIAS.to_string(), + real_ip, + FIXTURE_PORT, + TCP_DNS_PORT, + TRANSPARENT_LISTENER_PORT, + ) + }; + // Keep the fixture alive through the sandbox assertions. + match &fixture { + Fixture::Host(value) => assert_eq!(value.port, fixture_port), + Fixture::Network(value) => assert!(!value.alias.is_empty()), + } + let policy = write_policy(&policy_host, fixture_port, tcp_dns_port).expect("write policy"); let policy_path = policy.path().to_string_lossy().into_owned(); let mut sandbox = SandboxGuard::create_keep_with_args( &["--policy", &policy_path], @@ -327,12 +370,12 @@ assert denied({real_ip:?}, {port}) assert denied({real_ip:?}, {transparent_port}) print('transparent-tcp-e2e-ok') "#, - host = FIXTURE_ALIAS, - port = FIXTURE_PORT, - tcp_dns_port = TCP_DNS_PORT, - wrong_port = FIXTURE_PORT + 1, + host = policy_host, + port = fixture_port, + tcp_dns_port = tcp_dns_port, + wrong_port = fixture_port + 1, real_ip = real_ip, - transparent_port = TRANSPARENT_LISTENER_PORT, + transparent_port = transparent_port, ); let output = match sandbox.exec(&["python3", "-c", &script]).await { Ok(output) => output, @@ -355,13 +398,13 @@ print('transparent-tcp-e2e-ok') assert!(output.contains("transparent-tcp-e2e-ok"), "{output}"); let logs = wait_for_sandbox_logs(&sandbox.name, |logs| { - logs.contains(&format!("-> {FIXTURE_ALIAS}:{FIXTURE_PORT}")) + logs.contains(&format!("-> {policy_host}:{fixture_port}")) && logs.contains("Denied staged transparent connection") }) .await .expect("wait for sandbox logs"); assert!( - logs.contains(&format!("-> {FIXTURE_ALIAS}:{FIXTURE_PORT}")), + logs.contains(&format!("-> {policy_host}:{fixture_port}")), "{logs}" ); assert!( diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 5a8c904e6a..7334d31f72 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -524,20 +524,17 @@ if ! ensure_sandbox_image_available "${SANDBOX_IMAGE}"; then exit 2 fi -PKI_DIR="${WORKDIR}/pki" -e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" -export OPENSHELL_E2E_GATEWAY_CA_CERT="${PKI_DIR}/ca.crt" - HOST_PORT=$(e2e_pick_port) HEALTH_PORT=$(e2e_pick_port) STATE_DIR="${XDG_STATE_HOME}" mkdir -p "${STATE_DIR}" JWT_DIR="${STATE_DIR}/jwt" -GATEWAY_ENDPOINT="https://host.openshell.internal:${HOST_PORT}" E2E_NAMESPACE="e2e-docker-$$-${HOST_PORT}" DOCKER_NETWORK_NAME="${E2E_NAMESPACE}" GATEWAY_HOST_ALIAS_IP="" +GATEWAY_BIND_IP="127.0.0.1" +SUPERVISOR_GATEWAY_HOST="127.0.0.1" ensure_e2e_docker_network "${DOCKER_NETWORK_NAME}" export OPENSHELL_E2E_DOCKER_NETWORK_NAME="${DOCKER_NETWORK_NAME}" @@ -546,10 +543,27 @@ export OPENSHELL_E2E_SANDBOX_NAMESPACE="${E2E_NAMESPACE}" export OPENSHELL_E2E_DRIVER="docker" if connect_current_container_to_docker_network "${DOCKER_NETWORK_NAME}"; then echo "Connected CI job container to Docker network ${DOCKER_NETWORK_NAME} (${GATEWAY_HOST_ALIAS_IP})." + # Container jobs use the host Docker daemon. The host-networked supervisor + # therefore cannot reach the gateway through the job container's loopback; + # it reaches the gateway through the job container's address on this bridge. + GATEWAY_BIND_IP="0.0.0.0" + SUPERVISOR_GATEWAY_HOST="${GATEWAY_HOST_ALIAS_IP}" else GATEWAY_HOST_ALIAS_IP="" fi +PKI_DIR="${WORKDIR}/pki" +if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then + e2e_generate_pki \ + "${GATEWAY_BIN}" \ + "${PKI_DIR}" \ + "${SUPERVISOR_GATEWAY_HOST}" +else + e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" +fi +export OPENSHELL_E2E_GATEWAY_CA_CERT="${PKI_DIR}/ca.crt" +GATEWAY_ENDPOINT="https://${SUPERVISOR_GATEWAY_HOST}:${HOST_PORT}" + echo "Starting openshell-gateway on port ${HOST_PORT} (namespace: ${E2E_NAMESPACE})..." echo "Using sandbox image: ${SANDBOX_IMAGE} (pull policy: ${SANDBOX_IMAGE_PULL_POLICY})" e2e_generate_gateway_jwt "${JWT_DIR}" @@ -586,23 +600,18 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" - printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" printf 'enable_bind_mounts = true\n' printf 'sandbox_runtime_image = %s\n' "$(toml_string "${SANDBOX_RUNTIME_IMAGE}")" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then - printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" - fi fi } > "${GATEWAY_CONFIG}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then { printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" - printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" @@ -612,14 +621,11 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'enable_bind_mounts = true\n' printf 'sandbox_runtime_image = %s\n' "$(toml_string "${SANDBOX_RUNTIME_IMAGE}")" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then - printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" - fi } >"${DRIVER_CONFIG}" "${DRIVER_BIN}" \ --bind-socket "${DRIVER_SOCKET}" \ --config "${DRIVER_CONFIG}" \ - --gateway-bind "127.0.0.1:${HOST_PORT}" \ + --gateway-bind "${GATEWAY_BIND_IP}:${HOST_PORT}" \ >"${DRIVER_LOG}" 2>&1 & DRIVER_PID=$! e2e_wait_for_socket \ @@ -628,6 +634,7 @@ fi GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" + --bind-address "${GATEWAY_BIND_IP}" --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" --compute-driver docker diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 76750199b7..f851c92957 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -667,16 +667,9 @@ export OPENSHELL_E2E_GATEWAY_CA_CERT="${PKI_DIR}/ca.crt" HOST_PORT=$(e2e_pick_port) HEALTH_PORT=$(e2e_pick_port) -if [ "$(uname -s)" = "Darwin" ]; then - # Podman Machine reserves IPv4 loopback for its callback-only listener. - PRIMARY_BIND_IP="::1" - CLI_ENDPOINT_HOST="localhost" - HEALTH_ENDPOINT_HOST="[::1]" -else - PRIMARY_BIND_IP="127.0.0.1" - CLI_ENDPOINT_HOST="127.0.0.1" - HEALTH_ENDPOINT_HOST="127.0.0.1" -fi +PRIMARY_BIND_IP="127.0.0.1" +CLI_ENDPOINT_HOST="127.0.0.1" +HEALTH_ENDPOINT_HOST="127.0.0.1" STATE_DIR="${WORKDIR}/state" mkdir -p "${STATE_DIR}" export XDG_STATE_HOME="${STATE_DIR}" @@ -715,7 +708,7 @@ e2e_write_podman_gateway_config \ if [ -n "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE:-}" ]; then cp "${GATEWAY_CONFIG}" "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE}" fi -EXTERNAL_DRIVER_CALLBACK_ENDPOINT="https://host.containers.internal:${HOST_PORT}" +EXTERNAL_DRIVER_GRPC_ENDPOINT="https://127.0.0.1:${HOST_PORT}" EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS=10 EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS=true EXTERNAL_DRIVER_TLS_CA="${PKI_DIR}/ca.crt" @@ -732,7 +725,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then external_driver_environment=null if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then driver_transport=remote_uds - external_driver_grpc_endpoint="\"${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}\"" + external_driver_grpc_endpoint="\"${EXTERNAL_DRIVER_GRPC_ENDPOINT}\"" external_driver_host_gateway_ip='"host-gateway"' driver_tls_ca_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CA}" | cut -d' ' -f1)" driver_tls_cert_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CERT}" | cut -d' ' -f1)" @@ -744,7 +737,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${SANDBOX_IMAGE_REQUEST}" \ "${EXTERNAL_DRIVER_PULL_POLICY}" \ "${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ - "${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ + "${EXTERNAL_DRIVER_GRPC_ENDPOINT}" \ "${HOST_PORT}" \ "${PODMAN_NETWORK_NAME}" \ "${PODMAN_STOP_TIMEOUT_SECS}" \ @@ -807,7 +800,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE_REQUEST}" \ OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="${EXTERNAL_DRIVER_PULL_POLICY}" \ OPENSHELL_HEALTH_CHECK_INTERVAL_SECS="${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ - OPENSHELL_GRPC_ENDPOINT="${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ + OPENSHELL_GRPC_ENDPOINT="${EXTERNAL_DRIVER_GRPC_ENDPOINT}" \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ @@ -825,8 +818,8 @@ fi GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - # compute_driver comes from the RPM template. Override the loopback address - # and port so Podman Machine can keep its IPv4 callback listener distinct. + # compute_driver comes from the RPM template. Override the loopback port for + # this isolated test gateway. --bind-address "${PRIMARY_BIND_IP}" --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index 8c75a0a923..c2f4385137 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -31,13 +31,6 @@ service ComputeDriver { rpc AuthenticateSandbox(AuthenticateSandboxRequest) returns (AuthenticateSandboxResponse); - // Report additional gateway listeners required by this driver instance. - // - // A requirement is not authorization to expose the gateway. The gateway - // owns validation, authorization, and the authoritative bind. - rpc GetGatewayListenerRequirements(GetGatewayListenerRequirementsRequest) - returns (GetGatewayListenerRequirementsResponse); - // Validate a sandbox before create-time provisioning. rpc ValidateSandboxCreate(ValidateSandboxCreateRequest) returns (ValidateSandboxCreateResponse); @@ -138,32 +131,6 @@ message GpuResourceCapabilities { bool count_selection_supported = 2; } -message GetGatewayListenerRequirementsRequest {} - -message GatewayListenerRequirement { - // Untrusted human-readable driver rationale for diagnostics. - string reason = 1; - - oneof selector { - // Concrete IP:port address requested by the driver. The port must match - // the gateway's configured primary listener port. - string exact_bind_address = 2; - // Ask the gateway to bind the IPv4 address selected by its default route. - // This matches rootless pasta's default upstream-interface selection. - GatewayDefaultRouteInterfaceRequirement default_route_interface = 3; - // Ask the gateway to ensure an IPv4 loopback listener is present. This - // covers runtimes whose host forwarder terminates on gateway loopback. - GatewayLoopbackInterfaceRequirement loopback_interface = 4; - } -} - -message GatewayDefaultRouteInterfaceRequirement {} -message GatewayLoopbackInterfaceRequirement {} - -message GetGatewayListenerRequirementsResponse { - repeated GatewayListenerRequirement requirements = 1; -} - // Driver-owned sandbox model used for create requests and platform observations. // // This intentionally omits gateway-owned lifecycle fields such as the public diff --git a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py index 43b3b4aa2c..aace841cfb 100644 --- a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py +++ b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py @@ -141,7 +141,7 @@ def create_variant( "cli_trace_wrapper_sha256_before_execution": result[ "cli_trace_wrapper_sha256" ], - "external_driver_grpc_endpoint": "https://host.containers.internal:18181", + "external_driver_grpc_endpoint": "https://127.0.0.1:18181", "external_driver_host_gateway_ip": "host-gateway", "external_driver_userns": None, "external_driver_spiffe": False, @@ -154,7 +154,7 @@ def create_variant( "OPENSHELL_SANDBOX_IMAGE": "example.invalid/sandbox@" + IMAGE_DIGEST, "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": policy, "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10, - "OPENSHELL_GRPC_ENDPOINT": "https://host.containers.internal:18181", + "OPENSHELL_GRPC_ENDPOINT": "https://127.0.0.1:18181", "OPENSHELL_GATEWAY_PORT": 18181, "OPENSHELL_NETWORK_NAME": f"{variant}-network", "OPENSHELL_STOP_TIMEOUT": 15, diff --git a/python/openshell/rpm_gateway_config_migration_test.py b/python/openshell/rpm_gateway_config_migration_test.py index d622f36bb1..8dab835703 100644 --- a/python/openshell/rpm_gateway_config_migration_test.py +++ b/python/openshell/rpm_gateway_config_migration_test.py @@ -31,8 +31,7 @@ [openshell.gateway] # Keep the primary listener on the built-in 127.0.0.1:17670 default. The -# Podman driver reports the callback interface it needs, and the gateway -# adds a separate listener scoped to that interface. +# host-networked Podman supervisor uses this same loopback endpoint. # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index c833d2cfc8..8fccf3779d 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -252,15 +252,12 @@ Common findings: - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. - Local Docker gateway setup cannot copy `openshell-sandbox` after exporting a supervisor image: the sandbox runtime and supervisor are separate artifacts. The runtime image must provide `/openshell-sandbox`; the supervisor image provides `/openshell-supervisor`. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. -- Sandbox never registers: check gateway logs and supervisor callback endpoint. +- Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. - Calls to an external tool server fail while the sandbox is Ready: inspect `Tool server connections` in `openshell sandbox get `. For configured MCP-over-HTTP endpoints, JSON output exposes each address together with `last_result` and `last_reported_at` in `endpoint_statuses`. Select the endpoint by host, path, and ports, then check the reported failure boundary. `last_reported_at` records gateway acceptance time and can advance when retained evidence is accepted after a reset. Results do not expire or prove current availability; `HttpResponseReceived` can still contain a tool error. If several paths share a host and port, a failure before the path is known remains in logs. Verify the actual operation when current tool availability matters. -- On macOS, repeated `Policy fetch failed after 5 attempts` messages with a - Homebrew gateway bound to `[::1]:17670` indicate that the Docker - `host-gateway` IPv4 route has no matching callback listener. Current releases - leave `bind_address` unset in the Homebrew config, use the built-in - `127.0.0.1:17670` primary listener, and reuse it for authenticated sandbox - callbacks. On an older release, set `bind_address = "127.0.0.1:17670"` or - upgrade. +- On Docker Desktop, repeated `Policy fetch failed after 5 attempts` messages + can mean host networking is disabled. Enable host networking in Docker + Desktop, ensure Enhanced Container Isolation is disabled, and verify the + gateway's primary endpoint is reachable from a host-networked container. - Sandbox runtime image exits before printing `openshell-sandbox --version`: verify the configured image contains a static executable at `/openshell-sandbox`. - A sandbox with explicit `protocol: tcp` endpoints fails before workload readiness: confirm the selected isolation backend advertises TCP mediation, then inspect the sandbox and supervisor logs for protected-channel setup or listener failures. A driver that cannot supply the required outer egress fence and authenticated runtime channel must reject the policy before starting the agent. - Supervisor runtime validation fails: verify `supervisor_image` contains a static `/openshell-supervisor` executable from the same release as the sandbox runtime. @@ -291,26 +288,18 @@ Common findings: - Rootless networking unavailable: inspect Podman network configuration. - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. -- Supervisor cannot call back: check callback endpoint and gateway logs. +- Supervisor cannot connect: check its gateway endpoint and gateway logs. - Inspect both Podman containers for the sandbox: the `sandbox` isolation role - must have network mode `none`; the `supervisor` role owns gateway callbacks + must have network mode `none`; the `supervisor` role owns the gateway session and egress. Both run non-root with all capabilities dropped. Check the private channel volume and shared user-namespace mapping if authentication fails. - If a sandbox fails before readiness, inspect its unprivileged enforcement probe and the companion supervisor's private health check. Do not add capabilities, attach a workload network, or disable the runtime seccomp profile. There is no sandbox nftables or nested-network setup to repair. -- Gateway exits before becoming healthy with a callback-listener discovery - error: inspect `podman info --debug`, the configured Podman network, and the - host's IPv4 default route. Rootless pasta uses the private source address - selected by that route; rootful Podman uses the bridge gateway address. -- Current gateways reuse the primary listener when it covers Podman's callback - address. If the primary does not cover that address, inspect the gateway - startup logs for the additional callback-only listener and its provenance. -- Rootless slirp4netns, another named helper, or missing helper metadata - requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip` - cannot bypass slirp4netns host-loopback isolation. Do not work around - discovery failures by broadening the primary gateway listener to `0.0.0.0`. +- On Linux, verify that the host-networked Podman supervisor can reach the + gateway's primary loopback endpoint. On macOS, verify Podman Machine's + host-loopback forwarding or configure an explicit `grpc_endpoint`. When `userns` is configured (e.g. `userns = "auto"` or `userns = "keep-id"`): @@ -759,7 +748,7 @@ Use the VM driver logs and host diagnostics available in the user's environment. upper layer, prepared rootfs, explicit config, or current image. Do not assign `10001:10001` unless the persisted state reports that legacy identity. - Host virtualization support is enabled. -- The sandbox supervisor can establish its callback connection to the gateway. +- The sandbox supervisor can establish its authenticated gateway session. Then run: @@ -799,9 +788,7 @@ credential failures. | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | -| Gateway exits while resolving compute-driver listener requirements | The callback hostname is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | -| Admin, health, reflection, or HTTP request is denied on an additional Docker/Podman callback-only listener | Additional callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected | -| Docker or Podman sandbox never registers | Wrong callback endpoint or supervisor startup failure | Gateway logs and sandbox container logs | +| Docker or Podman sandbox never registers | Wrong gateway endpoint, unavailable host networking, or supervisor startup failure | Gateway logs and supervisor container logs | | Docker GPU sandbox fails before startup | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` | | Kubernetes gateway pod pending | PVC unbound, taint, selector, or insufficient resources | `kubectl -n openshell describe pod ` | | Kubernetes sandbox pod stuck pending, workspace PVC unbound | Cluster has no default `StorageClass` and OpenShell does not set `storageClassName` on the workspace PVC (clusters with a default `StorageClass` bind fine without it) | `kubectl -n openshell describe pvc`; set `server.workspaceStorageClass` (gateway config `workspace_storage_class`) to a valid `StorageClass` | diff --git a/snapcraft.yaml b/snapcraft.yaml index 126d05f263..0e2fc821cb 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -87,9 +87,10 @@ apps: # Operators must manually restart the service after a refresh if needed. refresh-mode: endure # The wrapper sets OPENSHELL_DISABLE_TLS=true and OPENSHELL_DB_URL to - # use $SNAP_COMMON/gateway.db. Before startup it validates the selected - # operator-provided config without creating or rewriting it. A nonempty - # OPENSHELL_GATEWAY_CONFIG takes precedence over gateway.toml. + # use $SNAP_COMMON/gateway.db. Before startup it bootstraps package-managed + # credentials and validates the selected operator-provided config without + # creating or rewriting it. A nonempty OPENSHELL_GATEWAY_CONFIG takes + # precedence over gateway.toml. environment: XDG_DATA_HOME: "$SNAP_COMMON" XDG_RUNTIME_DIR: "$SNAP_COMMON" diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 112319cca5..a98ff3ea2b 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -17,6 +17,7 @@ # OPENSHELL_SANDBOX_NAMESPACE=my-ns mise run gateway:docker # OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:docker # OPENSHELL_SUPERVISOR_IMAGE=ghcr.io/... mise run gateway:docker +# OPENSHELL_SANDBOX_RUNTIME_IMAGE=ghcr.io/... mise run gateway:docker # # After the gateway is running, point the CLI at it with either: # openshell --gateway docker-dev @@ -35,29 +36,11 @@ STATE_DIR="${OPENSHELL_DOCKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-docker}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-docker-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" SUPERVISOR_IMAGE="${OPENSHELL_SUPERVISOR_IMAGE:-openshell/supervisor:dev}" +SANDBOX_RUNTIME_IMAGE="${OPENSHELL_SANDBOX_RUNTIME_IMAGE:-openshell/sandbox:dev}" SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" -normalize_arch() { - case "$1" in - x86_64|amd64) echo "amd64" ;; - aarch64|arm64) echo "arm64" ;; - *) echo "$1" ;; - esac -} - -linux_target_triple() { - case "$1" in - amd64) echo "x86_64-unknown-linux-gnu" ;; - arm64) echo "aarch64-unknown-linux-gnu" ;; - *) - echo "ERROR: unsupported Docker daemon architecture '$1'" >&2 - exit 2 - ;; - esac -} - port_is_in_use() { local port=$1 if command -v lsof >/dev/null 2>&1; then @@ -71,6 +54,32 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +ensure_docker_runtime_image() { + local image=$1 + local configured_image=$2 + local build_target=$3 + local role=$4 + + if [[ -n "${configured_image}" ]]; then + if docker image inspect "${image}" >/dev/null 2>&1; then + return + fi + echo "ERROR: ${role} image '${image}' not found locally." >&2 + echo " Build it with Docker or unset its image override to build the local :dev image." >&2 + exit 1 + fi + + # Always run the build pipeline for default development images so source + # changes cannot leave a fixed :dev tag pointing at stale runtime code. + echo "Refreshing Docker ${role} image (${image})..." + CONTAINER_ENGINE=docker IMAGE_TAG=dev mise run "build:docker:${build_target}" + + if ! docker image inspect "${image}" >/dev/null 2>&1; then + echo "ERROR: expected ${role} image '${image}' after build" >&2 + exit 1 + fi +} + append_local_otlp_config_if_available() { local config_path=$1 if ! port_is_in_use 4317; then @@ -126,46 +135,18 @@ if port_is_in_use "${PORT}"; then exit 2 fi -if [[ -z "${OPENSHELL_SUPERVISOR_IMAGE:-}" ]]; then - # Keep the default development image aligned with this checkout. Cargo and - # BuildKit caches make an unchanged rebuild incremental. - echo "Building local supervisor image (${SUPERVISOR_IMAGE})..." - CONTAINER_ENGINE=docker IMAGE_TAG=dev mise run build:docker:supervisor -fi +ensure_docker_runtime_image \ + "${SUPERVISOR_IMAGE}" \ + "${OPENSHELL_SUPERVISOR_IMAGE:-}" \ + supervisor \ + supervisor +ensure_docker_runtime_image \ + "${SANDBOX_RUNTIME_IMAGE}" \ + "${OPENSHELL_SANDBOX_RUNTIME_IMAGE:-}" \ + sandbox \ + "sandbox runtime" -GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://host.openshell.internal:${PORT}}" - -DAEMON_ARCH="$(normalize_arch "$(docker info --format '{{.Architecture}}' 2>/dev/null || true)")" -HOST_OS="$(uname -s)" -HOST_ARCH="$(normalize_arch "$(uname -m)")" -SUPERVISOR_TARGET="$(linux_target_triple "${DAEMON_ARCH}")" -# Cache the supervisor binary alongside the gateway state. Reuses the same -# Docker pipeline used for the supervisor image. The Dockerfiles do not compile -# Rust; the cross-compile runs on the host via cargo zigbuild and the binary is -# staged into the build context. On macOS the host staging path raises the -# per-process file-descriptor limit automatically (see build-env.sh), so the -# static musl link no longer hits ProcessFdQuotaExceeded on this many rlibs. -SUPERVISOR_OUT_DIR="${STATE_DIR}/supervisor/${DAEMON_ARCH}" -SUPERVISOR_BIN="${SUPERVISOR_OUT_DIR}/openshell-sandbox" - -install_supervisor_binary() { - local source=$1 - local staged - - # A running sandbox may execute a bind-mounted copy of SUPERVISOR_BIN. - # Replacing its directory entry keeps that old inode alive for the running - # container without truncating it, while new containers see this build. - mkdir -p "${SUPERVISOR_OUT_DIR}" - staged="$(mktemp "${SUPERVISOR_OUT_DIR}/.openshell-sandbox.XXXXXX")" - if ! cp "${source}" "${staged}" || ! chmod 0755 "${staged}"; then - rm -f -- "${staged}" - return 1 - fi - if ! mv -f -- "${staged}" "${SUPERVISOR_BIN}"; then - rm -f -- "${staged}" - return 1 - fi -} +GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://127.0.0.1:${PORT}}" CARGO_BUILD_JOBS_ARG=() if [[ -n "${CARGO_BUILD_JOBS:-}" ]]; then @@ -184,42 +165,6 @@ echo "Generating local gateway credentials..." --server-san "localhost" \ --server-san "host.openshell.internal" -echo "Building openshell-sandbox for ${SUPERVISOR_TARGET}..." -if [[ "${HOST_OS}" == "Linux" && "${HOST_ARCH}" == "${DAEMON_ARCH}" ]]; then - # Native Linux build — no cross-toolchain required. - rustup target add "${SUPERVISOR_TARGET}" >/dev/null 2>&1 || true - cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ - -p openshell-sandbox --target "${SUPERVISOR_TARGET}" - install_supervisor_binary \ - "${ROOT}/target/${SUPERVISOR_TARGET}/debug/openshell-sandbox" -else - # Cross-compile through the prebuilt-binary staging helper, then use the - # sandbox stage to extract just the openshell-sandbox binary. - # - # This task is gated on a working Docker daemon above, so pin the - # container-engine helper to docker — otherwise it auto-detects podman - # whenever the binary happens to be on PATH. - mkdir -p "${SUPERVISOR_OUT_DIR}" - SUPERVISOR_BUILD_DIR="$(mktemp -d "${SUPERVISOR_OUT_DIR}/.build.XXXXXX")" - if ! CONTAINER_ENGINE=docker \ - DOCKER_PLATFORM="linux/${DAEMON_ARCH}" \ - DOCKER_OUTPUT="type=local,dest=${SUPERVISOR_BUILD_DIR}" \ - bash "${ROOT}/tasks/scripts/docker-build-image.sh" sandbox; then - rm -rf -- "${SUPERVISOR_BUILD_DIR}" - exit 1 - fi - if ! install_supervisor_binary "${SUPERVISOR_BUILD_DIR}/openshell-sandbox"; then - rm -rf -- "${SUPERVISOR_BUILD_DIR}" - exit 1 - fi - rm -rf -- "${SUPERVISOR_BUILD_DIR}" -fi - -if [[ ! -f "${SUPERVISOR_BIN}" ]]; then - echo "ERROR: expected supervisor binary at ${SUPERVISOR_BIN}" >&2 - exit 1 -fi - mkdir -p "${STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" < only on a Docker host with AppArmor enabled. app_armor_profile = "Unconfined" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index a45c1d35d4..745dcd542f 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -149,15 +149,6 @@ EOF printf '%s' "${name}" >"${config_home}/openshell/active_gateway" } -if [[ -z "${OPENSHELL_BIND_ADDRESS:-}" && "$(uname -s)" == "Darwin" ]]; then - # Podman Machine reserves IPv4 loopback for its callback-only listener. - # Keep the primary listener distinct while using a hostname that resolves - # to IPv6 loopback for local CLI connections. An explicit bind address - # overrides this platform default. - PRIMARY_BIND_IP="::1" - CLI_ENDPOINT_HOST="localhost" -fi - if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then echo "ERROR: OPENSHELL_PODMAN_GATEWAY_NAME must contain only letters, numbers, dots, underscores, or dashes" >&2 exit 2 diff --git a/tasks/scripts/snap-gateway-wrapper.sh b/tasks/scripts/snap-gateway-wrapper.sh index 83eebcdcb4..4a98962559 100755 --- a/tasks/scripts/snap-gateway-wrapper.sh +++ b/tasks/scripts/snap-gateway-wrapper.sh @@ -5,14 +5,15 @@ # Snap wrapper for openshell-gateway. Sets snap-specific defaults: # - OPENSHELL_DB_URL -> sqlite:$SNAP_COMMON/gateway.db (overridable) # - OPENSHELL_DISABLE_TLS -> true -# It validates, but never creates or rewrites, an operator-provided config -# before starting the gateway. +# It bootstraps package-managed credentials and validates, but never creates or +# rewrites, an operator-provided config before starting the gateway. set -eu CANONICAL_CONFIG_FILE="${SNAP_COMMON}/gateway.toml" export OPENSHELL_DB_URL="${OPENSHELL_DB_URL:-sqlite:${SNAP_COMMON}/gateway.db?mode=rwc}" export OPENSHELL_DISABLE_TLS="${OPENSHELL_DISABLE_TLS:-true}" +export OPENSHELL_LOCAL_TLS_DIR="${OPENSHELL_LOCAL_TLS_DIR:-${SNAP_COMMON}/tls}" # Mirror clap's CLI-over-environment precedence so preflight always inspects # the same file the daemon will load. Reject ambiguous duplicate selectors @@ -63,6 +64,13 @@ if [ "$expect_config_path" = true ] || { [ "$config_seen" = true ] && [ -z "$cli exit 2 fi +# Docker sandboxes require gateway-minted, launch-scoped credentials for the +# supervisor. Generate the local JWT bundle alongside the otherwise-unused TLS +# material; generate-certs is idempotent and preserves an existing bundle. +"${SNAP}/bin/openshell-gateway" generate-certs \ + --output-dir "$OPENSHELL_LOCAL_TLS_DIR" \ + --server-san host.openshell.internal + if [ "$config_seen" = true ]; then "${SNAP}/bin/openshell-gateway" config preflight -- "$@" exec "${SNAP}/bin/openshell-gateway" "$@" diff --git a/tasks/scripts/test-snap-gateway-wrapper.sh b/tasks/scripts/test-snap-gateway-wrapper.sh index 733ec796b5..a14076b121 100755 --- a/tasks/scripts/test-snap-gateway-wrapper.sh +++ b/tasks/scripts/test-snap-gateway-wrapper.sh @@ -19,6 +19,9 @@ mkdir -p "$snap/bin" "$common" cat >"$snap/bin/openshell-gateway" <<'EOF' #!/bin/sh printf '%s\n' "$*" >>"$FAKE_GATEWAY_LOG" +if [ "${1:-}" = generate-certs ]; then + exit 0 +fi printf 'env:%s|%s|%s\n' \ "${OPENSHELL_GATEWAY_CONFIG:-}" \ "${OPENSHELL_DB_URL:-}" \ @@ -63,7 +66,9 @@ run_wrapper() { } assert_log() { - printf '%s\n' "$1" >"$expected" + printf '%s\n' \ + "generate-certs --output-dir $common/tls --server-san host.openshell.internal" \ + "$1" >"$expected" if ! cmp -s "$expected" "$log"; then echo "FAIL: unexpected call sequence" >&2 diff -u "$expected" "$log" >&2 diff --git a/tests/ansible/roles/openshell_gateway/templates/gateway-docker.toml.j2 b/tests/ansible/roles/openshell_gateway/templates/gateway-docker.toml.j2 index 3e01d1562e..320d2979c7 100644 --- a/tests/ansible/roles/openshell_gateway/templates/gateway-docker.toml.j2 +++ b/tests/ansible/roles/openshell_gateway/templates/gateway-docker.toml.j2 @@ -7,7 +7,7 @@ socket_path = "{{ openshell_runtime_socket }}" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "if_not_present" sandbox_label = "tmachine" -grpc_endpoint = "http://host.openshell.internal:17670" +grpc_endpoint = "http://127.0.0.1:17670" sandbox_runtime_image = "docker.io/openshell/sandbox:tmachine" supervisor_image = "docker.io/openshell/supervisor:tmachine" {% endblock %}