From 23c1cf7d7534211bb37ed588cbe2bbc60d296c14 Mon Sep 17 00:00:00 2001 From: Akram Date: Thu, 10 Sep 2026 20:21:07 +0400 Subject: [PATCH 1/8] build(docker): add OpenShift in-cluster build variants for supervisor and gateway Add multi-stage Dockerfiles that build the OpenShell supervisor and gateway images entirely inside an OpenShift/Buildah cluster, for environments without the upstream Nix CI pipeline that stages prebuilt binaries under deploy/docker/.build/prebuilt-binaries. Both reproduce the exact upstream artifacts by running the project's own Nix devShells in a builder stage, then assembling a runtime stage identical to the existing Dockerfile.supervisor / Dockerfile.gateway: - Dockerfile.supervisor.multistage: builds the static musl openshell-sandbox binary via the musl devShell; runtime is alpine:3.22 with nftables/iptables and COPY --chmod=0555. - Dockerfile.gateway.multistage: builds openshell-gateway via the glibc-2-28 devShell, normalizes the ELF interpreter with patchelf and asserts z3 is statically embedded; runtime is distroless cc-debian13. Each builder collapses build and cleanup into a single RUN so the Nix store never enters the committed layer, keeping the intermediate commit small and within the node's ephemeral-storage budget. The upstream Dockerfiles and CI binary pipeline are unchanged. Signed-off-by: Akram Signed-off-by: Akram --- deploy/docker/Dockerfile.gateway.multistage | 80 +++++++++++++++++++ .../docker/Dockerfile.supervisor.multistage | 72 +++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 deploy/docker/Dockerfile.gateway.multistage create mode 100644 deploy/docker/Dockerfile.supervisor.multistage diff --git a/deploy/docker/Dockerfile.gateway.multistage b/deploy/docker/Dockerfile.gateway.multistage new file mode 100644 index 0000000000..7971a08657 --- /dev/null +++ b/deploy/docker/Dockerfile.gateway.multistage @@ -0,0 +1,80 @@ +# syntax=docker/dockerfile:1.4 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-cluster (OpenShift/Buildah) variant of Dockerfile.gateway. +# +# Upstream CI builds the GNU-linked openshell-gateway binary inside the +# project's Nix devShell with the gnu cross target (z3 and aws-lc statically +# embedded, standard ELF interpreter), and stages it under +# deploy/docker/.build/prebuilt-binaries. This multi-stage Dockerfile +# reproduces that exact artifact in a builder stage so clusters without the Nix +# CI pipeline can build the gateway image directly from source. The final stage +# is identical to Dockerfile.gateway. + +# In a multi-stage build, an ARG that feeds a `FROM` must be declared before the +# very first FROM (global pre-FROM scope). Declaring it between the two stages +# makes Buildah attach it to the builder stage and fail the second FROM with +# "no FROM statement found". +ARG GATEWAY_BASE_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:d97bc0a941b8d4be647dc0ee75b264ddbb772f1ac5ba690a4309c00723b23775 + +# ---- Builder: glibc base + Nix, reproduce upstream's cross build ------------- +# +# The build must run on a glibc/FHS base (like the CI runners), NOT on the +# minimal nixos/nix image: cross-compiling emits host build-scripts linked for +# x86_64-unknown-linux-gnu whose ELF interpreter is /lib64/ld-linux-x86-64.so.2, +# which the nixos/nix (Alpine, store-only) image does not provide. +FROM debian:bookworm-slim AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl xz-utils ca-certificates git && rm -rf /var/lib/apt/lists/* + +# Single-user Nix (no daemon) as root. NIX_CONFIG disables per-build users +# during the install itself (the installer reads config before our nix.conf +# exists); the written nix.conf carries that to later `nix develop` runs, plus +# flakes and the flake's cachix substituter for prebuilt toolchain/z3/aws-lc. +RUN export NIX_CONFIG="build-users-group =" && \ + mkdir -m 0755 /nix && \ + curl -L https://nixos.org/nix/install -o /tmp/nix-install.sh && \ + sh /tmp/nix-install.sh --no-daemon && \ + mkdir -p /etc/nix && \ + printf 'experimental-features = nix-command flakes\naccept-flake-config = true\nbuild-users-group =\nsandbox = false\n' > /etc/nix/nix.conf + +# Put single-user Nix on PATH directly (Buildah RUN is not a login shell, so the +# profile script is unreliable) and point it at Debian's CA bundle for HTTPS +# substituter fetches. +ENV HOME=/root \ + PATH=/root/.nix-profile/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +WORKDIR /build +COPY . . + +# Build the gateway exactly like CI: same devShell, same cargo command and +# target triple. The gnu cross toolchain emits a standard ELF interpreter, so +# the binary runs on distroless without post-processing. Everything runs in a +# single layer: build, stage the binary at /, then delete the Nix store and +# Cargo target so the committed builder layer stays small. +RUN nix develop -c bash -euo pipefail -c '\ + GIT_DIR=/nonexistent cargo auditable build --release \ + --target x86_64-unknown-linux-gnu \ + --package openshell-gateway --bin openshell-gateway' && \ + cp target/x86_64-unknown-linux-gnu/release/openshell-gateway /openshell-gateway && \ + cd / && rm -rf /nix /build /root/.cache /tmp/* + +# ---- Runtime: identical to deploy/docker/Dockerfile.gateway ------------------ +# +# Distroless Debian provides the glibc runtime required by the binary. +FROM ${GATEWAY_BASE_IMAGE} AS gateway + +ARG TARGETARCH + +WORKDIR /app + +COPY --from=builder /openshell-gateway /usr/local/bin/openshell-gateway + +USER 1000:1000 +EXPOSE 8080 + +ENTRYPOINT ["/usr/local/bin/openshell-gateway"] +CMD ["--bind-address", "0.0.0.0", "--port", "8080"] diff --git a/deploy/docker/Dockerfile.supervisor.multistage b/deploy/docker/Dockerfile.supervisor.multistage new file mode 100644 index 0000000000..4e6196d758 --- /dev/null +++ b/deploy/docker/Dockerfile.supervisor.multistage @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1.4 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-cluster (OpenShift/Buildah) variant of Dockerfile.supervisor. +# +# Upstream CI produces the static musl `openshell-sandbox` binary by building +# inside the project's Nix devShell with the musl cross target, and stages it +# under deploy/docker/.build/prebuilt-binaries. This multi-stage Dockerfile +# reproduces that exact binary in a builder stage so clusters without the Nix +# CI pipeline can build the supervisor image directly from source. The final +# stage is identical to Dockerfile.supervisor. + +# ---- Builder: glibc base + Nix, reproduce upstream's cross build ------------- +# +# The build must run on a glibc/FHS base (like the CI runners), NOT on the +# minimal nixos/nix image: cross-compiling emits host build-scripts linked for +# x86_64-unknown-linux-gnu whose ELF interpreter is /lib64/ld-linux-x86-64.so.2, +# which the nixos/nix (Alpine, store-only) image does not provide. +FROM debian:bookworm-slim AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl xz-utils ca-certificates git && rm -rf /var/lib/apt/lists/* + +# Single-user Nix (no daemon) as root. NIX_CONFIG disables per-build users +# during the install itself (the installer reads config before our nix.conf +# exists); the written nix.conf carries that to later `nix develop` runs, plus +# flakes and the flake's cachix substituter for prebuilt toolchain/z3/aws-lc. +RUN export NIX_CONFIG="build-users-group =" && \ + mkdir -m 0755 /nix && \ + curl -L https://nixos.org/nix/install -o /tmp/nix-install.sh && \ + sh /tmp/nix-install.sh --no-daemon && \ + mkdir -p /etc/nix && \ + printf 'experimental-features = nix-command flakes\naccept-flake-config = true\nbuild-users-group =\nsandbox = false\n' > /etc/nix/nix.conf + +# Put single-user Nix on PATH directly (Buildah RUN is not a login shell, so the +# profile script is unreliable) and point it at Debian's CA bundle for HTTPS +# substituter fetches. +ENV HOME=/root \ + PATH=/root/.nix-profile/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +WORKDIR /build +COPY . . + +# Build the sandbox binary exactly like CI: same devShell, same cargo command +# and target triple. Everything runs in a single layer: build, stage the static +# binary at /, then delete the Nix store and Cargo target. The binary is static +# musl (it needs nothing from /nix at runtime), so the committed builder layer +# shrinks to the binary alone, keeping the intermediate commit fast and within +# the node's ephemeral-storage budget. +RUN nix develop -c bash -euo pipefail -c '\ + GIT_DIR=/nonexistent cargo auditable build --release \ + --target x86_64-unknown-linux-musl \ + --package openshell-sandbox --bin openshell-sandbox' && \ + cp target/x86_64-unknown-linux-musl/release/openshell-sandbox /openshell-sandbox && \ + cd / && rm -rf /nix /build /root/.cache /tmp/* + +# ---- Runtime: identical to deploy/docker/Dockerfile.supervisor --------------- +# +# Alpine supplies nftables and iptables for pod-namespace egress enforcement. +FROM alpine:3.22 AS supervisor + +ARG TARGETARCH + +RUN apk add --no-cache nftables iptables iptables-legacy + +# Keep the binary root-owned for Podman image-volume mounts and executable by +# the Kubernetes network sidecar's non-root proxy UID. +COPY --from=builder --chmod=0555 /openshell-sandbox /openshell-sandbox + +ENTRYPOINT ["/openshell-sandbox"] From 5c8c5200d61f270c9f43849f4f864c08306962cb Mon Sep 17 00:00:00 2001 From: Akram Date: Thu, 10 Sep 2026 23:54:37 +0400 Subject: [PATCH 2/8] feat(sandbox): default to official Alpine sandbox image default_sandbox_image() now returns docker.io/library/alpine:3.22, a generic version-qualified official image, so a fresh install no longer depends on the community sandbox image catalog. All compute drivers (docker, podman, kubernetes, vm) inherit this fallback. Part of #3116. Signed-off-by: Akram Signed-off-by: Akram --- crates/openshell-core/src/image.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/openshell-core/src/image.rs b/crates/openshell-core/src/image.rs index e804afd60f..e1b242cbfa 100644 --- a/crates/openshell-core/src/image.rs +++ b/crates/openshell-core/src/image.rs @@ -13,13 +13,19 @@ /// Override at runtime with the `OPENSHELL_COMMUNITY_REGISTRY` env var. pub const DEFAULT_COMMUNITY_REGISTRY: &str = "ghcr.io/nvidia/openshell-community/sandboxes"; -/// Return the default sandbox image reference (`{registry}/base:latest`). +/// Default sandbox base image reference. +/// +/// A generic, version-qualified official Alpine image so a fresh install does +/// not depend on the community image catalog. +pub const DEFAULT_SANDBOX_BASE_IMAGE: &str = "docker.io/library/alpine:3.22"; + +/// Return the default sandbox image reference. /// /// Used by all compute drivers as the fallback image when none is specified in /// the sandbox spec. #[must_use] pub fn default_sandbox_image() -> String { - format!("{DEFAULT_COMMUNITY_REGISTRY}/base:latest") + DEFAULT_SANDBOX_BASE_IMAGE.to_string() } /// Resolve a user-supplied image string into a fully-qualified reference. From 1a43dcc99cc739b7e36b615a3e3624f097f002bb Mon Sep 17 00:00:00 2001 From: Akram Date: Fri, 11 Sep 2026 10:26:56 +0400 Subject: [PATCH 3/8] refactor(policy): drop community image /app path from default policy The restrictive default policy granted read-only access to /app, a directory that only existed in the community base image. A generic Alpine default has no /app, so remove it. Landlock best-effort already ignores absent paths; this just stops advertising a community-specific layout in the default. Part of #3116. Signed-off-by: Akram Signed-off-by: Akram --- crates/openshell-policy/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index d6c66ac597..efcb3de63c 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1274,7 +1274,6 @@ pub fn restrictive_default_policy() -> SandboxPolicy { "/lib".into(), "/proc".into(), "/dev/urandom".into(), - "/app".into(), "/etc".into(), "/var/log".into(), ], From bfb69871a94f11a2e85e8fcb8847b7e420734e59 Mon Sep 17 00:00:00 2001 From: Akram Date: Fri, 11 Sep 2026 15:57:23 +0400 Subject: [PATCH 4/8] feat(deploy): default deployment configs to the official Alpine sandbox image Update the shared gateway default_image, Helm chart values, the standalone Kubernetes manifest, and the dev gateway task scripts to use docker.io/library/alpine:3.22 instead of the community base image, consistent with default_sandbox_image(). GPU e2e image-build base is left unchanged (CUDA needs a glibc base). Part of #3116. Signed-off-by: Akram Signed-off-by: Akram --- deploy/docker/gateway.toml | 2 +- deploy/helm/openshell/values.yaml | 2 +- deploy/kube/manifests/openshell-helmchart.yaml | 2 +- tasks/scripts/gateway-docker.sh | 2 +- tasks/scripts/gateway-podman.sh | 2 +- tasks/scripts/gateway-vm.sh | 2 +- tasks/scripts/gateway.sh | 2 +- tasks/scripts/helm-k3s-local.sh | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index 1372bfc3fe..463bbd585b 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -35,7 +35,7 @@ disable_tls = true [openshell.drivers.docker] # Default image pulled for `openshell sandbox create` without --from. -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +default_image = "docker.io/library/alpine:3.22" # Sandbox runtime image from which the openshell-sandbox binary is extracted. sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:latest" # Image containing the external supervisor process. diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 67d5587ec9..6fedb52862 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -214,7 +214,7 @@ server: # `uri` key, e.g. postgresql://user:pass@host:5432/dbname. externalDbSecret: "" # -- Default sandbox image used when requests do not specify one. - sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" + sandboxImage: "docker.io/library/alpine:3.22" # -- Pull policy for sandbox pods. Leave unset to use the Kubernetes image # default (Always for :latest, IfNotPresent otherwise). Prefer always, # if_not_present, or never; the chart also accepts legacy Kubernetes spellings diff --git a/deploy/kube/manifests/openshell-helmchart.yaml b/deploy/kube/manifests/openshell-helmchart.yaml index 3ca6e3b902..8fd83c2796 100644 --- a/deploy/kube/manifests/openshell-helmchart.yaml +++ b/deploy/kube/manifests/openshell-helmchart.yaml @@ -29,7 +29,7 @@ spec: tag: latest pullPolicy: __IMAGE_PULL_POLICY__ server: - sandboxImage: ghcr.io/nvidia/openshell-community/sandboxes/base:latest + sandboxImage: docker.io/library/alpine:3.22 sandboxImagePullPolicy: __SANDBOX_IMAGE_PULL_POLICY__ supervisorImage: ghcr.io/nvidia/openshell/supervisor:latest dbUrl: __DB_URL__ diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 0b3edca729..bdcce93b76 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -33,7 +33,7 @@ PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_DOCKER_GATEWAY_NAME:-docker-dev}" STATE_DIR="${OPENSHELL_DOCKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-docker}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-docker-dev}" -SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-docker.io/library/alpine:3.22}" SUPERVISOR_IMAGE="${OPENSHELL_SUPERVISOR_IMAGE:-openshell/supervisor:dev}" SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index a45c1d35d4..4ef87107d5 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -31,7 +31,7 @@ PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_PODMAN_GATEWAY_NAME:-podman-dev}" STATE_DIR="${OPENSHELL_PODMAN_GATEWAY_STATE_DIR:-${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-podman}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-podman-dev}" -SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-docker.io/library/alpine:3.22}" SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index ff3f1e7dff..ca1e7aeaa9 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -39,7 +39,7 @@ PORT="${OPENSHELL_SERVER_PORT:-18081}" GATEWAY_NAME="${OPENSHELL_VM_GATEWAY_NAME:-vm-dev}" STATE_DIR="${OPENSHELL_VM_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-vm}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-vm-dev}" -SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-docker.io/library/alpine:3.22}}" VM_BOOTSTRAP_IMAGE="${OPENSHELL_VM_BOOTSTRAP_IMAGE:-}" SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" # VM currently has no image-pull-policy setting in its driver configuration; unlike diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 34bc143fb6..edacadb904 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -207,7 +207,7 @@ PORT="${OPENSHELL_SERVER_PORT:-8080}" GATEWAY_NAME="${OPENSHELL_GATEWAY_NAME:-${DRIVER}-dev}" STATE_DIR="${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-${DRIVER}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-${DRIVER}-dev}" -SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-docker.io/library/alpine:3.22}" SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index dc8adb9bdf..f93dbc3fb6 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -29,7 +29,7 @@ K3D_CLUSTER_NAME_MAX=32 HOST_LB_PORT="${HELM_K3S_LB_HOST_PORT:-8080}" # Preload the default community sandbox image so the first sandbox create does # not pay the full registry pull cost inside the cluster. -DEFAULT_SANDBOX_PRELOAD_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +DEFAULT_SANDBOX_PRELOAD_IMAGE="docker.io/library/alpine:3.22" PRELOAD_SANDBOX_IMAGE="${HELM_K3S_PRELOAD_SANDBOX_IMAGE-${DEFAULT_SANDBOX_PRELOAD_IMAGE}}" # Upstream agent-sandbox release pinned for both CRDs/controller and extensions. From 778ad02c46ba573f61de8f15194a4ed2df04bb0f Mon Sep 17 00:00:00 2001 From: Akram Date: Wed, 16 Sep 2026 19:38:06 +0400 Subject: [PATCH 5/8] refactor(cli)!: remove community image resolution Remove DEFAULT_COMMUNITY_REGISTRY, resolve_community_image, and the OPENSHELL_COMMUNITY_REGISTRY override. Bare --from values are no longer expanded into the OpenShell Community registry; the CLI and TUI now pass explicit OCI image references through to the gateway unchanged. The openshell-core image module is reduced to default_sandbox_image(). BREAKING CHANGE: community sandbox shorthand names and OPENSHELL_COMMUNITY_REGISTRY are no longer supported; pass a full OCI image reference to --from. Part of #3116. Signed-off-by: Akram --- crates/openshell-cli/src/main.rs | 10 +-- crates/openshell-cli/src/run.rs | 7 +- crates/openshell-core/src/image.rs | 115 +---------------------------- crates/openshell-tui/src/lib.rs | 3 +- 4 files changed, 10 insertions(+), 125 deletions(-) diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index fdec4ce7e5..5ff70b131e 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1275,13 +1275,9 @@ enum SandboxCommands { #[arg(long, conflicts_with_all = ["from", "gpu", "cpu", "memory", "driver_config_json", "envs"])] template: Option, - /// Sandbox source: a community sandbox name (e.g., `ollama`), a rootfs - /// tar archive (`.tar`, `.tar.gz`, or `.tgz`), or a full container - /// image reference (e.g., `myregistry.com/img:tag`). - /// - /// Community names are resolved to - /// `ghcr.io/nvidia/openshell-community/sandboxes/:latest` - /// (override the prefix with `OPENSHELL_COMMUNITY_REGISTRY`). + /// Sandbox source: a full container image reference (e.g., + /// `ghcr.io/owner/image:tag`, `myregistry.com/img:tag`) or a + /// rootfs tar archive (`.tar`, `.tar.gz`, or `.tgz`). /// /// To use a local Dockerfile, build and tag it with the container /// engine used by your local gateway, then pass the resulting image diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 243e10d1bf..4768dec055 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1234,11 +1234,8 @@ fn resolve_from(value: &str) -> Result { )); } - // Full image reference or community sandbox name — delegate to shared - // resolution in openshell-core. - Ok(ResolvedSource::Image( - openshell_core::image::resolve_community_image(value), - )) + // Explicit OCI image reference — passed through to the gateway unchanged. + Ok(ResolvedSource::Image(value.to_string())) } #[allow(clippy::case_sensitive_file_extension_comparisons)] // already lowercased diff --git a/crates/openshell-core/src/image.rs b/crates/openshell-core/src/image.rs index e1b242cbfa..8027322828 100644 --- a/crates/openshell-core/src/image.rs +++ b/crates/openshell-core/src/image.rs @@ -1,17 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Shared image-name resolution for community sandbox images. +//! Default sandbox image. //! -//! Both the CLI and TUI need to expand bare sandbox names (e.g. `"base"`) into -//! fully-qualified container image references. This module centralises that -//! logic so every client resolves names identically. - -/// Default registry prefix for community sandbox images. -/// -/// Bare sandbox names are expanded to `{prefix}/{name}:latest`. -/// Override at runtime with the `OPENSHELL_COMMUNITY_REGISTRY` env var. -pub const DEFAULT_COMMUNITY_REGISTRY: &str = "ghcr.io/nvidia/openshell-community/sandboxes"; +//! Provides the fallback image used by all compute drivers when a sandbox spec +//! does not specify one. User-supplied `--from` values are explicit OCI image +//! references passed through unchanged by the CLI and TUI. /// Default sandbox base image reference. /// @@ -27,104 +21,3 @@ pub const DEFAULT_SANDBOX_BASE_IMAGE: &str = "docker.io/library/alpine:3.22"; pub fn default_sandbox_image() -> String { DEFAULT_SANDBOX_BASE_IMAGE.to_string() } - -/// Resolve a user-supplied image string into a fully-qualified reference. -/// -/// Resolution rules (applied in order): -/// 1. If the value contains `/`, `:`, or `.` it is treated as a complete image -/// reference and returned as-is. -/// 2. Otherwise it is treated as a community sandbox name and expanded to -/// `{registry}/{value}:latest` where `{registry}` defaults to -/// [`DEFAULT_COMMUNITY_REGISTRY`] but can be overridden via the -/// `OPENSHELL_COMMUNITY_REGISTRY` environment variable. -/// -/// This function only handles image-name resolution. Dockerfile detection is -/// the responsibility of the caller (e.g. the CLI's `resolve_from()`). -pub fn resolve_community_image(value: &str) -> String { - // Already a fully-qualified reference. - if value.contains('/') || value.contains(':') || value.contains('.') { - return value.to_string(); - } - - // Community sandbox shorthand → expand with registry prefix. - let prefix = std::env::var("OPENSHELL_COMMUNITY_REGISTRY") - .unwrap_or_else(|_| DEFAULT_COMMUNITY_REGISTRY.to_string()); - let prefix = prefix.trim_end_matches('/'); - format!("{prefix}/{value}:latest") -} - -#[cfg(test)] -#[allow(unsafe_code)] -mod tests { - use super::*; - use std::sync::{Mutex, OnceLock}; - - fn env_lock() -> &'static Mutex<()> { - static ENV_LOCK: OnceLock> = OnceLock::new(); - ENV_LOCK.get_or_init(|| Mutex::new(())) - } - - #[test] - fn bare_name_expands_to_community_registry() { - let _guard = env_lock().lock().unwrap(); - let result = resolve_community_image("base"); - assert_eq!( - result, - "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" - ); - } - - #[test] - fn bare_name_with_env_override() { - let _guard = env_lock().lock().unwrap(); - // Use a temp env override. Safety: test-only, and these env-var tests - // are not run concurrently with other tests reading the same var. - let key = "OPENSHELL_COMMUNITY_REGISTRY"; - let prev = std::env::var(key).ok(); - // SAFETY: single-threaded test context; no other thread reads this var. - unsafe { std::env::set_var(key, "my-registry.example.com/sandboxes") }; - let result = resolve_community_image("python"); - assert_eq!(result, "my-registry.example.com/sandboxes/python:latest"); - // Restore. - match prev { - Some(v) => unsafe { std::env::set_var(key, v) }, - None => unsafe { std::env::remove_var(key) }, - } - } - - #[test] - fn full_reference_with_slash_passes_through() { - let _guard = env_lock().lock().unwrap(); - let input = "ghcr.io/myorg/myimage:v1"; - assert_eq!(resolve_community_image(input), input); - } - - #[test] - fn reference_with_colon_passes_through() { - let _guard = env_lock().lock().unwrap(); - let input = "myimage:latest"; - assert_eq!(resolve_community_image(input), input); - } - - #[test] - fn reference_with_dot_passes_through() { - let _guard = env_lock().lock().unwrap(); - let input = "registry.example.com"; - assert_eq!(resolve_community_image(input), input); - } - - #[test] - fn trailing_slash_in_env_is_trimmed() { - let _guard = env_lock().lock().unwrap(); - let key = "OPENSHELL_COMMUNITY_REGISTRY"; - let prev = std::env::var(key).ok(); - // SAFETY: single-threaded test context; no other thread reads this var. - unsafe { std::env::set_var(key, "my-registry.example.com/sandboxes/") }; - let result = resolve_community_image("base"); - assert_eq!(result, "my-registry.example.com/sandboxes/base:latest"); - match prev { - Some(v) => unsafe { std::env::set_var(key, v) }, - None => unsafe { std::env::remove_var(key) }, - } - } -} diff --git a/crates/openshell-tui/src/lib.rs b/crates/openshell-tui/src/lib.rs index 55fbcaff87..636a16893f 100644 --- a/crates/openshell-tui/src/lib.rs +++ b/crates/openshell-tui/src/lib.rs @@ -1444,9 +1444,8 @@ fn spawn_create_sandbox(app: &mut App, tx: mpsc::UnboundedSender) { tokio::spawn(async move { let has_custom_image = !image.is_empty(); let template = if has_custom_image { - let resolved = openshell_core::image::resolve_community_image(&image); Some(openshell_core::proto::SandboxTemplate { - image: resolved, + image, ..Default::default() }) } else { From 362fb753a47c9f7edfd3a3d2ef1475fb9af3447e Mon Sep 17 00:00:00 2001 From: Akram Date: Wed, 16 Sep 2026 19:47:32 +0400 Subject: [PATCH 6/8] feat(driver): default to numeric non-root identity for USER-less images With the default sandbox image now Alpine, images that declare no OCI USER must start instead of being rejected. When the image declares no USER and the policy requests none, the Podman and Docker drivers now supply a numeric non-root identity (DEFAULT_SANDBOX_UID/GID = 1000) instead of rejecting, matching the numeric-identity behavior of the Kubernetes and VM drivers. The supervisor's resolved-identity path runs the sandbox as a synthesized non-root account without the account existing in the image. Images that declare a USER keep the OCI resolution path unchanged. Part of #3116. Signed-off-by: Akram --- crates/openshell-core/src/sandbox_env.rs | 12 ++++++ crates/openshell-driver-docker/src/lib.rs | 15 +++++-- .../openshell-driver-podman/src/container.rs | 43 +++++++++++++------ 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index c1c91822b6..d298c5e99b 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -243,6 +243,18 @@ pub const SANDBOX_UID: &str = "OPENSHELL_SANDBOX_UID"; /// supervisor drops privileges to a group other than the UID's primary group. pub const SANDBOX_GID: &str = "OPENSHELL_SANDBOX_GID"; +/// Default numeric UID assigned to a sandbox when the image declares no OCI +/// `USER` (e.g. a plain Alpine base). +/// +/// Local container drivers (Docker, Podman) supply this in place of an empty +/// OCI declaration so the supervisor runs the sandbox as a synthesized non-root +/// account instead of rejecting the image, matching the numeric-identity +/// behavior of the Kubernetes and VM drivers. +pub const DEFAULT_SANDBOX_UID: u32 = 1000; + +/// Default numeric GID paired with [`DEFAULT_SANDBOX_UID`]. +pub const DEFAULT_SANDBOX_GID: u32 = 1000; + /// Raw OCI `Config.User` declaration from the immutable image selected by a /// local container driver. /// diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index b3f5f337a7..0f28eb6820 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -654,9 +654,18 @@ fn resolve_docker_identity_from_accounts( requested_user }; if user_selector.is_empty() { - return Err(Status::failed_precondition( - "the pinned image defaults to root; configure a non-root process.run_as_user", - )); + // The image declares no USER (e.g. a plain Alpine base) and the policy + // requested none. Synthesize a numeric non-root identity instead of + // rejecting, matching the Podman driver's USER-less default and the + // numeric-identity behavior of the Kubernetes and VM drivers. + return ResolvedWorkloadIdentity::new( + openshell_core::sandbox_env::DEFAULT_SANDBOX_UID, + openshell_core::sandbox_env::DEFAULT_SANDBOX_GID, + Vec::new(), + "default".to_string(), + image.id.clone(), + ) + .map_err(|error| Status::failed_precondition(error.to_string())); } let (uid, passwd_entry) = resolve_numeric_or_named_user(user_selector, &passwd)?; let username = passwd_entry.map(|entry| entry.name.as_str()); diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 4378d0a6bb..b9c1682743 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -593,18 +593,37 @@ fn build_env( // hostname could otherwise present a certificate for a name they control // and intercept the sandbox JWT. env.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - env.insert( - openshell_core::sandbox_env::OCI_IMAGE_USER.into(), - oci_user.to_string(), - ); - env.insert( - openshell_core::sandbox_env::SANDBOX_UID.into(), - String::new(), - ); - env.insert( - openshell_core::sandbox_env::SANDBOX_GID.into(), - String::new(), - ); + if oci_user.is_empty() { + // The image declares no OCI USER (e.g. a plain Alpine base). Assign a + // numeric non-root identity like the Kubernetes and VM drivers so the + // supervisor synthesizes the account instead of rejecting the image. + env.insert( + openshell_core::sandbox_env::OCI_IMAGE_USER.into(), + String::new(), + ); + env.insert( + openshell_core::sandbox_env::SANDBOX_UID.into(), + openshell_core::sandbox_env::DEFAULT_SANDBOX_UID.to_string(), + ); + env.insert( + openshell_core::sandbox_env::SANDBOX_GID.into(), + openshell_core::sandbox_env::DEFAULT_SANDBOX_GID.to_string(), + ); + } else { + // The image declares a USER; preserve the OCI resolution path. + env.insert( + openshell_core::sandbox_env::OCI_IMAGE_USER.into(), + oci_user.to_string(), + ); + env.insert( + openshell_core::sandbox_env::SANDBOX_UID.into(), + String::new(), + ); + env.insert( + openshell_core::sandbox_env::SANDBOX_GID.into(), + String::new(), + ); + } // 4. Gateway-minted sandbox JWT. Keep the raw bearer out of container // metadata; the supervisor reads it from a driver-owned bind mount. From 243dc1e114183097b6d2b66b5e8f4da1d748eafa Mon Sep 17 00:00:00 2001 From: Akram Date: Wed, 16 Sep 2026 20:13:42 +0400 Subject: [PATCH 7/8] refactor(image): retire residual community image references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the remaining OpenShell Community sandbox image references (RPM/Helm docs, SDK doc examples, and test fixtures) with the Alpine default, and update the resolve_from test that asserted community-name expansion — bare values are now passed through unchanged as explicit OCI references. The VM runtime image pin (driver-vm/runtime/pins.env) and the GPU e2e build base (tasks/scripts/e2e-gpu-build-images.sh) reference a community image for their own runtime/build needs and are intentionally left for separate follow-ups. Part of #3116. Signed-off-by: Akram --- crates/openshell-cli/src/run.rs | 11 +++++------ crates/openshell-driver-docker/src/tests.rs | 2 +- crates/openshell-ocsf/tests/roundtrip.rs | 2 +- crates/openshell-sdk/README.md | 2 +- crates/openshell-sdk/src/types.rs | 2 +- crates/openshell-server/src/compute/mod.rs | 2 +- deploy/helm/openshell/README.md | 2 +- deploy/rpm/CONFIGURATION.md | 12 ++++++------ deploy/rpm/TROUBLESHOOTING.md | 6 +++--- 9 files changed, 20 insertions(+), 21 deletions(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 4768dec055..e72bc84bf3 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -6563,7 +6563,7 @@ mod tests { } #[test] - fn resolve_from_keeps_bare_community_name_when_local_directory_matches() { + fn resolve_from_keeps_bare_name_as_image_when_local_directory_matches() { let _lock = TEST_ENV_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -6575,11 +6575,10 @@ mod tests { let result = resolve_from("python"); std::env::set_current_dir(original_dir).expect("restore current directory"); - match result.expect("bare community name should not be a local path") { - super::ResolvedSource::Image(image) => assert_eq!( - image, - "ghcr.io/nvidia/openshell-community/sandboxes/python:latest" - ), + match result.expect("bare name should be treated as an image, not a local path") { + // Bare values are passed through unchanged as explicit OCI references + // (community-name expansion was removed). + super::ResolvedSource::Image(image) => assert_eq!(image, "python"), other @ super::ResolvedSource::RootfsTar { .. } => { panic!("expected image source, got {other:?}"); } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 1f11dafb72..4a4f63fe96 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -62,7 +62,7 @@ fn test_sandbox() -> DriverSandbox { log_level: "debug".to_string(), environment: HashMap::from([("SPEC_ENV".to_string(), "spec".to_string())]), template: Some(DriverSandboxTemplate { - image: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), + image: "docker.io/library/alpine:3.22".to_string(), agent_socket_path: String::new(), labels: HashMap::new(), environment: HashMap::from([("TEMPLATE_ENV".to_string(), "template".to_string())]), diff --git a/crates/openshell-ocsf/tests/roundtrip.rs b/crates/openshell-ocsf/tests/roundtrip.rs index 42664400ab..6d8a9f639d 100644 --- a/crates/openshell-ocsf/tests/roundtrip.rs +++ b/crates/openshell-ocsf/tests/roundtrip.rs @@ -22,7 +22,7 @@ fn ctx() -> EventContext { EventContext { sandbox_id: "sb-7f3a9c2e14b8".to_string(), sandbox_name: "agent-workspace-01".to_string(), - container_image: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), + container_image: "docker.io/library/alpine:3.22".to_string(), hostname: "openshell-sb-7f3a9c2e14b8".to_string(), product_version: "0.42.1".to_string(), proxy_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), diff --git a/crates/openshell-sdk/README.md b/crates/openshell-sdk/README.md index 3adbcba375..2df7c91f03 100644 --- a/crates/openshell-sdk/README.md +++ b/crates/openshell-sdk/README.md @@ -92,7 +92,7 @@ client }), spec: Some(SandboxWorkloadTemplateSpec { workload: Some(SandboxWorkloadConfig { - image: "ghcr.io/nvidia/openshell-community/sandboxes/python:latest".to_string(), + image: "docker.io/library/alpine:3.22".to_string(), ..Default::default() }), ..Default::default() diff --git a/crates/openshell-sdk/src/types.rs b/crates/openshell-sdk/src/types.rs index a165c37124..37598d727d 100644 --- a/crates/openshell-sdk/src/types.rs +++ b/crates/openshell-sdk/src/types.rs @@ -100,7 +100,7 @@ impl From for SandboxPhase { pub struct SandboxSpec { /// Optional user-supplied sandbox name. When empty the server generates one. pub name: Option, - /// Container image reference (e.g. `ghcr.io/nvidia/openshell-community/sandboxes/python:latest`). + /// Container image reference (e.g. `docker.io/library/alpine:3.22`). pub image: Option, /// Labels attached to the sandbox. pub labels: HashMap, diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 2286ad89a7..f88dc35efc 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -11585,7 +11585,7 @@ mod tests { ..Default::default() }), template: Some(SandboxTemplate { - image: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), + image: "docker.io/library/alpine:3.22".to_string(), driver_config: Some(prost_types::Struct { fields: [ ( diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 231983aca7..6f8a84b651 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -288,7 +288,7 @@ discovery endpoint or its TLS CA. | server.policyValidationFailureMode | string | `"fail_closed"` | Posture when a candidate sandbox policy fails validation. `fail_closed` deactivates the previous policy; `retain_last_valid` keeps it active. | | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | -| server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | +| server.sandboxImage | string | `"docker.io/library/alpine:3.22"` | Default sandbox image used when requests do not specify one. | | server.sandboxImagePullPolicy | string | `nil` | Pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index af2e97a94f..8bcc28574d 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -221,7 +221,7 @@ overrides that persist across package upgrades. |-------------|---------|-------------| | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | | `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | -| `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | +| `[openshell.drivers.podman].default_image` | `docker.io/library/alpine:3.22` | Default sandbox image. | | `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:latest` | Static musl sandbox runtime image mounted into Podman workloads. | | `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Dynamic glibc supervisor image used outside the workload. | | `[openshell.gateway].guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Gateway-owned client TLS material injected into the selected local driver and mounted into sandbox containers. | @@ -244,7 +244,7 @@ version = 2 compute_driver = "podman" [openshell.drivers.podman] -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +default_image = "docker.io/library/alpine:3.22" image_pull_policy = "if_not_present" health_check_interval_secs = 10 network_name = "openshell" @@ -261,7 +261,7 @@ To update cached images: ```shell podman pull ghcr.io/nvidia/openshell/supervisor:latest -podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest +podman pull docker.io/library/alpine:3.22 ``` Or set `image_pull_policy = "always"` in @@ -273,7 +273,7 @@ To pin specific image versions instead of `:latest`, set these values in ```toml sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:v0.0.37" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:v0.0.37" -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:v0.0.37" +default_image = "docker.io/library/alpine:3.22" ``` For air-gapped environments: @@ -282,9 +282,9 @@ For air-gapped environments: ```shell podman pull ghcr.io/nvidia/openshell/supervisor:latest - podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest + podman pull docker.io/library/alpine:3.22 podman save -o supervisor.tar ghcr.io/nvidia/openshell/supervisor:latest - podman save -o sandbox.tar ghcr.io/nvidia/openshell-community/sandboxes/base:latest + podman save -o sandbox.tar docker.io/library/alpine:3.22 ``` 1. Transfer the tarballs to the air-gapped host and load them: diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index 29b27ad209..6f002c6cc2 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -176,7 +176,7 @@ sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USER **Image pull failure.** Verify ghcr.io is reachable: ```shell -podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest +podman pull docker.io/library/alpine:3.22 ``` ### Images not updating @@ -185,7 +185,7 @@ The default image pull policy is `if_not_present` -- images are pulled once and cached. To update: ```shell -podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest +podman pull docker.io/library/alpine:3.22 podman pull ghcr.io/nvidia/openshell/supervisor:latest ``` @@ -239,7 +239,7 @@ To pick up new container images after an upgrade: ```shell podman pull ghcr.io/nvidia/openshell/supervisor:latest -podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest +podman pull docker.io/library/alpine:3.22 ``` ### Migrating a TLS-enabled local driver to schema version 2 From fd27bc329846a201b02fae474553468ec1795f3a Mon Sep 17 00:00:00 2001 From: Akram Date: Wed, 16 Sep 2026 20:53:07 +0400 Subject: [PATCH 8/8] build(docker): split in-cluster build variants for the RFC-0012 three-image layout RFC-0012 (#2942) split the runtime into three binaries. Match the in-cluster (OpenShift/Buildah) multi-stage variants to the image set the Helm chart and official Dockerfiles expect: - Dockerfile.sandbox.multistage (new): static-musl openshell-sandbox on a scratch base (USER 65532), mirroring Dockerfile.sandbox. - Dockerfile.supervisor.multistage: now builds the dynamically linked GNU openshell-supervisor on a debian base with ca-certificates, mirroring Dockerfile.supervisor (previously it built openshell-sandbox, which predated the RFC-0012 split). - Dockerfile.gateway.multistage: unchanged (openshell-gateway). Part of #3116. Signed-off-by: Akram --- deploy/docker/Dockerfile.sandbox.multistage | 75 +++++++++++++++++++ .../docker/Dockerfile.supervisor.multistage | 45 +++++------ 2 files changed, 98 insertions(+), 22 deletions(-) create mode 100644 deploy/docker/Dockerfile.sandbox.multistage diff --git a/deploy/docker/Dockerfile.sandbox.multistage b/deploy/docker/Dockerfile.sandbox.multistage new file mode 100644 index 0000000000..b29af359fc --- /dev/null +++ b/deploy/docker/Dockerfile.sandbox.multistage @@ -0,0 +1,75 @@ +# syntax=docker/dockerfile:1.4 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-cluster (OpenShift/Buildah) variant of Dockerfile.sandbox. +# +# Upstream CI produces the static musl `openshell-sandbox` binary by building +# inside the project's Nix devShell with the musl cross target, and stages it +# under deploy/docker/.build/prebuilt-binaries. This multi-stage Dockerfile +# reproduces that exact binary in a builder stage so clusters without the Nix +# CI pipeline can build the sandbox image directly from source. The final stage +# is identical to Dockerfile.sandbox. + +# ---- Builder: glibc base + Nix, reproduce upstream's cross build ------------- +# +# The build must run on a glibc/FHS base (like the CI runners), NOT on the +# minimal nixos/nix image: cross-compiling emits host build-scripts linked for +# x86_64-unknown-linux-gnu whose ELF interpreter is /lib64/ld-linux-x86-64.so.2, +# which the nixos/nix (Alpine, store-only) image does not provide. +FROM debian:bookworm-slim AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl xz-utils ca-certificates git && rm -rf /var/lib/apt/lists/* + +# Single-user Nix (no daemon) as root. NIX_CONFIG disables per-build users +# during the install itself (the installer reads config before our nix.conf +# exists); the written nix.conf carries that to later `nix develop` runs, plus +# flakes and the flake's cachix substituter for prebuilt toolchain/z3/aws-lc. +RUN export NIX_CONFIG="build-users-group =" && \ + mkdir -m 0755 /nix && \ + curl -L https://nixos.org/nix/install -o /tmp/nix-install.sh && \ + sh /tmp/nix-install.sh --no-daemon && \ + mkdir -p /etc/nix && \ + printf 'experimental-features = nix-command flakes\naccept-flake-config = true\nbuild-users-group =\nsandbox = false\n' > /etc/nix/nix.conf + +# Put single-user Nix on PATH directly (Buildah RUN is not a login shell, so the +# profile script is unreliable) and point it at Debian's CA bundle for HTTPS +# substituter fetches. +ENV HOME=/root \ + PATH=/root/.nix-profile/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +WORKDIR /build +COPY . . + +# Build the sandbox binary exactly like CI: same devShell, same cargo command +# and target triple. Everything runs in a single layer: build, stage the static +# binary at /, then delete the Nix store and Cargo target. The binary is static +# musl (it needs nothing from /nix at runtime), so the committed builder layer +# shrinks to the binary alone, keeping the intermediate commit fast and within +# the node's ephemeral-storage budget. +RUN nix develop -c bash -euo pipefail -c '\ + GIT_DIR=/nonexistent cargo auditable build --release \ + --target x86_64-unknown-linux-musl \ + --package openshell-sandbox --bin openshell-sandbox' && \ + cp target/x86_64-unknown-linux-musl/release/openshell-sandbox /openshell-sandbox && \ + cd / && rm -rf /nix /build /root/.cache /tmp/* + +# ---- Runtime: identical to deploy/docker/Dockerfile.sandbox ------------------ +# +# Runtime-specific bootstrap tools belong to their compute drivers rather than +# this portable image, so the sandbox runtime image is a bare scratch layer. +FROM scratch AS sandbox + +ARG TARGETARCH + +# Keep the binary root-owned for image-volume mounts and executable by the +# sandbox runtime's non-root UID. +COPY --from=builder --chmod=0555 /openshell-sandbox /openshell-sandbox + +# Drivers may override this identity to match the admitted workload. Keep the +# standalone image non-root by default. +USER 65532:65532 + +ENTRYPOINT ["/openshell-sandbox"] diff --git a/deploy/docker/Dockerfile.supervisor.multistage b/deploy/docker/Dockerfile.supervisor.multistage index 4e6196d758..436e14915c 100644 --- a/deploy/docker/Dockerfile.supervisor.multistage +++ b/deploy/docker/Dockerfile.supervisor.multistage @@ -4,12 +4,13 @@ # In-cluster (OpenShift/Buildah) variant of Dockerfile.supervisor. # -# Upstream CI produces the static musl `openshell-sandbox` binary by building -# inside the project's Nix devShell with the musl cross target, and stages it -# under deploy/docker/.build/prebuilt-binaries. This multi-stage Dockerfile -# reproduces that exact binary in a builder stage so clusters without the Nix -# CI pipeline can build the supervisor image directly from source. The final -# stage is identical to Dockerfile.supervisor. +# Upstream CI builds the dynamically linked GNU `openshell-supervisor` binary +# inside the project's Nix devShell with the gnu cross target (z3 and aws-lc +# statically embedded, standard ELF interpreter), and stages it under +# deploy/docker/.build/prebuilt-binaries. This multi-stage Dockerfile reproduces +# that exact artifact in a builder stage so clusters without the Nix CI pipeline +# can build the supervisor image directly from source. The final stage is +# identical to Dockerfile.supervisor. # ---- Builder: glibc base + Nix, reproduce upstream's cross build ------------- # @@ -43,30 +44,30 @@ ENV HOME=/root \ WORKDIR /build COPY . . -# Build the sandbox binary exactly like CI: same devShell, same cargo command -# and target triple. Everything runs in a single layer: build, stage the static -# binary at /, then delete the Nix store and Cargo target. The binary is static -# musl (it needs nothing from /nix at runtime), so the committed builder layer -# shrinks to the binary alone, keeping the intermediate commit fast and within -# the node's ephemeral-storage budget. +# Build the supervisor exactly like CI: same devShell, same cargo command and +# target triple. The gnu cross toolchain emits a standard ELF interpreter, so +# the binary runs on a plain Debian base without post-processing. Everything +# runs in a single layer: build, stage the binary at /, then delete the Nix +# store and Cargo target so the committed builder layer stays small. RUN nix develop -c bash -euo pipefail -c '\ GIT_DIR=/nonexistent cargo auditable build --release \ - --target x86_64-unknown-linux-musl \ - --package openshell-sandbox --bin openshell-sandbox' && \ - cp target/x86_64-unknown-linux-musl/release/openshell-sandbox /openshell-sandbox && \ + --target x86_64-unknown-linux-gnu \ + --package openshell-supervisor --bin openshell-supervisor' && \ + cp target/x86_64-unknown-linux-gnu/release/openshell-supervisor /openshell-supervisor && \ cd / && rm -rf /nix /build /root/.cache /tmp/* # ---- Runtime: identical to deploy/docker/Dockerfile.supervisor --------------- # -# Alpine supplies nftables and iptables for pod-namespace egress enforcement. -FROM alpine:3.22 AS supervisor +# The dynamically linked GNU supervisor binary needs a glibc runtime plus a CA +# bundle for its outbound TLS connections. +FROM debian:bookworm-slim AS supervisor ARG TARGETARCH -RUN apk add --no-cache nftables iptables iptables-legacy +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && rm -rf /var/lib/apt/lists/* -# Keep the binary root-owned for Podman image-volume mounts and executable by -# the Kubernetes network sidecar's non-root proxy UID. -COPY --from=builder --chmod=0555 /openshell-sandbox /openshell-sandbox +COPY --from=builder --chmod=0555 /openshell-supervisor /openshell-supervisor -ENTRYPOINT ["/openshell-sandbox"] +ENTRYPOINT ["/openshell-supervisor"]