diff --git a/README.md b/README.md index f23ddb63c5..8954ede9f7 100644 --- a/README.md +++ b/README.md @@ -337,7 +337,7 @@ OpenShell is built agent-first. Issues should include a user story, problem stat OpenShell collects anonymous telemetry to help improve the project for developers. This data is not used to track individual user behavior. It helps us understand aggregate usage of sandbox, provider, and policy workflows so we can prioritize product improvements and share usage trends with the community. -Disable telemetry at runtime by setting `OPENSHELL_TELEMETRY_ENABLED=false` on the gateway deployment. For Helm installs, set `server.telemetryEnabled=false`. OpenShell propagates this deployment setting into sandbox supervisor environments so sandbox-side telemetry collection is disabled as well. +Disable telemetry at runtime by setting `OPENSHELL_TELEMETRY_ENABLED=false` on the gateway deployment. For Helm installs, set `server.telemetryEnabled=false`; the chart wires that deployment setting into the gateway and sandbox supervisor environments. You can also compile telemetry out entirely. Telemetry support is a default-on `telemetry` Cargo feature, and each crate that carries it also defines a `defaults-without-telemetry` alias covering every other default feature. Build telemetry-free artifacts with `--no-default-features --features defaults-without-telemetry`: diff --git a/architecture/gateway.md b/architecture/gateway.md index 45e934d610..50e05c8d47 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -55,6 +55,70 @@ the gateway listener uses TLS; package-managed local TLS can supply that bundle. Kubernetes instead projects guest credentials through its configured Secret. The gateway validates this requirement before constructing the selected driver. +### Helm configuration boundary + +The Kubernetes Helm chart exposes `gatewayConfig` as its non-secret gateway +application-configuration boundary. Each top-level map key names a TOML table, +and the chart serializes its fields into the mounted `gateway.toml` ConfigMap. +This keeps the chart independent of individual gateway fields: a new non-secret +gateway option does not require a chart template change. + +Secret material never belongs in `gatewayConfig` or the ConfigMap. Database +URLs, credentials, private keys, and equivalent values use Kubernetes Secrets +through the chart's supported environment-variable, file, or volume wiring. +Helm serializes unknown fields generically, so it cannot determine whether an +arbitrary string such as `api_token` is confidential. It rejects +`database_url`, inline URL credentials, and PEM private keys, but is not a +general secret scanner; operators must keep all secret material out of this +map. +The gateway's normal precedence still applies: CLI flags and `OPENSHELL_*` +environment variables override the mounted TOML file. + +The chart rejects the gateway's `database_url` file field and unambiguous +inline credential or PEM private-key strings before rendering a ConfigMap. +It validates Secret references as Kubernetes Secret names, but never reads or +copies referenced Secret data into `gateway.toml`. + +The serializer has a deterministic YAML-to-TOML contract. YAML `null` fields +are omitted; `null` array members are rejected because TOML has no equivalent. +Strings, booleans, integers, and floats preserve their types. Scalar arrays +become TOML arrays, maps become inline tables, and arrays of maps become arrays +of inline tables. Keys are ordered alphabetically, so equivalent input produces +the same ConfigMap checksum. Helm `tpl` expressions are evaluated only in +string values, never in keys or YAML structure. + +The implementation intentionally uses only long-standing Helm 3 template and +Sprig functions (tpl, kindIs, keys, sortAlpha, splitList, quote, and toJson); +it does not rely on a Helm-specific TOML encoder. This preserves the chart's +documented Helm 3 compatibility while making the serialization rules explicit +in the chart itself. + +Helm retains ownership of values that create or modify Kubernetes resources, +including Services, workloads, probes, Secrets, certificate resources, Routes, +RBAC, NetworkPolicies, and mounts. When one of those inputs also determines a +gateway runtime value, the chart derives one from the other rather than +exposing two independently configurable settings. + +### Legacy Helm value inventory + +The former hand-written ConfigMap template read the following values. This +inventory is the migration boundary for `gatewayConfig`; it prevents a legacy +knob from silently surviving as a second source of truth. + +| Classification | Legacy values read by the ConfigMap | Migration | +| --- | --- | --- | +| Application-only | `server.name`, `server.logLevel`, `server.enableLoopbackServiceHttp`, `server.policyValidationFailureMode`, `server.grpcRateLimit.requests`, `server.grpcRateLimit.windowSeconds` | `openshell.gateway`; defaulted runtime values are now in `gatewayConfig`, optional values are omitted unless the operator adds them. | +| Application-only | `server.otlp.endpoint`, `server.otlp.serviceName`, `server.auth.allowUnauthenticatedUsers`, `server.oidc.{issuer,audience,jwksTtl,rolesClaim,adminRole,userRole,scopesClaim}` | `openshell.gateway.{otlp,auth,oidc}`. Empty optional tables are not rendered. | +| Application-only | `server.sandboxImage`, `server.sandboxImagePullPolicy`, `server.sandboxImagePullSecrets`, `server.workspaceDefaultStorageSize`, `server.workspaceStorageClass`, `server.defaultRuntimeClassName`, `server.enableUserNamespaces` | `openshell.drivers.kubernetes`; `server.appArmorProfile` is removed by RFC 0012. | +| Application-only | `server.drivers.kubernetes.{workspaceMode,operatorNamespaceLabel,operatorNamespaceFile}`, `server.sandboxJwt.{gatewayId,ttlSecs,k8sSaTokenTtlSecs}` | `openshell.drivers.kubernetes`, `.managed_ssh_ingress`, and `openshell.gateway.gateway_jwt`. `supervisor.topology` and `supervisor.sidecar.*` are removed by RFC 0012. | +| Application-only | `server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace`, `server.credentialDrivers.vault.{address,mount,kvVersion,authMethod,role,kubernetesAuthMount,serviceAccountTokenPath,tokenPath,timeoutSecs}`, `server.providerTokenGrants.spiffe.{enabled,workloadApiSocketPath}` | The corresponding `openshell.drivers.kubernetes` or `openshell.credential_drivers.*` table. Credential-driver and SPIFFE runtime configuration selects any required Helm resources; Secret names and keys remain references, never Secret data. | +| Application-only | `upstreamProxy.{url,noProxy,authSecret.name,authSecret.key,authAllowInsecure,connectByHostname}`, `sandboxRuntime.image.*`, `supervisor.image.*`, `supervisor.sandboxRuntime.*` | `openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*,sandbox_runtime_image,supervisor_image,sandbox_runtime}`. | +| Deployment-only | `server.dbUrl`, `server.externalDbSecret` | Gateway process args and `OPENSHELL_DB_URL` Secret reference. They are never TOML. | +| Deployment-only | `server.credentialStorage.existingSecret`, `server.sandboxJwt.signingSecretName`, `server.tls.certSecretName`, `server.tls.clientCaSecretName` | Secret creation, mounting, and environment wiring. TOML contains only stable paths or an environment-variable name. | +| Dual-use | `service.{port,healthPort,metricsPort}`, `server.disableTls`, `server.tls.clientTlsSecretName` | The chart owns the Service, workload ports, mounts, and Secret references; `gatewayConfig` derives listener addresses and runtime references from them. | +| Dual-use | `certManager.{enabled,serverIssuerRef.name,serverDnsNames}`, `pkiInitJob.{enabled,serverDnsNames}` | Certificate resources and mounts remain chart-owned; the TLS table and server SANs are derived from their selected certificate source. | +| Dual-use | `networkPolicy.enabled`, `server.hostGatewayIP` | The chart owns NetworkPolicies and pod host aliases. `server.hostGatewayIP` is the sole input for both host aliases and the derived `host_gateway_ip` runtime field. | + ## Protocol and Auth Gateway validation and concurrency errors use the standard rich gRPC error diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 6bddddc4c2..28d25b0801 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -683,6 +683,17 @@ mod tests { } } + #[test] + fn gateway_rate_limits_reject_negative_values_during_toml_parsing() { + for field in ["grpc_rate_limit_requests", "grpc_rate_limit_window_seconds"] { + let tmp = write_tmp(&format!("[openshell.gateway]\n{field} = -1\n")); + assert!( + matches!(load(tmp.path()), Err(ConfigFileError::Parse { .. })), + "{field} must reject negative values because gateway rate limits are unsigned" + ); + } + } + #[test] fn canonical_compute_driver_is_singular() { let file: ConfigFile = toml::from_str( diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 003475df96..9b4acf0174 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -36,7 +36,7 @@ where Helm cannot discover cluster APIs. ```shell helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ``` ## Install on OpenShift @@ -44,15 +44,12 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ``` On OpenShift 4.22+, end-to-end TLS is supported via `BackendTLSPolicy`. See the @@ -110,7 +107,7 @@ Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ -n openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set workload.kind=deployment \ --set server.externalDbSecret=my-pg-credentials ``` @@ -128,16 +125,43 @@ database. The chart creates a retained Kubernetes Secret with the shared key-encryption key and injects that key into every gateway pod, so the same default works for single-replica and external database-backed HA deployments. -Use `kubernetes-secrets` or `vault` instead when credentials should live in a -cluster or external secret backend. Enabling one external credential driver -disables the default credential-storage key-encryption key Secret and env injection. +Use `gatewayConfig` to select `kubernetes-secrets` or `vault` when credentials +should live in a cluster or external secret backend. Selecting an external +credential driver disables the default credential-storage key-encryption-key +Secret and environment injection. The map is rendered directly as gateway TOML, +so it uses the gateway's snake_case field names: + +```yaml +gatewayConfig: + openshell.gateway: + credential_drivers: + - vault + openshell.credential_drivers.vault: + address: http://vault.vault.svc.cluster.local:8200 + mount: secret + kv_version: "2" + auth_method: kubernetes + role: openshell-gateway +``` + +> `gatewayConfig` must contain only non-secret values. Helm serializes unknown +> fields generically and cannot determine whether an arbitrary string, such as +> `api_token`, is confidential. Do not put passwords, tokens, private keys, +> database URLs, or other secret material in this map. Use Secret-backed +> environment variables, files, volumes, or gateway credential drivers instead. +> The chart rejects known unsafe forms such as `database_url`, inline URL +> credentials, and PEM private keys; it is not a general secret scanner. + +For the Kubernetes Secret driver, use +`openshell.credential_drivers.kubernetes-secrets.namespace` in the same map. +The chart derives any required RBAC from the selected driver; use a dedicated +namespace to limit access to OpenShell-managed Secrets. #### OpenShift Append these flags to any of the PostgreSQL commands above for OpenShift: ``` ---set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null ``` @@ -158,12 +182,16 @@ JWT signing Secret. ## SPIFFE/SPIRE provider token grants -Set `server.providerTokenGrants.spiffe.enabled=true` to let the gateway and -sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token grants. The -chart keeps supervisor-to-gateway authentication on gateway-minted sandbox JWTs, -mounts the SPIFFE CSI socket into the gateway pod, exports -`OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`, and passes the socket path to -the Kubernetes driver so sandbox pods can mount the same socket. +Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` +to let sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token +grants. The chart keeps supervisor-to-gateway authentication on gateway-minted +sandbox JWTs and passes the configured socket path to the Kubernetes driver. + +```yaml +gatewayConfig: + openshell.drivers.kubernetes: + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock +``` For local development, uncomment the SPIRE Helm releases in `skaffold.yaml` and add `ci/values-spire.yaml` to the OpenShell release values files. @@ -186,7 +214,9 @@ discovery endpoint or its TLS CA. | certManager.serverDnsNames | list | `["openshell","openshell.openshell.svc","openshell.openshell.svc.cluster.local","localhost","openshell.localhost","*.openshell.localhost","host.docker.internal"]` | DNS SANs on the cert-manager-issued server certificate. | | certManager.serverIpAddresses | list | `["127.0.0.1"]` | IP SANs on the cert-manager-issued server certificate. | | certManager.serverIssuerRef | object | `{"group":"","kind":"","name":""}` | Override the issuerRef for the external server Certificate (e.g. a real LetsEncrypt/ACME ClusterIssuer for a publicly-trusted cert on an external hostname). When set, the chart creates a second server certificate from this issuer with only the hostnames in serverDnsNames; the internal server certificate is always signed by the chart's own CA. Leave name empty to use the chart CA for all server certificates (default). Requires certManager.enabled=true. | +| credentialDrivers.vault.caConfigMapName | string | `""` | ConfigMap containing the private Vault/OpenBao CA certificate under the ca.crt key. Helm mounts it only when the Vault driver is selected. | | fullnameOverride | string | `""` | Override the full generated resource name. | +| gatewayConfig | object | `{"openshell":{"version":2},"openshell.drivers.kubernetes":{"client_tls_secret_name":"{{ .Values.server.tls.clientTlsSecretName }}","default_image":"ghcr.io/nvidia/openshell-community/sandboxes/base:latest","gateway_id":"{{ include \"openshell.fullname\" . }}","grpc_endpoint":"{{ include \"openshell.grpcEndpoint\" . }}","namespace":"{{ include \"openshell.sandboxNamespace\" . }}","sa_token_ttl_secs":3600,"sandbox_runtime":{"boundary_port":5500,"network_policy_enforced":false},"sandbox_runtime_image":"ghcr.io/nvidia/openshell/sandbox:{{ .Chart.AppVersion }}","service_account_name":"{{ include \"openshell.sandboxServiceAccountName\" . }}","supervisor_image":"ghcr.io/nvidia/openshell/supervisor:{{ .Chart.AppVersion }}","workspace_mode":"shared"},"openshell.drivers.kubernetes.managed_ssh_ingress":{"enabled":true,"gateway_namespace":"{{ .Release.Namespace }}","gateway_pod_selector":{"app.kubernetes.io/instance":"{{ .Release.Name }}","app.kubernetes.io/name":"{{ include \"openshell.name\" . }}"}},"openshell.gateway":{"bind_address":"0.0.0.0:{{ .Values.service.port }}","compute_driver":"kubernetes","enable_loopback_service_http":true,"health_bind_address":"0.0.0.0:{{ .Values.service.healthPort }}","log_level":"info","metrics_bind_address":"0.0.0.0:{{ .Values.service.metricsPort }}","name":"{{ include \"openshell.fullname\" . }}","policy_validation_failure_mode":"fail_closed"},"openshell.gateway.credential_storage":{"key_encryption_key_env":"{{ include \"openshell.credentialStorageKeyEncryptionKeyEnvName\" . }}"},"openshell.gateway.gateway_jwt":{"gateway_id":"{{ include \"openshell.fullname\" . }}","kid_path":"/etc/openshell-jwt/kid","public_key_path":"/etc/openshell-jwt/public.pem","signing_key_path":"/etc/openshell-jwt/signing.pem","ttl_secs":3600},"openshell.gateway.tls":{"cert_path":"/etc/openshell-tls/server/tls.crt","client_ca_path":"/etc/openshell-tls/client-ca/ca.crt","key_path":"/etc/openshell-tls/server/tls.key"}}` | Non-secret gateway application configuration. Top-level keys name TOML tables and are rendered into the mounted gateway.toml file. Kubernetes resource inputs remain outside this map; template expressions derive the corresponding runtime values from their resource owner. | | grpcRoute.backendTLSPolicy.caCertificateConfigMapName | string | `""` | Name of the ConfigMap containing the CA certificate (key: ca.crt) used to validate the gateway pod's TLS certificate. Defaults to `-backend-ca` when empty. The certgen hook auto-creates this: with pkiInitJob (default), immediately on install/upgrade; with cert-manager, the hook polls for pkiInitJob.timeoutSeconds seconds waiting for cert-manager to issue the server certificate, then creates the ConfigMap. A single install usually succeeds; if cert-manager takes longer, increase pkiInitJob.timeoutSeconds. By default (pkiInitJob.failOnTimeout=true), the install fails if the timeout is reached; set failOnTimeout=false to allow the install to succeed and run `helm upgrade` after the certificate is issued. | | grpcRoute.backendTLSPolicy.enabled | bool | `false` | Create a BackendTLSPolicy resource for end-to-end TLS between the Gateway proxy and the OpenShell gateway pod. The traffic flow is: client → HTTPS → Gateway (terminate) → TLS (re-encrypt) → gateway pod. Requires server.disableTls=false and server.tls.enableMtls=false. The certgen hook auto-creates the backend CA ConfigMap. | | grpcRoute.backendTLSPolicy.hostname | string | `""` | Hostname the Gateway proxy validates against the backend's TLS certificate SAN. Defaults to the service FQDN (`..svc.cluster.local`) when empty, which matches the SAN included by both cert-manager and the pkiInitJob. | @@ -207,6 +237,7 @@ discovery endpoint or its TLS CA. | nameOverride | string | `"openshell"` | Override the chart name used in generated resource names. | | networkPolicy.enabled | bool | `true` | Restrict SSH ingress on sandbox pods to the gateway. In managed mode, the driver applies the equivalent policy to each workspace namespace. | | nodeSelector | object | `{}` | Node selector for the gateway pod. | +| oidc.caConfigMapName | string | `""` | | | openshiftRoute.annotations | object | `{}` | Extra annotations on the Route (e.g. haproxy.router.openshift.io/*). | | openshiftRoute.enabled | bool | `false` | Create an OpenShift Route with TLS passthrough. | | openshiftRoute.host | string | `""` | Hostname for the Route. Must match a SAN on the gateway's server cert. | @@ -232,9 +263,6 @@ discovery endpoint or its TLS CA. | probes.startup.timeoutSeconds | int | `1` | Startup probe timeout, in seconds. | | replicaCount | int | `1` | Number of OpenShell gateway replicas. Values greater than 1 require server.externalDbSecret because the default SQLite backend is per pod. | | resources | object | `{}` | Gateway pod resource requests and limits. | -| sandboxRuntime.image.pullPolicy | string | `""` | Sandbox runtime image pull policy. Defaults to the gateway image pull policy when empty. | -| sandboxRuntime.image.repository | string | `"ghcr.io/nvidia/openshell/sandbox"` | Sandbox runtime image repository. Changing it uses the effective gateway image tag unless tag is also set. | -| sandboxRuntime.image.tag | string | `""` | Sandbox runtime image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | @@ -242,69 +270,19 @@ discovery endpoint or its TLS CA. | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | | securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. | -| server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. | -| server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace | bool | `false` | Deprecated compatibility field. Credential storage no longer supports user-authored namespace references. | -| server.credentialDrivers.kubernetesSecrets.enabled | bool | `false` | Enable the in-tree Kubernetes Secret credential driver. WARNING: The RBAC Role grants read/write access to ALL Secrets in the configured namespace. Use a dedicated namespace to limit blast radius. | -| server.credentialDrivers.kubernetesSecrets.namespace | string | `""` | Namespace where OpenShell-managed provider Secret objects are stored. Empty = Helm release namespace. A dedicated namespace is RECOMMENDED to isolate OpenShell-managed Secrets from other workloads. | -| server.credentialDrivers.kubernetesSecrets.rbac.create | bool | `true` | Create a Role/RoleBinding granting the gateway ServiceAccount read/write access to managed provider Secrets. | -| server.credentialDrivers.vault.address | string | `""` | Vault service base URL. Non-loopback endpoints must use HTTPS, for example https://vault.vault.svc.cluster.local:8200. | -| server.credentialDrivers.vault.authMethod | string | `"kubernetes"` | Authentication method. Use "kubernetes" in-cluster or "token_file" for local/dev validation. | -| server.credentialDrivers.vault.caConfigMapName | string | `""` | ConfigMap containing the private Vault CA certificate bundle in the ca.crt key. Leave empty to use platform trust roots. | -| server.credentialDrivers.vault.enabled | bool | `false` | Enable the in-tree Vault credential driver. | -| server.credentialDrivers.vault.kubernetesAuthMount | string | `"kubernetes"` | Vault Kubernetes auth mount. | -| server.credentialDrivers.vault.kvVersion | string | `"2"` | Default KV engine version. Use "1" or "2". | -| server.credentialDrivers.vault.mount | string | `"secret"` | Default KV mount name. | -| server.credentialDrivers.vault.role | string | `""` | Vault Kubernetes auth role when authMethod is kubernetes. | -| server.credentialDrivers.vault.serviceAccountTokenPath | string | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | ServiceAccount token path used for Kubernetes auth. | -| server.credentialDrivers.vault.timeoutSecs | string | `""` | HTTP request timeout in seconds. Empty = driver default. | -| server.credentialDrivers.vault.tokenPath | string | `""` | Mounted token file path when authMethod is token_file. | | server.credentialStorage.existingSecret | string | `""` | Name of a pre-existing Secret containing the key-encryption key. When set, the chart does NOT generate a new Secret; it references this one instead. The Secret must contain a key named "key-encryption-key" with a base64-encoded 32-byte value. Required for GitOps workflows that render manifests with `helm template` (where `lookup` is unavailable). | | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | -| server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | -| server.drivers.kubernetes.operatorNamespaceFile | string | `""` | Path to a JSON file containing an array of namespace names allowed in operator mode. Hot-reloaded on change. | -| server.drivers.kubernetes.operatorNamespaceLabel | string | `""` | K8s label selector for namespace discovery in operator mode. The driver watches namespaces matching this label. | -| server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | -| server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | -| server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | | server.externalDbSecret | string | `""` | Name of a pre-existing Opaque Secret containing a PostgreSQL connection URI (key: uri). When set, the gateway reads OPENSHELL_DB_URL from this Secret instead of using dbUrl. The Secret must contain a `uri` key, e.g. postgresql://user:pass@host:5432/dbname. | -| server.grpcEndpoint | string | `""` | gRPC endpoint sandboxes call back into the gateway. Leave empty to derive it from the chart fullname, release namespace, service port, and disableTls flag, for example https://openshell.openshell.svc.cluster.local:8080. Override only when sandboxes must reach the gateway via a different hostname (e.g. an external ingress or a host alias). | -| server.grpcRateLimit.requests | int | `0` | Maximum gRPC requests allowed per window. Must be positive (alongside windowSeconds) to enable rate limiting; 0 (default) disables it. | -| server.grpcRateLimit.windowSeconds | int | `0` | gRPC rate-limit window length in seconds. Must be positive (alongside requests) to enable rate limiting; 0 (default) disables it. | | server.hostGatewayIP | string | `""` | Host gateway IP for sandbox pod hostAliases. When set, sandbox pods get hostAliases entries mapping host.docker.internal and host.openshell.internal to this IP, allowing them to reach services running on the Docker host. Auto-detected by the cluster entrypoint script. | -| server.logLevel | string | `"info"` | Gateway log level. | -| server.name | string | `""` | Operator-facing gateway name. Defaults to the chart fullname so all replicas in one installation share an identity. Set explicitly when one telemetry collector receives spans from multiple namespaces or clusters. | -| server.oidc.adminRole | string | `""` | Role name for admin access. Leave empty (with userRole also empty) for authentication-only mode. Both must be set or both empty. | -| server.oidc.audience | string | `"openshell-cli"` | Expected audience claim for the API resource server. This should match the server's --oidc-audience, NOT the CLI client ID. | -| server.oidc.caConfigMapName | string | `""` | Name of a ConfigMap containing a CA certificate bundle (key: ca.crt) for verifying the OIDC issuer's TLS certificate. Required when the issuer uses a non-public CA (e.g. OpenShift ingress, private PKI). | -| server.oidc.dangerouslyAllowInsecureHttp | bool | `false` | Development only: permit cleartext OIDC requests to numeric loopback addresses. This never permits HTTP to hostnames or non-loopback addresses. | -| server.oidc.issuer | string | `""` | OIDC issuer URL (e.g. https://keycloak.example.com/realms/openshell). | -| server.oidc.jwksAllowedOrigins | list | `[]` | Additional trusted HTTPS origins allowed to serve JWKS. The issuer origin is always allowed. Entries must not include a path or query. | -| server.oidc.jwksTtl | int | `3600` | JWKS key cache TTL in seconds. Must be greater than zero. | -| server.oidc.rolesClaim | string | `""` | Dot-separated path to the roles array in the JWT claims. Keycloak: "realm_access.roles", Entra ID: "roles", Okta: "groups". | -| server.oidc.scopesClaim | string | `""` | Dot-separated path to the scopes array in the JWT claims. | -| server.oidc.userRole | string | `""` | Role name for standard user access. | -| server.otlp.endpoint | string | `""` | OTLP/gRPC collector endpoint, conventionally using port 4317. | -| server.otlp.serviceName | string | `""` | Gateway OpenTelemetry service name. Empty uses openshell-gateway. | -| server.policyValidationFailureMode | string | `"fail_closed"` | Posture when a candidate sandbox policy fails validation. `fail_closed` deactivates the previous policy; `retain_last_valid` keeps it active. | -| server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | -| server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | -| server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | -| server.sandboxImagePullPolicy | string | `nil` | Pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | -| server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | -| server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | -| server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | | server.sandboxJwt.secretDefaultMode | string | `""` | File mode for the mounted JWT signing key Secret. Default 0400 (owner-read only). Override to 0440 or 0444 if the container UID does not match the volume file owner. | | server.sandboxJwt.signingSecretName | string | `""` | Name of the Opaque Secret holding the signing key material. Empty falls back to the chart fullname with "-jwt-keys" appended. | -| server.sandboxJwt.ttlSecs | int | `3600` | Token TTL in seconds. Defaults to 3600 (1h). | | server.sandboxNamespace | string | `""` | Namespace where sandbox pods are created. Defaults to the Helm release namespace (.Release.Namespace) when left empty. | | server.telemetryEnabled | bool | `true` | Enable anonymous OpenShell telemetry from the gateway and the sandbox supervisors it launches. | | server.tls.certSecretName | string | `"openshell-server-tls"` | K8s secret (type kubernetes.io/tls) with tls.crt and tls.key for the server. | | server.tls.clientCaSecretName | string | `"openshell-server-client-ca"` | K8s secret with ca.crt for client certificate verification (mTLS). Only used when enableMtls is true. Set to "" to disable client certificate verification for HTTPS-only mode. | | server.tls.clientTlsSecretName | string | `"openshell-client-tls"` | K8s secret mounted into sandbox pods for mTLS to the server. | | server.tls.enableMtls | bool | `true` | Enable mTLS client certificate authentication. When false, the gateway runs HTTPS-only without requiring client certificates (use OIDC for auth instead). Must be false when using BackendTLSPolicy because ingress proxies cannot present client certificates to the backend. | -| server.workspaceDefaultStorageSize | string | `""` | Default storage size for the workspace PVC in sandbox pods. Uses Kubernetes quantity syntax (e.g. "2Gi", "10Gi", "500Mi"). Empty = built-in default (2Gi). | -| server.workspaceStorageClass | string | `""` | Kubernetes StorageClass for the workspace PVC in sandbox pods. Empty (default) = omit storageClassName, using the cluster's default StorageClass. Set this on clusters with no default StorageClass, otherwise the workspace PVC stays Pending and the sandbox never starts. | | service.healthPort | int | `8081` | Gateway health service port. | | service.metricsPort | int | `9090` | Gateway metrics service port. | | service.port | int | `8080` | Gateway gRPC/HTTP service port. | @@ -312,19 +290,7 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | -| supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | -| supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | -| supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. | -| supervisor.sandboxRuntime.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. | | tolerations | list | `[]` | Tolerations for the gateway pod. | -| upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | -| upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | -| upstreamProxy.authSecret.key | string | `""` | Secret key containing the proxy credential. | -| upstreamProxy.authSecret.name | string | `""` | Existing Secret in the sandbox namespace containing a user:pass value. | -| upstreamProxy.connectByHostname | bool | `false` | Last-resort option for hostname-filtering proxy ACLs. It lets the proxy resolve CONNECT targets. | -| upstreamProxy.noProxy | string | `""` | Comma-separated destinations that bypass only the corporate proxy. | -| upstreamProxy.url | string | `""` | HTTP proxy URL in http://host:port form. HTTPS-to-proxy is not supported. | | workload.allowMultiReplicaStatefulSet | bool | `false` | Allow replicaCount > 1 while rendering a StatefulSet. Prefer workload.kind=deployment for external database-backed multi-replica gateways; this override exists for operators who explicitly require StatefulSet identity or storage semantics. | | workload.kind | string | `"statefulset"` | Gateway workload controller kind. Use `statefulset` for the default SQLite database, or `deployment` when server.externalDbSecret points at an external database. | | workspaceResources.enabled | bool | `true` | Create the sandbox ServiceAccount, Role, RoleBinding, and NetworkPolicy from this chart. Disable for a gateway-only release. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 75e651e214..da6d2888ce 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -36,7 +36,7 @@ where Helm cannot discover cluster APIs. ```shell helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ``` ## Install on OpenShift @@ -44,15 +44,12 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ``` On OpenShift 4.22+, end-to-end TLS is supported via `BackendTLSPolicy`. See the @@ -110,7 +107,7 @@ Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ -n openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set workload.kind=deployment \ --set server.externalDbSecret=my-pg-credentials ``` @@ -128,16 +125,43 @@ database. The chart creates a retained Kubernetes Secret with the shared key-encryption key and injects that key into every gateway pod, so the same default works for single-replica and external database-backed HA deployments. -Use `kubernetes-secrets` or `vault` instead when credentials should live in a -cluster or external secret backend. Enabling one external credential driver -disables the default credential-storage key-encryption key Secret and env injection. +Use `gatewayConfig` to select `kubernetes-secrets` or `vault` when credentials +should live in a cluster or external secret backend. Selecting an external +credential driver disables the default credential-storage key-encryption-key +Secret and environment injection. The map is rendered directly as gateway TOML, +so it uses the gateway's snake_case field names: + +```yaml +gatewayConfig: + openshell.gateway: + credential_drivers: + - vault + openshell.credential_drivers.vault: + address: http://vault.vault.svc.cluster.local:8200 + mount: secret + kv_version: "2" + auth_method: kubernetes + role: openshell-gateway +``` + +> `gatewayConfig` must contain only non-secret values. Helm serializes unknown +> fields generically and cannot determine whether an arbitrary string, such as +> `api_token`, is confidential. Do not put passwords, tokens, private keys, +> database URLs, or other secret material in this map. Use Secret-backed +> environment variables, files, volumes, or gateway credential drivers instead. +> The chart rejects known unsafe forms such as `database_url`, inline URL +> credentials, and PEM private keys; it is not a general secret scanner. + +For the Kubernetes Secret driver, use +`openshell.credential_drivers.kubernetes-secrets.namespace` in the same map. +The chart derives any required RBAC from the selected driver; use a dedicated +namespace to limit access to OpenShell-managed Secrets. #### OpenShift Append these flags to any of the PostgreSQL commands above for OpenShift: ``` ---set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null ``` @@ -158,12 +182,16 @@ JWT signing Secret. ## SPIFFE/SPIRE provider token grants -Set `server.providerTokenGrants.spiffe.enabled=true` to let the gateway and -sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token grants. The -chart keeps supervisor-to-gateway authentication on gateway-minted sandbox JWTs, -mounts the SPIFFE CSI socket into the gateway pod, exports -`OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`, and passes the socket path to -the Kubernetes driver so sandbox pods can mount the same socket. +Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` +to let sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token +grants. The chart keeps supervisor-to-gateway authentication on gateway-minted +sandbox JWTs and passes the configured socket path to the Kubernetes driver. + +```yaml +gatewayConfig: + openshell.drivers.kubernetes: + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock +``` For local development, uncomment the SPIRE Helm releases in `skaffold.yaml` and add `ci/values-spire.yaml` to the OpenShell release values files. diff --git a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml index 70fdec8d7b..5b3092ae07 100644 --- a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml +++ b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml @@ -3,8 +3,11 @@ # The Kubernetes corporate-proxy e2e wrapper supplies the generated proxy URL # and creates `openshell-e2e-proxy-auth` before Helm installs the gateway. -upstreamProxy: - authSecret: - name: openshell-e2e-proxy-auth - key: proxy-auth - authAllowInsecure: true +gatewayConfig: + openshell.drivers.kubernetes: + # The e2e wrapper replaces this endpoint with its dynamically allocated + # port. Keep the overlay valid when rendered independently as well. + https_proxy: http://host.openshell.internal:8080 + proxy_auth_secret_name: openshell-e2e-proxy-auth + proxy_auth_secret_key: proxy-auth + proxy_auth_allow_insecure: true diff --git a/deploy/helm/openshell/ci/values-credential-driver-kubernetes-secrets.yaml b/deploy/helm/openshell/ci/values-credential-driver-kubernetes-secrets.yaml index 096ce46e29..440a7ff2ac 100644 --- a/deploy/helm/openshell/ci/values-credential-driver-kubernetes-secrets.yaml +++ b/deploy/helm/openshell/ci/values-credential-driver-kubernetes-secrets.yaml @@ -6,8 +6,9 @@ # Use with: # skaffold run -p credential-driver-kubernetes-secrets # -server: - credentialDrivers: - kubernetesSecrets: - enabled: true - namespace: openshell +gatewayConfig: + openshell.gateway: + credential_drivers: + - kubernetes-secrets + openshell.credential_drivers.kubernetes-secrets: + namespace: openshell diff --git a/deploy/helm/openshell/ci/values-credential-driver-vault.yaml b/deploy/helm/openshell/ci/values-credential-driver-vault.yaml index 5158d48d78..83a891dd2a 100644 --- a/deploy/helm/openshell/ci/values-credential-driver-vault.yaml +++ b/deploy/helm/openshell/ci/values-credential-driver-vault.yaml @@ -8,15 +8,17 @@ # # The profile assumes another process has already deployed a Vault-compatible # backend. Local e2e validation deploys OpenBao in the `openbao` namespace with -# TLS enabled, publishes its private CA in the `openbao-ca` ConfigMap, and -# creates an `openbao-0` DNS alias matching the OpenBao dev certificate. It also -# configures a Kubernetes auth role named `openshell-gateway` bound to the -# OpenShell gateway ServiceAccount in the `openshell` namespace. +# a Kubernetes auth role named `openshell-gateway` bound to the OpenShell +# gateway ServiceAccount in the `openshell` namespace. -server: - credentialDrivers: - vault: - enabled: true - address: https://openbao-0:8200 - caConfigMapName: openbao-ca - role: openshell-gateway +gatewayConfig: + openshell.gateway: + credential_drivers: + - vault + openshell.credential_drivers.vault: + address: https://openbao.openbao.svc.cluster.local:8200 + role: openshell-gateway + +credentialDrivers: + vault: + caConfigMapName: openbao-ca diff --git a/deploy/helm/openshell/ci/values-gateway-tls.yaml b/deploy/helm/openshell/ci/values-gateway-tls.yaml index a776760141..21bde40b85 100644 --- a/deploy/helm/openshell/ci/values-gateway-tls.yaml +++ b/deploy/helm/openshell/ci/values-gateway-tls.yaml @@ -28,6 +28,8 @@ grpcRoute: server: # Envoy terminates TLS at the edge; the gateway listens plaintext behind it. disableTls: true - oidc: + +gatewayConfig: + openshell.gateway.oidc: issuer: "https://keycloak.example.com/realms/openshell" audience: "openshell-cli" diff --git a/deploy/helm/openshell/ci/values-keycloak.yaml b/deploy/helm/openshell/ci/values-keycloak.yaml index ae1810f414..1f0efc0798 100644 --- a/deploy/helm/openshell/ci/values-keycloak.yaml +++ b/deploy/helm/openshell/ci/values-keycloak.yaml @@ -9,7 +9,7 @@ # Then layer this file on top of values.yaml when deploying: # helm upgrade --install openshell . \ # -f values.yaml -f ci/values-skaffold.yaml -f ci/values-keycloak.yaml \ -# --set supervisor.sandboxRuntime.networkPolicyEnforced=true +# --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' # # Or add this file to skaffold.yaml valuesFiles for iterative dev. # @@ -22,18 +22,22 @@ # CLI token acquisition: keep a port-forward running while using openshell login: # kubectl -n keycloak port-forward svc/keycloak 9090:80 -server: - oidc: +gatewayConfig: + openshell.gateway.oidc: # Must match KC_HOSTNAME set by keycloak:k8s:setup (in-cluster service hostname). issuer: "https://keycloak.keycloak.svc.cluster.local:443/realms/openshell" - caConfigMapName: "openshell-keycloak-ca" # Must match the client ID in the imported realm (openshell-cli). audience: "openshell-cli" # Short TTL for dev so JWKS key rotation is picked up quickly. # Use 3600 (default) in production. - jwksTtl: 60 + jwks_ttl_secs: 60 # Keycloak puts realm roles at realm_access.roles in the JWT. - rolesClaim: "realm_access.roles" + roles_claim: "realm_access.roles" # Leave both empty for authentication-only mode (any valid token is accepted). - adminRole: "openshell-admin" - userRole: "openshell-user" + admin_role: "openshell-admin" + user_role: "openshell-user" + +# The ConfigMap is a Kubernetes resource reference. Helm mounts it when the +# OIDC issuer above is configured and derives the runtime file path. +oidc: + caConfigMapName: "openshell-keycloak-ca" diff --git a/deploy/helm/openshell/ci/values-openshift-e2e.yaml b/deploy/helm/openshell/ci/values-openshift-e2e.yaml index d2aaf0fac8..8ac78d9bd9 100644 --- a/deploy/helm/openshell/ci/values-openshift-e2e.yaml +++ b/deploy/helm/openshell/ci/values-openshift-e2e.yaml @@ -25,20 +25,21 @@ # is true and mTLS is MANDATORY at the TLS handshake — a caller with only the # Route URL and no client certificate is rejected before any RPC. The passthrough # Route terminates TLS at the gateway pod, so this holds end-to-end. -# `allowUnauthenticatedUsers` only promotes the already cert-verified caller to a +# `allow_unauthenticated_users` only promotes the already cert-verified caller to a # dev principal at the app layer (mtls_auth is unsupported with the Kubernetes # driver). Both are required together; the client certificate is the access gate. image: pullPolicy: Always -supervisor: - image: - pullPolicy: Always - server: disableTls: false - auth: - allowUnauthenticatedUsers: true + +gatewayConfig: + openshell.gateway.auth: + allow_unauthenticated_users: true + openshell.drivers.kubernetes: + sandbox_runtime_image_pull_policy: always + supervisor_image_pull_policy: always openshiftRoute: enabled: true diff --git a/deploy/helm/openshell/ci/values-openshift-scc.yaml b/deploy/helm/openshell/ci/values-openshift-scc.yaml index 8f1a8d07d6..0ec3d2a40c 100644 --- a/deploy/helm/openshell/ci/values-openshift-scc.yaml +++ b/deploy/helm/openshell/ci/values-openshift-scc.yaml @@ -5,7 +5,7 @@ # fsGroup so that OpenShift's restricted-v2 SCC can inject the namespace- # assigned UID/GID range. Layer after values.yaml: # helm install openshell deploy/helm/openshell -f ci/values-openshift-scc.yaml \ -# --set supervisor.sandboxRuntime.networkPolicyEnforced=true +# --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' # # The e2e Kubernetes harness applies this automatically when it detects an # OpenShift cluster (route.openshift.io API present). diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 6f72265039..1c4d768f7a 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -2,19 +2,19 @@ # SPDX-License-Identifier: Apache-2.0 # Merge with values.yaml for Skaffold-driven local image builds (see skaffold.yaml). -server: - sandboxImagePullPolicy: if_not_present - otlp: +gatewayConfig: + openshell.drivers.kubernetes: + image_pull_policy: if_not_present + supervisor_image_pull_policy: if_not_present + sandbox_runtime: + # The local k3s cluster enables its NetworkPolicy controller for sandbox + # namespaces. + network_policy_enforced: true + openshell.gateway.otlp: endpoint: http://openshell-collector.observability.svc.cluster.local:4317 + openshell.gateway.auth: + allow_unauthenticated_users: true + +server: # Comment out to enforce mTLS (uses PKI secrets generated by pkiInitJob). disableTls: true - auth: - allowUnauthenticatedUsers: true - -supervisor: - image: - pullPolicy: if_not_present - # The local k3s cluster created by `mise run helm:k3s:create` enables its - # built-in NetworkPolicy controller for sandbox namespaces. - sandboxRuntime: - networkPolicyEnforced: true diff --git a/deploy/helm/openshell/ci/values-spire.yaml b/deploy/helm/openshell/ci/values-spire.yaml index 201520e817..f71cea7134 100644 --- a/deploy/helm/openshell/ci/values-spire.yaml +++ b/deploy/helm/openshell/ci/values-spire.yaml @@ -2,8 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # OpenShell overlay for local SPIRE-backed provider token grants. -server: - providerTokenGrants: - spiffe: - enabled: true - workloadApiSocketPath: /spiffe-workload-api/spire-agent.sock +gatewayConfig: + openshell.drivers.kubernetes: + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock diff --git a/deploy/helm/openshell/ci/values-workspace-managed.yaml b/deploy/helm/openshell/ci/values-workspace-managed.yaml index e9f88846c4..3c3ef72526 100644 --- a/deploy/helm/openshell/ci/values-workspace-managed.yaml +++ b/deploy/helm/openshell/ci/values-workspace-managed.yaml @@ -3,9 +3,8 @@ # # E2E overlay: deploy the gateway in managed workspace mode. # Sandbox namespaces are auto-created as openshell-{gateway_id}-{workspace}. -server: - sandboxImagePullSecrets: - - name: e2e-regcred - drivers: - kubernetes: - workspaceMode: "managed" +gatewayConfig: + openshell.drivers.kubernetes: + image_pull_secrets: + - e2e-regcred + workspace_mode: managed diff --git a/deploy/helm/openshell/ci/values-workspace-operator.yaml b/deploy/helm/openshell/ci/values-workspace-operator.yaml index 8d895e4e98..a543923683 100644 --- a/deploy/helm/openshell/ci/values-workspace-operator.yaml +++ b/deploy/helm/openshell/ci/values-workspace-operator.yaml @@ -3,8 +3,7 @@ # # E2E overlay: deploy the gateway in operator workspace mode. # Namespaces must be pre-provisioned and labeled before sandbox creation. -server: - drivers: - kubernetes: - workspaceMode: "operator" - operatorNamespaceLabel: "openshell.ai/e2e-operator-workspace=true" +gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: operator + operator_namespace_label: "openshell.ai/e2e-operator-workspace=true" diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index 63037b075b..aecc645fe2 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Local dev: builds gateway, sandbox, and supervisor images via tasks/scripts/docker-build-image.sh, +# Local dev: builds gateway + supervisor images via tasks/scripts/docker-build-image.sh, # which first stages Rust binaries natively on the host (using cargo / cargo-zigbuild # when cross-compiling) and then builds the image from the prebuilt binary. This # mirrors CI and is faster than compiling inside Docker on every rebuild because @@ -56,23 +56,6 @@ build: - deploy/docker/Dockerfile.supervisor - tasks/scripts/docker-build-image.sh - tasks/scripts/stage-prebuilt-binaries.sh - - image: openshell/sandbox - context: ../../.. - custom: - buildCommand: | - CONTAINER_ENGINE_TARGET=local-k8s-cluster \ - IMAGE_NAME="${IMAGE%:*}" \ - IMAGE_TAG="${IMAGE##*:}" \ - tasks/scripts/docker-build-image.sh sandbox - dependencies: - paths: - - Cargo.toml - - Cargo.lock - - crates/** - - proto/** - - deploy/docker/Dockerfile.sandbox - - tasks/scripts/docker-build-image.sh - - tasks/scripts/stage-prebuilt-binaries.sh deploy: helm: releases: @@ -141,10 +124,7 @@ deploy: setValueTemplates: image.repository: '{{.IMAGE_REPO_openshell_gateway}}' image.tag: '{{.IMAGE_TAG_openshell_gateway}}' - supervisor.image.repository: '{{.IMAGE_REPO_openshell_supervisor}}' - supervisor.image.tag: '{{.IMAGE_TAG_openshell_supervisor}}' - sandboxRuntime.image.repository: '{{.IMAGE_REPO_openshell_sandbox}}' - sandboxRuntime.image.tag: '{{.IMAGE_TAG_openshell_sandbox}}' + gatewayConfig.openshell\.drivers\.kubernetes.supervisor_image: '{{.IMAGE_REPO_openshell_supervisor}}:{{.IMAGE_TAG_openshell_supervisor}}' profiles: - name: credential-driver-kubernetes-secrets patches: diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 485fb0241a..53b16880ae 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -5,6 +5,16 @@ Gateway pod template shared by the StatefulSet and Deployment workload shapes. */}} {{- define "openshell.gatewayPodTemplate" -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $gatewayRuntimeConfig := get $gatewayConfig "openshell.gateway" | default dict -}} +{{- $oidcRuntimeConfig := get $gatewayConfig "openshell.gateway.oidc" | default dict -}} +{{- $kubernetesRuntimeConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $spiffeSocketPath := get $kubernetesRuntimeConfig "provider_spiffe_workload_api_socket_path" -}} +{{- $hasExternalCredentialDriver := or (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") -}} +{{- $vaultCredentialDriverEnabled := eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true" -}} +{{- $credentialDrivers := .Values.credentialDrivers | default dict -}} +{{- $vaultResources := get $credentialDrivers "vault" | default dict -}} +{{- $vaultCaConfigMapName := get $vaultResources "caConfigMapName" -}} metadata: annotations: # Roll the gateway workload when the rendered gateway TOML changes - the @@ -52,7 +62,7 @@ spec: - {{ .Values.server.dbUrl | quote }} {{- end }} env: - {{- if not (or .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.vault.enabled) }} + {{- if not $hasExternalCredentialDriver }} - name: {{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }} valueFrom: secretKeyRef: @@ -70,7 +80,7 @@ spec: # mounted at /etc/openshell/gateway.toml. Secret-bearing settings use # env vars that the TOML references by name. Some process-level # settings consumed by libraries outside gateway code also remain here. - {{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} # OIDC issuer custom-CA: rustls/reqwest read SSL_CERT_FILE for # outbound TLS verification. This is a process-level env var # consumed by the TLS stack itself, not by gateway code, so it @@ -80,9 +90,9 @@ spec: {{- end }} - name: OPENSHELL_TELEMETRY_ENABLED value: {{ .Values.server.telemetryEnabled | quote }} - {{- if .Values.server.providerTokenGrants.spiffe.enabled }} + {{- if $spiffeSocketPath }} - name: OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET - value: {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} + value: {{ $spiffeSocketPath | quote }} {{- end }} volumeMounts: {{- if eq (include "openshell.workloadKind" .) "statefulset" }} @@ -114,19 +124,19 @@ spec: readOnly: true {{- end }} {{- end }} - {{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} - name: oidc-ca mountPath: /etc/openshell-tls/oidc-ca readOnly: true {{- end }} - {{- if and .Values.server.credentialDrivers.vault.enabled .Values.server.credentialDrivers.vault.caConfigMapName }} + {{- if and $vaultCredentialDriverEnabled $vaultCaConfigMapName }} - name: vault-ca - mountPath: /etc/openshell-tls/vault-ca + mountPath: /etc/openshell-tls/vault readOnly: true {{- end }} - {{- if .Values.server.providerTokenGrants.spiffe.enabled }} + {{- if $spiffeSocketPath }} - name: spiffe-workload-api - mountPath: {{ dir .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} + mountPath: {{ dir $spiffeSocketPath | quote }} readOnly: true {{- end }} ports: @@ -196,20 +206,20 @@ spec: {{- end }} {{- end }} {{- end }} - {{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }} + {{- if and (get $oidcRuntimeConfig "issuer") .Values.oidc.caConfigMapName }} - name: oidc-ca configMap: - name: {{ .Values.server.oidc.caConfigMapName }} + name: {{ .Values.oidc.caConfigMapName }} {{- end }} - {{- if and .Values.server.credentialDrivers.vault.enabled .Values.server.credentialDrivers.vault.caConfigMapName }} + {{- if and $vaultCredentialDriverEnabled $vaultCaConfigMapName }} - name: vault-ca configMap: - name: {{ .Values.server.credentialDrivers.vault.caConfigMapName }} + name: {{ $vaultCaConfigMapName }} items: - key: ca.crt path: ca.crt {{- end }} - {{- if .Values.server.providerTokenGrants.spiffe.enabled }} + {{- if $spiffeSocketPath }} - name: spiffe-workload-api csi: driver: csi.spiffe.io diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index dd2d103f17..185ddcb6aa 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -94,30 +94,6 @@ so a released chart automatically pulls the matching image without extra overrid {{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} {{- end }} -{{/* Official sandbox runtime repository used by the gateway's built-in default. */}} -{{- define "openshell.defaultSandboxRuntimeRepository" -}} -ghcr.io/nvidia/openshell/sandbox -{{- end }} - -{{/* Whether Helm must propagate a sandbox runtime image override. */}} -{{- define "openshell.sandboxRuntimeImageOverrideEnabled" -}} -{{- $defaultRepository := include "openshell.defaultSandboxRuntimeRepository" . -}} -{{- $repository := .Values.sandboxRuntime.image.repository | default $defaultRepository -}} -{{- if or (ne $repository $defaultRepository) .Values.sandboxRuntime.image.tag -}}true{{- end -}} -{{- end }} - -{{/* Sandbox runtime image override. */}} -{{- define "openshell.sandboxRuntimeImage" -}} -{{- $repository := .Values.sandboxRuntime.image.repository | default (include "openshell.defaultSandboxRuntimeRepository" .) -}} -{{- $tag := .Values.sandboxRuntime.image.tag | default .Values.image.tag | default .Chart.AppVersion -}} -{{- printf "%s:%s" $repository $tag }} -{{- end }} - -{{/* Official supervisor repository used by the gateway's built-in default. */}} -{{- define "openshell.defaultSupervisorRepository" -}} -ghcr.io/nvidia/openshell/supervisor -{{- end }} - {{/* Whether the gateway listener should verify client certificates (mTLS). An explicit empty server.tls.clientCaSecretName disables client-CA wiring in @@ -133,26 +109,6 @@ true {{- end -}} {{- end -}} -{{/* -Whether Helm must propagate a supervisor image override into gateway.toml. -The chart's documented repository and empty tag are the gateway-owned default. -*/}} -{{- define "openshell.supervisorImageOverrideEnabled" -}} -{{- $defaultRepository := include "openshell.defaultSupervisorRepository" . -}} -{{- $repository := .Values.supervisor.image.repository | default $defaultRepository -}} -{{- if or (ne $repository $defaultRepository) .Values.supervisor.image.tag -}}true{{- end -}} -{{- end }} - -{{/* -Supervisor image override. A tag-only override uses the official repository; -a repository-only override uses the effective gateway image tag. -*/}} -{{- define "openshell.supervisorImage" -}} -{{- $repository := .Values.supervisor.image.repository | default (include "openshell.defaultSupervisorRepository" .) -}} -{{- $tag := .Values.supervisor.image.tag | default .Values.image.tag | default .Chart.AppVersion -}} -{{- printf "%s:%s" $repository $tag }} -{{- end }} - {{/* Namespaced Issuer (selfSigned) for cert-manager CA bootstrap. */}} @@ -173,7 +129,18 @@ Namespace where sandbox pods are created. An explicit Namespace where Kubernetes Secret-backed provider credentials live. */}} {{- define "openshell.credentialKubernetesSecretsNamespace" -}} -{{- .Values.server.credentialDrivers.kubernetesSecrets.namespace | default .Release.Namespace -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $config := get $gatewayConfig "openshell.credential_drivers.kubernetes-secrets" | default dict -}} +{{- get $config "namespace" | default .Release.Namespace -}} +{{- end }} + +{{/* Whether a credential driver is enabled in the generic gateway config. */}} +{{- define "openshell.credentialDriverEnabled" -}} +{{- $root := index . 0 -}} +{{- $driver := index . 1 -}} +{{- $gatewayConfig := $root.Values.gatewayConfig | default dict -}} +{{- $gateway := get $gatewayConfig "openshell.gateway" | default dict -}} +{{- if has $driver (get $gateway "credential_drivers" | default list) -}}true{{- end -}} {{- end }} {{/* @@ -210,20 +177,10 @@ Name of the Secret holding gateway-minted sandbox JWT signing material. {{- .Values.server.sandboxJwt.signingSecretName | default (printf "%s-jwt-keys" (include "openshell.fullname" .)) -}} {{- end }} -{{/* -gRPC endpoint sandbox pods use to call back into the gateway. An explicit -.Values.server.grpcEndpoint is used verbatim. Otherwise it is derived from -the in-cluster Service DNS, release namespace, service port, and disableTls -flag — so the default value works for any release name or namespace without -override. -*/}} +{{/* Derive the in-cluster callback endpoint from the chart-owned TLS state. */}} {{- define "openshell.grpcEndpoint" -}} -{{- if .Values.server.grpcEndpoint -}} -{{- .Values.server.grpcEndpoint -}} -{{- else -}} {{- $scheme := ternary "http" "https" (default false .Values.server.disableTls) -}} {{- printf "%s://%s.%s.svc.cluster.local:%d" $scheme (include "openshell.fullname" .) .Release.Namespace (int .Values.service.port) -}} -{{- end -}} {{- end }} {{/* @@ -283,6 +240,26 @@ never {{- end -}} {{- end }} +{{/* +Validate a non-empty, user-provided Kubernetes Secret name. Secret data never +passes through Helm values into gateway.toml; only this reference is rendered. +*/}} +{{- define "openshell.validateSecretReference" -}} +{{- $path := index . 0 -}} +{{- $name := index . 1 -}} +{{- if and (ne $name nil) (ne $name "") -}} +{{- if not (kindIs "string" $name) -}} +{{- fail (printf "%s must be a Kubernetes Secret name, got %s" $path (kindOf $name)) -}} +{{- end -}} +{{- if gt (len $name) 253 -}} +{{- fail (printf "%s must be no more than 253 characters" $path) -}} +{{- end -}} +{{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" $name) -}} +{{- fail (printf "%s must be a valid Kubernetes Secret name" $path) -}} +{{- end -}} +{{- end -}} +{{- end }} + {{/* Validate chart values that Helm would otherwise accept silently. */}} @@ -293,6 +270,9 @@ Validate chart values that Helm would otherwise accept silently. {{- if and (hasKey .Values "postgres") (kindIs "map" .Values.postgres) (hasKey .Values.postgres "enabled") -}} {{- fail "postgres.enabled was removed; the OpenShell chart no longer deploys PostgreSQL. Provision PostgreSQL separately and set server.externalDbSecret to a Secret containing a PostgreSQL URI." -}} {{- end -}} +{{- if and .Values.certManager.serverIssuerRef.name (not .Values.certManager.enabled) -}} +{{- fail "certManager.serverIssuerRef.name is set but certManager.enabled is false — the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." -}} +{{- end -}} {{- if not (or (eq $workloadKind "statefulset") (eq $workloadKind "deployment")) -}} {{- fail "workload.kind must be one of: statefulset, deployment." -}} {{- end -}} @@ -305,19 +285,15 @@ Validate chart values that Helm would otherwise accept silently. {{- if and (eq $workloadKind "statefulset") (gt $replicaCount 1) (not (get $workload "allowMultiReplicaStatefulSet" | default false)) -}} {{- fail "replicaCount > 1 with workload.kind=statefulset requires workload.allowMultiReplicaStatefulSet=true; use workload.kind=deployment for external database-backed multi-replica gateways." -}} {{- end -}} -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- include "openshell.validateSecretReference" (list "server.externalDbSecret" .Values.server.externalDbSecret) -}} +{{- include "openshell.validateSecretReference" (list "server.credentialStorage.existingSecret" .Values.server.credentialStorage.existingSecret) -}} +{{- include "openshell.validateSecretReference" (list "server.sandboxJwt.signingSecretName" .Values.server.sandboxJwt.signingSecretName) -}} +{{- include "openshell.validateSecretReference" (list "server.tls.certSecretName" .Values.server.tls.certSecretName) -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}} -{{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}} -{{- end -}} -{{- $credentialDrivers := list -}} -{{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} -{{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} -{{- end -}} -{{- if .Values.server.credentialDrivers.vault.enabled -}} -{{- $credentialDrivers = append $credentialDrivers "vault" -}} -{{- end -}} -{{- if gt (len $credentialDrivers) 1 -}} -{{- fail "only one external server.credentialDrivers backend can be enabled at a time." -}} +{{- fail "gatewayConfig.openshell.drivers.kubernetes.workspace_mode must be one of: shared, managed, operator." -}} {{- end -}} {{- if kindIs "invalid" .Values.server.tls.clientCaSecretName -}} {{- fail "server.tls.clientCaSecretName cannot be null; omit the key to use the chart default (openshell-server-client-ca), or set to \"\" to disable client certificate verification for HTTPS-only mode" -}} diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl new file mode 100644 index 0000000000..018ee474cc --- /dev/null +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -0,0 +1,194 @@ +{{/* +Render gatewayConfig as TOML. + +The chart deliberately treats the top-level keys as TOML table names. Nested +maps are TOML inline tables, and maps in arrays are inline-table array items. +This keeps the YAML-to-TOML boundary generic: adding a non-secret gateway +field must not require a Helm template change. +*/}} + +{{/* Render a TOML key. Bare keys keep ordinary output readable. */}} +{{- define "openshell.toml.key" -}} +{{- $key := . | toString -}} +{{- if regexMatch "^[A-Za-z0-9_-]+$" $key -}} +{{- $key -}} +{{- else -}} +{{- $key | quote -}} +{{- end -}} +{{- end -}} + +{{/* Render a scalar. Strings alone are Helm-templated. */}} +{{- define "openshell.toml.scalar" -}} +{{- $root := index . 0 -}} +{{- $value := index . 1 -}} +{{- if kindIs "string" $value -}} +{{- $rendered := tpl $value $root -}} +{{- if regexMatch "-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----" $rendered -}} +{{- fail "gatewayConfig must not contain an inline private key; provide it through a Secret-backed file mount" -}} +{{- end -}} +{{- if regexMatch "^[A-Za-z][A-Za-z0-9+.-]*://[^/@[:space:]]*@" $rendered -}} +{{- fail "gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume" -}} +{{- end -}} +{{- $rendered | quote -}} +{{- else if or + (kindIs "bool" $value) + (kindIs "int" $value) + (kindIs "int8" $value) + (kindIs "int16" $value) + (kindIs "int32" $value) + (kindIs "int64" $value) + (kindIs "uint" $value) + (kindIs "uint8" $value) + (kindIs "uint16" $value) + (kindIs "uint32" $value) + (kindIs "uint64" $value) + (kindIs "float32" $value) + (kindIs "float64" $value) -}} +{{- $value | toJson -}} +{{- else -}} +{{- fail (printf "gatewayConfig values must be strings, booleans, numbers, maps, or arrays; got %s" (kindOf $value)) -}} +{{- end -}} +{{- end -}} + +{{/* Render a TOML inline table, omitting YAML null values. */}} +{{- define "openshell.toml.inlineTable" -}} +{{- $root := index . 0 -}} +{{- $table := index . 1 -}} +{{- $entries := list -}} +{{- range $key := keys $table | sortAlpha -}} +{{- $value := get $table $key -}} +{{- if ne $value nil -}} +{{- if eq $key "database_url" -}} +{{- fail "gatewayConfig must not contain database_url; provide database credentials through the chart's Secret-backed OPENSHELL_DB_URL environment variable" -}} +{{- end -}} +{{- $entry := printf "%s = %s" (include "openshell.toml.key" $key) (include "openshell.toml.value" (list $root $value)) -}} +{{- $entries = append $entries $entry -}} +{{- end -}} +{{- end -}} +{{- printf "{ %s }" (join ", " $entries) -}} +{{- end -}} + +{{/* Render an array. Maps become TOML inline-table entries. */}} +{{- define "openshell.toml.array" -}} +{{- $root := index . 0 -}} +{{- $array := index . 1 -}} +{{- $entries := list -}} +{{- range $value := $array -}} +{{- if eq $value nil -}} +{{- fail "gatewayConfig arrays cannot contain null values" -}} +{{- end -}} +{{- $entries = append $entries (include "openshell.toml.value" (list $root $value)) -}} +{{- end -}} +{{- printf "[%s]" (join ", " $entries) -}} +{{- end -}} + +{{/* Render any supported YAML value as TOML. */}} +{{- define "openshell.toml.value" -}} +{{- $root := index . 0 -}} +{{- $value := index . 1 -}} +{{- if kindIs "map" $value -}} +{{- include "openshell.toml.inlineTable" (list $root $value) -}} +{{- else if kindIs "slice" $value -}} +{{- include "openshell.toml.array" (list $root $value) -}} +{{- else -}} +{{- include "openshell.toml.scalar" (list $root $value) -}} +{{- end -}} +{{- end -}} + +{{/* Render the top-level gatewayConfig map as deterministic TOML tables. */}} +{{- define "openshell.gatewayConfigToml" -}} +{{- $root := . -}} +{{- $config := deepCopy (.Values.gatewayConfig | default dict) -}} +{{/* External credential drivers own their storage. Do not configure the +chart-managed encrypted database store when any driver is selected: its KEK +environment variable is intentionally not mounted in that mode. */}} +{{- $configuredGateway := get $config "openshell.gateway" | default dict -}} +{{- $configuredCredentialDrivers := get $configuredGateway "credential_drivers" | default list -}} +{{- if gt (len $configuredCredentialDrivers) 0 -}} +{{- $_ := unset $config "openshell.gateway.credential_storage" -}} +{{- end -}} +{{/* Kubernetes packaging owns host aliases. Do not permit a second runtime +source to make sandbox callback hostnames disagree with the pod spec. */}} +{{- $kubernetes := get $config "openshell.drivers.kubernetes" | default dict -}} +{{- if .Values.server.hostGatewayIP -}} +{{- $_ := set $kubernetes "host_gateway_ip" .Values.server.hostGatewayIP -}} +{{- else -}} +{{- $_ := unset $kubernetes "host_gateway_ip" -}} +{{- end -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} + +{{/* A Vault CA is a Kubernetes resource reference, not a free-form runtime +path. Derive its mounted path only from the chart-owned ConfigMap reference. */}} +{{- $credentialDrivers := .Values.credentialDrivers | default dict -}} +{{- $vaultResources := get $credentialDrivers "vault" | default dict -}} +{{- if hasKey $config "openshell.credential_drivers.vault" -}} +{{- $vaultConfig := get $config "openshell.credential_drivers.vault" | default dict -}} +{{- $_ := unset $vaultConfig "ca_bundle" -}} +{{- if and (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") (get $vaultResources "caConfigMapName") -}} +{{- $_ := set $vaultConfig "ca_bundle" "/etc/openshell-tls/vault/ca.crt" -}} +{{- end -}} +{{- $_ := set $config "openshell.credential_drivers.vault" $vaultConfig -}} +{{- end -}} + +{{/* TLS resources, mounts, and their corresponding runtime fields have one +owner: server.*. Override any gatewayConfig copies before serializing TOML. */}} +{{- $gateway := get $config "openshell.gateway" | default dict -}} +{{- $_ := set $gateway "disable_tls" .Values.server.disableTls -}} +{{- $_ := set $config "openshell.gateway" $gateway -}} +{{- if .Values.server.disableTls -}} +{{- $_ := unset $config "openshell.gateway.tls" -}} +{{- $_ := unset $kubernetes "client_tls_secret_name" -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} +{{- else -}} +{{- if .Values.server.tls.clientTlsSecretName -}} +{{- $_ := set $kubernetes "client_tls_secret_name" .Values.server.tls.clientTlsSecretName -}} +{{- else -}} +{{- $_ := unset $kubernetes "client_tls_secret_name" -}} +{{- end -}} +{{- $_ := set $config "openshell.drivers.kubernetes" $kubernetes -}} +{{- $gatewayTls := get $config "openshell.gateway.tls" | default dict -}} +{{- $_ := set $gatewayTls "cert_path" "/etc/openshell-tls/server/tls.crt" -}} +{{- $_ := set $gatewayTls "key_path" "/etc/openshell-tls/server/tls.key" -}} +{{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" -}} +{{- $_ := set $gatewayTls "client_ca_path" "/etc/openshell-tls/client-ca/ca.crt" -}} +{{- else -}} +{{- $_ := unset $gatewayTls "client_ca_path" -}} +{{- end -}} +{{- if .Values.certManager.serverIssuerRef.name -}} +{{- $_ := set $gatewayTls "external_cert_path" "/etc/openshell-tls/server-external/tls.crt" -}} +{{- $_ := set $gatewayTls "external_key_path" "/etc/openshell-tls/server-external/tls.key" -}} +{{- $_ := set $gatewayTls "external_server_names" (deepCopy (.Values.certManager.serverDnsNames | default list)) -}} +{{- else -}} +{{- $_ := unset $gatewayTls "external_cert_path" -}} +{{- $_ := unset $gatewayTls "external_key_path" -}} +{{- $_ := unset $gatewayTls "external_server_names" -}} +{{- end -}} +{{- $_ := set $config "openshell.gateway.tls" $gatewayTls -}} +{{- end -}} +{{- range $tableName := keys $config | sortAlpha -}} +{{- $fields := get $config $tableName -}} +{{- if ne $fields nil -}} +{{- if not (kindIs "map" $fields) -}} +{{- fail (printf "gatewayConfig table %q must be a map, got %s" $tableName (kindOf $fields)) -}} +{{- end -}} +{{- $header := list -}} +{{- $segments := splitList "." $tableName -}} +{{- range $index, $segment := $segments -}} +{{- if eq $segment "" -}} +{{- fail (printf "gatewayConfig table %q contains an empty TOML key segment" $tableName) -}} +{{- end -}} +{{- $header = append $header (include "openshell.toml.key" $segment) -}} +{{- end -}} +{{ printf "[%s]\n" (join "." $header) }} +{{- range $fieldName := keys $fields | sortAlpha }} +{{- $value := get $fields $fieldName -}} +{{- if ne $value nil }} +{{- if eq $fieldName "database_url" -}} +{{- fail "gatewayConfig must not contain database_url; provide database credentials through the chart's Secret-backed OPENSHELL_DB_URL environment variable" -}} +{{- end -}} +{{ printf "%s = %s\n" (include "openshell.toml.key" $fieldName) (include "openshell.toml.value" (list $root $value)) }} +{{- end }} +{{- end }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/openshell/templates/backend-tls-policy.yaml b/deploy/helm/openshell/templates/backend-tls-policy.yaml index 8d7a4fd557..1c7e59ec3d 100644 --- a/deploy/helm/openshell/templates/backend-tls-policy.yaml +++ b/deploy/helm/openshell/templates/backend-tls-policy.yaml @@ -1,7 +1,7 @@ +{{- if .Values.grpcRoute.backendTLSPolicy.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if .Values.grpcRoute.backendTLSPolicy.enabled }} {{- if .Values.server.disableTls }} {{- fail "grpcRoute.backendTLSPolicy requires the gateway pod to serve TLS; set server.disableTls=false" }} {{- end }} diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index b2ce8d2a82..01d4595bd2 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -1,7 +1,7 @@ +{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} {{- $hookName := printf "%s-certgen" (include "openshell.fullname" .) }} {{- $ns := .Release.Namespace }} apiVersion: v1 diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index e48c27cdd2..cec8c9ed23 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -1,7 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" }} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" }} +{{- $managedSshIngress := get $gatewayConfig "openshell.drivers.kubernetes.managed_ssh_ingress" | default dict -}} +{{- $managedSshIngressEnabled := true -}} +{{- if hasKey $managedSshIngress "enabled" -}} +{{- $managedSshIngressEnabled = get $managedSshIngress "enabled" -}} +{{- end }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -83,9 +90,9 @@ rules: {{- $copiedSecretNames = append $copiedSecretNames .Values.server.tls.clientTlsSecretName }} {{- end }} {{- if eq $workspaceMode "managed" }} - {{- range .Values.server.sandboxImagePullSecrets }} - {{- if .name }} - {{- $copiedSecretNames = append $copiedSecretNames .name }} + {{- range (get $kubernetesConfig "image_pull_secrets" | default list) }} + {{- if . }} + {{- $copiedSecretNames = append $copiedSecretNames . }} {{- end }} {{- end }} {{- end }} @@ -130,7 +137,7 @@ rules: verbs: - create {{- end }} - {{- if and (ne $workspaceMode "shared") .Values.server.credentialDrivers.kubernetesSecrets.enabled }} + {{- if and (ne $workspaceMode "shared") (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") }} # The kubernetes-secrets credential driver uses dynamic hashed names in # workspace namespaces, so Kubernetes RBAC cannot restrict resourceNames. - apiGroups: @@ -152,7 +159,7 @@ rules: verbs: - create - get - {{- if .Values.networkPolicy.enabled }} + {{- if $managedSshIngressEnabled }} # Apply gateway-only SSH ingress isolation in managed namespaces. - apiGroups: - networking.k8s.io diff --git a/deploy/helm/openshell/templates/credential-secrets-role.yaml b/deploy/helm/openshell/templates/credential-secrets-role.yaml index f6187c9acb..1fcb97fd90 100644 --- a/deploy/helm/openshell/templates/credential-secrets-role.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-role.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} +{{- if eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true" }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 @@ -13,7 +13,7 @@ metadata: # runtime. Kubernetes RBAC does not support label-based or prefix-based # filtering for resourceNames. To limit blast radius, deploy the gateway # with a dedicated namespace for credential Secrets -# (server.credentialDrivers.kubernetesSecrets.namespace). +# (openshell.credential_drivers.kubernetes-secrets.namespace). rules: - apiGroups: - "" diff --git a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml index 3a9ee0bddc..0df76a9ddc 100644 --- a/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml +++ b/deploy/helm/openshell/templates/credential-secrets-rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.kubernetesSecrets.rbac.create }} +{{- if eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true" }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/helm/openshell/templates/credential-storage-key-encryption-key-secret.yaml b/deploy/helm/openshell/templates/credential-storage-key-encryption-key-secret.yaml index 1e53d84bfb..f57ef56307 100644 --- a/deploy/helm/openshell/templates/credential-storage-key-encryption-key-secret.yaml +++ b/deploy/helm/openshell/templates/credential-storage-key-encryption-key-secret.yaml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if not (or .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.vault.enabled) }} +{{- if not (or (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true")) }} {{- if not .Values.server.credentialStorage.existingSecret }} {{- $secretName := include "openshell.credentialStorageKeyEncryptionKeySecretName" . -}} {{- $secretKey := include "openshell.credentialStorageKeyEncryptionKeySecretKey" . -}} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 0652db4b07..bd7fefdc07 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -7,22 +7,10 @@ The gateway reads `/etc/openshell/gateway.toml` (mounted from this ConfigMap) at startup. CLI flags and OPENSHELL_* env vars on the gateway workload container still override anything in this file. -One value is intentionally NOT rendered here: - - server.dbUrl → passed via OPENSHELL_DB_URL env var (from Secret) - when server.externalDbSecret is set, otherwise - --db-url arg for SQLite +gatewayConfig is the complete non-secret application configuration boundary. +Database URLs and other credentials remain injected through Secret-backed +environment variables, files, or volumes outside this ConfigMap. */}} -{{- $credentialDrivers := list -}} -{{- $otlp := .Values.server.otlp | default dict -}} -{{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} -{{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} -{{- end -}} -{{- if .Values.server.credentialDrivers.vault.enabled -}} -{{- $credentialDrivers = append $credentialDrivers "vault" -}} -{{- end -}} -{{- if and .Values.certManager.serverIssuerRef.name (not .Values.certManager.enabled) }} -{{- fail "certManager.serverIssuerRef.name is set but certManager.enabled is false \u2014 the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." }} -{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -31,237 +19,4 @@ metadata: {{- include "openshell.labels" . | nindent 4 }} data: gateway.toml: | - [openshell] - version = 2 - - [openshell.gateway] - name = {{ .Values.server.name | default (include "openshell.fullname" .) | quote }} - bind_address = "0.0.0.0:{{ .Values.service.port }}" - {{- if .Values.service.healthPort }} - health_bind_address = "0.0.0.0:{{ .Values.service.healthPort }}" - {{- end }} - {{- if .Values.service.metricsPort }} - metrics_bind_address = "0.0.0.0:{{ .Values.service.metricsPort }}" - {{- end }} - log_level = {{ .Values.server.logLevel | quote }} - compute_driver = "kubernetes" - {{- if $credentialDrivers }} - credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] - {{- end }} - {{- $policyValidationFailureMode := .Values.server.policyValidationFailureMode }} - {{- if not (has $policyValidationFailureMode (list "fail_closed" "retain_last_valid")) }} - {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} - {{- end }} - policy_validation_failure_mode = {{ $policyValidationFailureMode | quote }} - {{- if .Values.server.disableTls }} - disable_tls = true - {{- end }} - enable_loopback_service_http = {{ .Values.server.enableLoopbackServiceHttp }} - {{- $sans := list -}} - {{- if and .Values.certManager.enabled .Values.certManager.serverDnsNames }} - {{- $sans = .Values.certManager.serverDnsNames }} - {{- else if and .Values.pkiInitJob.enabled .Values.pkiInitJob.serverDnsNames }} - {{- $sans = .Values.pkiInitJob.serverDnsNames }} - {{- end }} - {{- if $sans }} - server_sans = [{{- range $i, $san := $sans }}{{ if $i }}, {{ end }}{{ $san | quote }}{{- end }}] - {{- end }} - {{- $rlRequests := int .Values.server.grpcRateLimit.requests }} - {{- $rlWindowSeconds := int .Values.server.grpcRateLimit.windowSeconds }} - {{- if or (lt $rlRequests 0) (lt $rlWindowSeconds 0) }} - {{- fail "server.grpcRateLimit.requests and server.grpcRateLimit.windowSeconds must not be negative; they map to unsigned gateway settings" }} - {{- end }} - {{- if and (gt $rlRequests 0) (gt $rlWindowSeconds 0) }} - grpc_rate_limit_requests = {{ $rlRequests }} - grpc_rate_limit_window_seconds = {{ $rlWindowSeconds }} - {{- else if or (gt $rlRequests 0) (gt $rlWindowSeconds 0) }} - {{- fail "server.grpcRateLimit requires both requests and windowSeconds to be positive to enable rate limiting, or both 0/unset to disable it" }} - {{- end }} - - {{- if $otlp.endpoint }} - - [openshell.gateway.otlp] - endpoint = {{ $otlp.endpoint | quote }} - {{- if $otlp.serviceName }} - service_name = {{ $otlp.serviceName | quote }} - {{- end }} - {{- end }} - - {{- if not .Values.server.disableTls }} - - [openshell.gateway.tls] - cert_path = "/etc/openshell-tls/server/tls.crt" - key_path = "/etc/openshell-tls/server/tls.key" - {{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }} - client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" - {{- end }} - {{- if .Values.certManager.serverIssuerRef.name }} - external_cert_path = "/etc/openshell-tls/server-external/tls.crt" - external_key_path = "/etc/openshell-tls/server-external/tls.key" - external_server_names = [{{- range $i, $name := .Values.certManager.serverDnsNames }}{{ if $i }}, {{ end }}{{ $name | quote }}{{- end }}] - {{- end }} - {{- end }} - - {{- if .Values.server.auth.allowUnauthenticatedUsers }} - - [openshell.gateway.auth] - allow_unauthenticated_users = true - {{- end }} - - [openshell.gateway.gateway_jwt] - signing_key_path = "/etc/openshell-jwt/signing.pem" - public_key_path = "/etc/openshell-jwt/public.pem" - kid_path = "/etc/openshell-jwt/kid" - gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} - ttl_secs = {{ .Values.server.sandboxJwt.ttlSecs | default 3600 }} - - {{- if .Values.server.oidc.issuer }} - - [openshell.gateway.oidc] - issuer = {{ .Values.server.oidc.issuer | quote }} - dangerously_allow_insecure_http = {{ .Values.server.oidc.dangerouslyAllowInsecureHttp }} - jwks_allowed_origins = {{ .Values.server.oidc.jwksAllowedOrigins | toJson }} - audience = {{ .Values.server.oidc.audience | quote }} - jwks_ttl_secs = {{ .Values.server.oidc.jwksTtl }} - {{- if .Values.server.oidc.rolesClaim }} - roles_claim = {{ .Values.server.oidc.rolesClaim | quote }} - {{- end }} - {{- if .Values.server.oidc.adminRole }} - admin_role = {{ .Values.server.oidc.adminRole | quote }} - {{- end }} - {{- if .Values.server.oidc.userRole }} - user_role = {{ .Values.server.oidc.userRole | quote }} - {{- end }} - {{- if .Values.server.oidc.scopesClaim }} - scopes_claim = {{ .Values.server.oidc.scopesClaim | quote }} - {{- end }} - {{- end }} - - [openshell.drivers.kubernetes] - namespace = {{ include "openshell.sandboxNamespace" . | quote }} - default_image = {{ .Values.server.sandboxImage | quote }} - {{- if include "openshell.sandboxRuntimeImageOverrideEnabled" . }} - sandbox_runtime_image = {{ include "openshell.sandboxRuntimeImage" . | quote }} - {{- end }} - {{- if include "openshell.supervisorImageOverrideEnabled" . }} - supervisor_image = {{ include "openshell.supervisorImage" . | quote }} - {{- end }} - {{- if .Values.server.hostGatewayIP }} - host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} - {{- end }} - {{- if not .Values.server.disableTls }} - client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} - {{- end }} - workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} - gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} - grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} - service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} - {{- if .Values.server.enableUserNamespaces }} - enable_user_namespaces = true - {{- end }} - {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} - operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} - {{- end }} - {{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }} - operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }} - {{- end }} - sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} - {{- if .Values.upstreamProxy.url }} - https_proxy = {{ .Values.upstreamProxy.url | quote }} - {{- end }} - {{- if .Values.upstreamProxy.noProxy }} - no_proxy = {{ .Values.upstreamProxy.noProxy | quote }} - {{- end }} - {{- if .Values.upstreamProxy.authSecret.name }} - proxy_auth_secret_name = {{ .Values.upstreamProxy.authSecret.name | quote }} - {{- end }} - {{- if .Values.upstreamProxy.authSecret.key }} - proxy_auth_secret_key = {{ .Values.upstreamProxy.authSecret.key | quote }} - {{- end }} - {{- if and .Values.upstreamProxy.authSecret.name .Values.upstreamProxy.authSecret.key }} - proxy_auth_allow_insecure = {{ .Values.upstreamProxy.authAllowInsecure }} - {{- end }} - {{- if .Values.upstreamProxy.connectByHostname }} - proxy_connect_by_hostname = true - {{- end }} - {{- if .Values.server.providerTokenGrants.spiffe.enabled }} - provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} - {{- end }} - {{- if .Values.server.sandboxImagePullPolicy }} - image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.server.sandboxImagePullPolicy | quote }} - {{- end }} - {{- $sandboxImagePullSecretNames := list -}} - {{- range .Values.server.sandboxImagePullSecrets }} - {{- if .name }} - {{- $sandboxImagePullSecretNames = append $sandboxImagePullSecretNames .name }} - {{- end }} - {{- end }} - {{- if $sandboxImagePullSecretNames }} - image_pull_secrets = [{{- range $i, $name := $sandboxImagePullSecretNames }}{{ if $i }}, {{ end }}{{ $name | quote }}{{- end }}] - {{- end }} - {{- if .Values.server.workspaceDefaultStorageSize }} - workspace_default_storage_size = {{ .Values.server.workspaceDefaultStorageSize | quote }} - {{- end }} - {{- if .Values.server.workspaceStorageClass }} - workspace_storage_class = {{ .Values.server.workspaceStorageClass | quote }} - {{- end }} - {{- if .Values.server.defaultRuntimeClassName }} - default_runtime_class_name = {{ .Values.server.defaultRuntimeClassName | quote }} - {{- end }} - {{- if .Values.supervisor.image.pullPolicy }} - supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.supervisor.image.pullPolicy | quote }} - {{- end }} - {{- if .Values.sandboxRuntime.image.pullPolicy }} - sandbox_runtime_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.sandboxRuntime.image.pullPolicy | quote }} - {{- end }} - - [openshell.drivers.kubernetes.managed_ssh_ingress] - enabled = {{ .Values.networkPolicy.enabled }} - gateway_namespace = {{ .Release.Namespace | quote }} - gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } - - [openshell.drivers.kubernetes.sandbox_runtime] - network_policy_enforced = {{ .Values.supervisor.sandboxRuntime.networkPolicyEnforced }} - boundary_port = {{ .Values.supervisor.sandboxRuntime.boundaryPort | default 5500 }} - - {{- if not $credentialDrivers }} - - [openshell.gateway.credential_storage] - key_encryption_key_env = {{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . | quote }} - {{- end }} - - {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled }} - - [openshell.credential_drivers.kubernetes-secrets] - namespace = {{ include "openshell.credentialKubernetesSecretsNamespace" . | quote }} - allow_reference_namespace = {{ .Values.server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace }} - workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} - gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} - {{- end }} - - {{- if .Values.server.credentialDrivers.vault.enabled }} - - [openshell.credential_drivers.vault] - address = {{ .Values.server.credentialDrivers.vault.address | quote }} - {{- if .Values.server.credentialDrivers.vault.caConfigMapName }} - ca_bundle = "/etc/openshell-tls/vault-ca/ca.crt" - {{- end }} - mount = {{ .Values.server.credentialDrivers.vault.mount | quote }} - kv_version = {{ .Values.server.credentialDrivers.vault.kvVersion | quote }} - auth_method = {{ .Values.server.credentialDrivers.vault.authMethod | quote }} - {{- if .Values.server.credentialDrivers.vault.role }} - role = {{ .Values.server.credentialDrivers.vault.role | quote }} - {{- end }} - {{- if .Values.server.credentialDrivers.vault.kubernetesAuthMount }} - kubernetes_auth_mount = {{ .Values.server.credentialDrivers.vault.kubernetesAuthMount | quote }} - {{- end }} - {{- if .Values.server.credentialDrivers.vault.serviceAccountTokenPath }} - service_account_token_path = {{ .Values.server.credentialDrivers.vault.serviceAccountTokenPath | quote }} - {{- end }} - {{- if .Values.server.credentialDrivers.vault.tokenPath }} - token_path = {{ .Values.server.credentialDrivers.vault.tokenPath | quote }} - {{- end }} - {{- if .Values.server.credentialDrivers.vault.timeoutSecs }} - timeout_secs = {{ .Values.server.credentialDrivers.vault.timeoutSecs }} - {{- end }} - {{- end }} +{{ include "openshell.gatewayConfigToml" . | nindent 4 }} diff --git a/deploy/helm/openshell/templates/network-policy-ack.yaml b/deploy/helm/openshell/templates/network-policy-ack.yaml index 780d4314c9..d3da7523db 100644 --- a/deploy/helm/openshell/templates/network-policy-ack.yaml +++ b/deploy/helm/openshell/templates/network-policy-ack.yaml @@ -1,5 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if not .Values.supervisor.sandboxRuntime.networkPolicyEnforced }} -{{- fail "supervisor.sandboxRuntime.networkPolicyEnforced must be true after you verify that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace" }} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $sandboxRuntime := get $kubernetesConfig "sandbox_runtime" | default dict -}} +{{- if not (get $sandboxRuntime "network_policy_enforced") }} +{{- fail "gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced must be true after you verify that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace" }} {{- end }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index c781b46e3d..468d0fbb44 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,4 +1,6 @@ -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if and (eq $workspaceMode "shared") (include "openshell.workspaceResourcesEnabled" .) }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index 32f11644bf..6c1d57de90 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,4 +1,6 @@ -{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}} +{{- $gatewayConfig := .Values.gatewayConfig | default dict -}} +{{- $kubernetesConfig := get $gatewayConfig "openshell.drivers.kubernetes" | default dict -}} +{{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- if and (eq $workspaceMode "shared") (include "openshell.workspaceResourcesEnabled" .) }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 diff --git a/deploy/helm/openshell/templates/route.yaml b/deploy/helm/openshell/templates/route.yaml index 459a0ac462..018e32085b 100644 --- a/deploy/helm/openshell/templates/route.yaml +++ b/deploy/helm/openshell/templates/route.yaml @@ -1,7 +1,7 @@ +{{- if .Values.openshiftRoute.enabled }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if .Values.openshiftRoute.enabled }} {{- if .Values.server.disableTls }} {{- fail "openshiftRoute.enabled=true requires TLS (server.disableTls must be false) \u2014 a passthrough Route forwards encrypted traffic by SNI, so the gateway must terminate its own TLS." }} {{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 9639232c32..a824a611af 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -15,9 +15,13 @@ tests: path: metadata.name value: openshell-node-reader-my-namespace - - it: grants managed namespace NetworkPolicy apply permissions + - it: grants managed namespace NetworkPolicy apply permissions when managed SSH ingress is enabled set: - server.drivers.kubernetes.workspaceMode: managed + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed + openshell.drivers.kubernetes.managed_ssh_ingress: + enabled: true asserts: - contains: path: rules @@ -28,8 +32,11 @@ tests: - it: preserves sandbox-runtime fence permissions when gateway isolation is disabled set: - server.drivers.kubernetes.workspaceMode: managed - networkPolicy.enabled: false + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed + openshell.drivers.kubernetes.managed_ssh_ingress: + enabled: false asserts: - contains: path: rules @@ -44,10 +51,29 @@ tests: resources: ["networkpolicies"] verbs: ["get", "create", "patch", "update"] + - it: grants managed NetworkPolicy permissions independently of the shared-namespace policy toggle + set: + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed + openshell.drivers.kubernetes.managed_ssh_ingress: + enabled: true + networkPolicy.enabled: false + asserts: + - contains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create", "patch", "update"] + - it: grants broad secret access when credential driver is enabled (operator) set: - server.drivers.kubernetes.workspaceMode: operator - server.credentialDrivers.kubernetesSecrets.enabled: true + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: operator + openshell.gateway: + credential_drivers: [kubernetes-secrets] asserts: - contains: path: rules @@ -58,7 +84,9 @@ tests: - it: restricts operator TLS sync to the configured secret set: - server.drivers.kubernetes.workspaceMode: operator + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: operator server.tls.clientTlsSecretName: custom-client-tls asserts: - contains: @@ -71,7 +99,9 @@ tests: - it: grants operator bootstrap secret lifecycle permissions when TLS and credential storage are disabled set: - server.drivers.kubernetes.workspaceMode: operator + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: operator server.disableTls: true asserts: - contains: @@ -89,11 +119,11 @@ tests: - it: restricts managed copies to TLS and configured image-pull secrets set: - server.drivers.kubernetes.workspaceMode: managed + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed + image_pull_secrets: [registry-one, registry-two] server.tls.clientTlsSecretName: custom-client-tls - server.sandboxImagePullSecrets: - - name: registry-one - - name: registry-two asserts: - contains: path: rules @@ -111,10 +141,11 @@ tests: - it: restricts managed copies to image-pull secrets when TLS is disabled set: - server.drivers.kubernetes.workspaceMode: managed + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed + image_pull_secrets: [registry-one] server.disableTls: true - server.sandboxImagePullSecrets: - - name: registry-one asserts: - contains: path: rules @@ -132,7 +163,9 @@ tests: - it: omits secrets rule entirely in shared mode set: - server.drivers.kubernetes.workspaceMode: shared + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: shared asserts: - notContains: path: rules @@ -143,7 +176,9 @@ tests: - it: grants managed sandbox-runtime companion permissions set: - server.drivers.kubernetes.workspaceMode: managed + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed asserts: - contains: path: rules @@ -166,7 +201,9 @@ tests: - it: grants operator sandbox-runtime companion permissions set: - server.drivers.kubernetes.workspaceMode: operator + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: operator asserts: - contains: path: rules diff --git a/deploy/helm/openshell/tests/credential_drivers_test.yaml b/deploy/helm/openshell/tests/credential_drivers_test.yaml index df6de36479..8400e843f9 100644 --- a/deploy/helm/openshell/tests/credential_drivers_test.yaml +++ b/deploy/helm/openshell/tests/credential_drivers_test.yaml @@ -59,15 +59,19 @@ tests: - it: renders Kubernetes Secrets credential driver config template: templates/gateway-config.yaml set: - server.credentialDrivers.kubernetesSecrets.enabled: true - server.credentialDrivers.kubernetesSecrets.namespace: provider-secrets + gatewayConfig: + openshell.gateway: + credential_drivers: [kubernetes-secrets] + openshell.credential_drivers.kubernetes-secrets: + namespace: provider-secrets + allow_reference_namespace: false asserts: - matchRegex: path: data["gateway.toml"] pattern: 'credential_drivers\s*=\s*\["kubernetes-secrets"\]' - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.credential_drivers\.kubernetes-secrets\].*?namespace\s*=\s*"provider-secrets".*?allow_reference_namespace\s*=\s*false' + pattern: '(?ms)\[openshell\.credential_drivers\.kubernetes-secrets\].*?allow_reference_namespace\s*=\s*false.*?namespace\s*=\s*"provider-secrets"' - notMatchRegex: path: data["gateway.toml"] pattern: 'transport\s*=\s*"in_tree"' @@ -78,52 +82,55 @@ tests: - it: renders Vault credential driver config template: templates/gateway-config.yaml set: - server.credentialDrivers.vault.enabled: true - server.credentialDrivers.vault.address: https://vault.vault.svc.cluster.local:8200 - server.credentialDrivers.vault.caConfigMapName: vault-ca - server.credentialDrivers.vault.role: openshell-gateway + gatewayConfig: + openshell.gateway: + credential_drivers: [vault] + openshell.credential_drivers.vault: + address: https://vault.vault.svc.cluster.local:8200 + ca_bundle: /operator-supplied/path/that-must-not-be-used.pem + auth_method: kubernetes + role: openshell-gateway + credentialDrivers.vault.caConfigMapName: vault-private-ca asserts: - matchRegex: path: data["gateway.toml"] pattern: 'credential_drivers\s*=\s*\["vault"\]' - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.credential_drivers\.vault\].*?address\s*=\s*"https://vault\.vault\.svc\.cluster\.local:8200".*?ca_bundle\s*=\s*"/etc/openshell-tls/vault-ca/ca\.crt".*?auth_method\s*=\s*"kubernetes".*?role\s*=\s*"openshell-gateway"' + pattern: '(?ms)\[openshell\.credential_drivers\.vault\].*?address\s*=\s*"https://vault\.vault\.svc\.cluster\.local:8200".*?auth_method\s*=\s*"kubernetes".*?ca_bundle\s*=\s*"/etc/openshell-tls/vault/ca\.crt".*?role\s*=\s*"openshell-gateway"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '/operator-supplied/path/that-must-not-be-used\.pem' - notMatchRegex: path: data["gateway.toml"] pattern: 'transport\s*=\s*"in_tree"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '\[openshell\.gateway\.credential_storage\]' - - it: rejects multiple enabled credential drivers - template: templates/statefulset.yaml - set: - server.credentialDrivers.kubernetesSecrets.enabled: true - server.credentialDrivers.vault.enabled: true - server.credentialDrivers.vault.address: https://vault.vault.svc.cluster.local:8200 - server.credentialDrivers.vault.role: openshell-gateway - asserts: - - failedTemplate: - errorPattern: "only one external server.credentialDrivers backend can be enabled at a time" - - - it: mounts the Vault CA ConfigMap into the gateway + - it: mounts the chart-owned Vault CA only when the Vault driver is selected template: templates/statefulset.yaml set: - server.credentialDrivers.vault.enabled: true - server.credentialDrivers.vault.address: https://vault.vault.svc.cluster.local:8200 - server.credentialDrivers.vault.caConfigMapName: vault-ca - server.credentialDrivers.vault.role: openshell-gateway + gatewayConfig: + openshell.gateway: + credential_drivers: [vault] + openshell.credential_drivers.vault: + address: https://vault.vault.svc.cluster.local:8200 + role: openshell-gateway + credentialDrivers.vault.caConfigMapName: vault-private-ca asserts: - contains: path: spec.template.spec.containers[0].volumeMounts content: name: vault-ca - mountPath: /etc/openshell-tls/vault-ca + mountPath: /etc/openshell-tls/vault readOnly: true - contains: path: spec.template.spec.volumes content: name: vault-ca configMap: - name: vault-ca + name: vault-private-ca items: - key: ca.crt path: ca.crt @@ -131,8 +138,11 @@ tests: - it: creates namespaced Kubernetes Secret manager RBAC template: templates/credential-secrets-role.yaml set: - server.credentialDrivers.kubernetesSecrets.enabled: true - server.credentialDrivers.kubernetesSecrets.namespace: provider-secrets + gatewayConfig: + openshell.gateway: + credential_drivers: [kubernetes-secrets] + openshell.credential_drivers.kubernetes-secrets: + namespace: provider-secrets asserts: - equal: path: metadata.namespace @@ -156,8 +166,11 @@ tests: - it: binds Kubernetes Secret manager RBAC to the gateway ServiceAccount template: templates/credential-secrets-rolebinding.yaml set: - server.credentialDrivers.kubernetesSecrets.enabled: true - server.credentialDrivers.kubernetesSecrets.namespace: provider-secrets + gatewayConfig: + openshell.gateway: + credential_drivers: [kubernetes-secrets] + openshell.credential_drivers.kubernetes-secrets: + namespace: provider-secrets asserts: - equal: path: metadata.namespace diff --git a/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml b/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml new file mode 100644 index 0000000000..7465a3327f --- /dev/null +++ b/deploy/helm/openshell/tests/fixtures/parser-validation-values.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Inputs used by deploy/helm/test-gateway-config-parser.sh. The driver table is +# intentionally opaque to the gateway loader, which makes it a safe way to +# exercise every TOML shape supported by the chart serializer. +gatewayConfig: + openshell.drivers.parser-validation: + rendered_name: '{{ .Release.Namespace }}/{{ .Release.Name }} "quoted" \\ path' + boolean_value: true + integer_value: 42 + float_value: 1.5 + empty_value: "" + scalar_array: + - first + - second + inline_map: + alpha: false + zebra: 2 + map_array: + - name: first + enabled: true + - name: second + enabled: false + escaped_string: 'quotes " and backslash \\ and a newline + are preserved' + omitted_value: null diff --git a/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml new file mode 100644 index 0000000000..776b5f0075 --- /dev/null +++ b/deploy/helm/openshell/tests/gateway_config_serializer_test.yaml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: gatewayConfig TOML serializer +templates: + - templates/gateway-config.yaml +release: + name: serializer-test + namespace: serializer-namespace + +tests: + - it: renders all supported YAML shapes with deterministic TOML output + set: + gatewayConfig: + example.config: + string_value: plain text + boolean_value: true + integer_value: 42 + float_value: 1.5 + scalar_array: + - first + - second + nested_map: + zebra: 2 + alpha: false + map_array: + - name: first + enabled: true + - name: second + enabled: false + omitted_value: null + empty_value: "" + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)^\[example\.config\]\n.*?boolean_value = true\n.*?empty_value = ""\n.*?float_value = 1\.5\n.*?integer_value = 42\n.*?map_array = \[\{ enabled = true, name = "first" \}, \{ enabled = false, name = "second" \}\]\n.*?nested_map = \{ alpha = false, zebra = 2 \}\n.*?scalar_array = \["first", "second"\]\n.*?string_value = "plain text"$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: omitted_value + + - it: evaluates templates only in string values + set: + gatewayConfig: + example: + rendered_string: '{{ .Release.Namespace }}/{{ .Release.Name }}' + boolean_value: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^rendered_string = "serializer-namespace/serializer-test"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^boolean_value = true$' + + - it: quotes non-bare TOML keys safely + set: + gatewayConfig: + example: + "field with spaces": value + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^"field with spaces" = "value"$' + + - it: rejects a top-level table that is not a map + set: + gatewayConfig: + example: invalid + asserts: + - failedTemplate: + errorMessage: 'gatewayConfig table "example" must be a map, got string' + + - it: rejects null members in arrays + set: + gatewayConfig: + example: + values: + - valid + - null + asserts: + - failedTemplate: + errorMessage: gatewayConfig arrays cannot contain null values + + - it: rejects table names with empty TOML key segments + set: + gatewayConfig: + "example..config": + value: valid + asserts: + - failedTemplate: + errorMessage: 'gatewayConfig table "example..config" contains an empty TOML key segment' diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index a9186d3339..2f445c3d14 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -21,7 +21,9 @@ tests: - it: renders an explicit gateway name template: templates/gateway-config.yaml set: - server.name: production-us-west + gatewayConfig: + openshell.gateway: + name: production-us-west asserts: - matchRegex: path: data["gateway.toml"] @@ -108,8 +110,10 @@ tests: - it: renders OTLP tracing configuration when configured template: templates/gateway-config.yaml set: - server.otlp.endpoint: http://otel-collector.observability.svc:4317 - server.otlp.serviceName: production-gateway + gatewayConfig: + openshell.gateway.otlp: + endpoint: http://otel-collector.observability.svc:4317 + service_name: production-gateway asserts: - matchRegex: path: data["gateway.toml"] @@ -125,9 +129,12 @@ tests: - it: renders OIDC transport security settings template: templates/gateway-config.yaml set: - server.oidc.issuer: https://issuer.example.com - server.oidc.dangerouslyAllowInsecureHttp: false - server.oidc.jwksAllowedOrigins[0]: https://keys.example.com + gatewayConfig: + openshell.gateway.oidc: + issuer: https://issuer.example.com + dangerously_allow_insecure_http: false + jwks_allowed_origins: + - https://keys.example.com asserts: - matchRegex: path: data["gateway.toml"] @@ -139,7 +146,8 @@ tests: - it: treats a null OTLP map as disabled template: templates/gateway-config.yaml set: - server.otlp: null + gatewayConfig: + openshell.gateway.otlp: null asserts: - notMatchRegex: path: data["gateway.toml"] @@ -149,8 +157,10 @@ tests: template: templates/statefulset.yaml set: server.disableTls: true - server.oidc.issuer: https://issuer.example.com - server.oidc.caConfigMapName: openshell-oidc-ca + oidc.caConfigMapName: openshell-oidc-ca + gatewayConfig: + openshell.gateway.oidc: + issuer: https://issuer.example.com asserts: - equal: path: spec.template.spec.containers[0].volumeMounts[3].name @@ -189,49 +199,43 @@ tests: - it: renders canonical image pull policies in the Kubernetes driver table template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: if_not_present - supervisor.image.pullPolicy: never + gatewayConfig: + openshell.drivers.kubernetes: + image_pull_policy: if_not_present + supervisor_image_pull_policy: never asserts: - matchRegex: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"if_not_present".*?supervisor_image_pull_policy\s*=\s*"never"' - - it: translates legacy Kubernetes pull policy values to canonical gateway values + - it: preserves Kubernetes driver image pull policy values template: templates/gateway-config.yaml set: - server.sandboxImagePullPolicy: Always - supervisor.image.pullPolicy: IfNotPresent + gatewayConfig: + openshell.drivers.kubernetes: + image_pull_policy: always + supervisor_image_pull_policy: if_not_present asserts: - matchRegex: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"always".*?supervisor_image_pull_policy\s*=\s*"if_not_present"' - - it: rejects unsupported sandbox image pull policies - template: templates/statefulset.yaml - set: - server.sandboxImagePullPolicy: Sometimes - asserts: - - failedTemplate: - errorMessage: 'image pull policy "Sometimes" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' - - - it: rejects unsupported supervisor image pull policies - template: templates/statefulset.yaml - set: - supervisor.image.pullPolicy: newer - asserts: - - failedTemplate: - errorMessage: 'image pull policy "newer" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' - - - it: renders driver-owned Kubernetes settings only in its driver table + - it: derives the Kubernetes host gateway IP from the pod host alias owner template: templates/gateway-config.yaml set: - server.hostGatewayIP: 10.0.0.1 - server.enableUserNamespaces: true - supervisor.image.tag: test + server.hostGatewayIP: 10.23.45.67 + gatewayConfig: + openshell.drivers.kubernetes: + host_gateway_ip: 192.0.2.1 + enable_user_namespaces: true + supervisor_image: ghcr.io/nvidia/openshell/supervisor:test asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace".*?default_image\s*=.*?supervisor_image\s*=.*?host_gateway_ip\s*=\s*"10\.0\.0\.1".*?client_tls_secret_name\s*=.*?service_account_name\s*=\s*"openshell-sandbox".*?enable_user_namespaces\s*=\s*true.*?sa_token_ttl_secs\s*=' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?client_tls_secret_name\s*=.*?default_image\s*=.*?enable_user_namespaces\s*=\s*true.*?host_gateway_ip\s*=\s*"10\.23\.45\.67".*?namespace\s*=\s*"my-namespace".*?sa_token_ttl_secs\s*=.*?service_account_name\s*=\s*"openshell-sandbox".*?supervisor_image\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'host_gateway_ip\s*=\s*"192\.0\.2\.1"' - notMatchRegex: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?(sandbox_namespace|default_image|supervisor_image|client_tls_secret_name|service_account_name|host_gateway_ip|enable_user_namespaces|sa_token_ttl_secs)\s*=' @@ -239,10 +243,44 @@ tests: path: data["gateway.toml"] pattern: 'guest_tls_(ca|cert|key)\s*=' + - it: derives TLS runtime fields from chart-owned TLS resources + template: templates/gateway-config.yaml + set: + server.disableTls: false + server.tls.clientTlsSecretName: chart-client-tls + gatewayConfig: + openshell.gateway: + disable_tls: true + openshell.drivers.kubernetes: + client_tls_secret_name: conflicting-client-tls + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\].*?disable_tls\s*=\s*false' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?client_tls_secret_name\s*=\s*"chart-client-tls"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'conflicting-client-tls' + + - it: restores chart-owned TLS runtime paths when gatewayConfig omits the TLS table + template: templates/gateway-config.yaml + set: + server.disableTls: false + gatewayConfig: + openshell.gateway.tls: null + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\.tls\].*?cert_path\s*=\s*"/etc/openshell-tls/server/tls\.crt".*?key_path\s*=\s*"/etc/openshell-tls/server/tls\.key"' + - it: renders user namespace enablement under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: - server.enableUserNamespaces: true + gatewayConfig: + openshell.drivers.kubernetes: + enable_user_namespaces: true asserts: - matchRegex: path: data["gateway.toml"] @@ -251,15 +289,17 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?enable_user_namespaces' - - it: renders operator-owned upstream proxy settings under the Kubernetes driver + - it: renders proxy settings from the Kubernetes driver configuration template: templates/gateway-config.yaml set: - upstreamProxy.url: http://proxy.corp.example:8080 - upstreamProxy.noProxy: .svc.cluster.local,10.96.0.0/12 - upstreamProxy.authSecret.name: corporate-proxy-auth - upstreamProxy.authSecret.key: credentials - upstreamProxy.authAllowInsecure: true - upstreamProxy.connectByHostname: true + gatewayConfig: + openshell.drivers.kubernetes: + https_proxy: http://proxy.corp.example:8080 + no_proxy: .svc.cluster.local,10.96.0.0/12 + proxy_auth_secret_name: corporate-proxy-auth + proxy_auth_secret_key: credentials + proxy_auth_allow_insecure: true + proxy_connect_by_hostname: true asserts: - matchRegex: path: data["gateway.toml"] @@ -280,28 +320,28 @@ tests: path: data["gateway.toml"] pattern: 'proxy_connect_by_hostname\s*=\s*true' - - it: uses the gateway built-in supervisor image by default + - it: renders the default supervisor image from gatewayConfig template: templates/gateway-config.yaml asserts: - - notMatchRegex: + - matchRegex: path: data["gateway.toml"] - pattern: 'supervisor_image\s*=' + pattern: 'supervisor_image\s*=\s*"ghcr\.io/nvidia/openshell/supervisor:0\.0\.0"' - - it: uses the gateway built-in sandbox runtime image by default + - it: renders the default sandbox runtime image from gatewayConfig template: templates/gateway-config.yaml asserts: - - notMatchRegex: + - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_runtime_image\s*=' + pattern: 'sandbox_runtime_image\s*=\s*"ghcr\.io/nvidia/openshell/sandbox:0\.0\.0"' - - it: renders independent sandbox runtime and supervisor image overrides + - it: renders independent sandbox runtime and supervisor image overrides from gatewayConfig template: templates/gateway-config.yaml set: - sandboxRuntime.image.repository: registry.example.com/openshell/sandbox - sandboxRuntime.image.tag: sandbox-build - sandboxRuntime.image.pullPolicy: Always - supervisor.image.repository: registry.example.com/openshell/supervisor - supervisor.image.tag: supervisor-build + gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime_image: registry.example.com/openshell/sandbox:sandbox-build + sandbox_runtime_image_pull_policy: always + supervisor_image: registry.example.com/openshell/supervisor:supervisor-build asserts: - matchRegex: path: data["gateway.toml"] @@ -313,30 +353,34 @@ tests: path: data["gateway.toml"] pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:supervisor-build"' - - it: renders a supervisor tag override with the official repository + - it: renders a supervisor image override template: templates/gateway-config.yaml set: - supervisor.image.tag: 1.2.3 + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_image: ghcr.io/nvidia/openshell/supervisor:1.2.3 asserts: - matchRegex: path: data["gateway.toml"] pattern: 'supervisor_image\s*=\s*"ghcr\.io/nvidia/openshell/supervisor:1\.2\.3"' - - it: renders a supervisor repository override with the effective gateway tag + - it: renders a supervisor repository image override template: templates/gateway-config.yaml set: - image.tag: gateway-build - supervisor.image.repository: registry.example.com/openshell/supervisor + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_image: registry.example.com/openshell/supervisor:gateway-build asserts: - matchRegex: path: data["gateway.toml"] pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:gateway-build"' - - it: renders complete supervisor repository and tag overrides + - it: renders a complete supervisor image override template: templates/gateway-config.yaml set: - supervisor.image.repository: registry.example.com/openshell/supervisor - supervisor.image.tag: supervisor-build + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_image: registry.example.com/openshell/supervisor:supervisor-build asserts: - matchRegex: path: data["gateway.toml"] @@ -352,9 +396,11 @@ tests: - it: renders sandbox image pull secrets under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: - server.sandboxImagePullSecrets: - - name: regcred - - name: backup-regcred + gatewayConfig: + openshell.drivers.kubernetes: + image_pull_secrets: + - regcred + - backup-regcred asserts: - matchRegex: path: data["gateway.toml"] @@ -387,7 +433,9 @@ tests: - it: renders explicit unauthenticated user dev mode when enabled template: templates/gateway-config.yaml set: - server.auth.allowUnauthenticatedUsers: true + gatewayConfig: + openshell.gateway.auth: + allow_unauthenticated_users: true asserts: - matchRegex: path: data["gateway.toml"] @@ -413,7 +461,9 @@ tests: - it: renders retain-last-valid policy validation posture template: templates/gateway-config.yaml set: - server.policyValidationFailureMode: retain_last_valid + gatewayConfig: + openshell.gateway: + policy_validation_failure_mode: retain_last_valid asserts: - matchRegex: path: data["gateway.toml"] @@ -422,8 +472,10 @@ tests: - it: renders the gRPC rate limit under [openshell.gateway] when both values are positive template: templates/gateway-config.yaml set: - server.grpcRateLimit.requests: 120 - server.grpcRateLimit.windowSeconds: 60 + gatewayConfig: + openshell.gateway: + grpc_rate_limit_requests: 120 + grpc_rate_limit_window_seconds: 60 asserts: - matchRegex: path: data["gateway.toml"] @@ -432,41 +484,6 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\].*?grpc_rate_limit_window_seconds\s*=\s*60' - # The validation lives in gateway-config.yaml but surfaces through the - # statefulset checksum include, so the failure is asserted against that - # template (mirrors the postgres serviceBindings failure test below). - - it: fails to render when only requests is positive - template: templates/statefulset.yaml - set: - server.grpcRateLimit.requests: 120 - asserts: - - failedTemplate: - errorMessage: "server.grpcRateLimit requires both requests and windowSeconds to be positive to enable rate limiting, or both 0/unset to disable it" - - - it: fails to render when only windowSeconds is positive - template: templates/statefulset.yaml - set: - server.grpcRateLimit.windowSeconds: 60 - asserts: - - failedTemplate: - errorMessage: "server.grpcRateLimit requires both requests and windowSeconds to be positive to enable rate limiting, or both 0/unset to disable it" - - - it: fails to render when requests is negative - template: templates/statefulset.yaml - set: - server.grpcRateLimit.requests: -1 - asserts: - - failedTemplate: - errorMessage: "server.grpcRateLimit.requests and server.grpcRateLimit.windowSeconds must not be negative; they map to unsigned gateway settings" - - - it: fails to render when windowSeconds is negative - template: templates/statefulset.yaml - set: - server.grpcRateLimit.windowSeconds: -5 - asserts: - - failedTemplate: - errorMessage: "server.grpcRateLimit.requests and server.grpcRateLimit.windowSeconds must not be negative; they map to unsigned gateway settings" - - it: uses the configured existing sandbox service account name template: templates/gateway-config.yaml set: @@ -489,6 +506,10 @@ tests: server.disableTls: true certManager.enabled: false pkiInitJob.enabled: false + gatewayConfig: + openshell.gateway: + disable_tls: true + openshell.gateway.tls: null template: templates/gateway-config.yaml asserts: - matchRegex: @@ -502,6 +523,9 @@ tests: template: templates/gateway-config.yaml set: server.tls.enableMtls: false + gatewayConfig: + openshell.gateway.tls: + client_ca_path: null asserts: - matchRegex: path: data["gateway.toml"] @@ -527,6 +551,9 @@ tests: template: templates/gateway-config.yaml set: server.tls.clientCaSecretName: "" + gatewayConfig: + openshell.gateway.tls: + client_ca_path: null asserts: - matchRegex: path: data["gateway.toml"] @@ -549,6 +576,13 @@ tests: certManager.serverDnsNames: - gateway.example.com server.tls.clientCaSecretName: "" + gatewayConfig: + openshell.gateway.tls: + client_ca_path: null + external_cert_path: /etc/openshell-tls/server-external/tls.crt + external_key_path: /etc/openshell-tls/server-external/tls.key + external_server_names: + - gateway.example.com asserts: - notMatchRegex: path: data["gateway.toml"] @@ -569,6 +603,9 @@ tests: certManager.enabled: true certManager.clientCaFromServerTlsSecret: true server.tls.clientCaSecretName: "" + gatewayConfig: + openshell.gateway.tls: + client_ca_path: null asserts: - matchRegex: path: data["gateway.toml"] @@ -590,6 +627,9 @@ tests: certManager.clientCaFromServerTlsSecret: false pkiInitJob.enabled: true server.tls.clientCaSecretName: "" + gatewayConfig: + openshell.gateway.tls: + client_ca_path: null asserts: - matchRegex: path: data["gateway.toml"] @@ -611,6 +651,11 @@ tests: - openshell - "*.dev.openshell.localhost" pkiInitJob.enabled: false + gatewayConfig: + openshell.gateway.tls: + server_sans: + - openshell + - "*.dev.openshell.localhost" template: templates/gateway-config.yaml asserts: - matchRegex: @@ -654,7 +699,9 @@ tests: workload.kind: deployment replicaCount: 2 server.externalDbSecret: my-pg-secret - server.credentialDrivers.kubernetesSecrets.enabled: true + gatewayConfig: + openshell.gateway: + credential_drivers: [kubernetes-secrets] asserts: - equal: path: kind @@ -709,7 +756,9 @@ tests: replicaCount: 2 server.externalDbSecret: my-pg-secret workload.allowMultiReplicaStatefulSet: true - server.credentialDrivers.kubernetesSecrets.enabled: true + gatewayConfig: + openshell.gateway: + credential_drivers: [kubernetes-secrets] asserts: - equal: path: kind @@ -775,7 +824,9 @@ tests: key: uri - it: renders provider SPIFFE token grants while keeping gateway JWT auth set: - server.providerTokenGrants.spiffe.enabled: true + gatewayConfig: + openshell.drivers.kubernetes: + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock template: templates/gateway-config.yaml asserts: - matchRegex: @@ -790,7 +841,9 @@ tests: - it: mounts the gateway SPIFFE socket while keeping sandbox JWT auth set: - server.providerTokenGrants.spiffe.enabled: true + gatewayConfig: + openshell.drivers.kubernetes: + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock template: templates/statefulset.yaml asserts: - contains: @@ -846,12 +899,14 @@ tests: asserts: - matchRegex: path: data["gateway.toml"] - pattern: "(?ms)name\\s*=\\s*\\\"openshell\\\".*?\\[openshell\\.drivers\\.kubernetes\\].*?namespace\\s*=\\s*\\\"my-namespace\\\".*?grpc_endpoint\\s*=\\s*\\\"https://openshell\\.my-namespace\\.svc\\.cluster\\.local:8080\\\"" + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?grpc_endpoint\\s*=\\s*\\\"https://openshell\\.my-namespace\\.svc\\.cluster\\.local:8080\\\".*?namespace\\s*=\\s*\\\"my-namespace\\\".*?\\[openshell\\.gateway\\].*?name\\s*=\\s*\\\"openshell\\\"" - it: uses an explicit server grpc endpoint verbatim template: templates/gateway-config.yaml set: - server.grpcEndpoint: https://gateway.example.test:9443 + gatewayConfig: + openshell.drivers.kubernetes: + grpc_endpoint: https://gateway.example.test:9443 asserts: - matchRegex: path: data["gateway.toml"] @@ -861,6 +916,12 @@ tests: template: templates/gateway-config.yaml set: server.disableTls: true + gatewayConfig: + openshell.gateway: + disable_tls: true + openshell.gateway.tls: null + openshell.drivers.kubernetes: + client_tls_secret_name: null asserts: - matchRegex: path: data["gateway.toml"] @@ -868,63 +929,3 @@ tests: - notMatchRegex: path: data["gateway.toml"] pattern: "client_tls_secret_name\\s*=" - - - it: accepts Always pull policy spelling for sandbox and supervisor - template: templates/gateway-config.yaml - set: - server.sandboxImagePullPolicy: Always - supervisor.image.pullPolicy: Always - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"always\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"always\\\"" - - - it: accepts IfNotPresent pull policy spelling for sandbox and supervisor - template: templates/gateway-config.yaml - set: - server.sandboxImagePullPolicy: IfNotPresent - supervisor.image.pullPolicy: IfNotPresent - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"if_not_present\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"if_not_present\\\"" - - - it: accepts Never pull policy spelling for sandbox and supervisor - template: templates/gateway-config.yaml - set: - server.sandboxImagePullPolicy: Never - supervisor.image.pullPolicy: Never - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"never\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"never\\\"" - - - it: accepts canonical lowercase pull policies for sandbox and supervisor - template: templates/gateway-config.yaml - set: - server.sandboxImagePullPolicy: always - supervisor.image.pullPolicy: always - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"always\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"always\\\"" - - - it: accepts canonical if_not_present pull policies for sandbox and supervisor - template: templates/gateway-config.yaml - set: - server.sandboxImagePullPolicy: if_not_present - supervisor.image.pullPolicy: if_not_present - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"if_not_present\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"if_not_present\\\"" - - - it: accepts canonical never pull policies for sandbox and supervisor - template: templates/gateway-config.yaml - set: - server.sandboxImagePullPolicy: never - supervisor.image.pullPolicy: never - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"never\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"never\\\"" diff --git a/deploy/helm/openshell/tests/gateway_pod_security_context_test.yaml b/deploy/helm/openshell/tests/gateway_pod_security_context_test.yaml index cdb70a05eb..6c7c45bb28 100644 --- a/deploy/helm/openshell/tests/gateway_pod_security_context_test.yaml +++ b/deploy/helm/openshell/tests/gateway_pod_security_context_test.yaml @@ -34,3 +34,18 @@ tests: asserts: - notExists: path: spec.template.spec.securityContext + + - it: derives host aliases from server.hostGatewayIP and not gatewayConfig + template: templates/statefulset.yaml + set: + server.hostGatewayIP: 10.23.45.67 + gatewayConfig: + openshell.drivers.kubernetes: + host_gateway_ip: 192.0.2.1 + asserts: + - equal: + path: spec.template.spec.hostAliases[0].ip + value: 10.23.45.67 + - contains: + path: spec.template.spec.hostAliases[0].hostnames + content: host.openshell.internal diff --git a/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml new file mode 100644 index 0000000000..71d5ea1810 --- /dev/null +++ b/deploy/helm/openshell/tests/gateway_secret_boundary_test.yaml @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: gateway ConfigMap secret boundary +templates: + - templates/gateway-config.yaml + - templates/statefulset.yaml + +tests: + - it: rejects a database URL in gatewayConfig before creating the ConfigMap + template: templates/statefulset.yaml + set: + gatewayConfig.example.database_url: postgresql://gateway:password@database:5432/openshell + asserts: + - failedTemplate: + errorMessage: "gatewayConfig must not contain database_url; provide database credentials through the chart's Secret-backed OPENSHELL_DB_URL environment variable" + + - it: rejects an inline PEM private key before creating the ConfigMap + template: templates/statefulset.yaml + set: + gatewayConfig.example.private_key: "-----BEGIN PRIVATE KEY-----" + asserts: + - failedTemplate: + errorMessage: gatewayConfig must not contain an inline private key; provide it through a Secret-backed file mount + + - it: rejects inline credentials embedded in a URL + template: templates/statefulset.yaml + set: + gatewayConfig.example.endpoint: https://gateway:password@database.example.com + asserts: + - failedTemplate: + errorMessage: gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume + + - it: rejects URL credentials with an empty username + template: templates/statefulset.yaml + set: + gatewayConfig.example.endpoint: https://:password@database.example.com + asserts: + - failedTemplate: + errorMessage: gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume + + - it: rejects URL userinfo without a password + template: templates/statefulset.yaml + set: + gatewayConfig.example.endpoint: https://username@database.example.com + asserts: + - failedTemplate: + errorMessage: gatewayConfig must not contain inline URL credentials; provide them through a Secret-backed environment variable, file, or volume + + - it: keeps external database credentials out of gateway.toml + template: templates/gateway-config.yaml + set: + server.externalDbSecret: gateway-database + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?i)postgresql://|gateway-database|password' + + - it: references external database credentials through an environment variable + template: templates/statefulset.yaml + set: + server.externalDbSecret: gateway-database + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: gateway-database + key: uri + + - it: rejects malformed Secret references + template: templates/statefulset.yaml + set: + server.credentialStorage.existingSecret: contains spaces + asserts: + - failedTemplate: + errorMessage: server.credentialStorage.existingSecret must be a valid Kubernetes Secret name + + - it: rejects Secret references longer than the Kubernetes limit + template: templates/statefulset.yaml + set: + server.credentialStorage.existingSecret: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + asserts: + - failedTemplate: + errorMessage: server.credentialStorage.existingSecret must be no more than 253 characters diff --git a/deploy/helm/openshell/tests/network_policy_ack_test.yaml b/deploy/helm/openshell/tests/network_policy_ack_test.yaml index 12ca399e73..16678eb81c 100644 --- a/deploy/helm/openshell/tests/network_policy_ack_test.yaml +++ b/deploy/helm/openshell/tests/network_policy_ack_test.yaml @@ -7,14 +7,20 @@ templates: tests: - it: rejects an install without operator acknowledgement set: - supervisor.sandboxRuntime.networkPolicyEnforced: false + gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + network_policy_enforced: false asserts: - failedTemplate: - errorMessage: supervisor.sandboxRuntime.networkPolicyEnforced must be true after you verify that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace + errorMessage: gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced must be true after you verify that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace - it: accepts an acknowledged CNI set: - supervisor.sandboxRuntime.networkPolicyEnforced: true + gatewayConfig: + openshell.drivers.kubernetes: + sandbox_runtime: + network_policy_enforced: true asserts: - hasDocuments: count: 0 diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 67d5587ec9..81c7ec214c 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -26,50 +26,6 @@ image: # -- Gateway image tag. Defaults to the chart appVersion when empty. tag: "" -# Trusted workload-side runtime image. -sandboxRuntime: - image: - # -- Sandbox runtime image repository. Changing it uses the effective gateway image tag unless tag is also set. - repository: ghcr.io/nvidia/openshell/sandbox - # -- Sandbox runtime image pull policy. Defaults to the gateway image pull policy when empty. - pullPolicy: "" - # -- Sandbox runtime image tag override. Empty uses the version pinned into the gateway unless repository is changed. - tag: "" - -# Trusted control-side runtime image. -supervisor: - image: - # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. - repository: ghcr.io/nvidia/openshell/supervisor - # -- Sandbox supervisor pull policy. Leave unset to use the Kubernetes - # image default. Prefer always, if_not_present, or never; the chart also - # accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. - pullPolicy: null - # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. - tag: "" - sandboxRuntime: - # -- Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. - networkPolicyEnforced: false - # -- Workload boundary TLS listener port. - boundaryPort: 5500 - -# -- Operator-owned corporate forward proxy for policy-approved TLS egress -# from Kubernetes sandboxes. The workload cannot select or override it. -upstreamProxy: - # -- HTTP proxy URL in http://host:port form. HTTPS-to-proxy is not supported. - url: "" - # -- Comma-separated destinations that bypass only the corporate proxy. - noProxy: "" - authSecret: - # -- Existing Secret in the sandbox namespace containing a user:pass value. - name: "" - # -- Secret key containing the proxy credential. - key: "" - # -- Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. - authAllowInsecure: false - # -- Last-resort option for hostname-filtering proxy ACLs. It lets the proxy resolve CONNECT targets. - connectByHostname: false - # -- Image pull secrets attached to gateway and helper pods. imagePullSecrets: [] # -- Override the chart name used in generated resource names. @@ -186,20 +142,57 @@ tolerations: [] # -- Affinity rules for the gateway pod. affinity: {} +# -- Non-secret gateway application configuration. Top-level keys name TOML +# tables and are rendered into the mounted gateway.toml file. Kubernetes +# resource inputs remain outside this map; template expressions derive the +# corresponding runtime values from their resource owner. +gatewayConfig: + openshell: + version: 2 + openshell.gateway: + name: '{{ include "openshell.fullname" . }}' + bind_address: '0.0.0.0:{{ .Values.service.port }}' + health_bind_address: '0.0.0.0:{{ .Values.service.healthPort }}' + metrics_bind_address: '0.0.0.0:{{ .Values.service.metricsPort }}' + log_level: info + compute_driver: kubernetes + enable_loopback_service_http: true + policy_validation_failure_mode: fail_closed + openshell.gateway.gateway_jwt: + signing_key_path: /etc/openshell-jwt/signing.pem + public_key_path: /etc/openshell-jwt/public.pem + kid_path: /etc/openshell-jwt/kid + gateway_id: '{{ include "openshell.fullname" . }}' + ttl_secs: 3600 + openshell.gateway.tls: + cert_path: /etc/openshell-tls/server/tls.crt + key_path: /etc/openshell-tls/server/tls.key + client_ca_path: /etc/openshell-tls/client-ca/ca.crt + openshell.drivers.kubernetes: + namespace: '{{ include "openshell.sandboxNamespace" . }}' + default_image: ghcr.io/nvidia/openshell-community/sandboxes/base:latest + client_tls_secret_name: '{{ .Values.server.tls.clientTlsSecretName }}' + workspace_mode: shared + gateway_id: '{{ include "openshell.fullname" . }}' + grpc_endpoint: '{{ include "openshell.grpcEndpoint" . }}' + service_account_name: '{{ include "openshell.sandboxServiceAccountName" . }}' + sandbox_runtime_image: ghcr.io/nvidia/openshell/sandbox:{{ .Chart.AppVersion }} + supervisor_image: ghcr.io/nvidia/openshell/supervisor:{{ .Chart.AppVersion }} + sandbox_runtime: + network_policy_enforced: false + boundary_port: 5500 + sa_token_ttl_secs: 3600 + openshell.drivers.kubernetes.managed_ssh_ingress: + enabled: true + gateway_namespace: '{{ .Release.Namespace }}' + gateway_pod_selector: + app.kubernetes.io/name: '{{ include "openshell.name" . }}' + app.kubernetes.io/instance: '{{ .Release.Name }}' + openshell.gateway.credential_storage: + key_encryption_key_env: '{{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }}' + # Server configuration server: - # -- Operator-facing gateway name. Defaults to the chart fullname so all - # replicas in one installation share an identity. Set explicitly when one - # telemetry collector receives spans from multiple namespaces or clusters. - name: "" - # -- Gateway log level. - logLevel: info - # OpenTelemetry trace export over OTLP/gRPC. Leave endpoint empty to disable. - otlp: - # -- OTLP/gRPC collector endpoint, conventionally using port 4317. - endpoint: "" - # -- Gateway OpenTelemetry service name. Empty uses openshell-gateway. - serviceName: "" # -- Enable anonymous OpenShell telemetry from the gateway and the sandbox # supervisors it launches. telemetryEnabled: true @@ -213,37 +206,6 @@ server: # from this Secret instead of using dbUrl. The Secret must contain a # `uri` key, e.g. postgresql://user:pass@host:5432/dbname. externalDbSecret: "" - # -- Default sandbox image used when requests do not specify one. - sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" - # -- Pull policy for sandbox pods. Leave unset to use the Kubernetes image - # default (Always for :latest, IfNotPresent otherwise). Prefer always, - # if_not_present, or never; the chart also accepts legacy Kubernetes spellings - # Always, IfNotPresent, and Never. - sandboxImagePullPolicy: null - # -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist - # in the sandbox namespace. - sandboxImagePullSecrets: [] - # -- Default storage size for the workspace PVC in sandbox pods. - # Uses Kubernetes quantity syntax (e.g. "2Gi", "10Gi", "500Mi"). - # Empty = built-in default (2Gi). - workspaceDefaultStorageSize: "" - # -- Kubernetes StorageClass for the workspace PVC in sandbox pods. - # Empty (default) = omit storageClassName, using the cluster's default - # StorageClass. Set this on clusters with no default StorageClass, otherwise - # the workspace PVC stays Pending and the sandbox never starts. - workspaceStorageClass: "" - # -- Default Kubernetes runtimeClassName for sandbox pods. - # Applied when a CreateSandbox request does not specify one. - # Empty (default) = omit the field, using the cluster's default RuntimeClass. - # Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it - # to all sandboxes that don't explicitly override it. - defaultRuntimeClassName: "" - # -- gRPC endpoint sandboxes call back into the gateway. Leave empty to derive - # it from the chart fullname, release namespace, service port, and - # disableTls flag, for example https://openshell.openshell.svc.cluster.local:8080. - # Override only when sandboxes must reach the gateway via a different - # hostname (e.g. an external ingress or a host alias). - grpcEndpoint: "" # The gateway terminates TLS directly. Its client CA authenticates sandbox # callbacks; OIDC-enabled listeners permit bearer-only user clients while # still validating any client certificate they present. @@ -252,46 +214,9 @@ server: # to this IP, allowing them to reach services running on the Docker host. # Auto-detected by the cluster entrypoint script. hostGatewayIP: "" - # -- Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox - # pods. Requires Kubernetes 1.33+ with user namespace support available - # (beta through 1.35, GA in 1.36+), plus a supporting container runtime and - # Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host - # UID and capabilities become namespaced. - enableUserNamespaces: false - # Kubernetes compute driver settings. - drivers: - kubernetes: - # -- How workspaces map to Kubernetes namespaces. - # "shared" (default): all sandboxes in a single namespace. - # "managed": auto-creates per-workspace namespaces. - # "operator": uses pre-provisioned namespaces. - workspaceMode: "shared" - # -- K8s label selector for namespace discovery in operator mode. - # The driver watches namespaces matching this label. - operatorNamespaceLabel: "" - # -- Path to a JSON file containing an array of namespace names - # allowed in operator mode. Hot-reloaded on change. - operatorNamespaceFile: "" # -- Disable TLS entirely - the server listens on plaintext HTTP. # Set to true when a reverse proxy / tunnel terminates TLS at the edge. disableTls: false - # -- Enable plaintext HTTP routing for loopback sandbox service URLs on - # TLS-enabled gateways. - enableLoopbackServiceHttp: true - # -- Posture when a candidate sandbox policy fails validation. `fail_closed` - # deactivates the previous policy; `retain_last_valid` keeps it active. - policyValidationFailureMode: fail_closed - # Optional gateway-wide gRPC request rate limit. Applies only to gRPC API - # traffic after protocol multiplexing; health, metrics, and loopback service - # HTTP routes are not rate limited. Both values must be positive to enable the - # limit, otherwise it is omitted from the rendered config and stays disabled. - grpcRateLimit: - # -- Maximum gRPC requests allowed per window. Must be positive (alongside - # windowSeconds) to enable rate limiting; 0 (default) disables it. - requests: 0 - # -- gRPC rate-limit window length in seconds. Must be positive (alongside - # requests) to enable rate limiting; 0 (default) disables it. - windowSeconds: 0 # Default credential storage settings (used when no credential driver is # enabled). The gateway encrypts provider credentials in the database using # AES-256-GCM with a key-encryption key (KEK). By default, the Helm chart @@ -304,53 +229,6 @@ server: # with a base64-encoded 32-byte value. Required for GitOps workflows that # render manifests with `helm template` (where `lookup` is unavailable). existingSecret: "" - # Provider credential drivers store provider credential secret material in an - # external or native backend. When no driver is enabled, the gateway uses its - # default encrypted database credential storage with a retained Kubernetes - # Secret for the shared key-encryption key. - credentialDrivers: - kubernetesSecrets: - # -- Enable the in-tree Kubernetes Secret credential driver. - # WARNING: The RBAC Role grants read/write access to ALL Secrets in the - # configured namespace. Use a dedicated namespace to limit blast radius. - enabled: false - # -- Namespace where OpenShell-managed provider Secret objects are stored. - # Empty = Helm release namespace. A dedicated namespace is RECOMMENDED - # to isolate OpenShell-managed Secrets from other workloads. - namespace: "" - # -- Deprecated compatibility field. Credential storage no longer supports user-authored namespace references. - allowReferenceNamespace: false - rbac: - # -- Create a Role/RoleBinding granting the gateway ServiceAccount read/write access to managed provider Secrets. - create: true - vault: - # -- Enable the in-tree Vault credential driver. - enabled: false - # -- Vault service base URL. Non-loopback endpoints must use HTTPS, for example https://vault.vault.svc.cluster.local:8200. - address: "" - # -- ConfigMap containing the private Vault CA certificate bundle in the ca.crt key. Leave empty to use platform trust roots. - caConfigMapName: "" - # -- Default KV mount name. - mount: secret - # -- Default KV engine version. Use "1" or "2". - kvVersion: "2" - # -- Authentication method. Use "kubernetes" in-cluster or "token_file" for local/dev validation. - authMethod: kubernetes - # -- Vault Kubernetes auth role when authMethod is kubernetes. - role: "" - # -- Vault Kubernetes auth mount. - kubernetesAuthMount: kubernetes - # -- ServiceAccount token path used for Kubernetes auth. - serviceAccountTokenPath: /var/run/secrets/kubernetes.io/serviceaccount/token - # -- Mounted token file path when authMethod is token_file. - tokenPath: "" - # -- HTTP request timeout in seconds. Empty = driver default. - timeoutSecs: "" - auth: - # -- UNSAFE: accept unauthenticated CLI/user requests as a local developer - # principal. Intended only for trusted local Skaffold/k3d development or a - # fully trusted fronting proxy. Leave false for shared or production clusters. - allowUnauthenticatedUsers: false tls: # -- K8s secret (type kubernetes.io/tls) with tls.crt and tls.key for the server. certSecretName: openshell-server-tls @@ -374,60 +252,21 @@ server: # -- Name of the Opaque Secret holding the signing key material. Empty # falls back to the chart fullname with "-jwt-keys" appended. signingSecretName: "" - # -- Stable gateway identity embedded in iss/aud of every minted token. - # Defaults to the release name so HA replicas share identity. - gatewayId: "" - # -- Token TTL in seconds. Defaults to 3600 (1h). - ttlSecs: 3600 - # -- Lifetime (seconds) of the projected ServiceAccount token kubelet - # writes into each sandbox pod for the IssueSandboxToken bootstrap - # exchange. Kubelet enforces a minimum of 600s; the driver clamps - # values outside [600, 86400]. Default 3600 — generous, since the - # supervisor consumes the token within seconds of pod start. - k8sSaTokenTtlSecs: 3600 # -- File mode for the mounted JWT signing key Secret. Default 0400 # (owner-read only). Override to 0440 or 0444 if the container UID # does not match the volume file owner. secretDefaultMode: "" - # Dynamic provider token grants. When SPIFFE is enabled here, both the - # gateway and sandbox supervisors mount the SPIFFE Workload API socket so - # token-exchange profiles can use gateway- and sandbox-scoped JWT-SVIDs. - # Supervisor-to-gateway authentication still uses gateway-minted sandbox JWTs. - providerTokenGrants: - spiffe: - # -- Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. - enabled: false - # -- Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. - workloadApiSocketPath: /spiffe-workload-api/spire-agent.sock - # OIDC (OpenID Connect) configuration for JWT-based authentication. - # When issuer is set, the server validates Bearer tokens on gRPC requests. - oidc: - # -- OIDC issuer URL (e.g. https://keycloak.example.com/realms/openshell). - issuer: "" - # -- Development only: permit cleartext OIDC requests to numeric loopback - # addresses. This never permits HTTP to hostnames or non-loopback addresses. - dangerouslyAllowInsecureHttp: false - # -- Additional trusted HTTPS origins allowed to serve JWKS. The issuer - # origin is always allowed. Entries must not include a path or query. - jwksAllowedOrigins: [] - # -- Expected audience claim for the API resource server. - # This should match the server's --oidc-audience, NOT the CLI client ID. - audience: "openshell-cli" - # -- JWKS key cache TTL in seconds. Must be greater than zero. - jwksTtl: 3600 - # -- Dot-separated path to the roles array in the JWT claims. - # Keycloak: "realm_access.roles", Entra ID: "roles", Okta: "groups". - rolesClaim: "" - # -- Role name for admin access. Leave empty (with userRole also empty) for - # authentication-only mode. Both must be set or both empty. - adminRole: "" - # -- Role name for standard user access. - userRole: "" - # -- Dot-separated path to the scopes array in the JWT claims. - scopesClaim: "" - # -- Name of a ConfigMap containing a CA certificate bundle (key: ca.crt) - # for verifying the OIDC issuer's TLS certificate. Required when the - # issuer uses a non-public CA (e.g. OpenShift ingress, private PKI). +# Kubernetes resource reference for a private-CA OIDC issuer. The OIDC +# runtime settings themselves belong in gatewayConfig.openshell.gateway.oidc. +oidc: + caConfigMapName: "" + +# Kubernetes resource references for external credential drivers. Runtime +# driver settings remain in gatewayConfig. +credentialDrivers: + vault: + # -- ConfigMap containing the private Vault/OpenBao CA certificate under + # the ca.crt key. Helm mounts it only when the Vault driver is selected. caConfigMapName: "" # NetworkPolicy restricting SSH ingress on sandbox pods to the gateway only. diff --git a/deploy/helm/test-gateway-config-parser.sh b/deploy/helm/test-gateway-config-parser.sh new file mode 100755 index 0000000000..0c4268c6e1 --- /dev/null +++ b/deploy/helm/test-gateway-config-parser.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Render the chart exactly as an operator would, then validate gateway.toml +# with the gateway binary. Helm unit tests cover template-level assertions; +# this test makes the Rust loader the compatibility authority. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +chart="${repo_root}/deploy/helm/openshell" +fixture="${chart}/tests/fixtures/parser-validation-values.yaml" +work_dir="$(mktemp -d)" +trap 'rm -rf "${work_dir}"' EXIT + +cargo build --quiet --package openshell-gateway +gateway_bin="${repo_root}/target/debug/openshell-gateway" + +render() { + local output="$1" + shift + helm template parser-validation "${chart}" \ + --namespace parser-namespace \ + --set agentSandbox.preflight.enabled=false \ + --set gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime.network_policy_enforced=true \ + "$@" >"${output}" +} + +extract_toml() { + local manifest="$1" + local toml="$2" + yq ea -e -r \ + 'select(.kind == "ConfigMap" and (.data | has("gateway.toml"))) | .data."gateway.toml"' \ + "${manifest}" >"${toml}" +} + +preflight() { + local toml="$1" + "${gateway_bin}" config preflight --path "${toml}" +} + +if helm template parser-validation "${chart}" --namespace parser-namespace \ + --set agentSandbox.preflight.enabled=false >"${work_dir}/unacknowledged-default.yaml" 2>"${work_dir}/unacknowledged-default.err"; then + echo "the chart must require explicit NetworkPolicy enforcement acknowledgement" >&2 + exit 1 +fi +grep -F 'gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced must be true' "${work_dir}/unacknowledged-default.err" >/dev/null + +# The runtime default is valid after the required infrastructure acknowledgement. +render "${work_dir}/default.yaml" +extract_toml "${work_dir}/default.yaml" "${work_dir}/default.toml" +preflight "${work_dir}/default.toml" + +# Exercise all serializer shapes through a raw driver table and prove that +# string-only tpl expansion, TOML escaping, and null omission survive parsing. +render "${work_dir}/shapes.yaml" --values "${fixture}" +extract_toml "${work_dir}/shapes.yaml" "${work_dir}/shapes.toml" +preflight "${work_dir}/shapes.toml" +grep -F 'rendered_name = "parser-namespace/parser-validation' "${work_dir}/shapes.toml" >/dev/null +grep -F 'empty_value = ""' "${work_dir}/shapes.toml" >/dev/null +if grep -Fq 'omitted_value' "${work_dir}/shapes.toml"; then + echo "null gatewayConfig values must be omitted from gateway.toml" >&2 + exit 1 +fi + +# The loader gives absent and empty credential-driver selection deliberately +# different meanings: absent retains encrypted storage, while an empty list is +# an invalid and ambiguous external-driver selection. +render "${work_dir}/credential-drivers-absent.yaml" \ + --set-json 'gatewayConfig.openshell\.gateway.credential_drivers=null' +extract_toml "${work_dir}/credential-drivers-absent.yaml" "${work_dir}/credential-drivers-absent.toml" +preflight "${work_dir}/credential-drivers-absent.toml" +if grep -Fq 'credential_drivers' "${work_dir}/credential-drivers-absent.toml"; then + echo "null credential_drivers must be absent from gateway.toml" >&2 + exit 1 +fi +render "${work_dir}/credential-drivers-empty.yaml" \ + --set-json 'gatewayConfig.openshell\.gateway.credential_drivers=[]' +extract_toml "${work_dir}/credential-drivers-empty.yaml" "${work_dir}/credential-drivers-empty.toml" +if preflight "${work_dir}/credential-drivers-empty.toml" >"${work_dir}/credential-drivers-empty.err" 2>&1; then + echo "the gateway loader accepted empty credential_drivers" >&2 + exit 1 +fi + +# Unknown non-secret values are intentionally serializable by Helm, but must +# fail at the Rust schema boundary rather than being silently ignored. +render "${work_dir}/unknown.yaml" \ + --set-string 'gatewayConfig.openshell\.gateway.unknown_non_secret=accepted-by-helm' +extract_toml "${work_dir}/unknown.yaml" "${work_dir}/unknown.toml" +if preflight "${work_dir}/unknown.toml" >"${work_dir}/unknown.err" 2>&1; then + echo "the gateway loader accepted an unknown non-secret field" >&2 + exit 1 +fi + +# Secrets remain outside the ConfigMap even when their Secret reference is +# rendered into the workload environment. +render "${work_dir}/secret-boundary.yaml" --set server.externalDbSecret=parser-database +extract_toml "${work_dir}/secret-boundary.yaml" "${work_dir}/secret-boundary.toml" +if grep -Eqi 'parser-database|postgresql:|password' "${work_dir}/secret-boundary.toml"; then + echo "gateway.toml contains Secret-backed database material" >&2 + exit 1 +fi + +# A ConfigMap-only mutation must change the StatefulSet checksum and trigger a +# rollout. Check this against full Helm output, not just a template fragment. +default_checksum="$(yq ea -e -r 'select(.kind == "StatefulSet") | .spec.template.metadata.annotations."checksum/gateway-config"' "${work_dir}/default.yaml")" +render "${work_dir}/checksum.yaml" \ + --set-string 'gatewayConfig.openshell\.gateway.log_level=debug' +changed_checksum="$(yq ea -e -r 'select(.kind == "StatefulSet") | .spec.template.metadata.annotations."checksum/gateway-config"' "${work_dir}/checksum.yaml")" +if [[ -z "${default_checksum}" || "${default_checksum}" == "${changed_checksum}" ]]; then + echo "gateway ConfigMap changes must update the StatefulSet checksum" >&2 + exit 1 +fi + +# All maintained CI/dev overlays must render a loader-valid configuration. +for values in "${chart}"/ci/values-*.yaml; do + name="$(basename "${values}" .yaml)" + render "${work_dir}/${name}.yaml" --values "${values}" + extract_toml "${work_dir}/${name}.yaml" "${work_dir}/${name}.toml" + preflight "${work_dir}/${name}.toml" +done + +echo "rendered gateway TOML passed Rust loader validation" diff --git a/deploy/helm/test-gateway-resource-coherence.sh b/deploy/helm/test-gateway-resource-coherence.sh new file mode 100755 index 0000000000..e23f14c941 --- /dev/null +++ b/deploy/helm/test-gateway-resource-coherence.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Assert relationships that span rendered Kubernetes objects and gateway.toml. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +chart="${repo_root}/deploy/helm/openshell" +work_dir="$(mktemp -d)" +trap 'rm -rf "${work_dir}"' EXIT + +render() { + local name="$1" + shift + helm template resource-coherence "${chart}" \ + --namespace resource-namespace \ + --set agentSandbox.preflight.enabled=false \ + --set gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime.network_policy_enforced=true \ + "$@" >"${work_dir}/${name}.yaml" + + if ! awk 'BEGIN { RS="---" } + /^\n?# Source:/ && $0 !~ /\napiVersion:/ && $0 !~ /network-policy-ack\.yaml/ { + print "rendered an empty or invalid Kubernetes document:" $0 > "/dev/stderr" + exit 1 + }' "${work_dir}/${name}.yaml"; then + echo "${name}: rendered an invalid Kubernetes document" >&2 + exit 1 + fi +} + +toml() { + yq ea -e -r \ + 'select(.kind == "ConfigMap" and (.data | has("gateway.toml"))) | .data."gateway.toml"' \ + "$1" >"$2" +} + +workload_value() { + local manifest="$1" + local expression="$2" + yq ea -e -r "select(.kind == \"StatefulSet\" or .kind == \"Deployment\") | ${expression}" "${manifest}" +} + +render default +default_manifest="${work_dir}/default.yaml" +default_toml="${work_dir}/default.toml" +toml "${default_manifest}" "${default_toml}" +service_name="$(yq ea -e -r 'select(.kind == "Service") | .metadata.name' "${default_manifest}")" +service_port="$(yq ea -e -r 'select(.kind == "Service") | .spec.ports[] | select(.name == "grpc") | .port' "${default_manifest}")" +workload_port="$(workload_value "${default_manifest}" '.spec.template.spec.containers[] | select(.name == "openshell-gateway") | .ports[] | select(.name == "grpc") | .containerPort')" +config_map="$(yq ea -e -r 'select(.kind == "ConfigMap" and (.data | has("gateway.toml"))) | .metadata.name' "${default_manifest}")" +mounted_config_map="$(workload_value "${default_manifest}" '.spec.template.spec.volumes[] | select(.name == "gateway-config") | .configMap.name')" +[[ "${service_port}" == "${workload_port}" && "${config_map}" == "${mounted_config_map}" ]] +grep -F "bind_address = \"0.0.0.0:${service_port}\"" "${default_toml}" >/dev/null +grep -F "grpc_endpoint = \"https://${service_name}.resource-namespace.svc.cluster.local:${service_port}\"" "${default_toml}" >/dev/null +grep -F '[openshell.gateway.tls]' "${default_toml}" >/dev/null +[[ "$(workload_value "${default_manifest}" '.spec.template.spec.volumes[] | select(.name == "tls-cert") | .secret.secretName')" == "openshell-server-tls" ]] +[[ "$(workload_value "${default_manifest}" '.spec.template.spec.volumes[] | select(.name == "tls-client-ca") | .secret.secretName')" == "openshell-server-tls" ]] + +render tls-disabled --values "${chart}/ci/values-tls-disabled.yaml" +tls_disabled_manifest="${work_dir}/tls-disabled.yaml" +tls_disabled_toml="${work_dir}/tls-disabled.toml" +toml "${tls_disabled_manifest}" "${tls_disabled_toml}" +grep -F 'disable_tls = true' "${tls_disabled_toml}" >/dev/null +if grep -Fq '[openshell.gateway.tls]' "${tls_disabled_toml}" \ + || workload_value "${tls_disabled_manifest}" '.spec.template.spec.volumes[]?.name' | grep -Eq '^(tls-cert|tls-client-ca)$'; then + echo "TLS-disabled runtime configuration and workload mounts disagree" >&2 + exit 1 +fi + +render openshift-route --values "${chart}/ci/values-openshift-route-cert-manager.yaml" +route_manifest="${work_dir}/openshift-route.yaml" +route_toml="${work_dir}/openshift-route.toml" +toml "${route_manifest}" "${route_toml}" +route_service="$(yq ea -e -r 'select(.kind == "Route") | .spec.to.name' "${route_manifest}")" +route_port="$(yq ea -e -r 'select(.kind == "Route") | .spec.port.targetPort' "${route_manifest}")" +[[ "${route_service}" == "$(yq ea -e -r 'select(.kind == "Service") | .metadata.name' "${route_manifest}")" && "${route_port}" == "grpc" ]] +[[ "$(workload_value "${route_manifest}" '.spec.template.spec.volumes[] | select(.name == "tls-external-cert") | .secret.secretName')" == "${route_service}-server-external-tls" ]] +[[ "$(yq ea -e -r 'select(.kind == "Certificate") | .spec.secretName' "${route_manifest}" | grep -Fx "${route_service}-server-external-tls")" == "${route_service}-server-external-tls" ]] +grep -F 'external_cert_path = "/etc/openshell-tls/server-external/tls.crt"' "${route_toml}" >/dev/null +grep -F 'external_key_path = "/etc/openshell-tls/server-external/tls.key"' "${route_toml}" >/dev/null +grep -F 'external_server_names = ["openshell.example.com"]' "${route_toml}" >/dev/null + +echo "gateway Service, TLS, PKI, Route, workload, and runtime config are coherent" diff --git a/deploy/helm/test-split-ownership.sh b/deploy/helm/test-split-ownership.sh index a7a865363b..e8f28d2bf2 100755 --- a/deploy/helm/test-split-ownership.sh +++ b/deploy/helm/test-split-ownership.sh @@ -11,7 +11,7 @@ trap 'rm -rf "${work_dir}"' EXIT helm template openshell "${repo_root}/deploy/helm/openshell" \ --namespace openshell \ --set agentSandbox.preflight.enabled=false \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime.network_policy_enforced=true \ --set workspaceResources.enabled=false \ >"${work_dir}/gateway.yaml" @@ -41,7 +41,7 @@ fi helm template openshell "${repo_root}/deploy/helm/openshell" \ --namespace openshell \ --set agentSandbox.preflight.enabled=false \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime.network_policy_enforced=true \ --set-json workspaceResources=null \ >"${work_dir}/legacy-reuse-values.yaml" diff --git a/docs/kubernetes/access-control.mdx b/docs/kubernetes/access-control.mdx index 251c67c054..3f7cc50cfe 100644 --- a/docs/kubernetes/access-control.mdx +++ b/docs/kubernetes/access-control.mdx @@ -23,7 +23,7 @@ For how the CLI resolves gateways and stores credentials, refer to [Gateway Auth Kubernetes sandbox supervisors authenticate back to the gateway as sandbox workloads. By default, the Kubernetes compute driver validates each projected ServiceAccount token and returns the authenticated sandbox ID to the gateway. The gateway verifies the sandbox still exists and mints its own sandbox JWT. -Dynamic provider token grants can use SPIFFE without changing supervisor-to-gateway authentication. Set `server.providerTokenGrants.spiffe.enabled=true` to mount the SPIFFE CSI Workload API socket into gateway and sandbox pods while keeping the projected ServiceAccount token bootstrap and gateway-minted sandbox JWT path. +Dynamic provider token grants can use SPIFFE without changing supervisor-to-gateway authentication. Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` to mount the SPIFFE CSI Workload API socket into gateway and sandbox pods while keeping the projected ServiceAccount token bootstrap and gateway-minted sandbox JWT path. Provider token grants require a SPIFFE implementation such as SPIRE and identities for the gateway and sandbox pods. The repository's local SPIRE overlay assigns sandbox IDs from the pod's `openshell.ai/sandbox-id` annotation, but the gateway validation path only requires the supervisor SVID to be valid and in the same SPIFFE trust domain as the gateway SVID. Provider profiles with `token_grant` metadata cause the sandbox supervisor to request JWT-SVIDs and exchange them for upstream OAuth2 access tokens. Token-exchange profiles also require a gateway SPIFFE identity because the gateway brokers the intermediate token exchange with its own JWT-SVID. @@ -31,15 +31,23 @@ The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIF ## OIDC User Authentication -Set `server.oidc.issuer` to enable OIDC. The gateway validates the `Authorization: Bearer ` header on every request against the issuer's JWKS endpoint. It accepts JWTs signed with RS256, RS384, RS512, PS256, PS384, PS512, ES256, ES384, or EdDSA (Ed25519) keys published in the issuer JWKS. Okta tenants that sign access tokens with ES256 work without additional gateway configuration. +Set `gatewayConfig.openshell.gateway.oidc.issuer` to enable OIDC. The gateway validates the `Authorization: Bearer ` header on every request against the issuer's JWKS endpoint. It accepts JWTs signed with RS256, RS384, RS512, PS256, PS384, PS512, ES256, ES384, or EdDSA (Ed25519) keys published in the issuer JWKS. Okta tenants that sign access tokens with ES256 work without additional gateway configuration. + +Create `oidc-values.yaml`: + +```yaml +gatewayConfig: + openshell.gateway.oidc: + issuer: https://your-idp.example.com/realms/openshell + audience: openshell-cli +``` ```shell helm upgrade openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set server.oidc.issuer=https://your-idp.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli \ + --values oidc-values.yaml \ --set server.tls.clientCaSecretName="" ``` @@ -51,16 +59,13 @@ The `audience` value must match the client ID configured in your identity provid | Value | Default | Purpose | |---|---|---| -| `server.oidc.issuer` | `""` | OIDC issuer URL. Empty disables OIDC. | -| `server.oidc.dangerouslyAllowInsecureHttp` | `false` | Development-only acknowledgement for numeric-loopback HTTP. It does not allow cluster-service or remote HTTP issuers. | -| `server.oidc.jwksAllowedOrigins` | `[]` | Additional trusted HTTPS origins allowed to serve JWKS. | -| `server.oidc.caConfigMapName` | `""` | ConfigMap containing the private issuer CA in `ca.crt`. | -| `server.oidc.audience` | `openshell-cli` | Expected `aud` claim in the JWT. | -| `server.oidc.jwksTtl` | `3600` | JWKS key cache TTL in seconds. Must be greater than zero. | -| `server.oidc.rolesClaim` | `""` | Dot-separated path to the roles array in JWT claims. | -| `server.oidc.adminRole` | `""` | Role name that grants admin access. | -| `server.oidc.userRole` | `""` | Role name that grants standard user access. | -| `server.oidc.scopesClaim` | `""` | Dot-separated path to the scopes array in JWT claims. | +| `gatewayConfig.openshell.gateway.oidc.issuer` | `""` | OIDC issuer URL. Empty disables OIDC. | +| `gatewayConfig.openshell.gateway.oidc.audience` | `openshell-cli` | Expected `aud` claim in the JWT. | +| `gatewayConfig.openshell.gateway.oidc.jwks_ttl_secs` | `3600` | JWKS key cache TTL in seconds. Must be greater than zero. | +| `gatewayConfig.openshell.gateway.oidc.roles_claim` | `""` | Dot-separated path to the roles array in JWT claims. | +| `gatewayConfig.openshell.gateway.oidc.admin_role` | `""` | Role name that grants admin access. | +| `gatewayConfig.openshell.gateway.oidc.user_role` | `""` | Role name that grants standard user access. | +| `gatewayConfig.openshell.gateway.oidc.scopes_claim` | `""` | Dot-separated path to the scopes array in JWT claims. | The issuer must use HTTPS. The gateway rejects discovery and JWKS redirects, limits response sizes, requires a JSON media type, and rejects a `jwks_uri` on @@ -69,7 +74,7 @@ a different origin unless that origin appears in `jwksAllowedOrigins`. Use ### Auth-only mode vs. RBAC mode -Leave both `adminRole` and `userRole` empty to use auth-only mode: any request with a valid JWT from the configured issuer is accepted, but no role distinction is enforced. +Leave both `admin_role` and `user_role` empty to use auth-only mode: any request with a valid JWT from the configured issuer is accepted, but no role distinction is enforced. Set both values to enable RBAC mode, where the gateway checks the role claim and enforces access based on the assigned role: @@ -78,15 +83,23 @@ helm upgrade openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set server.oidc.issuer=https://your-idp.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli \ - --set server.tls.clientCaSecretName="" \ - --set server.oidc.rolesClaim=realm_access.roles \ - --set server.oidc.adminRole=openshell-admin \ - --set server.oidc.userRole=openshell-user + --values oidc-rbac-values.yaml \ + --set server.tls.clientCaSecretName="" ``` -Both `adminRole` and `userRole` must be set, or both must be empty. Setting only one is not supported. +Create `oidc-rbac-values.yaml` with the OIDC values above plus: + +```yaml +gatewayConfig: + openshell.gateway.oidc: + issuer: https://your-idp.example.com/realms/openshell + audience: openshell-cli + roles_claim: realm_access.roles + admin_role: openshell-admin + user_role: openshell-user +``` + +Both `admin_role` and `user_role` must be set, or both must be empty. Setting only one is not supported. OIDC RBAC is method-level authorization. It controls which API operations a caller can perform, but provider and sandbox records are not owned by individual OIDC subjects. In shared clusters, treat provider credentials as gateway-wide resources and use separate gateways or external tenancy controls when users must not see or attach each other's providers and sandboxes. @@ -100,27 +113,28 @@ OIDC RBAC is method-level authorization. It controls which API operations a call ## Reverse-Proxy Auth Termination -When an access proxy, such as Cloudflare Access, ngrok, or a corporate SSO gateway, handles authentication in front of the OpenShell gateway, you can explicitly allow unauthenticated user calls at the gateway: +When an access proxy, such as Cloudflare Access, ngrok, or a corporate SSO gateway, terminates TLS and handles authentication in front of the OpenShell gateway, configure the gateway for plaintext traffic from that trusted proxy and explicitly allow unauthenticated user calls: ```shell helm upgrade openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set server.auth.allowUnauthenticatedUsers=true + --values reverse-proxy-values.yaml ``` -The gateway still serves TLS and sandbox supervisors still authenticate with gateway-minted sandbox JWTs. User-facing CLI/API calls without OIDC or mTLS credentials are accepted as an unauthenticated local developer principal. The proxy is responsible for authenticating callers and forwarding only authorized traffic. - -When the gateway terminates TLS directly and callers connect without client certificates, also set `server.tls.clientCaSecretName=""` as described in the OIDC section above. - -To also disable TLS entirely (when the proxy terminates TLS before the request reaches the gateway): +Create `reverse-proxy-values.yaml`: -```shell - --set server.disableTls=true \ - --set server.auth.allowUnauthenticatedUsers=true +```yaml +server: + disableTls: true +gatewayConfig: + openshell.gateway.auth: + allow_unauthenticated_users: true ``` +The gateway listens on plaintext traffic from the proxy, while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. User-facing CLI/API calls without OIDC or mTLS credentials are accepted as an unauthenticated local developer principal. The proxy is responsible for authenticating callers and forwarding only authorized traffic. + Only enable unauthenticated users when the gateway is not reachable from outside a trusted local development environment or the proxy path is fully trusted. Never expose a plaintext, auth-disabled gateway to a public network. diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 5af0589f20..ebc198c601 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -57,7 +57,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set grpcRoute.enabled=true \ --set grpcRoute.gateway.create=true \ --set grpcRoute.gateway.className=eg @@ -107,23 +107,30 @@ The Secret may also be issued by cert-manager, or you can reference the chart's ### Install with HTTPS termination -Enable an HTTPS listener, point it at the Secret, disable gateway-pod TLS so Envoy forwards plaintext, and configure an OIDC issuer for client identity: +Enable an HTTPS listener, point it at the Secret, configure the gateway for a plaintext backend, and provide OIDC settings in `oidc-values.yaml`: + +```yaml +server: + disableTls: true +gatewayConfig: + openshell.gateway.oidc: + issuer: https://keycloak.example.com/realms/openshell + audience: openshell-cli +``` ```shell helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set grpcRoute.enabled=true \ --set grpcRoute.gateway.create=true \ --set grpcRoute.gateway.className=eg \ --set grpcRoute.gateway.listener.protocol=HTTPS \ --set grpcRoute.gateway.listener.port=443 \ --set 'grpcRoute.gateway.listener.tls.certificateRefs[0].name=openshell-ingress-tls' \ - --set server.disableTls=true \ - --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli \ + --values oidc-values.yaml \ --set 'grpcRoute.hostnames[0]=gateway.example.com' ``` @@ -149,7 +156,7 @@ As an alternative to the plaintext backend path above, the chart can create a `B client → HTTPS → Gateway (terminate TLS) → TLS (re-encrypt) → openshell gateway pod ``` -This keeps TLS on the gateway pod rather than disabling it with `server.disableTls=true`. The Gateway proxy validates the backend's certificate against a CA ConfigMap that the certgen hook auto-creates. +This keeps TLS on the gateway pod. The Gateway proxy validates the backend's certificate against a CA ConfigMap that the certgen hook auto-creates. BackendTLSPolicy is a standard Gateway API resource. It is supported on OpenShift 4.22+ (via the OpenShift gateway controller) and on other platforms where the Gateway API implementation supports it (check your controller's documentation). @@ -170,12 +177,11 @@ helm upgrade --install openshell \ --set grpcRoute.gateway.listener.port=443 \ --set 'grpcRoute.gateway.listener.tls.certificateRefs[0].name=openshell-ingress-tls' \ --set grpcRoute.backendTLSPolicy.enabled=true \ - --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli \ + --values oidc-values.yaml \ --set 'grpcRoute.hostnames[0]=gateway.example.com' ``` -Note that `server.disableTls` is **not** set — the gateway pod continues to serve TLS — but `server.tls.enableMtls=false` disables mTLS client certificate authentication because the Gateway proxy cannot present a client certificate to the backend. The chart will fail the install if you try to enable both `grpcRoute.backendTLSPolicy.enabled=true` and `server.tls.enableMtls=true` simultaneously. The BackendTLSPolicy hostname defaults to the service FQDN, which matches the SAN on the server certificate. Use OIDC for authentication (configured via `server.oidc.issuer`). +The gateway pod continues to serve TLS, while `server.tls.enableMtls=false` disables mTLS client certificate authentication because the Gateway proxy cannot present a client certificate to the backend. The chart will fail the install if you try to enable both `grpcRoute.backendTLSPolicy.enabled=true` and `server.tls.enableMtls=true` simultaneously. The BackendTLSPolicy hostname defaults to the service FQDN, which matches the SAN on the server certificate. Use OIDC for authentication (configured in `gatewayConfig.openshell.gateway.oidc`). The example above uses the default `pkiInitJob` for TLS, which creates the backend CA ConfigMap immediately. If using cert-manager instead (`--set certManager.enabled=true`), the Certificate resources are regular release objects, and a separate post-install/post-upgrade Job (`-certgen-backend-ca`) polls for up to 120 seconds waiting for cert-manager to issue the server certificate, then creates the backend CA ConfigMap. This means a single `helm install` is sufficient in most cases. diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index efef147f0a..e1cad3c062 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -52,7 +52,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set certManager.enabled=true ``` @@ -74,7 +74,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set certManager.enabled=true \ --set certManager.serverIssuerRef.name=letsencrypt-prod \ --set certManager.serverIssuerRef.kind=ClusterIssuer \ @@ -105,9 +105,9 @@ validates this at install time and fails with an actionable error if `certManager.serverDnsNames` contains internal-only entries while `serverIssuerRef` is set. -You do **not** need to set `server.grpcEndpoint` to the external hostname. +You do **not** need to set `gatewayConfig.openshell.drivers.kubernetes.grpc_endpoint` to the external hostname. Supervisors connect via the internal service name automatically. Setting -`server.grpcEndpoint` to an external hostname would cause supervisors to +`grpc_endpoint` to an external hostname would cause supervisors to receive the ACME certificate (via SNI) which they cannot verify against the chart CA. diff --git a/docs/kubernetes/migrate-gateway-config.mdx b/docs/kubernetes/migrate-gateway-config.mdx new file mode 100644 index 0000000000..741913bbcc --- /dev/null +++ b/docs/kubernetes/migrate-gateway-config.mdx @@ -0,0 +1,76 @@ +# Migrate Helm gateway configuration to `gatewayConfig` + +Chart schema v2 replaces application-specific Helm values with one non-secret +`gatewayConfig` map. Its top-level keys are TOML tables; nested maps are TOML +inline tables. Kubernetes resource references (Secrets, certificates, Services, +and mounts) remain chart values. + +## Breaking upgrade + +Remove every application-only `server.*` setting before upgrading. There is no +compatibility translation. Runtime images, proxy settings, and NetworkPolicy +acknowledgement all live in `gatewayConfig`. Kubernetes pull +policy spellings such as `Always` are not aliases: use `always`, +`if_not_present`, or `never`. `gatewayConfig` is strictly a non-secret +configuration contract: do not put passwords, tokens, private keys, +`database_url`, or other secret material in it; use Secret-backed environment, +file, or volume configuration instead. Helm serializes unknown fields +generically and cannot determine whether an arbitrary string such as +`api_token` is confidential. It rejects `database_url`, inline URL credentials, +and PEM private keys, but is not a general secret scanner. + +Keep `server.disableTls`, TLS Secret names, `server.hostGatewayIP`, `server.externalDbSecret`, +`server.sandboxNamespace`, `oidc.caConfigMapName`, certificate settings and +Service settings: these own Kubernetes resources. The chart derives their +runtime counterparts. + +## Mapping + +| Removed value | New location | +| --- | --- | +| `server.name`, `server.logLevel`, `server.enableLoopbackServiceHttp`, `server.policyValidationFailureMode` | `openshell.gateway.{name,log_level,enable_loopback_service_http,policy_validation_failure_mode}` | +| `server.grpcRateLimit.{requests,windowSeconds}` | `openshell.gateway.{grpc_rate_limit_requests,grpc_rate_limit_window_seconds}` | +| `server.otlp.{endpoint,serviceName}` | `openshell.gateway.otlp.{endpoint,service_name}` | +| `server.auth.allowUnauthenticatedUsers` | `openshell.gateway.auth.allow_unauthenticated_users` | +| `server.oidc.*` | `openshell.gateway.oidc` with snake_case keys | +| `server.sandboxImage*`, workspace storage/runtime and user namespace settings | `openshell.drivers.kubernetes`. Kubernetes AppArmor configuration is removed by RFC 0012. | +| `server.drivers.kubernetes.*` | `openshell.drivers.kubernetes` | +| `server.sandboxJwt.{gatewayId,ttlSecs,k8sSaTokenTtlSecs}` | `openshell.gateway.gateway_jwt` or `openshell.drivers.kubernetes.sa_token_ttl_secs` | +| `supervisor.image.*`, `supervisor.sandboxRuntime.*` | `openshell.drivers.kubernetes.{supervisor_image,supervisor_image_pull_policy,sandbox_runtime}`. `supervisor.topology` and `supervisor.sidecar.*` are removed by RFC 0012 with no replacement. | +| `upstreamProxy.*` | `openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*,proxy_connect_by_hostname}`. | +| `server.credentialDrivers.{kubernetesSecrets,vault}.*` | `openshell.gateway.credential_drivers` and `openshell.credential_drivers.*` | +| `server.providerTokenGrants.spiffe.{enabled,workloadApiSocketPath}` | `openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path`; omit it to disable SPIFFE provider grants. | +| `server.hostGatewayIP` | Retained as the chart-owned host-alias input; the chart derives `openshell.drivers.kubernetes.host_gateway_ip`. | + +## Examples + +```yaml +gatewayConfig: + openshell.gateway.oidc: + issuer: https://idp.example/realms/openshell + audience: openshell-cli + jwks_ttl_secs: 3600 + openshell.gateway.otlp: + endpoint: http://otel-collector:4317 + service_name: production-gateway + openshell.drivers.kubernetes: + default_image: registry.example/sandbox:latest + image_pull_policy: if_not_present + supervisor_image: registry.example/openshell-supervisor:latest + sandbox_runtime: + network_policy_enforced: true + workspace_mode: managed + grpc_rate_limit_requests: null + openshell.gateway: + grpc_rate_limit_requests: 120 + grpc_rate_limit_window_seconds: 60 + openshell.credential_drivers.kubernetes-secrets: + namespace: provider-secrets +``` + +For Vault, set `openshell.gateway.credential_drivers: [vault]` and configure +`openshell.credential_drivers.vault`; use `credentialDrivers.vault.caConfigMapName` +for a private Vault CA. For a proxy, use the Kubernetes-driver fields in +`gatewayConfig`. TLS remains Secret-backed; set `server.disableTls: true` only for +trusted external termination. A null map field omits it; null array elements +are invalid. Strings evaluate Helm `tpl`; other values do not. diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index aaad7f6c53..61f6500077 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -3,68 +3,76 @@ # SPDX-License-Identifier: Apache-2.0 title: "OpenShift" sidebar-title: "OpenShift" -description: "Install the OpenShell Helm chart on OpenShift with capability-free sandbox workloads." -keywords: "Generative AI, Cybersecurity, Kubernetes, OpenShift, SCC, Security Context Constraints, Helm, Gateway, Installation" +description: "Install the OpenShell Helm chart on OpenShift with the Agent Sandbox controller and NetworkPolicy enforcement." +keywords: "Generative AI, Cybersecurity, Kubernetes, OpenShift, Helm, Gateway, Agent Sandbox, NetworkPolicy, Installation" position: 6 --- -The Kubernetes driver resolves the UID range assigned to each OpenShift -namespace and renders the sandbox and supervisor with a numeric non-root -identity from that range. OpenShell does not require the `privileged` SCC or any -added Linux capability. - -Verify that the selected OpenShift runtime profile permits an unprivileged -process to install a nested seccomp user-notification filter and use Landlock. -OpenShell fails sandbox startup when either capability-free runtime probe fails. +The OpenShift install path is experimental. Verify that your CNI enforces ingress and egress NetworkPolicy before acknowledging the isolated sandbox runtime. +OpenShell uses the Kubernetes Agent Sandbox controller and runs with the +restricted security posture supported by current OpenShift releases. The chart +requires an explicit acknowledgement only after the cluster CNI enforces the +NetworkPolicies that isolate sandbox workloads. + ## Prerequisites -- OpenShift 4.x cluster with `oc` configured. -- Helm 3.x. -- [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs. -- A CNI that enforces ingress and egress `NetworkPolicy` in sandbox namespaces. +- OpenShift 4.x cluster with `oc` configured +- Helm 3.x +- [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs installed + +## Install + + -## Install OpenShell +## Create the namespace -Pre-create the namespace, then install the chart. Keep the default restricted -security posture and acknowledge NetworkPolicy only after validating the CNI. +Pre-create the namespace before installing the chart: ```shell oc create ns openshell +``` + +## Install the chart + +```shell helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ``` -The driver reads the namespace's `openshift.io/sa.scc.uid-range` annotation and -uses the resulting UID/GID for the sandbox, agent, trusted init containers, and -supervisor. Each container sets `allowPrivilegeEscalation: false`, drops all -Linux capabilities, and uses `RuntimeDefault` seccomp. - -Wait for the gateway: +## Wait for the gateway to be ready ```shell oc -n openshell rollout status statefulset/openshell ``` -If you set `workload.kind=deployment`, wait for `deployment/openshell` instead. +If you set `workload.kind=deployment`, use +`oc -n openshell rollout status deployment/openshell` instead. -## Connect to the Gateway + -Forward the gateway port for local evaluation: +## Connect to the gateway + +The gateway serves its normal TLS listener. Connect with `oc port-forward`: ```shell oc -n openshell port-forward svc/openshell 8080:8080 +``` + +Register the gateway with the CLI: + +```shell openshell gateway add https://127.0.0.1:8080 --local --name openshift openshell status ``` ## Options for end-to-end TLS -The steps above run the gateway over plaintext HTTP for quick evaluation. For production deployments, choose one of the approaches below based on your OpenShift version and preferences. +The steps above retain the chart's TLS defaults. For externally exposed production deployments, choose one of the approaches below based on your OpenShift version and preferences. ### End-to-end TLS using Gateway API and BackendTLSPolicy (OpenShift 4.22+) @@ -74,7 +82,7 @@ OpenShift 4.22 and later support `BackendTLSPolicy` in the Gateway API, enabling client → HTTPS → OpenShift Gateway (terminate TLS) → TLS (re-encrypt) → openshell gateway pod ``` -This removes the requirement to run the gateway with `server.disableTls=true`. The OpenShift router terminates client-facing TLS at the listener and re-encrypts when connecting to the backend service, validating the backend's certificate against a CA you provide. +The OpenShift router terminates client-facing TLS at the listener and re-encrypts when connecting to the backend service, validating the backend's certificate against a CA you provide. #### Prerequisites @@ -139,28 +147,26 @@ Install the chart with the GRPCRoute and BackendTLSPolicy enabled. The certgen h helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set server.tls.enableMtls=false \ --set grpcRoute.enabled=true \ --set grpcRoute.gateway.name=openshell-gateway \ --set grpcRoute.gateway.namespace=openshift-ingress \ --set 'grpcRoute.hostnames[0]=gateway.example.com' \ --set grpcRoute.backendTLSPolicy.enabled=true \ - --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli + --values oidc-values.yaml ``` | Override | Reason | |---|---| -| `podSecurityContext.fsGroup=null` / `securityContext.runAsUser=null` | Let OpenShift's SCC admission assign UIDs. | +| `gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced=true` | Acknowledge that the cluster CNI enforces ingress and egress NetworkPolicy in sandbox namespaces. | | `server.tls.enableMtls=false` | Disable mTLS client certificate authentication. BackendTLSPolicy only validates the server certificate; the ingress proxy cannot present a client certificate to the backend. Use OIDC for authentication instead. | | `grpcRoute.enabled=true` | Create a GRPCRoute pointing at the external Gateway. | | `grpcRoute.gateway.name` / `namespace` | Reference the Gateway created above in `openshift-ingress`. | | `grpcRoute.backendTLSPolicy.enabled=true` | Create a BackendTLSPolicy for TLS re-encryption to the gateway pod. The certgen hook auto-creates the backend CA ConfigMap. The Gateway proxy validates the backend certificate against the service FQDN, which is already in the default server certificate SANs. | | `grpcRoute.hostnames` | External hostname for the GRPCRoute. This goes on the Gateway listener certificate, not the backend certificate. | -Note that `server.disableTls` is **not** set — the gateway pod serves TLS over HTTPS without requiring client certificates. Use OIDC for authentication (see [Access Control](/kubernetes/access-control)). +The gateway pod serves TLS over HTTPS without requiring client certificates. Use OIDC for authentication (see [Access Control](/kubernetes/access-control)). **Using cert-manager instead of pkiInitJob:** Add `--set certManager.enabled=true` to the install command. The default `certManager.serverDnsNames` already includes the service FQDN needed for BackendTLSPolicy validation. The Certificate resources are regular release objects, and a separate post-install/post-upgrade Job (`-certgen-backend-ca`) polls for up to 120 seconds waiting for cert-manager to issue the server certificate, then creates the backend CA ConfigMap. A single `helm install` is sufficient in most cases. @@ -191,24 +197,21 @@ Install the chart with: helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null \ - --set server.disableTls=false \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set certManager.enabled=true \ --set certManager.serverIssuerRef.name=letsencrypt-prod \ --set certManager.serverIssuerRef.kind=ClusterIssuer \ --set certManager.serverDnsNames[0]=gateway.example.com \ --set openshiftRoute.enabled=true \ --set openshiftRoute.host=gateway.example.com \ - --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ - --set server.oidc.audience=openshell-cli + --values oidc-values.yaml ``` | Override | Reason | |---|---| | `certManager.serverIssuerRef` | Creates a second server certificate from your Issuer or ClusterIssuer for external clients. The gateway uses SNI to present this cert for the external hostname while continuing to present the internal (chart CA) cert to supervisors. The internal certificate's `ca.crt` is the chart CA that also signed the client cert, so the default `clientCaFromServerTlsSecret=true` is correct. | | `openshiftRoute.enabled` / `openshiftRoute.host` | Creates an OpenShift Route with TLS passthrough — the router forwards the encrypted connection by SNI without decrypting, so the gateway uses the SNI hostname to select the external certificate. | -| `server.oidc.issuer` / `server.oidc.audience` | Configures server-side OIDC validation. Without these, the gateway expects mTLS client certificates and rejects OIDC-only CLI connections. See [Access Control](/kubernetes/access-control). | +| `gatewayConfig.openshell.gateway.oidc` | Configures server-side OIDC validation. Without these values, the gateway expects mTLS client certificates and rejects OIDC-only CLI connections. See [Access Control](/kubernetes/access-control). | Register the gateway with the CLI over OIDC. Remote gateways authenticate CLI users via OIDC, not mTLS — see [Access Control](/kubernetes/access-control): diff --git a/docs/kubernetes/sandbox-runtime.mdx b/docs/kubernetes/sandbox-runtime.mdx index aa039ad027..1e1014031d 100644 --- a/docs/kubernetes/sandbox-runtime.mdx +++ b/docs/kubernetes/sandbox-runtime.mdx @@ -82,7 +82,8 @@ restricts them. Kubernetes policies are additive. Keep sandbox namespaces under administrative control so another principal cannot add permissive policies, create Pods with OpenShell labels, or read bootstrap Secrets. Set -`supervisor.sandboxRuntime.networkPolicyEnforced: true` only after you verify that the +`gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced: true` +only after you verify that the cluster CNI enforces both ingress and egress policies for these namespaces. ## Bootstrap a Sandbox diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index c3f4e2416e..e6f363cbd2 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -52,7 +52,7 @@ The chart does not install or upgrade the cluster-scoped Agent Sandbox CRDs or controller. -**Air-gapped clusters:** mirror the manifest above and the `registry.k8s.io/agent-sandbox/agent-sandbox-controller` image referenced inside it to your internal registry, then point the manifest's image reference at your mirror before applying. You will also need to mirror the OpenShell gateway and sandbox images — see the chart's `image.repository` value for the gateway and `server.sandboxImage` / `server.supervisorImage` for the sandbox runtime. +**Air-gapped clusters:** mirror the manifest above and the `registry.k8s.io/agent-sandbox/agent-sandbox-controller` image referenced inside it to your internal registry, then point the manifest's image reference at your mirror before applying. You will also need to mirror the OpenShell gateway and sandbox images — see `image.repository` for the gateway and `gatewayConfig.openshell.drivers.kubernetes.{default_image,sandbox_runtime_image,supervisor_image}` for sandbox runtime images. Confirm the controller pod is running before proceeding: @@ -86,7 +86,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ``` To use the latest development build instead of a stable release: @@ -96,7 +96,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version 0.0.0-dev \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' ``` The chart automatically generates PKI secrets on first install using pre-install Helm hooks. No manual secret creation is required. @@ -111,7 +111,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set workspaceResources.enabled=false \ --set server.sandboxNamespace=app-a @@ -126,7 +126,7 @@ helm upgrade --install openshell-workspace \ The workspace chart does not create the namespace or deploy a gateway. It owns only the sandbox ServiceAccount, Role, RoleBinding, and NetworkPolicy in its release namespace. For one pre-provisioned namespace, keep -`server.drivers.kubernetes.workspaceMode=shared` and set +`gatewayConfig.openshell.drivers.kubernetes.workspace_mode=shared` and set `server.sandboxNamespace=app-a`. To map multiple workspaces to separately provisioned namespaces, use `workspaceMode=operator`, configure exactly one of `operatorNamespaceLabel` or `operatorNamespaceFile`, and install the workspace @@ -199,16 +199,15 @@ The most commonly changed values are: | `workspaceResources.enabled` | Create namespace-scoped sandbox prerequisites from the gateway chart. Disable when installing the workspace chart separately. | | `server.externalDbSecret` | Secret containing a PostgreSQL connection URI in the `uri` key. Use when the database is managed outside the chart. | | `server.telemetryEnabled` | Enable anonymous OpenShell telemetry from the gateway and its sandbox supervisors. Set to `false` to opt out. | -| `server.sandboxImage` | Default sandbox image used when a sandbox does not specify one. | -| `server.sandboxImagePullSecrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | -| `server.grpcEndpoint` | Endpoint that sandbox supervisors use to call back to the gateway. Must be reachable from inside the cluster. | -| `server.disableTls` | Run the gateway over plaintext HTTP. Use only behind a trusted transport. | -| `server.auth.allowUnauthenticatedUsers` | Accept user-facing calls without OIDC or mTLS credentials. Use only for trusted local development or a fully trusted access proxy. | -| `server.enableLoopbackServiceHttp` | Enable local plaintext HTTP for loopback sandbox service URLs. Defaults to `true`. | +| `gatewayConfig.openshell.drivers.kubernetes.default_image` | Default sandbox image used when a sandbox does not specify one. | +| `gatewayConfig.openshell.drivers.kubernetes.image_pull_secrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | +| `gatewayConfig.openshell.drivers.kubernetes.grpc_endpoint` | Endpoint that sandbox supervisors use to call back to the gateway. Helm derives it from the release Service unless overridden. | +| `gatewayConfig.openshell.gateway.auth.allow_unauthenticated_users` | Accept user-facing calls without OIDC or mTLS credentials. Use only for trusted local development or a fully trusted access proxy. | +| `gatewayConfig.openshell.gateway.enable_loopback_service_http` | Enable local plaintext HTTP for loopback sandbox service URLs. Defaults to `true`. | | `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. | -| `supervisor.sandboxRuntime.networkPolicyEnforced` | Required acknowledgement that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. | -| `supervisor.sandboxRuntime.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. | -| `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | +| `gatewayConfig.openshell.drivers.kubernetes.{sandbox_runtime_image,supervisor_image}` | Images for the isolated sandbox workload and its supervisor. | +| `gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced` | Explicit acknowledgement that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace. | +| `gatewayConfig.openshell.drivers.kubernetes.{https_proxy,no_proxy,proxy_auth_*}` | Corporate HTTP forward proxy settings. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | Use a values file for repeatable deployments: @@ -217,7 +216,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --values my-values.yaml ``` @@ -232,10 +231,11 @@ kubectl -n openshell create secret docker-registry regcred \ ``` ```yaml -server: - sandboxImage: registry.example.com/team/openshell-sandbox:latest - sandboxImagePullSecrets: - - name: regcred +gatewayConfig: + openshell.drivers.kubernetes: + default_image: registry.example.com/team/openshell-sandbox:latest + image_pull_secrets: + - regcred ``` ## Configure a Corporate Upstream Proxy @@ -249,23 +249,23 @@ kubectl -n openshell create secret generic corporate-proxy-auth \ --from-literal=credentials="$PROXY_USER:$PROXY_PASSWORD" ``` -Add the proxy settings to your Helm values file. Replace the DNS suffixes and CIDRs in `noProxy` with values for your cluster. `noProxy` bypasses only the corporate proxy. OpenShell policy evaluation still applies. +Add the proxy settings to your Helm values file. Replace the DNS suffixes and CIDRs in `no_proxy` with values for your cluster. `no_proxy` bypasses only the corporate proxy. OpenShell policy evaluation still applies. ```yaml -upstreamProxy: - url: http://proxy.corp.example:8080 - noProxy: .svc,.svc.cluster.local,10.96.0.0/12,10.244.0.0/16 - authSecret: - name: corporate-proxy-auth - key: credentials - authAllowInsecure: true - +gatewayConfig: + openshell.drivers.kubernetes: + https_proxy: http://proxy.corp.example:8080 + no_proxy: .svc,.svc.cluster.local,10.96.0.0/12,10.244.0.0/16 + proxy_auth_secret_name: corporate-proxy-auth + proxy_auth_secret_key: credentials + proxy_auth_allow_insecure: true ``` -Use `authAllowInsecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. The initial release supports `http://` proxy endpoints and TLS CONNECT egress. It does not support HTTPS-to-proxy, custom corporate CA bundles, or forwarding plain HTTP egress through the proxy. +Use `proxy_auth_allow_insecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. The initial release supports `http://` proxy endpoints and TLS CONNECT egress. It does not support HTTPS-to-proxy, custom corporate CA bundles, or forwarding plain HTTP egress through the proxy. The credential mounts only in the separately scheduled supervisor Pod. The sandbox workload cannot read it through its environment or volumes. +Use `authAllowInsecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. The initial release supports `http://` proxy endpoints and TLS CONNECT egress. It does not support HTTPS-to-proxy, custom corporate CA bundles, or forwarding plain HTTP egress through the proxy. The credential Secret is mounted only into the network supervisor. ## RBAC @@ -300,7 +300,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true' \ --set serviceAccount.create=false \ --set serviceAccount.name=my-existing-sa ``` @@ -317,6 +317,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency ## Next Steps - Kubernetes sandboxes use separate workload and directly managed supervisor Pods; refer to [Sandbox runtime](/kubernetes/sandbox-runtime). +- To understand the isolated workload and supervisor model, refer to [Sandbox Runtime](/kubernetes/sandbox-runtime). - To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress). - To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/reference/gateway-auth.mdx b/docs/reference/gateway-auth.mdx index 4d9e0841ad..dd0c694ad3 100644 --- a/docs/reference/gateway-auth.mdx +++ b/docs/reference/gateway-auth.mdx @@ -99,20 +99,18 @@ The same settings are available through environment variables: | `OPENSHELL_OIDC_USER_ROLE` | Role required for standard user operations. | `openshell-user` | | `OPENSHELL_OIDC_SCOPES_CLAIM` | Dot-separated claim path containing scopes. Empty disables scope enforcement. | Empty | -For Helm deployments, set the same values under `server.oidc`: +For Helm deployments, set the same values in `gatewayConfig` under the +`openshell.gateway.oidc` TOML table: ```yaml -server: - oidc: +gatewayConfig: + openshell.gateway.oidc: issuer: https://idp.example.com/realms/openshell audience: openshell-cli - # Only needed when discovery returns a deliberately separate JWKS origin. - jwksAllowedOrigins: - - https://keys.example.com - rolesClaim: realm_access.roles - adminRole: openshell-admin - userRole: openshell-user - scopesClaim: "" + roles_claim: realm_access.roles + admin_role: openshell-admin + user_role: openshell-user + scopes_claim: "" ``` The gateway requires HTTPS for OIDC discovery and JWKS retrieval, rejects @@ -270,7 +268,7 @@ This is transparent to the user. All CLI commands work the same regardless of wh ### Plaintext -When a gateway is deployed with `server.disableTls=true`, TLS is disabled entirely. The CLI connects over plain HTTP/2. This mode is intended for local port-forwarding or gateways behind a trusted reverse proxy or tunnel that handles TLS termination externally. +When a gateway is deployed without a TLS table in `gatewayConfig`, TLS is disabled entirely. The CLI connects over plain HTTP/2. This mode is intended for local port-forwarding or gateways behind a trusted reverse proxy or tunnel that handles TLS termination externally. Register a plaintext gateway with an explicit `http://` endpoint: diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index ead6f89f11..762d3adab0 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -18,7 +18,7 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa `database_url` is env-only. The loader rejects it when it appears in the file. When `OPENSHELL_DB_URL` is unset, the gateway stores its SQLite database under `$XDG_STATE_HOME/openshell/gateway/openshell.db`. -`name` assigns an operator-facing identity to the gateway installation. Set it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. It defaults to `openshell`; the Helm chart defaults it to the chart fullname so all replicas in one installation share a name. Chart fullnames are only unique within their Kubernetes namespace, so set `server.name` explicitly when one collector receives telemetry from multiple namespaces or clusters. This identity is independent of client-side gateway aliases, TLS names, and `gateway_jwt.gateway_id`. +`name` assigns an operator-facing identity to the gateway installation. Set it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. It defaults to `openshell`; the Helm chart defaults it to the chart fullname so all replicas in one installation share a name. Chart fullnames are only unique within their Kubernetes namespace, so set `gatewayConfig.openshell.gateway.name` explicitly when one collector receives telemetry from multiple namespaces or clusters. This identity is independent of client-side gateway aliases, TLS names, and `gateway_jwt.gateway_id`. ## Package-Managed Locations @@ -296,8 +296,8 @@ Only OpenTelemetry traces are exported. Inbound gRPC and HTTP requests produce s The gateway forwards the OTLP configuration, configured gateway name, configured compute driver, and W3C trace context to managed external drivers. Built-in drivers also export their spans to the same collector through dedicated in-process providers. Driver spans retain the gateway trace context, use a distinct service name such as `openshell-driver-docker` or `openshell-driver-podman`, and carry the same `openshell.gateway.name` and `openshell.gateway.compute_driver` resource attributes as gateway spans. Compute-driver client and server spans use the same fully qualified protobuf operation name, such as `openshell.compute.v1.ComputeDriver/CreateSandbox`, in both the span name and `rpc.method`. The service name and span kind distinguish each side. Backend-prefixed child spans identify implementation work. A streaming watch records a terminal status when observed; consumer teardown without a terminal status leaves the span status unset. Operator-run external drivers own their own telemetry configuration. -For Helm deployments, set `server.otlp.endpoint` to render this table. The -optional `server.otlp.serviceName` value overrides the gateway service name; +For Helm deployments, set `gatewayConfig.openshell.gateway.otlp.endpoint` to render this table. The +optional `gatewayConfig.openshell.gateway.otlp.service_name` value overrides the gateway service name; driver service names remain fixed. The local `mise run helm:k3s:create` workflow installs a trace collector and UI, @@ -445,7 +445,7 @@ credential_drivers = ["vault"] [openshell.credential_drivers.vault] address = "https://vault.vault.svc.cluster.local:8200" -ca_bundle = "/etc/openshell/vault/ca.pem" +ca_bundle = "/etc/openshell-tls/vault/ca.crt" mount = "secret" kv_version = "2" auth_method = "kubernetes" @@ -463,11 +463,11 @@ server: existingSecret: my-preprovisioned-kek-secret ``` -For `kubernetes-secrets`, `namespace` sets where OpenShell-managed provider Secret objects are stored. When omitted, the driver uses the in-cluster ServiceAccount namespace when available, otherwise `default`. The Helm chart creates a Role granting the gateway access to all Secrets in the credential namespace because OpenShell-managed Secret names are dynamic SHA-256 hashes that cannot be restricted with `resourceNames`. Deploy credential Secrets in a dedicated namespace (`server.credentialDrivers.kubernetesSecrets.namespace`) to limit the RBAC blast radius. +For `kubernetes-secrets`, `namespace` sets where OpenShell-managed provider Secret objects are stored. When omitted, the driver uses the in-cluster ServiceAccount namespace when available, otherwise `default`. The Helm chart creates a Role granting the gateway access to all Secrets in the credential namespace because OpenShell-managed Secret names are dynamic SHA-256 hashes that cannot be restricted with `resourceNames`. Deploy credential Secrets in a dedicated namespace by setting `gatewayConfig.openshell.credential_drivers.kubernetes-secrets.namespace` to limit the RBAC blast radius. For `vault`, `address` points at the Vault service. Non-loopback endpoints must use HTTPS; plaintext HTTP is accepted only for `localhost` or an IP loopback address during local development. Vault requests do not follow redirects, preventing credentials from being replayed to a downgraded or substituted endpoint. HTTPS uses platform trust roots by default. Set `ca_bundle` to a certificate-only PEM bundle for a private Vault CA; the bundle augments platform roots and normal hostname verification remains enabled. `mount` and `kv_version` describe the KV engine where OpenShell-managed provider secrets are stored, and `auth_method = "kubernetes"` logs in with the gateway Pod's ServiceAccount token. For local or development validation, use `auth_method = "token_file"` with `token_path = "/path/to/token"`. Do not put literal Vault tokens in TOML. -For Helm deployments, set `server.credentialDrivers.vault.caConfigMapName` to a ConfigMap containing the private CA bundle under the `ca.crt` key. The chart mounts that key and renders `ca_bundle` automatically. +For Helm deployments, set `credentialDrivers.vault.caConfigMapName` to a ConfigMap containing the private CA bundle under the `ca.crt` key. The chart mounts that key and derives `ca_bundle` automatically. Provider records that already contain inline database credentials remain readable for upgrade compatibility. New provider create/update requests still submit credential values through the normal API, but the gateway stores those values through the active credential storage path and persists only handles. Before OpenShell 0.1.0, OpenShell does not automatically migrate inline refresh material or credential handles between drivers. Reconfigure refresh grants after an upgrade. Before changing credential drivers, remove affected credentials while the original driver is still available, then select the new driver and create them again. Do not run mixed gateway versions against the same refresh records. @@ -529,8 +529,8 @@ compute_driver = "kubernetes" [openshell.gateway.tls] cert_path = "/etc/openshell-tls/server/tls.crt" key_path = "/etc/openshell-tls/server/tls.key" -# client_ca_path is only rendered when server.tls.enableMtls is true (the -# default). When enableMtls is false — required for BackendTLSPolicy — the +# client_ca_path is only rendered when the client CA Secret is configured (the +# default). When it is disabled — required for BackendTLSPolicy — the # gateway runs HTTPS-only and this line is omitted by Helm. client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" # When cert-manager serverIssuerRef is configured, these are populated by Helm: @@ -640,8 +640,9 @@ already exist in the sandbox namespace. For token-exchange provider profiles, the gateway also needs access to its own SPIFFE Workload API socket. In Helm deployments, set -`server.providerTokenGrants.spiffe.enabled=true`; the chart mounts the socket -into the gateway pod and sets `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`. +`gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path`; +the chart mounts the socket into the gateway pod and sets +`OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`. The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so this validation path does not require gateway access to the SPIRE OIDC discovery endpoint or its TLS CA. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index fab785e91f..ce9b131275 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -40,9 +40,36 @@ an exited canonical process remains a terminal sandbox result. Exit code zero produces `Completed`; a nonzero or signal-normalized exit produces `Error` with the exact exit code. Driver and supervisor failures remain `Error`. +## Build with Selected Compute Drivers + +Source builds of `openshell-gateway` can include any subset of the Docker, +Podman, Kubernetes, VM, and MXC drivers. Enable the corresponding +`compute-driver-docker`, `compute-driver-podman`, `compute-driver-kubernetes`, +`compute-driver-vm`, or `compute-driver-mxc` Cargo features. For example, build +a Docker-only gateway with telemetry support: + +```shell +cargo build --release -p openshell-gateway --no-default-features --features telemetry,compute-driver-docker +``` + +On Windows, select only MXC with: + +```shell +cargo build --release -p openshell-gateway --no-default-features --features telemetry,compute-driver-mxc,bundled-z3 +``` + +The default `in-tree-compute-drivers` feature retains the full platform driver +set, including MXC on Windows. MXC links only on Windows. The other four +features link drivers on non-Windows platforms; on Windows they install +registrations that report the driver as unsupported. A build with no default +features and no driver features connects to external drivers only. +Auto-detection probes only compiled registrations. +To select a driver omitted from a custom build, configure its external +`socket_path` as described below. + ## Configure a Compute Driver -Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_driver` in the gateway TOML file: +Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set the singular `compute_driver` key in the gateway TOML file: ```toml [openshell.gateway] @@ -56,13 +83,15 @@ Non-reserved names select an extension driver and require a When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_COMPUTE_DRIVER=vm` in the launch environment. +`compute_driver` accepts exactly one scalar driver name. The legacy `compute_drivers` list is rejected by schema version 2. + Common gateway options: | Gateway TOML option | Description | |---|---| | `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, gateway endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. +Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. A TLS-enabled gateway-managed Docker, Podman, or VM driver requires a complete `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` bundle in `[openshell.gateway]`; package-managed local TLS supplies it automatically. Driver tables reject those gateway-owned fields. Kubernetes projects guest TLS through a Secret instead. See the [Gateway Configuration File](./gateway-config) reference for the full schema and migration steps. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -81,8 +110,8 @@ socket path. The endpoint replaces normal driver construction for that name, including canonical built-in names: ```shell -openshell-gateway --drivers kyma --compute-driver-socket /run/openshell/kyma.sock -openshell-gateway --drivers docker --compute-driver-socket /run/openshell/docker.sock +openshell-gateway --compute-driver kyma --compute-driver-socket /run/openshell/kyma.sock +openshell-gateway --compute-driver docker --compute-driver-socket /run/openshell/docker.sock ``` The gateway connects to the operator-provided endpoint; it does not provision @@ -157,16 +186,11 @@ gateway's primary loopback listener. Sandbox JWT authentication restricts each supervisor to the sandbox-callable RPC allowlist; no additional gateway listener is created. -The published supervisor container uses a shell-free distroless Debian 13 image. -Inspect its container logs and health status with your runtime tools; it does not -provide a shell or package manager for interactive debugging. This does not -change the tools available inside your workload image. - ## Docker Driver [Docker](https://www.docker.com/get-started/)-backed sandboxes run as containers on the gateway host. Use Docker for local development, single-machine gateways, and hosts that already use Docker Desktop or Docker Engine. -The gateway talks to the Docker daemon to create sandbox containers. Docker is also required for local image builds from directories or Dockerfiles. +The gateway talks to the Docker daemon to create sandbox containers. The trusted supervisor companion uses Docker host networking; the agent workload retains `network=none`. On Linux the supervisor reaches the gateway at @@ -177,7 +201,7 @@ daemon host. For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `sandbox_label`, `sandbox_runtime_image`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -244,7 +268,7 @@ OpenShell rejects mount `source`, `target`, and Docker volume `subpath` values with surrounding whitespace. OpenShell also rejects mount targets that replace the workspace root or container root, or contain or are contained by the configured SSH socket or reserved `/opt/openshell`, `/etc/openshell`, -`/etc/openshell-tls`, `/run/openshell`, and network +`/etc/openshell-tls`, `/run/openshell`, `/run/openshell-sidecar`, and network namespace roots. These checks do not make host bind mounts safe. ## Podman Driver @@ -254,10 +278,29 @@ namespace roots. These checks do not make host bind mounts safe. The gateway talks to the Podman API socket. The Podman driver requires Podman 5.x, cgroups v2, rootless networking, and an active Podman user socket. When `socket_path` is not set, the driver probes known socket paths, then uses the `podman` CLI to resolve the active native or machine-backed connection. It fails to start if neither method finds a socket. The agent workload uses `network=none`. Its trusted supervisor companion uses Podman's host network for its gateway session and policy-approved upstream connections. - For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.podman]`. + +### macOS Podman Socket Path + +On macOS, Homebrew-installed Podman does not create the default socket path +that the driver probes (`~/.local/share/containers/podman/machine/podman.sock`). +The actual API socket lives under `/var/folders/` in a path that macOS can +rotate after a reboot. + +If the gateway fails with `Podman socket not found; is podman machine running?` +while `podman machine list` shows a running machine, set the +`OPENSHELL_PODMAN_SOCKET` environment variable to the dynamic socket path: + +```shell +export OPENSHELL_PODMAN_SOCKET="$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}')" +``` + +Add this to your shell profile or gateway launch environment so it resolves +correctly after each reboot. Alternatively, set `socket_path` in +`[openshell.drivers.podman]` to the current path, but note that the path may +change when macOS rotates `/var/folders/`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -271,11 +314,13 @@ stopped sandboxes alone. For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. -On Linux, the host-networked supervisor uses the gateway's primary loopback -endpoint. On macOS with `podman machine`, the driver uses gvproxy's -host-loopback IP, `192.168.127.254`, by default. Set `host_gateway_ip` only when -your Podman machine uses a non-standard host-loopback address, or set -`grpc_endpoint` explicitly when the gateway is remote. +Podman preserves its runtime-selected AppArmor profile when +`app_armor_profile` is omitted. Set `Unconfined` explicitly only when the +supervisor's mount setup requires it. Explicit `RuntimeDefault` and +`Localhost/` selections fail startup when Podman reports that AppArmor +is unavailable. + +On Linux, the host-networked supervisor uses the gateway's primary loopback endpoint. On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. ### Podman Driver Config Mounts @@ -364,10 +409,16 @@ compute_driver = "vm" For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. -Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. +Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, and `krun_log_level` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. The gateway starts `openshell-driver-vm` over a private Unix socket and passes its process ID so the driver can reject unexpected local clients. The driver's standalone TCP listener is disabled unless `--allow-unauthenticated-tcp` is set for local development. +Scripts that invoke the experimental standalone driver directly must use +`--grpc-endpoint` and the `--upstream-proxy*` option family. Schema v2 removes +the previous `--openshell-endpoint`, `--https-proxy`, `--no-proxy`, and +`--proxy-*` spellings; gateway-managed deployments do not use those options +directly. + ### Local image resolution The VM driver resolves sandbox images from a local container engine before falling back to registry pulls. It tries Docker first, then uses the same Podman socket discovery as the Podman driver. On Linux with Podman, enable the API socket so the driver can find local images: @@ -376,28 +427,23 @@ The VM driver resolves sandbox images from a local container engine before falli systemctl --user start podman.socket ``` -### Network isolation +### Host Firewall + +The VM driver creates nftables rules on the host for each sandbox VM's TAP network interface. These rules provide NAT for VM connectivity and defense-in-depth isolation: unsolicited inbound connections to the VM are dropped, and the VM can only reach the gateway port on the host. Primary security enforcement (proxy-only egress and bypass detection) is handled by the sandbox supervisor inside the VM guest. -VM sandboxes boot without a virtual NIC. `openshell-sandbox` intercepts workload -network syscalls inside the guest and carries mediated streams over virtio-vsock -to the host `openshell-supervisor`, which owns DNS, policy evaluation, and -external connections. The driver does not create TAP interfaces or host -nftables rules. +On hosts with restrictive firewalls (e.g. firewalld), the host firewall may additionally block VM traffic that the driver's rules accept. If VM sandboxes cannot reach the network, verify that the host firewall allows forwarding and input for `vmtap-*` interfaces. See the [VM driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-vm/README.md#host-side-nftables-rules) for details. ### Corporate Proxy Egress -For proxy-required networks, the VM driver accepts the same corporate egress proxy keys as the Podman driver: `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, and `proxy_ca_bundle`. The host supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. +For proxy-required networks, the VM driver accepts the same corporate egress proxy keys as the Podman driver: `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, and `proxy_ca_bundle`. The in-guest supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. -The settings reach the host supervisor through driver-owned arguments, so a sandbox cannot select, alter, or disable the proxy from inside the guest. +The settings reach the guest supervisor on its command line through a per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox cannot select, alter, or disable the proxy from inside the guest — including through image `ENV`, the sandbox environment, or files baked into the image at the paths the driver uses. A proxy on the corporate network needs no special address and works on every VM sandbox. The guest's callback to the gateway never traverses the proxy. -A proxy on the gateway host works for both libkrun and QEMU sandboxes. Configure -`https_proxy = "http://host.openshell.internal:"`; the host supervisor -normalizes that name to host loopback. +A proxy on the gateway host itself works only for libkrun-backed (non-GPU) sandboxes, whose egress leaves through gvproxy: configure `https_proxy = "http://host.openshell.internal:"` rather than a `127.0.0.1` URL, because gvproxy NATs that alias to the host's `127.0.0.1`. GPU sandboxes use the QEMU/TAP backend, where `host.openshell.internal` resolves to the TAP host address and the driver's [host firewall rules](#host-firewall) allow the guest to reach only the gateway port on the host. The driver rejects a gateway-host proxy URL when a sandbox launches on QEMU instead of letting every CONNECT time out, so give GPU sandboxes a proxy address routable from the guest's masqueraded egress. -The credential and private CA material stay with the host supervisor rather -than being staged into the guest. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. +Because a microVM has no bind mounts or container secrets, the driver stages the credential (root-only) and the CA bundle into the per-sandbox overlay disk and removes them with the sandbox. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. ## Kubernetes Driver @@ -414,7 +460,7 @@ owner references or use the sandbox ServiceAccount. The operator namespace allowlist is a trust grant, not a tenant isolation mechanism. -Helm deployments set Kubernetes driver values through the chart. +Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`; schema version 2 rejects their historical `[openshell.gateway]` locations. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). @@ -422,28 +468,21 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA |---|---|---| | `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | -| `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | -| `default_image` | `server.sandboxImage` | Set the default sandbox image. | -| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | -| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | +| `[openshell.drivers.kubernetes].service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].enable_user_namespaces` | `gatewayConfig.openshell.drivers.kubernetes.enable_user_namespaces` | Enable Kubernetes user namespaces for sandbox pods. | +| `default_image` | `gatewayConfig.openshell.drivers.kubernetes.default_image` | Set the default sandbox image. | +| `image_pull_policy` | `gatewayConfig.openshell.drivers.kubernetes.image_pull_policy` | Set the canonical Kubernetes image pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | +| `image_pull_secrets` | `gatewayConfig.openshell.drivers.kubernetes.image_pull_secrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | -| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway endpoint reachable from sandbox pods. | +| `grpc_endpoint` | `gatewayConfig.openshell.drivers.kubernetes.grpc_endpoint` | Set the gateway callback endpoint reachable from sandbox pods. Raw TOML and the standalone Kubernetes driver require an explicit endpoint because the sandbox namespace does not identify the gateway Service. Helm derives it from the release's gateway Service when the value is omitted. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | -| `sandbox_runtime_image` | `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` | Override the image that provides `openshell-sandbox`. The default repository with an empty tag uses the version pinned into the gateway. | -| `sandbox_runtime_image_pull_policy` | `sandboxRuntime.image.pullPolicy` | Set the Kubernetes image pull policy for the sandbox runtime image. | -| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the image that provides `openshell-supervisor`. The default repository with an empty tag uses the version pinned into the gateway. | -| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | -| `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | -| `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | -| `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | -| `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. | -| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. The Secret mounts only in the supervisor Pod. | -| `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. | -| `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret. | -| `proxy_connect_by_hostname` | `upstreamProxy.connectByHostname` | Send hostnames rather than validated IPs in CONNECT requests. Use only when proxy ACLs require hostname targets. | -| `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | -| `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | -| `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | +| `supervisor_image` | `gatewayConfig.openshell.drivers.kubernetes.supervisor_image` | Set the isolated supervisor image. | +| `supervisor_image_pull_policy` | `gatewayConfig.openshell.drivers.kubernetes.supervisor_image_pull_policy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | +| `sandbox_runtime` | `gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime` | Configure the isolated runtime's NetworkPolicy acknowledgement and boundary port. | +| `https_proxy`, `no_proxy`, `proxy_auth_secret_name`, `proxy_auth_secret_key`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname` | `gatewayConfig.openshell.drivers.kubernetes` | Configure the corporate proxy URL, bypass list, credential Secret reference, cleartext-auth acknowledgement, and hostname CONNECT option. | +| `workspace_default_storage_size` | `gatewayConfig.openshell.drivers.kubernetes.workspace_default_storage_size` | Set the default workspace PVC size for new sandboxes. | +| `workspace_storage_class` | `gatewayConfig.openshell.drivers.kubernetes.workspace_storage_class` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | +| `sa_token_ttl_secs` | `gatewayConfig.openshell.drivers.kubernetes.sa_token_ttl_secs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | Managed-mode Secret copying requires the gateway ServiceAccount to create Secrets. Kubernetes RBAC cannot restrict Secret `create` by resource name, so @@ -452,17 +491,11 @@ remain limited to the explicitly configured TLS and image-pull Secret names. The driver creates copies only in gateway-owned managed namespaces. Do not reuse the gateway ServiceAccount for unrelated workloads. -The Kubernetes driver always places the sandbox runtime in the workload Pod and -the supervisor in a separate, directly managed Pod. The -workload Pod runs `openshell-sandbox` as the same non-root UID/GID as the agent -and requests no added Linux capabilities. The supervisor Pod runs -`openshell-supervisor`, authenticates to the gateway with a sandbox JWT, and -owns upstream connections. Both containers disable privilege escalation, drop -all capabilities, and use `RuntimeDefault` seccomp. The sandbox adds a nested -seccomp user-notification filter and Landlock restrictions before it launches -the agent. One namespace-wide, empty-egress `NetworkPolicy` is the mandatory -outer fence for all OpenShell workload Pods. It permits supervisor Pods to -reach sandbox listeners; TLS and JWT identity enforce the exact pairing. +RFC 0012 uses the Agent Sandbox controller to create a separate isolated +workload and network supervisor. The former `combined` and `sidecar` topology +knobs, supervisor sideload method, and Kubernetes AppArmor override no longer +exist. Verify that the cluster CNI enforces NetworkPolicy before setting +`gatewayConfig.openshell.drivers.kubernetes.sandbox_runtime.network_policy_enforced: true`. The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. @@ -607,7 +640,7 @@ The resolved UID/GID appear in: ### VM Driver -The VM driver injects the sandbox UID into the rootfs guest's `/etc/passwd`, `/etc/group`, and `/etc/gshadow` during rootfs preparation. Default UID is `10001`; configure `sandbox_uid` in `[openshell.drivers.vm]` to use a different value. +The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created. An unmarked overlay recovers identity from concrete overlay or prepared-image state, an explicit override, or the current image; the driver never assigns legacy `10001:10001` without persisted evidence. ### Custom Images diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index 740b5c997f..8af1d0d308 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -92,7 +92,7 @@ To override the default image references, use Helm values: | Helm value | Purpose | |---|---| | `image.repository` / `image.tag` | Override the gateway image reference. | -| `server.sandboxImage` | Override the default sandbox image. | +| `gatewayConfig.openshell.drivers.kubernetes.default_image` | Override the default sandbox image. | ## Kernel Requirements diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index a011677c23..d96397290b 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -71,7 +71,7 @@ This provides defense-in-depth: even if a container escape vulnerability exists, | Aspect | Detail | |---|---| -| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in `[openshell.drivers.kubernetes]` to enable cluster-wide. | +| Default | Disabled. Set `gatewayConfig.openshell.drivers.kubernetes.enable_user_namespaces: true` in Helm values to enable cluster-wide. | | What you can change | Enable cluster-wide through Helm or gateway config. Override per-sandbox through the `user_namespaces` field on `SandboxTemplate` in the API. | | Prerequisites | Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), a container runtime that supports user namespaces (containerd 2.0+, CRI-O 1.25+), and Linux 5.12+ for ID-mapped mounts. | | Risk if enabled with GPU | NVIDIA device plugin compatibility with user namespaces is unverified. OpenShell logs a warning when both GPU and user namespaces are active on the same sandbox. | @@ -264,7 +264,7 @@ Gateway transport uses TLS, with client certificate checks available where the d | Aspect | Detail | |---|---| | Default | Local TLS bundles enable mTLS user authentication for single-user local gateways. Helm deployments generate mTLS certificates for transport, while sandbox supervisors authenticate API calls with gateway-minted sandbox JWTs. TLS-enabled loopback gateways also accept plaintext HTTP for sandbox service hostnames by default. | -| What you can change | Configure OIDC or a trusted access proxy for multi-user gateways, set `OPENSHELL_ENABLE_MTLS_AUTH=true` for local single-user gateways, enable `server.auth.allowUnauthenticatedUsers=true` only for trusted local Kubernetes development or a fully trusted proxy, disable TLS only for trusted reverse-proxy setups, or disable loopback service HTTP with `--enable-loopback-service-http=false`. | +| What you can change | Configure OIDC or a trusted access proxy for multi-user gateways, set `OPENSHELL_ENABLE_MTLS_AUTH=true` for local single-user gateways, enable `gatewayConfig.openshell.gateway.auth.allow_unauthenticated_users=true` only for trusted local Kubernetes development or a fully trusted proxy, disable TLS only for trusted reverse-proxy setups, or disable loopback service HTTP with `--enable-loopback-service-http=false`. | | Risk if relaxed | Disabling TLS removes transport-level protection entirely. Allowing unauthenticated users removes the gateway user-auth boundary and must not be exposed to shared or public networks. Treating transport certificates as shared user identity in Kubernetes would collapse user and sandbox trust boundaries. Loopback service HTTP is local-only and rejects cross-origin browser requests, but any local process can still reach exposed service URLs directly. | | Recommendation | Use local mTLS user authentication only for single-user Docker, Podman, and VM gateways. Use OIDC or a trusted access proxy for Kubernetes and shared deployments. | diff --git a/e2e/parity/kubernetes-options-test.sh b/e2e/parity/kubernetes-options-test.sh index 4f14687e7a..ff455ef4fc 100644 --- a/e2e/parity/kubernetes-options-test.sh +++ b/e2e/parity/kubernetes-options-test.sh @@ -9,6 +9,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" SCRIPT="${ROOT}/e2e/parity/kubernetes-options.sh" TMP="$(mktemp -d)" trap 'rm -rf "${TMP}"' EXIT +TRUE_BIN="$(type -P true)" OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 bash "${SCRIPT}" --print-config baseline >"${TMP}/baseline.toml" OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 bash "${SCRIPT}" --print-config candidate >"${TMP}/candidate.toml" @@ -28,6 +29,13 @@ check(shared.isdisjoint(candidate['gateway'].keys()),'candidate leaked driver fi check(shared <= candidate['drivers']['kubernetes'].keys(),'candidate driver fields missing') b=dict(baseline['drivers']['kubernetes']); b.update({key:baseline['gateway'][key] for key in shared}) c=dict(candidate['drivers']['kubernetes']) +# These fields belong only to the frozen schema-v1 baseline. RFC 0012 removed +# their configuration surface; the Agent Sandbox controller owns its internals. +legacy_v1_only={'supervisor_sideload_method','topology','app_armor_profile'} +check(legacy_v1_only <= b.keys(),'baseline legacy fields missing') +check(legacy_v1_only.isdisjoint(c.keys()),'candidate retained RFC 0012 fields') +for field in legacy_v1_only: + b.pop(field) for projection in (b,c): projection['gateway_id']='' projection['grpc_endpoint']='http://host.openshell.internal:' @@ -39,9 +47,9 @@ PY : >"${TMP}/kubeconfig" set +e OPENSHELL_PARITY_BASELINE_ROOT="${TMP}/not-a-worktree" \ -OPENSHELL_PARITY_BASELINE_GATEWAY=/bin/true \ -OPENSHELL_PARITY_CANDIDATE_GATEWAY=/bin/true \ -OPENSHELL_PARITY_CLI=/bin/true \ +OPENSHELL_PARITY_BASELINE_GATEWAY="${TRUE_BIN}" \ +OPENSHELL_PARITY_CANDIDATE_GATEWAY="${TRUE_BIN}" \ +OPENSHELL_PARITY_CLI="${TRUE_BIN}" \ OPENSHELL_PARITY_KUBECONFIG="${TMP}/kubeconfig" \ OPENSHELL_PARITY_KUBE_CONTEXT=default/external-production-cluster \ OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 \ diff --git a/e2e/parity/kubernetes-options.sh b/e2e/parity/kubernetes-options.sh index 658e751c3e..6120fc7792 100644 --- a/e2e/parity/kubernetes-options.sh +++ b/e2e/parity/kubernetes-options.sh @@ -52,6 +52,14 @@ write_config() { local run_dir=$5 local gateway_id="step8-${variant}-${RUN_ID}" local pull_policy + local print_to_stdout=false + + # macOS does not permit redirecting a heredoc directly to /dev/stdout. + # Keep --print-config portable for the deterministic contract test. + if [ "${path}" = /dev/stdout ]; then + path="$(mktemp "${TMPDIR:-/tmp}/openshell-parity-config.XXXXXX")" + print_to_stdout=true + fi if [ "${variant}" = baseline ]; then pull_policy=IfNotPresent @@ -132,8 +140,6 @@ image_pull_secrets = ["parity-pull-secret"] service_account_name = "parity-sandbox" supervisor_image = "${SUPERVISOR_IMAGE}" supervisor_image_pull_policy = "${pull_policy}" -supervisor_sideload_method = "init-container" -topology = "combined" grpc_endpoint = "http://host.openshell.internal:${port}" ssh_socket_path = "/run/openshell/parity-kubernetes-ssh.sock" client_tls_secret_name = "parity-client-tls" @@ -143,11 +149,15 @@ sa_token_ttl_secs = 600 workspace_default_storage_size = "64Mi" workspace_storage_class = "standard" default_runtime_class_name = "${RUNTIME_CLASS}" -app_armor_profile = "Unconfined" sandbox_uid = 1000 sandbox_gid = 1000 EOF fi + + if ${print_to_stdout}; then + cat "${path}" + rm -f "${path}" + fi } if [ "${1:-}" = --print-config ]; then @@ -358,7 +368,6 @@ check(env['OPENSHELL_SSH_SOCKET_PATH']=='/run/openshell/parity-kubernetes-ssh.so check(env['OPENSHELL_SANDBOX_UID']=='1000' and env['OPENSHELL_SANDBOX_GID']=='1000','sandbox identity differs') check(('host.openshell.internal',host_ip) in hosts and ('host.docker.internal',host_ip) in hosts,'host aliases differ') check(spec['runtimeClassName']==runtime_class and spec.get('hostUsers',True) is not False,'RuntimeClass or user namespace posture differs') -check(agent['securityContext']['appArmorProfile']['type']=='Unconfined','AppArmor profile differs') check(agent['resources']['requests']=={'cpu':'250m','memory':'128Mi'},'resource requests differ') check(agent['resources']['limits']=={'cpu':'250m','memory':'128Mi'},'resource limits differ') check(vols['openshell-sa-token']['projected']['sources'][0]['serviceAccountToken']['expirationSeconds']==600,'ServiceAccount token TTL differs') @@ -369,22 +378,17 @@ check(pvc['spec']['resources']['requests']['storage']=='64Mi','PVC storage reque labels=sb['metadata']['labels'] for key in ('openshell.ai/sandbox-id','openshell.ai/sandbox-name','openshell.ai/sandbox-workspace','openshell.ai/gateway-id','openshell.ai/managed-by'): check(labels.get(key),f'managed label {key} missing') -observed_sideload='init-container' if 'openshell-supervisor-install' in inits else 'unknown' -observed_topology='combined' if [c['name'] for c in spec['containers']]==['agent'] else 'other' observed_workspace_mode='shared' if pvc['metadata']['namespace']==pod['metadata']['namespace'] and pvc['metadata']['name'].startswith('workspace-default--') else 'other' -check(observed_sideload=='init-container','supervisor sideload method differs') -check(observed_topology=='combined','supervisor topology differs') check(observed_workspace_mode=='shared','workspace placement differs') normalized={ 'scenario':'kubernetes-core-options','pod_phase':'Running','sandbox_ready':True, 'sandbox_image':agent['image'],'sandbox_image_pull_policy':agent['imagePullPolicy'], 'image_pull_secrets':['parity-pull-secret'],'service_account':'parity-sandbox', 'supervisor_image':install['image'],'supervisor_image_pull_policy':install['imagePullPolicy'], - 'supervisor_sideload_method':observed_sideload,'topology':observed_topology, 'callback_endpoint_host':'host.openshell.internal','callback_exec':True, 'ssh_socket_path':env['OPENSHELL_SSH_SOCKET_PATH'],'client_tls_secret':'parity-client-tls', 'host_gateway_ip':host_ip,'sa_token_ttl_secs':600,'runtime_class_handler':'runc', - 'enable_user_namespaces':False,'app_armor_profile':'Unconfined','sandbox_uid':1000,'sandbox_gid':1000, + 'enable_user_namespaces':False,'sandbox_uid':1000,'sandbox_gid':1000, 'workspace_mode':observed_workspace_mode,'workspace_storage':'64Mi','workspace_storage_class':'standard','pvc_phase':'Bound', 'cpu':'250m','memory':'128Mi','managed_labels':True, } diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 333acd0fec..50bda35d31 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -20,9 +20,10 @@ publish = false e2e = [] # Selects tests that rely on `host.openshell.internal` (the sandbox's stable # alias to the host running test fixtures). docker, podman, and vm wire the -# alias unconditionally; the kube driver only does so when the chart's -# `server.hostGatewayIP` is set, so `e2e-kubernetes` does NOT imply this and -# the helm wrapper opts in explicitly when it has resolved an IP. +# alias unconditionally; the kube driver only does so when +# `gatewayConfig.openshell.drivers.kubernetes.host_gateway_ip` is set, so +# `e2e-kubernetes` does NOT imply this and the Helm wrapper opts in explicitly +# when it has resolved an IP. e2e-host-gateway = ["e2e"] e2e-local-container-driver = ["e2e"] e2e-docker = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index e49c7a3b7b..95570d4bc8 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -10,10 +10,11 @@ # # Features: the default set includes `e2e-host-gateway` so tests that rely on # the sandbox-side `host.openshell.internal` alias compile and run. The -# wrapper detects the cluster's host-routable IP and wires it into the chart -# via `server.hostGatewayIP`. Targeting a cluster where the test host is -# unreachable from pods? Set OPENSHELL_E2E_KUBERNETES_FEATURES=e2e to drop the -# alias-dependent tests entirely. +# wrapper detects the cluster's host-routable IP and passes it through the +# chart-owned server.hostGatewayIP input, which derives the runtime field. +# Targeting a +# cluster where the test host is unreachable from pods? Set +# OPENSHELL_E2E_KUBERNETES_FEATURES=e2e to drop the alias-dependent tests. # # Results: `run_suite` writes a JUnit + HTML report under `results/`. Set # `OPENSHELL_E2E_REPORT_NAME` to name it per run when invoking this script repeatedly. diff --git a/e2e/rust/tests/credential_drivers.rs b/e2e/rust/tests/credential_drivers.rs index 6c4088a2f3..fe59941528 100644 --- a/e2e/rust/tests/credential_drivers.rs +++ b/e2e/rust/tests/credential_drivers.rs @@ -182,8 +182,19 @@ async fn provider_identity(provider_name: &str) -> Result = serde_json::from_str(&clean) + let listing: serde_json::Value = serde_json::from_str(&clean) .map_err(|err| format!("failed to parse provider list JSON: {err}\n{clean}"))?; + let next_page_token = listing["next_page_token"] + .as_str() + .ok_or_else(|| format!("provider list response omitted next_page_token:\n{clean}"))?; + if !next_page_token.is_empty() { + return Err(format!( + "provider list response was incomplete; received continuation token:\n{clean}" + )); + } + let providers = listing["providers"] + .as_array() + .ok_or_else(|| format!("provider list response omitted providers array:\n{clean}"))?; let provider = providers .iter() .find(|provider| provider["name"].as_str() == Some(provider_name)) diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index c4ed805293..1fee43a660 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -149,6 +149,53 @@ kube_workload_ref() { return 1 } +# Verify the ConfigMap checksum causes a live gateway rollout. This belongs in +# the harness, before port-forwards are established, because replacing a pod +# necessarily interrupts any existing port-forward to it. +verify_gateway_config_rollout() { + local workload_ref old_checksum new_checksum old_pod_uid new_pod_uid attempt + local pod_selector="app.kubernetes.io/instance=${RELEASE_NAME},app.kubernetes.io/name=openshell" + + workload_ref="$(kube_workload_ref "${RELEASE_NAME}")" + old_checksum="$(kctl -n "${NAMESPACE}" get "${workload_ref}" -o jsonpath='{.spec.template.metadata.annotations.checksum/gateway-config}')" + old_pod_uid="$(kctl -n "${NAMESPACE}" get pods -l "${pod_selector}" -o jsonpath='{.items[0].metadata.uid}')" + if [[ -z "${old_checksum}" || -z "${old_pod_uid}" ]]; then + echo "ERROR: gateway workload is missing its ConfigMap checksum or ready pod" >&2 + return 1 + fi + + echo "Verifying ConfigMap-only gateway configuration rollout..." + helmctl upgrade "${RELEASE_NAME}" "${ROOT}/deploy/helm/openshell" \ + --namespace "${NAMESPACE}" \ + --reuse-values \ + "${helm_values_args[@]}" \ + --set "fullnameOverride=openshell" \ + --set "image.repository=${REGISTRY_VALUE}/gateway" \ + --set "image.tag=${IMAGE_TAG_VALUE}" \ + --set-string "gatewayConfig.openshell\\.drivers\\.kubernetes.supervisor_image=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" \ + "${helm_extra_args[@]}" \ + "${helm_post_renderer_args[@]}" \ + --set-string 'gatewayConfig.openshell\.gateway.log_level=debug' \ + --wait --timeout 5m + + new_checksum="$(kctl -n "${NAMESPACE}" get "${workload_ref}" -o jsonpath='{.spec.template.metadata.annotations.checksum/gateway-config}')" + if [[ -z "${new_checksum}" || "${new_checksum}" == "${old_checksum}" ]]; then + echo "ERROR: ConfigMap-only gateway configuration change did not update workload checksum" >&2 + return 1 + fi + kctl -n "${NAMESPACE}" rollout status "${workload_ref}" --timeout=5m || return 1 + + for attempt in $(seq 1 60); do + new_pod_uid="$(kctl -n "${NAMESPACE}" get pods -l "${pod_selector}" -o jsonpath='{.items[0].metadata.uid}')" + if [[ -n "${new_pod_uid}" && "${new_pod_uid}" != "${old_pod_uid}" ]]; then + return 0 + fi + sleep 1 + done + echo "ERROR: gateway workload rolled out without replacing its pod" >&2 + return 1 +} + deploy_postgres_fixture() { local secret_name="$1" local pg_uri @@ -519,10 +566,8 @@ run_scenario() { --set "fullnameOverride=openshell" \ --set "image.repository=${REGISTRY_VALUE}/gateway" \ --set "image.tag=${IMAGE_TAG_VALUE}" \ - --set "sandboxRuntime.image.repository=${REGISTRY_VALUE}/sandbox" \ - --set "sandboxRuntime.image.tag=${IMAGE_TAG_VALUE}" \ - --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ - --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ + --set-string "gatewayConfig.openshell\\.drivers\\.kubernetes.sandbox_runtime_image=${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" \ + --set-string "gatewayConfig.openshell\\.drivers\\.kubernetes.supervisor_image=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" \ "${helm_post_renderer_args[@]}" \ "$@" \ --wait --timeout 5m @@ -1009,6 +1054,10 @@ fi helm_extra_args=() helm_post_renderer_args=() helm_extra_args+=(--set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}") +helm_extra_args+=(--set 'gatewayConfig.openshell\.drivers\.kubernetes.sandbox_runtime.network_policy_enforced=true') +# Keep the runtime configuration aligned with the locally built/imported image +# without creating a second Helm values API for driver configuration. +helm_extra_args+=(--set-string "gatewayConfig.openshell\\.drivers\\.kubernetes.supervisor_image=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}") if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" != "1" ]; then echo "ERROR: external Kubernetes driver e2e requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=1." >&2 @@ -1020,7 +1069,10 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then ) fi if [ -n "${HOST_GATEWAY_IP}" ]; then - helm_extra_args+=(--set "server.hostGatewayIP=${HOST_GATEWAY_IP}") + # server.hostGatewayIP owns both the gateway runtime field and sandbox-pod + # hostAliases. Exercise the public chart input rather than bypassing it with + # a direct gatewayConfig override. + helm_extra_args+=(--set-string "server.hostGatewayIP=${HOST_GATEWAY_IP}") fi helm_values_args=(--values "${ROOT}/deploy/helm/openshell/ci/values-skaffold.yaml") @@ -1066,13 +1118,14 @@ if [ "${OPENSHELL_E2E_KUBE_CORPORATE_PROXY:-0}" = "1" ]; then fi CORPORATE_PROXY_VALUES="${WORKDIR}/corporate-proxy-values.yaml" cat >"${CORPORATE_PROXY_VALUES}" <>"${CORPORATE_PROXY_VALUES}" <