diff --git a/deploy/k8s-dev/operator-deployment.yaml b/deploy/k8s-dev/operator-deployment.yaml index 963ea8d..62048da 100755 --- a/deploy/k8s-dev/operator-deployment.yaml +++ b/deploy/k8s-dev/operator-deployment.yaml @@ -61,6 +61,8 @@ spec: value: "true" - name: OPERATOR_TENANT_MONITOR_INTERVAL_SECONDS value: "300" + - name: OPERATOR_CLUSTER_DOMAIN + value: "cluster.local" - name: POD_NAME valueFrom: fieldRef: diff --git a/deploy/rustfs-operator/README.md b/deploy/rustfs-operator/README.md index fbcfb83..74f4e9f 100755 --- a/deploy/rustfs-operator/README.md +++ b/deploy/rustfs-operator/README.md @@ -53,11 +53,14 @@ The following table lists the configurable parameters of the RustFS Operator cha | `operator.prometheusRule.enabled` | Create Prometheus alert rules for operator and tenant storage health | `false` | | `operator.tenantMonitor.enabled` | Poll RustFS tenant storage health and capacity metrics | `true` | | `operator.tenantMonitor.intervalSeconds` | Tenant storage monitor interval | `300` | +| `clusterDomain` | Kubernetes cluster DNS domain used for Tenant peer URLs, generated TLS SANs, and operator STS auto TLS | `cluster.local` | | `operator.env` | Environment variables | `[{name: RUST_LOG, value: info}]` | | `operator.nodeSelector` | Node selector for pod placement | `{}` | | `operator.tolerations` | Tolerations for pod scheduling | `[]` | | `operator.affinity` | Affinity rules for pod scheduling | `{}` | +Use `clusterDomain` for custom Kubernetes DNS domains; `operator.env` must not set `OPERATOR_CLUSTER_DOMAIN`. + ### Operator STS Configuration | Parameter | Description | Default | diff --git a/deploy/rustfs-operator/templates/deployment.yaml b/deploy/rustfs-operator/templates/deployment.yaml index 77637cf..4495a18 100755 --- a/deploy/rustfs-operator/templates/deployment.yaml +++ b/deploy/rustfs-operator/templates/deployment.yaml @@ -3,6 +3,11 @@ {{- if and (not .Values.operator.metrics.enabled) (or (hasKey $livenessProbe "httpGet") (hasKey $readinessProbe "httpGet")) -}} {{- fail "operator.metrics.enabled=false requires overriding operator.livenessProbe and operator.readinessProbe because the chart defaults use the metrics port" -}} {{- end -}} +{{- range $env := .Values.operator.env }} +{{- if eq $env.name "OPERATOR_CLUSTER_DOMAIN" -}} +{{- fail "operator.env must not set OPERATOR_CLUSTER_DOMAIN; use clusterDomain instead" -}} +{{- end -}} +{{- end -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -81,6 +86,8 @@ spec: value: {{ .Values.operator.tenantMonitor.enabled | quote }} - name: OPERATOR_TENANT_MONITOR_INTERVAL_SECONDS value: {{ .Values.operator.tenantMonitor.intervalSeconds | quote }} + - name: OPERATOR_CLUSTER_DOMAIN + value: {{ .Values.clusterDomain | default "cluster.local" | quote }} - name: OPERATOR_STS_ENABLED value: {{ .Values.sts.enabled | quote }} - name: OPERATOR_STS_AUDIENCE diff --git a/deploy/rustfs-operator/values.schema.json b/deploy/rustfs-operator/values.schema.json new file mode 100644 index 0000000..9c19afc --- /dev/null +++ b/deploy/rustfs-operator/values.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "clusterDomain": { + "type": "string", + "minLength": 1, + "oneOf": [ + { + "maxLength": 253, + "pattern": "^([A-Za-z0-9]([-A-Za-z0-9]{0,61}[A-Za-z0-9])?)(\\.([A-Za-z0-9]([-A-Za-z0-9]{0,61}[A-Za-z0-9])?))*$" + }, + { + "maxLength": 254, + "pattern": "^([A-Za-z0-9]([-A-Za-z0-9]{0,61}[A-Za-z0-9])?)(\\.([A-Za-z0-9]([-A-Za-z0-9]{0,61}[A-Za-z0-9])?))*\\.$" + } + ], + "description": "Kubernetes cluster DNS domain used for Tenant peer URLs, generated TLS SANs, and operator STS auto TLS." + } + } +} diff --git a/deploy/rustfs-operator/values.yaml b/deploy/rustfs-operator/values.yaml index 15b49da..db0cb90 100755 --- a/deploy/rustfs-operator/values.yaml +++ b/deploy/rustfs-operator/values.yaml @@ -1,5 +1,8 @@ # Default values for rustfs-operator. +# Kubernetes cluster DNS domain used for Tenant peer URLs and generated TLS SANs. +clusterDomain: cluster.local + # Operator deployment configuration operator: # Number of operator replicas diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index 5cd46bd..4043641 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -119,12 +119,15 @@ Common chart sections: | `sts` | Operator STS endpoint, service port, TokenReview audience, and TLS handling. | | `serviceAccount` / `rbac` | Operator ServiceAccount and RBAC creation. | | `console` | Operator Console backend/UI Deployment, service, session cookie secret, ingress, resources, and optional split frontend. | +| `clusterDomain` | Kubernetes cluster DNS domain used for Tenant peer URLs, generated TLS SANs, and operator STS auto TLS. Defaults to `cluster.local`. | | `namespace` | Namespace override for chart resources; defaults to the Helm release namespace. | | `commonLabels` / `commonAnnotations` | Labels and annotations added to chart-managed resources. | Example production-oriented values: ```yaml +clusterDomain: cluster.local + operator: replicas: 2 image: @@ -348,7 +351,7 @@ The operator reserves these environment variables and manages them automatically - `RUSTFS_KMS_*` variables; use `spec.encryption` instead. - TLS-related RustFS variables when Tenant TLS is enabled. -For a single-pool single-node single-disk Tenant, `RUSTFS_VOLUMES` is rendered as the local data path, for example `/data/rustfs0`. Multi-pool tenants and other layouts render peer DNS URLs through the Tenant headless Service and are validated by RustFS at runtime. +For a single-pool single-node single-disk Tenant, `RUSTFS_VOLUMES` is rendered as the local data path, for example `/data/rustfs0`. Multi-pool tenants and other layouts render peer DNS URLs through the Tenant headless Service and are validated by RustFS at runtime. Set the Helm chart `clusterDomain` value when the Kubernetes cluster DNS domain is not `cluster.local`; the same domain is used for generated TLS SANs. `podDeletionPolicyWhenNodeIsDown` accepts: @@ -437,7 +440,7 @@ spec: The default certificate is projected to `rustfs_cert.pem` and `rustfs_key.pem` at `mountPath`, so RustFS can use it as the fallback certificate and for internode HTTPS. Each `hosts` value is projected as a RustFS SNI directory, for example `s3.example.com/rustfs_cert.pem` and `s3.example.com/rustfs_key.pem`. When `certificates` is set, omitted `includeGeneratedDnsNames` is treated as `true` only on the `default: true` certificate. Non-default entries include only `hosts` and `certManager.dnsNames` unless they explicitly set `includeGeneratedDnsNames: true`. -When `enableInternodeHttps: true`, the default managed certificate must cover the generated RustFS peer DNS names. Keep `includeGeneratedDnsNames` enabled, or list the generated peer names explicitly in `hosts` or `certManager.dnsNames`. +When `enableInternodeHttps: true`, the default managed certificate must cover the RustFS peer DNS names used by the operator and `RUSTFS_VOLUMES`. Keep `includeGeneratedDnsNames` enabled, or list the required names explicitly in `hosts` or `certManager.dnsNames`. Clusters with a custom Kubernetes DNS domain should set the Helm chart `clusterDomain`; required names then use `.svc.`. External certificates must cover the headless Service FQDN (`-hl..svc.`) and pod FQDNs; a wildcard such as `*.-hl..svc.` covers the generated pod FQDNs. When `certificates` is set, configure process-wide trust with top-level `caTrust` or the `caTrust` on the `default: true` certificate. The legacy `certManager.caTrust` field is only used by the single-certificate form, and `certManager.caTrust` on non-default entries is rejected. ### 7.6 Logging diff --git a/docs/operator-user-guide.zh-CN.md b/docs/operator-user-guide.zh-CN.md index 58fa598..72b4e8d 100644 --- a/docs/operator-user-guide.zh-CN.md +++ b/docs/operator-user-guide.zh-CN.md @@ -121,12 +121,15 @@ helm upgrade --install rustfs-operator deploy/rustfs-operator/ \ | `sts` | Operator STS 端点、Service 端口、TokenReview audience 和 TLS。 | | `serviceAccount` / `rbac` | Operator ServiceAccount 和 RBAC 创建策略。 | | `console` | Operator Console 后端/UI Deployment、Service、session cookie 密钥、Ingress、资源和可选独立前端。 | +| `clusterDomain` | Kubernetes 集群 DNS 域，用于 Tenant peer URL、自动生成的 TLS SAN 和 operator STS 自动 TLS。默认 `cluster.local`。 | | `namespace` | Chart 资源命名空间覆盖；默认使用 Helm release namespace。 | | `commonLabels` / `commonAnnotations` | 添加到 Chart 管理资源上的统一 label 和 annotation。 | 生产风格 values 示例： ```yaml +clusterDomain: cluster.local + operator: replicas: 2 image: @@ -350,7 +353,7 @@ Operator 会自动管理以下环境变量： - `RUSTFS_KMS_*` 变量；请改用 `spec.encryption` 配置 - 启用 TLS 时的 RustFS TLS 相关变量 -对于单 pool 的单节点单盘 Tenant，`RUSTFS_VOLUMES` 会渲染为本地数据路径，例如 `/data/rustfs0`。多 pool Tenant 和其他布局仍会通过 Tenant headless Service 渲染 peer DNS URL，并由 RustFS 在运行时校验。 +对于单 pool 的单节点单盘 Tenant，`RUSTFS_VOLUMES` 会渲染为本地数据路径，例如 `/data/rustfs0`。多 pool Tenant 和其他布局仍会通过 Tenant headless Service 渲染 peer DNS URL，并由 RustFS 在运行时校验。当 Kubernetes 集群 DNS 域不是 `cluster.local` 时，请设置 Helm chart 的 `clusterDomain`；自动生成的 TLS SAN 也会使用同一个域。 `podDeletionPolicyWhenNodeIsDown` 支持以下值： @@ -439,7 +442,7 @@ spec: 默认条目会被投影到 `mountPath` 根目录下的 `rustfs_cert.pem` 和 `rustfs_key.pem`，供 RustFS 作为 fallback 证书和节点间 HTTPS 证书使用。每个 `hosts` 值会被投影为 RustFS SNI 子目录，例如 `s3.example.com/rustfs_cert.pem` 和 `s3.example.com/rustfs_key.pem`。 配置了 `certificates` 时，省略 `includeGeneratedDnsNames` 只有在 `default: true` 证书条目中才按 `true` 处理。非默认条目只包含 `hosts` 和 `certManager.dnsNames`，除非显式设置 `includeGeneratedDnsNames: true`。 -当 `enableInternodeHttps: true` 时，默认的托管证书必须覆盖 RustFS 自动生成的节点间 DNS 名称。应保持 `includeGeneratedDnsNames` 启用，或在 `hosts` / `certManager.dnsNames` 中显式列出这些节点间名称。 +当 `enableInternodeHttps: true` 时，默认托管证书必须覆盖 operator 和 `RUSTFS_VOLUMES` 实际使用的 RustFS peer DNS 名称。应保持 `includeGeneratedDnsNames` 启用，或在 `hosts` / `certManager.dnsNames` 中显式列出必需名称。使用自定义 Kubernetes DNS 域的集群应设置 Helm chart 的 `clusterDomain`；必需名称会使用 `.svc.`。外部证书必须覆盖 headless Service FQDN（`-hl..svc.`）和 pod FQDN；类似 `*.-hl..svc.` 的 wildcard 可以覆盖生成的 pod FQDN。 配置了 `certificates` 时，进程级 trust 应放在顶层 `caTrust` 或 `default: true` 证书条目的 `caTrust` 中。旧的 `certManager.caTrust` 只对单证书写法生效，非默认条目上的 `certManager.caTrust` 会被拒绝。 ### 7.6 日志配置 diff --git a/e2e/src/bin/rustfs-e2e.rs b/e2e/src/bin/rustfs-e2e.rs index 46d8252..27170ba 100644 --- a/e2e/src/bin/rustfs-e2e.rs +++ b/e2e/src/bin/rustfs-e2e.rs @@ -118,7 +118,7 @@ fn create_kind_cluster(config: &E2eConfig) -> Result<()> { live::require_live_enabled(config)?; let kind = KindCluster::new(config.clone()); kind.reset_host_storage_dirs()?; - kind.create_command().run_checked()?; + kind.create_command()?.run_checked()?; Ok(()) } diff --git a/e2e/src/framework/cert_manager_tls.rs b/e2e/src/framework/cert_manager_tls.rs index bbf4e69..acf108e 100644 --- a/e2e/src/framework/cert_manager_tls.rs +++ b/e2e/src/framework/cert_manager_tls.rs @@ -30,6 +30,7 @@ use std::path::Path; use std::time::Duration; use tempfile::TempDir; +use crate::framework::cluster_dns; use crate::framework::{ assertions, command::CommandSpec, config::E2eConfig, kubectl::Kubectl, resources, storage, tenant_factory::TenantTemplate, wait, @@ -171,17 +172,29 @@ fn tls_certificate_dns_names(config: &E2eConfig, tenant: &Tenant) -> Vec if cert_manager.include_generated_dns_names.unwrap_or(true) { let tenant_name = &config.tenant_name; let namespace = &config.test_namespace; + let cluster_domain = &config.cluster_domain; let io_service = format!("{tenant_name}-io"); let headless_service = format!("{tenant_name}-hl"); names.insert(format!("{io_service}.{namespace}.svc")); - names.insert(format!("{io_service}.{namespace}.svc.cluster.local")); + names.insert(cluster_dns::service_fqdn( + &io_service, + namespace, + cluster_domain, + )); names.insert(format!("{headless_service}.{namespace}.svc")); - names.insert(format!("{headless_service}.{namespace}.svc.cluster.local")); + names.insert(cluster_dns::service_fqdn( + &headless_service, + namespace, + cluster_domain, + )); for pool in &tenant.spec.pools { for ordinal in 0..pool.servers.max(0) { - names.insert(format!( - "{tenant_name}-{}-{ordinal}.{headless_service}.{namespace}.svc.cluster.local", - pool.name + let pod_name = format!("{tenant_name}-{}-{ordinal}", pool.name); + names.insert(cluster_dns::pod_fqdn( + &pod_name, + &headless_service, + namespace, + cluster_domain, )); } } diff --git a/e2e/src/framework/cluster_dns.rs b/e2e/src/framework/cluster_dns.rs new file mode 100644 index 0000000..86b9693 --- /dev/null +++ b/e2e/src/framework/cluster_dns.rs @@ -0,0 +1,84 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub const DEFAULT_CLUSTER_DOMAIN: &str = "cluster.local"; +const E2E_CLUSTER_DOMAIN_ENV: &str = "RUSTFS_E2E_CLUSTER_DOMAIN"; +const MAX_DNS_NAME_LENGTH: usize = 253; + +pub fn configured_cluster_domain(value: &str) -> String { + normalize_cluster_domain(value).unwrap_or_else(|| { + panic!( + "{E2E_CLUSTER_DOMAIN_ENV} must be a valid DNS domain, for example 'cluster.local' or 'k8s.mse.cloud'" + ) + }) +} + +pub fn service_fqdn(service_name: &str, namespace: &str, cluster_domain: &str) -> String { + format!("{service_name}.{namespace}.svc.{cluster_domain}") +} + +pub fn pod_fqdn( + pod_name: &str, + headless_service: &str, + namespace: &str, + cluster_domain: &str, +) -> String { + format!("{pod_name}.{headless_service}.{namespace}.svc.{cluster_domain}") +} + +fn normalize_cluster_domain(value: &str) -> Option { + let domain = value + .strip_suffix('.') + .unwrap_or(value) + .to_ascii_lowercase(); + if domain.is_empty() + || domain.len() > MAX_DNS_NAME_LENGTH + || !domain.split('.').all(valid_dns_label) + { + return None; + } + Some(domain) +} + +fn valid_dns_label(label: &str) -> bool { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && label + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && label + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn configured_cluster_domain_normalizes_supported_values() { + assert_eq!(configured_cluster_domain("K8S.MSE.Cloud."), "k8s.mse.cloud"); + } + + #[test] + #[should_panic(expected = "RUSTFS_E2E_CLUSTER_DOMAIN must be a valid DNS domain")] + fn configured_cluster_domain_rejects_invalid_values() { + configured_cluster_domain(&format!("{}.local", "a".repeat(64))); + } +} diff --git a/e2e/src/framework/config.rs b/e2e/src/framework/config.rs index c0c079f..89161b4 100644 --- a/e2e/src/framework/config.rs +++ b/e2e/src/framework/config.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::framework::cluster_dns; use operator::types::v1alpha1::k8s::PodManagementPolicy; use std::ops::{Deref, DerefMut}; use std::path::PathBuf; @@ -45,6 +46,7 @@ pub struct E2eConfig { pub operator_image: String, pub console_web_image: String, pub cert_manager_version: String, + pub cluster_domain: String, pub kind_config: PathBuf, pub live_enabled: bool, } @@ -112,6 +114,11 @@ impl E2eConfig { "RUSTFS_E2E_CERT_MANAGER_VERSION", DEFAULT_CERT_MANAGER_VERSION, ), + cluster_domain: cluster_dns::configured_cluster_domain(&env_or( + &get_env, + "RUSTFS_E2E_CLUSTER_DOMAIN", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + )), kind_config: PathBuf::from(env_or( &get_env, "RUSTFS_E2E_KIND_CONFIG", @@ -190,6 +197,7 @@ mod tests { assert_eq!(config.pv_count, 12); assert_eq!(config.rustfs_image, DEFAULT_RUSTFS_IMAGE); assert_eq!(config.cert_manager_version, "v1.16.2"); + assert_eq!(config.cluster_domain, "cluster.local"); assert_eq!( config.kind_config, std::path::PathBuf::from("e2e/manifests/kind-rustfs-e2e.yaml") @@ -207,6 +215,7 @@ mod tests { "RUSTFS_E2E_CONSOLE_WEB_IMAGE" => Some("rustfs/console-web:other".to_string()), "RUSTFS_E2E_SERVER_IMAGE" => Some("rustfs/rustfs:dev".to_string()), "RUSTFS_E2E_CERT_MANAGER_VERSION" => Some("v9.9.9".to_string()), + "RUSTFS_E2E_CLUSTER_DOMAIN" => Some("K8S.MSE.Cloud.".to_string()), "RUSTFS_E2E_LIVE" => Some("true".to_string()), _ => None, }); @@ -217,6 +226,7 @@ mod tests { assert_eq!(config.console_web_image, "rustfs/console-web:e2e"); assert_eq!(config.rustfs_image, "rustfs/rustfs:dev"); assert_eq!(config.cert_manager_version, "v9.9.9"); + assert_eq!(config.cluster_domain, "k8s.mse.cloud"); assert!(config.live_enabled); } } diff --git a/e2e/src/framework/deploy.rs b/e2e/src/framework/deploy.rs index 832f2c1..28d169a 100644 --- a/e2e/src/framework/deploy.rs +++ b/e2e/src/framework/deploy.rs @@ -62,7 +62,7 @@ pub fn deploy_dev(config: &E2eConfig) -> Result<()> { kubectl .apply_yaml_command(patch_images_and_tags( - OPERATOR_DEPLOYMENT, + &patch_operator_cluster_domain(OPERATOR_DEPLOYMENT, &config.cluster_domain), &config.operator_image, E2E_OPERATOR_IMAGE_TAG_DEFAULT, )) @@ -140,6 +140,15 @@ stringData: ) } +fn patch_operator_cluster_domain(yaml: &str, cluster_domain: &str) -> String { + yaml.replace( + " - name: OPERATOR_CLUSTER_DOMAIN\n value: \"cluster.local\"", + &format!( + " - name: OPERATOR_CLUSTER_DOMAIN\n value: \"{cluster_domain}\"" + ), + ) +} + fn patch_images_and_tags(manifest: &str, image: &str, fallback: &str) -> String { if image == fallback { manifest.to_string() @@ -152,7 +161,7 @@ fn patch_images_and_tags(manifest: &str, image: &str, fallback: &str) -> String mod tests { use super::{ E2E_CONSOLE_WEB_IMAGE_TAG_DEFAULT, E2E_CONTROL_PLANE_DEPLOYMENTS, - E2E_OPERATOR_IMAGE_TAG_DEFAULT, patch_images_and_tags, + E2E_OPERATOR_IMAGE_TAG_DEFAULT, patch_images_and_tags, patch_operator_cluster_domain, }; #[test] @@ -174,6 +183,17 @@ mod tests { assert!(!web.contains(E2E_CONSOLE_WEB_IMAGE_TAG_DEFAULT)); } + #[test] + fn patch_operator_cluster_domain_updates_managed_env() { + let patched = patch_operator_cluster_domain( + "env:\n - name: OPERATOR_CLUSTER_DOMAIN\n value: \"cluster.local\"\n", + "k8s.mse.cloud", + ); + + assert!(patched.contains("value: \"k8s.mse.cloud\"")); + assert!(!patched.contains("value: \"cluster.local\"")); + } + #[test] fn rollout_deployments_are_explicit_and_stable() { assert_eq!( diff --git a/e2e/src/framework/kind.rs b/e2e/src/framework/kind.rs index fab4158..805fe1c 100644 --- a/e2e/src/framework/kind.rs +++ b/e2e/src/framework/kind.rs @@ -13,6 +13,7 @@ // limitations under the License. use anyhow::{Context, Result, bail}; +use serde_yaml_ng::Value; use std::fs; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; @@ -36,15 +37,20 @@ impl KindCluster { Self { config } } - pub fn create_command(&self) -> CommandSpec { - CommandSpec::new("kind").args([ - "create".to_string(), - "cluster".to_string(), - "--name".to_string(), - self.config.cluster_name.clone(), - "--config".to_string(), - self.config.kind_config.display().to_string(), - ]) + pub fn create_command(&self) -> Result { + let source = fs::read_to_string(&self.config.kind_config) + .with_context(|| format!("read Kind config {}", self.config.kind_config.display()))?; + let rendered = render_kind_config(&source, &self.config.cluster_domain)?; + + Ok(CommandSpec::new("kind") + .args([ + "create".to_string(), + "cluster".to_string(), + "--name".to_string(), + self.config.cluster_name.clone(), + "--config=-".to_string(), + ]) + .stdin(rendered)) } pub fn host_storage_dirs(&self) -> Vec { @@ -245,6 +251,32 @@ impl KindCluster { } } +fn render_kind_config(source: &str, cluster_domain: &str) -> Result { + let mut config: Value = serde_yaml_ng::from_str(source).context("parse Kind config")?; + let nodes = config + .get_mut("nodes") + .and_then(Value::as_sequence_mut) + .context("Kind config must contain a nodes list")?; + let control_plane = nodes + .iter_mut() + .find(|node| node.get("role").and_then(Value::as_str) == Some("control-plane")) + .context("Kind config must contain a control-plane node")?; + let control_plane = control_plane + .as_mapping_mut() + .context("Kind control-plane node must be an object")?; + let patches = control_plane + .entry(Value::String("kubeadmConfigPatches".to_string())) + .or_insert_with(|| Value::Sequence(Vec::new())) + .as_sequence_mut() + .context("Kind control-plane kubeadmConfigPatches must be a list")?; + + patches.push(Value::String(format!( + "kind: ClusterConfiguration\nnetworking:\n dnsDomain: \"{cluster_domain}\"\n" + ))); + + serde_yaml_ng::to_string(&config).context("serialize Kind config") +} + fn ensure_host_storage_dir_is_empty(dir: &Path) -> Result<()> { let mut entries = fs::read_dir(dir) .with_context(|| format!("read e2e host storage dir {}", dir.display()))?; @@ -356,6 +388,7 @@ mod tests { use super::{ KindCluster, ensure_dedicated_host_storage_dir, ensure_host_storage_dir_is_absent, ensure_host_storage_dir_is_empty, ensure_host_storage_dir_owner_allows_cleanup, + render_kind_config, }; use crate::framework::config::E2eConfig; @@ -384,6 +417,38 @@ mod tests { ); } + #[test] + fn rendered_kind_config_uses_custom_cluster_domain() { + let rendered = render_kind_config( + include_str!("../../manifests/kind-rustfs-e2e.yaml"), + "k8s.mse.cloud", + ) + .expect("Kind config should render"); + let config: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&rendered).expect("rendered Kind config should parse"); + let patches = config["nodes"] + .as_sequence() + .and_then(|nodes| { + nodes.iter().find(|node| { + node.get("role").and_then(serde_yaml_ng::Value::as_str) == Some("control-plane") + }) + }) + .and_then(|node| node.get("kubeadmConfigPatches")) + .and_then(serde_yaml_ng::Value::as_sequence) + .expect("control-plane patches should exist"); + let cluster_config = patches + .iter() + .filter_map(serde_yaml_ng::Value::as_str) + .filter_map(|patch| serde_yaml_ng::from_str::(patch).ok()) + .find(|patch| patch["kind"].as_str() == Some("ClusterConfiguration")) + .expect("ClusterConfiguration patch should exist"); + + assert_eq!( + cluster_config["networking"]["dnsDomain"].as_str(), + Some("k8s.mse.cloud") + ); + } + #[test] fn host_storage_cleanup_only_allows_dedicated_tmp_dirs() { assert!( diff --git a/e2e/src/framework/mod.rs b/e2e/src/framework/mod.rs index 4d8f32f..e2a48e7 100644 --- a/e2e/src/framework/mod.rs +++ b/e2e/src/framework/mod.rs @@ -15,6 +15,7 @@ pub mod artifacts; pub mod assertions; pub mod cert_manager_tls; +pub mod cluster_dns; pub mod command; pub mod config; pub mod console_client; diff --git a/e2e/tests/cert_manager_tls.rs b/e2e/tests/cert_manager_tls.rs index 9777b03..a4586c6 100644 --- a/e2e/tests/cert_manager_tls.rs +++ b/e2e/tests/cert_manager_tls.rs @@ -26,7 +26,7 @@ use rustfs_operator_e2e::{ cases::cert_manager_tls, framework::{ artifacts::ArtifactCollector, - assertions, cert_manager_tls as tls_e2e, + assertions, cert_manager_tls as tls_e2e, cluster_dns, command::CommandSpec, config::{E2eConfig, KIND_WORKER_COUNT}, kube_client, live, @@ -1154,7 +1154,7 @@ fn assert_positive_tls_fixture_uses_minimal_four_volume_https_erasure_set( let pod_spec = statefulset_pod_spec(&statefulset)?; let rustfs_container = rustfs_container(pod_spec)?; let rustfs_volumes = env_value(rustfs_container, "RUSTFS_VOLUMES")?; - require_single_volume_https_rustfs_volumes(case_name, rustfs_volumes)?; + require_single_volume_https_rustfs_volumes(config, case_name, rustfs_volumes)?; let expected_sans = expected_tls_dns_names(config, tenant); for host in expand_rustfs_volume_hosts(rustfs_volumes)? { @@ -1201,7 +1201,11 @@ fn env_value<'a>(container: &'a corev1::Container, name: &str) -> Result<&'a str .with_context(|| format!("rustfs container should set {name}")) } -fn require_single_volume_https_rustfs_volumes(case_name: &str, volumes_value: &str) -> Result<()> { +fn require_single_volume_https_rustfs_volumes( + config: &E2eConfig, + case_name: &str, + volumes_value: &str, +) -> Result<()> { let specs = volumes_value.split_whitespace().collect::>(); ensure!( specs.len() == 1, @@ -1218,8 +1222,9 @@ fn require_single_volume_https_rustfs_volumes(case_name: &str, volumes_value: &s .with_context(|| { format!("{case_name} RUSTFS_VOLUMES entry should include :9000: {spec}") })?; + let expected_cluster_suffix = format!(".svc.{}", config.cluster_domain); ensure!( - host_pattern.contains(".svc.cluster.local"), + host_pattern.contains(&expected_cluster_suffix), "{case_name} RUSTFS_VOLUMES should use peer FQDNs, got host pattern {host_pattern}" ); ensure!( @@ -1258,17 +1263,29 @@ fn expected_tls_dns_names(config: &E2eConfig, tenant: &Tenant) -> BTreeSet Result { + match std::env::var(OPERATOR_CLUSTER_DOMAIN_ENV) { + Ok(value) => Self::parse(&value), + Err(_) => Ok(Self::default()), + } + } + + pub(crate) fn parse(value: &str) -> Result { + normalize_cluster_domain(value) + .map(Self) + .ok_or_else(|| ClusterDomainError::new(value)) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for ClusterDomain { + fn default() -> Self { + Self(DEFAULT_CLUSTER_DOMAIN.to_string()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ClusterDomainError { + value: String, +} + +impl ClusterDomainError { + fn new(value: &str) -> Self { + Self { + value: value.to_string(), + } + } +} + +impl fmt::Display for ClusterDomainError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{OPERATOR_CLUSTER_DOMAIN_ENV} must be a valid DNS domain, for example 'cluster.local' or 'k8s.mse.cloud' (got {:?})", + self.value + ) + } +} + +impl Error for ClusterDomainError {} + +fn normalize_cluster_domain(value: &str) -> Option { + let domain = value + .strip_suffix('.') + .unwrap_or(value) + .to_ascii_lowercase(); + if domain.is_empty() + || domain.len() > MAX_DNS_NAME_LENGTH + || !domain.split('.').all(valid_dns_label) + { + return None; + } + Some(domain) +} + +pub(crate) fn service_fqdn(service_name: &str, namespace: &str, cluster_domain: &str) -> String { + format!("{service_name}.{namespace}.svc.{cluster_domain}") +} + +pub(crate) fn pod_fqdn( + pod_name: &str, + headless_service: &str, + namespace: &str, + cluster_domain: &str, +) -> String { + format!("{pod_name}.{headless_service}.{namespace}.svc.{cluster_domain}") +} + +fn valid_dns_label(label: &str) -> bool { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && label + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && label + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_cluster_domain_lowercases_and_removes_trailing_dot() { + assert_eq!( + normalize_cluster_domain("K8S.MSE.Cloud."), + Some("k8s.mse.cloud".to_string()) + ); + } + + #[test] + fn normalize_cluster_domain_rejects_invalid_values() { + assert_eq!(normalize_cluster_domain(""), None); + assert_eq!(normalize_cluster_domain("."), None); + assert_eq!(normalize_cluster_domain("bad..domain"), None); + assert_eq!(normalize_cluster_domain("-bad.domain"), None); + assert_eq!(normalize_cluster_domain("bad_domain"), None); + assert_eq!(normalize_cluster_domain(" cluster.local"), None); + assert_eq!(normalize_cluster_domain("cluster.local.."), None); + assert_eq!( + normalize_cluster_domain(&format!("{}.local", "a".repeat(64))), + None + ); + + let max_length_domain = format!( + "{}.{}.{}.{}", + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(61), + ); + assert_eq!(max_length_domain.len(), MAX_DNS_NAME_LENGTH); + assert_eq!( + normalize_cluster_domain(&max_length_domain), + Some(max_length_domain.clone()) + ); + assert_eq!( + normalize_cluster_domain(&format!("{max_length_domain}.")), + Some(max_length_domain) + ); + + let oversized_domain = format!( + "{}.{}.{}.{}", + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(62), + ); + assert_eq!(oversized_domain.len(), MAX_DNS_NAME_LENGTH + 1); + assert_eq!(normalize_cluster_domain(&oversized_domain), None); + } + + #[test] + fn service_and_pod_fqdns_use_given_cluster_domain() { + assert_eq!( + service_fqdn("tenant-a-io", "storage", "k8s.mse.cloud"), + "tenant-a-io.storage.svc.k8s.mse.cloud" + ); + assert_eq!( + pod_fqdn( + "tenant-a-pool-0-0", + "tenant-a-hl", + "storage", + "k8s.mse.cloud" + ), + "tenant-a-pool-0-0.tenant-a-hl.storage.svc.k8s.mse.cloud" + ); + } + + #[test] + fn parse_cluster_domain_returns_normalized_value() { + let domain = ClusterDomain::parse("K8S.MSE.Cloud.").unwrap(); + + assert_eq!(domain.as_str(), "k8s.mse.cloud"); + } + + #[test] + fn parse_cluster_domain_rejects_invalid_values() { + let error = ClusterDomain::parse("bad_domain").unwrap_err(); + + assert!(error.to_string().contains(OPERATOR_CLUSTER_DOMAIN_ENV)); + } +} diff --git a/src/console/state.rs b/src/console/state.rs index b77474e..87540b4 100755 --- a/src/console/state.rs +++ b/src/console/state.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::cluster_dns; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use kube::Client; use ring::{ @@ -40,6 +41,9 @@ pub struct AppState { /// /// Most unit tests run without a live cluster, so this is optional. pub kube_client: Option, + + /// Kubernetes cluster DNS domain used by handlers that call Tenant services. + pub cluster_domain: Arc, } impl AppState { @@ -48,6 +52,7 @@ impl AppState { Self { jwt_secret: Arc::new(jwt_secret), kube_client: None, + cluster_domain: Arc::new(cluster_dns::DEFAULT_CLUSTER_DOMAIN.to_string()), } } @@ -57,6 +62,11 @@ impl AppState { self } + pub fn with_cluster_domain(mut self, cluster_domain: &str) -> Self { + self.cluster_domain = Arc::new(cluster_domain.to_string()); + self + } + pub fn create_session(&self, k8s_token: String) -> Result { let iat = current_timestamp(); let exp = iat.saturating_add(SESSION_TTL_SECONDS); diff --git a/src/context.rs b/src/context.rs index 940285d..ca730f2 100755 --- a/src/context.rs +++ b/src/context.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::cluster_dns::ClusterDomain; use crate::types; use crate::types::v1alpha1::encryption::LocalKmsMasterKeySecretRef; use crate::types::v1alpha1::tenant::Tenant; @@ -325,17 +326,33 @@ pub(crate) fn map_secret_get_error( pub struct Context { pub(crate) client: kube::Client, pub(crate) recorder: Recorder, + cluster_domain: ClusterDomain, } impl Context { pub fn new(client: kube::Client) -> Self { + Self::new_with_cluster_domain(client, ClusterDomain::default()) + } + + pub(crate) fn new_with_cluster_domain( + client: kube::Client, + cluster_domain: ClusterDomain, + ) -> Self { let reporter = Reporter { controller: "rustfs-operator".into(), instance: std::env::var("HOSTNAME").ok(), }; let recorder = Recorder::new(client.clone(), reporter); - Self { client, recorder } + Self { + client, + recorder, + cluster_domain, + } + } + + pub(crate) fn cluster_domain(&self) -> &str { + self.cluster_domain.as_str() } /// send event diff --git a/src/lib.rs b/src/lib.rs index 1b4bde1..b8b76af 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,6 +83,7 @@ pub fn init_tracing() { }); } +mod cluster_dns; mod context; pub mod metrics; pub mod reconcile; @@ -102,6 +103,12 @@ pub async fn run(options: ServerOptions) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box::all(client.clone()); - let context = Context::new(client.clone()); + let context = Context::new_with_cluster_domain(client.clone(), cluster_domain); let controller = Controller::new(tenant_client, watcher::Config::default()) .watches( Api::::all(client.clone()), @@ -269,18 +284,29 @@ async fn instrumented_reconcile_rustfs( result } -async fn run_active_leader_tasks(client: Client, cancel: CancellationToken) { +async fn run_active_leader_tasks( + client: Client, + cancel: CancellationToken, + cluster_domain: cluster_dns::ClusterDomain, +) { let tasks_cancel = CancellationToken::new(); let controller_client = client.clone(); let controller_cancel = tasks_cancel.clone(); + let controller_cluster_domain = cluster_domain.clone(); let mut controller_handle = tokio::spawn(async move { - run_controller(controller_client, controller_cancel).await; + run_controller( + controller_client, + controller_cancel, + controller_cluster_domain, + ) + .await; }); let mut monitor_handle = if tenant_monitor::is_enabled() { let monitor_cancel = tasks_cancel.clone(); + let monitor_cluster_domain = cluster_domain.as_str().to_string(); Some(tokio::spawn(async move { - tenant_monitor::run(client, monitor_cancel).await; + tenant_monitor::run(client, monitor_cancel, monitor_cluster_domain).await; })) } else { info!("tenant storage monitor disabled by OPERATOR_TENANT_MONITOR_ENABLED=false"); @@ -325,6 +351,7 @@ async fn stop_task(name: &str, mut handle: JoinHandle<()>) { /// Callbacks for running the controller inside leader election. struct ControllerCallbacks { client: Client, + cluster_domain: cluster_dns::ClusterDomain, } #[async_trait::async_trait] @@ -332,7 +359,7 @@ impl LeaderCallbacks for ControllerCallbacks { async fn on_started_leading(&self, cancel: CancellationToken) { info!("acquired leader lease, starting active leader tasks"); metrics::set_operator_leader(true); - run_active_leader_tasks(self.client.clone(), cancel).await; + run_active_leader_tasks(self.client.clone(), cancel, self.cluster_domain.clone()).await; metrics::set_operator_leader(false); } diff --git a/src/reconcile/phases.rs b/src/reconcile/phases.rs index 3475506..56d2a4b 100644 --- a/src/reconcile/phases.rs +++ b/src/reconcile/phases.rs @@ -647,7 +647,12 @@ async fn reconcile_existing_pool_statefulset( "checking existing pool StatefulSet" ); - if let Err(e) = tenant.validate_statefulset_update_with_tls_plan(&existing_ss, pool, tls_plan) { + if let Err(e) = tenant.validate_statefulset_update_with_tls_plan_and_cluster_domain( + &existing_ss, + pool, + tls_plan, + ctx.cluster_domain(), + ) { warn!( tenant = %tenant.name(), namespace = %namespace, @@ -675,7 +680,12 @@ async fn reconcile_existing_pool_statefulset( } if types_result( - tenant.statefulset_needs_update_with_tls_plan(&existing_ss, pool, tls_plan), + tenant.statefulset_needs_update_with_tls_plan_and_cluster_domain( + &existing_ss, + pool, + tls_plan, + ctx.cluster_domain(), + ), ctx, tenant, ) @@ -699,7 +709,11 @@ async fn reconcile_existing_pool_statefulset( .await; let desired = types_result( - tenant.new_statefulset_with_tls_plan(pool, tls_plan), + tenant.new_statefulset_with_tls_plan_and_cluster_domain( + pool, + tls_plan, + ctx.cluster_domain(), + ), ctx, tenant, ) @@ -767,7 +781,11 @@ async fn reconcile_missing_pool_statefulset( .await; let desired = types_result( - tenant.new_statefulset_with_tls_plan(pool, tls_plan), + tenant.new_statefulset_with_tls_plan_and_cluster_domain( + pool, + tls_plan, + ctx.cluster_domain(), + ), ctx, tenant, ) diff --git a/src/reconcile/pool_lifecycle.rs b/src/reconcile/pool_lifecycle.rs index bdce995..39a5bd4 100644 --- a/src/reconcile/pool_lifecycle.rs +++ b/src/reconcile/pool_lifecycle.rs @@ -213,13 +213,19 @@ async fn reconcile_single_pool_lifecycle( } }; - let status = - match verify_decommissioned_pool_for_cleanup(&client, tenant, namespace, pool, &status) - .await - { - Ok(status) => status, - Err(decision) => return decision, - }; + let status = match verify_decommissioned_pool_for_cleanup( + &client, + tenant, + namespace, + pool, + &status, + ctx.cluster_domain(), + ) + .await + { + Ok(status) => status, + Err(decision) => return decision, + }; return cleanup_decommissioned_pool(ctx, tenant, namespace, pool, status).await; } @@ -264,41 +270,42 @@ async fn reconcile_single_pool_lifecycle( } }; - let matched_pool = match find_rustfs_pool(&client, tenant, namespace, pool).await { - Ok(pool_item) => pool_item, - Err(error) if error.is_retriable() => { - warn!( - tenant = %tenant.name(), - namespace = %namespace, - pool = %pool.name, - request_id = %request.request_id, - reason = error.reason(), - message = error.message(), - "RustFS pool mapping is not ready" - ); - return retriable_decision( - Some(request.request_id.clone()), - error.reason(), - error.message(), - ); - } - Err(error) => { - warn!( - tenant = %tenant.name(), - namespace = %namespace, - pool = %pool.name, - request_id = %request.request_id, - reason = error.reason(), - message = error.message(), - "RustFS pool mapping failed" - ); - return failed_decision( - Some(request.request_id.clone()), - error.reason(), - error.message(), - ); - } - }; + let matched_pool = + match find_rustfs_pool(&client, tenant, namespace, pool, ctx.cluster_domain()).await { + Ok(pool_item) => pool_item, + Err(error) if error.is_retriable() => { + warn!( + tenant = %tenant.name(), + namespace = %namespace, + pool = %pool.name, + request_id = %request.request_id, + reason = error.reason(), + message = error.message(), + "RustFS pool mapping is not ready" + ); + return retriable_decision( + Some(request.request_id.clone()), + error.reason(), + error.message(), + ); + } + Err(error) => { + warn!( + tenant = %tenant.name(), + namespace = %namespace, + pool = %pool.name, + request_id = %request.request_id, + reason = error.reason(), + message = error.message(), + "RustFS pool mapping failed" + ); + return failed_decision( + Some(request.request_id.clone()), + error.reason(), + error.message(), + ); + } + }; let pool_id = matched_pool.item.id.to_string(); if cancel_without_decommission_info_is_noop(request, matched_pool.item.decommission.as_ref()) { @@ -448,7 +455,13 @@ async fn rustfs_admin_client( ) -> Result { let credentials = RustfsAdminClient::load_tenant_credentials(&ctx.client, tenant).await?; if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - RustfsAdminClient::from_tls_tenant_for_sts(&ctx.client, tenant, credentials).await + RustfsAdminClient::from_tls_tenant_for_sts( + &ctx.client, + tenant, + credentials, + ctx.cluster_domain(), + ) + .await } else { RustfsAdminClient::from_tenant(tenant, credentials) } @@ -459,8 +472,9 @@ async fn find_rustfs_pool( tenant: &Tenant, namespace: &str, pool: &Pool, + cluster_domain: &str, ) -> Result { - let expected_cmd_line = expected_pool_cmd_line(tenant, namespace, pool) + let expected_cmd_line = expected_pool_cmd_line(tenant, namespace, pool, cluster_domain) .map_err(PoolMappingError::TenantNamespace)?; let expected_endpoint_set_hash = endpoint_set_hash(&expected_cmd_line); let pools = client @@ -496,7 +510,12 @@ async fn find_rustfs_pool( }) } -fn expected_pool_cmd_line(tenant: &Tenant, namespace: &str, pool: &Pool) -> Result { +fn expected_pool_cmd_line( + tenant: &Tenant, + namespace: &str, + pool: &Pool, + cluster_domain: &str, +) -> Result { let scheme = if tenant .spec .tls @@ -514,7 +533,7 @@ fn expected_pool_cmd_line(tenant: &Tenant, namespace: &str, pool: &Pool) -> Resu )); } - Ok(tenant.rustfs_pool_volume_spec(pool, scheme, namespace)) + Ok(tenant.rustfs_pool_volume_spec(pool, scheme, namespace, cluster_domain)) } fn same_cmd_line(left: &str, right: &str) -> bool { @@ -550,8 +569,10 @@ async fn verify_decommissioned_pool_for_cleanup( namespace: &str, pool: &Pool, existing: &PoolDecommissionStatus, + cluster_domain: &str, ) -> Result { - let matched_pool = match find_rustfs_pool(client, tenant, namespace, pool).await { + let matched_pool = match find_rustfs_pool(client, tenant, namespace, pool, cluster_domain).await + { Ok(matched_pool) => matched_pool, Err(error) if error.is_retriable() => { return Err(cleanup_retriable_decision( @@ -1404,9 +1425,19 @@ mod tests { let tenant = test_tenant(pool.clone()); assert_eq!( - expected_pool_cmd_line(&tenant, "rustfs-system", &pool).unwrap(), + expected_pool_cmd_line( + &tenant, + "rustfs-system", + &pool, + crate::cluster_dns::DEFAULT_CLUSTER_DOMAIN + ) + .unwrap(), "http://logs-pool-a-{0...3}.logs-hl.rustfs-system.svc.cluster.local:9000/data/rustfs{0...1}" ); + assert_eq!( + expected_pool_cmd_line(&tenant, "rustfs-system", &pool, "k8s.mse.cloud").unwrap(), + "http://logs-pool-a-{0...3}.logs-hl.rustfs-system.svc.k8s.mse.cloud:9000/data/rustfs{0...1}" + ); } #[test] diff --git a/src/reconcile/provisioning.rs b/src/reconcile/provisioning.rs index 275bcc2..5254b2f 100644 --- a/src/reconcile/provisioning.rs +++ b/src/reconcile/provisioning.rs @@ -410,7 +410,13 @@ async fn rustfs_admin_client( ) -> Result { let credentials = RustfsAdminClient::load_tenant_credentials(&ctx.client, tenant).await?; if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - RustfsAdminClient::from_tls_tenant_for_sts(&ctx.client, tenant, credentials).await + RustfsAdminClient::from_tls_tenant_for_sts( + &ctx.client, + tenant, + credentials, + ctx.cluster_domain(), + ) + .await } else { RustfsAdminClient::from_tenant(tenant, credentials) } diff --git a/src/reconcile/tls.rs b/src/reconcile/tls.rs index 8109f17..5de6d6d 100644 --- a/src/reconcile/tls.rs +++ b/src/reconcile/tls.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::{Error, patch_status_and_record, patch_status_error}; +use crate::cluster_dns; use crate::context::{self, Context}; use crate::status::{StatusBuilder, StatusError}; use crate::types::v1alpha1::status::Reason; @@ -196,6 +197,7 @@ async fn reconcile_cert_manager_tls( let mut observed = Vec::with_capacity(entries.len()); let mut desired_managed_certificate_names = BTreeSet::new(); + let cluster_domain = ctx.cluster_domain(); for entry in entries { let cert_manager = &entry.cert_manager; let Some(secret_name) = cert_manager @@ -235,9 +237,13 @@ async fn reconcile_cert_manager_tls( let certificate_name = certificate_name(tenant, &entry); desired_managed_certificate_names.insert(certificate_name.clone()); - if let Err(failure) = - validate_managed_certificate_san_config(tenant, namespace, config, &entry) - { + if let Err(failure) = validate_managed_certificate_san_config( + tenant, + namespace, + config, + &entry, + cluster_domain, + ) { return tls_validation_blocked(ctx, tenant, config, failure).await; } @@ -262,8 +268,11 @@ async fn reconcile_cert_manager_tls( cert_manager, &entry.hosts, include_generated_dns_names(&entry), - &secret_name, - &certificate_name, + cluster_domain, + CertManagerCertificateNames { + secret: &secret_name, + certificate: &certificate_name, + }, ); let observed_certificate = match apply_cert_manager_certificate( ctx, @@ -360,7 +369,8 @@ async fn reconcile_cert_manager_tls( ) .await?; - let san_dns_names = san_validation_dns_names(tenant, namespace, config, &entry); + let san_dns_names = + san_validation_dns_names(tenant, namespace, config, &entry, cluster_domain); if config.require_san_match && let Err(failure) = validate_tls_secret_san_match(&secret_name, &cert_bytes, &san_dns_names) @@ -886,17 +896,22 @@ async fn cert_manager_prerequisite_failed( Err(error.into()) } +struct CertManagerCertificateNames<'a> { + secret: &'a str, + certificate: &'a str, +} + fn build_cert_manager_certificate( tenant: &Tenant, namespace: &str, cert_manager: &CertManagerTlsConfig, hosts: &[String], include_generated_dns_names: bool, - secret_name: &str, - certificate_name: &str, + cluster_domain: &str, + names: CertManagerCertificateNames<'_>, ) -> DynamicObject { let mut spec = Map::new(); - spec.insert("secretName".to_string(), json!(secret_name)); + spec.insert("secretName".to_string(), json!(names.secret)); if let Some(issuer_ref) = cert_manager.issuer_ref.as_ref() { spec.insert("issuerRef".to_string(), issuer_ref_value(issuer_ref)); } @@ -915,6 +930,7 @@ fn build_cert_manager_certificate( cert_manager, hosts, include_generated_dns_names, + cluster_domain, )), ); spec.insert( @@ -944,7 +960,7 @@ fn build_cert_manager_certificate( ); let resource = certificate_api_resource(); - let mut certificate = DynamicObject::new(certificate_name, &resource) + let mut certificate = DynamicObject::new(names.certificate, &resource) .within(namespace) .data(json!({ "spec": Value::Object(spec) })); certificate.metadata.labels = Some(tenant.common_labels()); @@ -1104,6 +1120,7 @@ fn certificate_dns_names( cert_manager: &CertManagerTlsConfig, hosts: &[String], include_generated_dns_names: bool, + cluster_domain: &str, ) -> Vec { let mut names = BTreeSet::new(); names.extend(hosts.iter().filter(|name| !name.is_empty()).cloned()); @@ -1119,14 +1136,25 @@ fn certificate_dns_names( let io_service = format!("{tenant_name}-io"); let headless_service = tenant.headless_service_name(); names.insert(format!("{io_service}.{namespace}.svc")); - names.insert(format!("{io_service}.{namespace}.svc.cluster.local")); + names.insert(cluster_dns::service_fqdn( + &io_service, + namespace, + cluster_domain, + )); names.insert(format!("{headless_service}.{namespace}.svc")); - names.insert(format!("{headless_service}.{namespace}.svc.cluster.local")); + names.insert(cluster_dns::service_fqdn( + &headless_service, + namespace, + cluster_domain, + )); for pool in &tenant.spec.pools { for ordinal in 0..pool.servers.max(0) { - names.insert(format!( - "{tenant_name}-{}-{ordinal}.{headless_service}.{namespace}.svc.cluster.local", - pool.name + let pod_name = format!("{tenant_name}-{}-{ordinal}", pool.name); + names.insert(cluster_dns::pod_fqdn( + &pod_name, + &headless_service, + namespace, + cluster_domain, )); } } @@ -1146,19 +1174,30 @@ fn validate_managed_certificate_san_config( namespace: &str, config: &TlsConfig, entry: &TlsCertificateEntry, + cluster_domain: &str, ) -> Result<(), TlsValidationFailure> { if !config.enable_internode_https || !entry.default || include_generated_dns_names(entry) { return Ok(()); } - let generated_names = generated_dns_names(tenant, namespace); - let configured_names = - certificate_dns_names(tenant, namespace, &entry.cert_manager, &entry.hosts, false) - .into_iter() - .collect::>(); + let generated_names = required_tls_dns_names(tenant, namespace, cluster_domain); + let configured_names = certificate_dns_names( + tenant, + namespace, + &entry.cert_manager, + &entry.hosts, + false, + cluster_domain, + ) + .into_iter() + .collect::>(); let missing_names = generated_names .iter() - .filter(|name| !configured_names.contains(*name)) + .filter(|name| { + !configured_names + .iter() + .any(|configured| dns_name_covers(configured, name)) + }) .cloned() .collect::>(); @@ -1180,38 +1219,103 @@ fn san_validation_dns_names( namespace: &str, config: &TlsConfig, entry: &TlsCertificateEntry, + cluster_domain: &str, ) -> Vec { - let include_generated_dns_names = - include_generated_dns_names(entry) || (entry.default && config.enable_internode_https); - certificate_dns_names( - tenant, + let mut names = BTreeSet::new(); + names.extend(entry.hosts.iter().filter(|name| !name.is_empty()).cloned()); + names.extend( + entry + .cert_manager + .dns_names + .iter() + .filter(|name| !name.is_empty()) + .cloned(), + ); + if entry.default && config.enable_internode_https { + names.extend(required_tls_dns_names(tenant, namespace, cluster_domain)); + } else if include_generated_dns_names(entry) { + names.extend(generated_dns_names(tenant, namespace, cluster_domain)); + } + names.into_iter().collect() +} + +fn required_tls_dns_names(tenant: &Tenant, namespace: &str, cluster_domain: &str) -> Vec { + let mut names = BTreeSet::new(); + let tenant_name = tenant.name(); + let headless_service = tenant.headless_service_name(); + names.insert(cluster_dns::service_fqdn( + &headless_service, namespace, - &entry.cert_manager, - &entry.hosts, - include_generated_dns_names, - ) + cluster_domain, + )); + for pool in &tenant.spec.pools { + if tenant.spec.pools.len() == 1 && pool.is_single_node_single_disk() { + continue; + } + for ordinal in 0..pool.servers.max(0) { + let pod_name = format!("{tenant_name}-{}-{ordinal}", pool.name); + names.insert(cluster_dns::pod_fqdn( + &pod_name, + &headless_service, + namespace, + cluster_domain, + )); + } + } + names.into_iter().collect() } -fn generated_dns_names(tenant: &Tenant, namespace: &str) -> Vec { +fn generated_dns_names(tenant: &Tenant, namespace: &str, cluster_domain: &str) -> Vec { let mut names = BTreeSet::new(); let tenant_name = tenant.name(); let io_service = format!("{tenant_name}-io"); let headless_service = tenant.headless_service_name(); names.insert(format!("{io_service}.{namespace}.svc")); - names.insert(format!("{io_service}.{namespace}.svc.cluster.local")); + names.insert(cluster_dns::service_fqdn( + &io_service, + namespace, + cluster_domain, + )); names.insert(format!("{headless_service}.{namespace}.svc")); - names.insert(format!("{headless_service}.{namespace}.svc.cluster.local")); + names.insert(cluster_dns::service_fqdn( + &headless_service, + namespace, + cluster_domain, + )); for pool in &tenant.spec.pools { for ordinal in 0..pool.servers.max(0) { - names.insert(format!( - "{tenant_name}-{}-{ordinal}.{headless_service}.{namespace}.svc.cluster.local", - pool.name + let pod_name = format!("{tenant_name}-{}-{ordinal}", pool.name); + names.insert(cluster_dns::pod_fqdn( + &pod_name, + &headless_service, + namespace, + cluster_domain, )); } } names.into_iter().collect() } +fn dns_name_covers(pattern: &str, dns_name: &str) -> bool { + if pattern == dns_name { + return true; + } + let Some(suffix) = pattern.strip_prefix("*.") else { + return false; + }; + let Some(label) = dns_name + .strip_suffix(suffix) + .and_then(|prefix| prefix.strip_suffix('.')) + else { + return false; + }; + !label.is_empty() && !label.contains('.') +} + +fn is_dns_wildcard_pattern(name: &str) -> bool { + name.starts_with("*.") +} + fn rustfs_certificate_domains(certificate: &ObservedTlsCertificate) -> Vec> { let mut domains = Vec::new(); if certificate.entry.default { @@ -1417,9 +1521,19 @@ fn validate_tls_secret_san_match( secret_name, TLS_CERT_KEY ), })?; + let presented_dns_names = cert.valid_dns_names().collect::>(); let mut missing = Vec::new(); for dns_name in expected_dns_names { + if is_dns_wildcard_pattern(dns_name) { + if !presented_dns_names + .iter() + .any(|presented| presented.eq_ignore_ascii_case(dns_name)) + { + missing.push(dns_name.clone()); + } + continue; + } let server_name = ServerName::try_from(dns_name.as_str()).map_err(|_| TlsValidationFailure { reason: Reason::CertificateSanMismatch, @@ -2317,8 +2431,11 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj cert_manager, &[], true, - "tenant-a-server-tls", - "tenant-a-server", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + CertManagerCertificateNames { + secret: "tenant-a-server-tls", + certificate: "tenant-a-server", + }, ); let dns_names = certificate .data @@ -2392,6 +2509,52 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj ); } + #[test] + fn managed_certificate_manifest_uses_custom_cluster_domain_for_generated_dns() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("prod-rustfs".to_string()); + tenant.metadata.namespace = Some("mse".to_string()); + tenant.spec.pools[0].name = "mse-nvme-500".to_string(); + tenant.spec.pools[0].servers = 3; + + let certificate = build_cert_manager_certificate( + &tenant, + "mse", + &CertManagerTlsConfig::default(), + &[], + true, + "k8s.mse.cloud", + CertManagerCertificateNames { + secret: "prod-rustfs-private-certificate-secret", + certificate: "prod-rustfs-private-certificate", + }, + ); + let dns_names = certificate + .data + .pointer("/spec/dnsNames") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(serde_json::Value::as_str) + .map(ToString::to_string) + .collect::>() + }) + .expect("dnsNames should render"); + + assert!(dns_names.contains(&"prod-rustfs-io.mse.svc".to_string())); + assert!(dns_names.contains(&"prod-rustfs-io.mse.svc.k8s.mse.cloud".to_string())); + assert!(dns_names.contains(&"prod-rustfs-hl.mse.svc.k8s.mse.cloud".to_string())); + assert!(dns_names.contains( + &"prod-rustfs-mse-nvme-500-2.prod-rustfs-hl.mse.svc.k8s.mse.cloud".to_string() + )); + assert!( + !dns_names + .iter() + .any(|name| name.ends_with(".svc.cluster.local")) + ); + } + #[test] fn managed_certificate_manifest_includes_sni_hosts() { let mut tenant = crate::tests::create_test_tenant(None, None); @@ -2409,8 +2572,11 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj &cert_manager, &["s3.example.com".to_string()], false, - "tenant-a-public-tls", - "tenant-a-public-tls", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + CertManagerCertificateNames { + secret: "tenant-a-public-tls", + certificate: "tenant-a-public-tls", + }, ); let dns_names = certificate .data @@ -2512,8 +2678,11 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj &entry.cert_manager, &entry.hosts, include_generated_dns_names(&entry), - "tenant-a-public-tls", - "tenant-a-public-tls", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + CertManagerCertificateNames { + secret: "tenant-a-public-tls", + certificate: "tenant-a-public-tls", + }, ); let dns_names = certificate .data @@ -2553,8 +2722,11 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj &entry.cert_manager, &entry.hosts, include_generated_dns_names(&entry), - "tenant-a-public-tls", - "tenant-a-public-tls", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + CertManagerCertificateNames { + secret: "tenant-a-public-tls", + certificate: "tenant-a-public-tls", + }, ); let dns_names = certificate .data @@ -2584,10 +2756,11 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj enable_internode_https: true, ..Default::default() }; - let generated_pod_dns = generated_dns_names(&tenant, "storage") - .into_iter() - .find(|name| name.contains("-0.")) - .expect("generated names should include a pod DNS name"); + let generated_pod_dns = + generated_dns_names(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN) + .into_iter() + .find(|name| name.contains("-0.")) + .expect("generated names should include a pod DNS name"); let default_entry = TlsCertificateEntry { name: "internal".to_string(), @@ -2599,7 +2772,13 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj }, legacy: false, }; - let names = san_validation_dns_names(&tenant, "storage", &config, &default_entry); + let names = san_validation_dns_names( + &tenant, + "storage", + &config, + &default_entry, + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ); assert!(names.contains(&generated_pod_dns)); let non_default_entry = TlsCertificateEntry { @@ -2612,7 +2791,13 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj }, legacy: false, }; - let names = san_validation_dns_names(&tenant, "storage", &config, &non_default_entry); + let names = san_validation_dns_names( + &tenant, + "storage", + &config, + &non_default_entry, + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ); assert!(names.contains(&"s3.example.com".to_string())); assert!(names.contains(&generated_pod_dns)); @@ -2620,7 +2805,13 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj cert_manager: CertManagerTlsConfig::default(), ..non_default_entry }; - let names = san_validation_dns_names(&tenant, "storage", &config, &public_default); + let names = san_validation_dns_names( + &tenant, + "storage", + &config, + &public_default, + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ); assert_eq!(names, vec!["s3.example.com".to_string()]); } @@ -2646,8 +2837,14 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj legacy: false, }; - let failure = validate_managed_certificate_san_config(&tenant, "storage", &config, &entry) - .expect_err("managed internode cert must cover generated peer DNS names"); + let failure = validate_managed_certificate_san_config( + &tenant, + "storage", + &config, + &entry, + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ) + .expect_err("managed internode cert must cover generated peer DNS names"); assert_eq!(failure.reason, Reason::CertificateInvalid); assert!( @@ -2658,11 +2855,97 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj failure.message ); - entry.cert_manager.dns_names = generated_dns_names(&tenant, "storage"); + entry.cert_manager.dns_names = + generated_dns_names(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); + assert_eq!( + validate_managed_certificate_san_config( + &tenant, + "storage", + &config, + &entry, + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ), + Ok(()) + ); + } + + #[test] + fn managed_default_certificate_accepts_issue_152_headless_wildcard_dns() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("prod-rustfs".to_string()); + tenant.metadata.namespace = Some("mse".to_string()); + tenant.spec.pools[0].name = "mse-nvme-500".to_string(); + tenant.spec.pools[0].servers = 3; + let config = TlsConfig { + enable_internode_https: true, + ..Default::default() + }; + let entry = TlsCertificateEntry { + name: "private".to_string(), + default: true, + hosts: Vec::new(), + cert_manager: CertManagerTlsConfig { + manage_certificate: true, + include_generated_dns_names: Some(false), + dns_names: vec![ + "prod-rustfs-hl.mse.svc.k8s.mse.cloud".to_string(), + "*.prod-rustfs-hl.mse.svc.k8s.mse.cloud".to_string(), + ], + ..Default::default() + }, + legacy: false, + }; + assert_eq!( - validate_managed_certificate_san_config(&tenant, "storage", &config, &entry), + validate_managed_certificate_san_config( + &tenant, + "mse", + &config, + &entry, + "k8s.mse.cloud", + ), + Ok(()) + ); + + let expected_dns_names = + san_validation_dns_names(&tenant, "mse", &config, &entry, "k8s.mse.cloud"); + assert!( + expected_dns_names + .iter() + .any(|name| is_dns_wildcard_pattern(name)), + "SAN validation must retain configured wildcard names" + ); + + let certificate = rcgen::generate_simple_self_signed(entry.cert_manager.dns_names.clone()) + .expect("issue #152 DNS names should generate a certificate"); + assert_eq!( + validate_tls_secret_san_match( + "prod-rustfs-private-certificate-secret", + certificate.cert.pem().as_bytes(), + &expected_dns_names, + ), Ok(()) ); + + let concrete_certificate = rcgen::generate_simple_self_signed( + expected_dns_names + .iter() + .filter(|name| !is_dns_wildcard_pattern(name)) + .cloned() + .collect::>(), + ) + .expect("concrete issue #152 DNS names should generate a certificate"); + let failure = validate_tls_secret_san_match( + "prod-rustfs-private-certificate-secret", + concrete_certificate.cert.pem().as_bytes(), + &expected_dns_names, + ) + .expect_err("certificate missing configured wildcard SAN should fail validation"); + assert!( + failure + .message + .contains("*.prod-rustfs-hl.mse.svc.k8s.mse.cloud") + ); } #[test] diff --git a/src/sts/rustfs_client.rs b/src/sts/rustfs_client.rs index 3f0aa60..c5d38b4 100644 --- a/src/sts/rustfs_client.rs +++ b/src/sts/rustfs_client.rs @@ -19,6 +19,7 @@ use kube::{Api, Client}; use reqwest::{Certificate, Client as HttpClient, Response, StatusCode}; use crate::Tenant; +use crate::cluster_dns; use crate::utils::sanitize::redact_sensitive_pairs; /// admin_ops: tenant admin operations (user/policy APIs). @@ -419,6 +420,15 @@ fn default_http_client() -> HttpClient { .unwrap_or_else(|_| HttpClient::new()) } +fn tls_tenant_base_url(tenant: &Tenant, cluster_domain: &str) -> Result { + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let service_fqdn = + cluster_dns::service_fqdn(&tenant.headless_service_name(), &namespace, cluster_domain); + Ok(format!("https://{service_fqdn}:9000")) +} + impl RustfsAdminClient { pub const STS_VERSION: &'static str = "2011-06-15"; pub const STS_ACTION: &'static str = "AssumeRole"; @@ -501,6 +511,7 @@ impl RustfsAdminClient { kube_client: &Client, tenant: &Tenant, credentials: RustfsCredentials, + cluster_domain: &str, ) -> Result { if !helpers::tenant_tls_enabled(tenant) { return Err(RustfsClientError::TenantTlsRequired); @@ -509,15 +520,7 @@ impl RustfsAdminClient { return Err(RustfsClientError::TenantTlsClientCertificateRequired); } - let namespace = tenant - .namespace() - .map_err(|_| RustfsClientError::MissingTenantNamespace)?; - let service_name = tenant - .new_io_service() - .metadata - .name - .unwrap_or_else(|| format!("{}-io", tenant.name())); - let base_url = format!("https://{service_name}.{namespace}.svc:9000"); + let base_url = tls_tenant_base_url(tenant, cluster_domain)?; match Self::load_tenant_tls_ca(kube_client, tenant).await? { Some(ca_pem) => Self::new_with_base_url_and_ca_pem( diff --git a/src/sts/server.rs b/src/sts/server.rs index a76e2e7..8e260e3 100644 --- a/src/sts/server.rs +++ b/src/sts/server.rs @@ -88,6 +88,7 @@ async fn assume_role_with_web_identity( let runtime = RealStsRuntime { kube_client: kube_client.clone(), + cluster_domain: state.cluster_domain.as_ref().clone(), }; process_assume_role_request(&runtime, parsed_request).await @@ -133,6 +134,7 @@ trait StsRuntime { struct RealStsRuntime { kube_client: Client, + cluster_domain: String, } #[async_trait] @@ -164,7 +166,7 @@ impl StsRuntime for RealStsRuntime { &self, tenant: &Tenant, ) -> Result { - create_rustfs_admin_client(&self.kube_client, tenant).await + create_rustfs_admin_client(&self.kube_client, tenant, &self.cluster_domain).await } async fn fetch_canned_policy( @@ -541,6 +543,7 @@ async fn select_tenant( async fn create_rustfs_admin_client( client: &Client, tenant: &Tenant, + cluster_domain: &str, ) -> Result { if !tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { return Err(StsError::InvalidParameterValue { @@ -552,7 +555,7 @@ async fn create_rustfs_admin_client( .await .map_err(|_| StsError::InternalError)?; - RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials) + RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain) .await .map_err(map_rustfs_client_creation_error) } diff --git a/src/sts/tests.rs b/src/sts/tests.rs index 986372c..72f0482 100644 --- a/src/sts/tests.rs +++ b/src/sts/tests.rs @@ -31,6 +31,7 @@ use super::{ POOLS_DECOMMISSION_PATH, POOLS_LIST_PATH, POOLS_STATUS_PATH, RustfsAdminClient, RustfsClientError, SERVER_INFO_PATH, SET_POLICY_PATH, USER_INFO_PATH, helpers::{extract_canned_policy_document, extract_credentials, parse_assume_role_response}, + tls_tenant_base_url, }; fn secret_with_fields(fields: Vec<(&str, &[u8])>) -> corev1::Secret { @@ -54,6 +55,18 @@ fn assert_oversized_upstream_body_hidden(err: RustfsClientError) { ); } +#[test] +fn tls_tenant_base_url_uses_custom_cluster_domain() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("prod-rustfs".to_string()); + tenant.metadata.namespace = Some("mse".to_string()); + + assert_eq!( + tls_tenant_base_url(&tenant, "k8s.mse.cloud").unwrap(), + "https://prod-rustfs-hl.mse.svc.k8s.mse.cloud:9000" + ); +} + #[test] fn extract_credentials_reports_missing_access_key() { let secret = secret_with_fields(vec![("secretkey", b"sekret")]); diff --git a/src/sts/tls.rs b/src/sts/tls.rs index 18f1e70..a77331f 100644 --- a/src/sts/tls.rs +++ b/src/sts/tls.rs @@ -17,6 +17,7 @@ use std::io::Cursor; use std::net::Ipv4Addr; use std::time::Duration; +use crate::cluster_dns; use k8s_openapi::ByteString; use k8s_openapi::api::core::v1 as corev1; use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; @@ -102,6 +103,7 @@ pub struct OperatorStsTlsConfig { pub auto_generate: bool, pub namespace: String, pub service_name: String, + pub cluster_domain: String, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -119,6 +121,14 @@ impl OperatorStsTlsConfig { auto_generate: env_bool("OPERATOR_STS_TLS_AUTO", true), namespace: operator_namespace(), service_name: env_string("OPERATOR_STS_SERVICE_NAME", DEFAULT_STS_SERVICE_NAME), + cluster_domain: cluster_dns::DEFAULT_CLUSTER_DOMAIN.to_string(), + } + } + + pub fn from_env_with_cluster_domain(cluster_domain: &str) -> Self { + Self { + cluster_domain: cluster_domain.to_string(), + ..Self::from_env() } } } @@ -265,7 +275,11 @@ async fn wait_for_secret_material( } fn generated_sts_tls_secret(config: &OperatorStsTlsConfig) -> TlsResult { - let generated = generate_sts_tls_material(&config.namespace, &config.service_name)?; + let generated = generate_sts_tls_material( + &config.namespace, + &config.service_name, + &config.cluster_domain, + )?; let mut data = BTreeMap::new(); data.insert(TLS_CERT_KEY.to_string(), ByteString(generated.cert_pem)); data.insert(TLS_KEY_KEY.to_string(), ByteString(generated.key_pem)); @@ -298,6 +312,7 @@ fn generated_sts_tls_secret(config: &OperatorStsTlsConfig) -> TlsResult TlsResult { let ca_key = KeyPair::generate().context(GenerateCertificateSnafu)?; let mut ca_params = CertificateParams::default(); @@ -312,7 +327,7 @@ fn generate_sts_tls_material( .context(GenerateCertificateSnafu)?; let server_key = KeyPair::generate().context(GenerateCertificateSnafu)?; - let mut server_names = service_dns_names(namespace, service_name); + let mut server_names = service_dns_names(namespace, service_name, cluster_domain); server_names.push("localhost".to_string()); server_names.push(Ipv4Addr::LOCALHOST.to_string()); let mut server_params = @@ -378,12 +393,12 @@ fn secret_data( }) } -fn service_dns_names(namespace: &str, service_name: &str) -> Vec { +fn service_dns_names(namespace: &str, service_name: &str, cluster_domain: &str) -> Vec { vec![ service_name.to_string(), format!("{service_name}.{namespace}"), format!("{service_name}.{namespace}.svc"), - format!("{service_name}.{namespace}.svc.cluster.local"), + cluster_dns::service_fqdn(service_name, namespace, cluster_domain), ] } @@ -438,7 +453,11 @@ mod tests { #[test] fn service_dns_names_cover_short_and_cluster_forms() { assert_eq!( - service_dns_names("rustfs-system", "rustfs-operator-sts"), + service_dns_names( + "rustfs-system", + "rustfs-operator-sts", + cluster_dns::DEFAULT_CLUSTER_DOMAIN + ), vec![ "rustfs-operator-sts", "rustfs-operator-sts.rustfs-system", @@ -448,9 +467,22 @@ mod tests { ); } + #[test] + fn service_dns_names_use_custom_cluster_domain() { + assert_eq!( + service_dns_names("rustfs-system", "rustfs-operator-sts", "k8s.mse.cloud")[3], + "rustfs-operator-sts.rustfs-system.svc.k8s.mse.cloud" + ); + } + #[test] fn generated_material_builds_rustls_server_config() { - let material = generate_sts_tls_material("rustfs-system", "rustfs-operator-sts").unwrap(); + let material = generate_sts_tls_material( + "rustfs-system", + "rustfs-operator-sts", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ) + .unwrap(); assert!(!material.cert_pem.is_empty()); assert!(!material.key_pem.is_empty()); @@ -465,8 +497,14 @@ mod tests { auto_generate: true, namespace: "rustfs-system".to_string(), service_name: "rustfs-operator-sts".to_string(), + cluster_domain: cluster_dns::DEFAULT_CLUSTER_DOMAIN.to_string(), }; - let generated = generate_sts_tls_material(&config.namespace, &config.service_name).unwrap(); + let generated = generate_sts_tls_material( + &config.namespace, + &config.service_name, + &config.cluster_domain, + ) + .unwrap(); let mut data = BTreeMap::new(); data.insert(TLS_CERT_KEY.to_string(), ByteString(generated.cert_pem)); data.insert(TLS_KEY_KEY.to_string(), ByteString(generated.key_pem)); diff --git a/src/tenant_monitor.rs b/src/tenant_monitor.rs index 61d47f3..6b428e8 100644 --- a/src/tenant_monitor.rs +++ b/src/tenant_monitor.rs @@ -47,7 +47,7 @@ pub fn interval() -> Duration { } } -pub async fn run(client: Client, cancel: CancellationToken) { +pub async fn run(client: Client, cancel: CancellationToken, cluster_domain: String) { let interval = interval(); info!( interval_seconds = interval.as_secs(), @@ -55,7 +55,7 @@ pub async fn run(client: Client, cancel: CancellationToken) { ); loop { - poll_all_tenants(client.clone()).await; + poll_all_tenants(client.clone(), &cluster_domain).await; tokio::select! { _ = cancel.cancelled() => { @@ -67,7 +67,7 @@ pub async fn run(client: Client, cancel: CancellationToken) { } } -async fn poll_all_tenants(client: Client) { +async fn poll_all_tenants(client: Client, cluster_domain: &str) { let started = Instant::now(); let tenants = match list_all_tenants(client.clone()).await { Ok(tenants) => tenants, @@ -91,14 +91,15 @@ async fn poll_all_tenants(client: Client) { stream::iter(tenants) .for_each_concurrent(MAX_CONCURRENT_TENANT_POLLS, |tenant| { let client = client.clone(); + let cluster_domain = cluster_domain.to_string(); async move { - poll_tenant(client, tenant).await; + poll_tenant(client, tenant, &cluster_domain).await; } }) .await; } -async fn poll_tenant(client: Client, tenant: Tenant) { +async fn poll_tenant(client: Client, tenant: Tenant, cluster_domain: &str) { let started = Instant::now(); let tenant_name = tenant.name(); let namespace = match tenant.namespace() { @@ -120,7 +121,7 @@ async fn poll_tenant(client: Client, tenant: Tenant) { return; } - match poll_tenant_storage(&client, &tenant).await { + match poll_tenant_storage(&client, &tenant, cluster_domain).await { Ok(storage) => { metrics::record_tenant_storage(&namespace, &tenant_name, storage); metrics::record_tenant_monitor_poll("success", started.elapsed()); @@ -164,10 +165,12 @@ async fn list_all_tenants(client: Client) -> Result, kube::Error> { async fn poll_tenant_storage( client: &Client, tenant: &Tenant, + cluster_domain: &str, ) -> Result> { let credentials = RustfsAdminClient::load_tenant_credentials(client, tenant).await?; let rustfs_client = if tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { - RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials).await? + RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials, cluster_domain) + .await? } else { RustfsAdminClient::from_tenant(tenant, credentials)? }; diff --git a/src/types/v1alpha1/tenant/workloads.rs b/src/types/v1alpha1/tenant/workloads.rs index 2ea549b..39f2ee8 100755 --- a/src/types/v1alpha1/tenant/workloads.rs +++ b/src/types/v1alpha1/tenant/workloads.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::Tenant; +use crate::cluster_dns; use crate::types; use crate::types::v1alpha1::encryption::KmsBackendType; use crate::types::v1alpha1::persistence::{ @@ -87,6 +88,7 @@ impl Tenant { pool: &Pool, scheme: &str, namespace: &str, + cluster_domain: &str, ) -> String { let tenant_name = self.name(); let headless_service = self.headless_service_name(); @@ -101,10 +103,11 @@ impl Tenant { return format!("{base_path}/rustfs0"); } + let pod_name = format!("{tenant_name}-{}-{{0...{}}}", pool.name, pool.servers - 1); + let peer_host = + cluster_dns::pod_fqdn(&pod_name, &headless_service, namespace, cluster_domain); format!( - "{scheme}://{tenant_name}-{}-{{0...{}}}.{headless_service}.{namespace}.svc.cluster.local:9000{}/rustfs{{0...{}}}", - pool.name, - pool.servers - 1, + "{scheme}://{peer_host}:9000{}/rustfs{{0...{}}}", base_path, pool.persistence.volumes_per_server - 1 ) @@ -113,13 +116,17 @@ impl Tenant { /// Constructs the RUSTFS_VOLUMES environment variable value /// Distributed and multi-pool tenants use peer DNS entries, while a single-pool /// single-node single-disk tenant uses its local data path. - fn rustfs_volumes_env_value(&self, scheme: &str) -> Result { + fn rustfs_volumes_env_value( + &self, + scheme: &str, + cluster_domain: &str, + ) -> Result { let namespace = self.namespace()?; let volume_specs = self .spec .pools .iter() - .map(|pool| self.rustfs_pool_volume_spec(pool, scheme, &namespace)) + .map(|pool| self.rustfs_pool_volume_spec(pool, scheme, &namespace, cluster_domain)) .collect::>(); Ok(volume_specs.join(" ")) @@ -420,6 +427,19 @@ impl Tenant { &self, pool: &Pool, tls_plan: &TlsPlan, + ) -> Result { + self.new_statefulset_with_tls_plan_and_cluster_domain( + pool, + tls_plan, + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ) + } + + pub(crate) fn new_statefulset_with_tls_plan_and_cluster_domain( + &self, + pool: &Pool, + tls_plan: &TlsPlan, + cluster_domain: &str, ) -> Result { let labels = self.pool_labels(pool); let selector_labels = self.pool_selector_labels(pool); @@ -443,7 +463,8 @@ impl Tenant { let mut env_vars = Vec::new(); // Add RUSTFS_VOLUMES environment variable for the inferred storage layout. - let rustfs_volumes = self.rustfs_volumes_env_value(tls_plan.internode_scheme)?; + let rustfs_volumes = + self.rustfs_volumes_env_value(tls_plan.internode_scheme, cluster_domain)?; env_vars.push(corev1::EnvVar { name: "RUSTFS_VOLUMES".to_owned(), value: Some(rustfs_volumes), @@ -675,7 +696,23 @@ impl Tenant { pool: &Pool, tls_plan: &TlsPlan, ) -> Result { - let desired = self.new_statefulset_with_tls_plan(pool, tls_plan)?; + self.statefulset_needs_update_with_tls_plan_and_cluster_domain( + existing, + pool, + tls_plan, + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ) + } + + pub(crate) fn statefulset_needs_update_with_tls_plan_and_cluster_domain( + &self, + existing: &v1::StatefulSet, + pool: &Pool, + tls_plan: &TlsPlan, + cluster_domain: &str, + ) -> Result { + let desired = + self.new_statefulset_with_tls_plan_and_cluster_domain(pool, tls_plan, cluster_domain)?; // Compare key spec fields that should trigger updates let existing_spec = existing @@ -927,7 +964,23 @@ impl Tenant { pool: &Pool, tls_plan: &TlsPlan, ) -> Result<(), types::error::Error> { - let desired = self.new_statefulset_with_tls_plan(pool, tls_plan)?; + self.validate_statefulset_update_with_tls_plan_and_cluster_domain( + existing, + pool, + tls_plan, + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ) + } + + pub(crate) fn validate_statefulset_update_with_tls_plan_and_cluster_domain( + &self, + existing: &v1::StatefulSet, + pool: &Pool, + tls_plan: &TlsPlan, + cluster_domain: &str, + ) -> Result<(), types::error::Error> { + let desired = + self.new_statefulset_with_tls_plan_and_cluster_domain(pool, tls_plan, cluster_domain)?; let existing_spec = existing .spec @@ -1690,6 +1743,23 @@ mod tests { )); } + #[test] + fn rustfs_pool_volume_spec_uses_custom_cluster_domain() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("prod-rustfs".to_string()); + tenant.spec.pools[0].name = "mse-nvme-500".to_string(); + tenant.spec.pools[0].servers = 3; + tenant.spec.pools[0].persistence.volumes_per_server = 1; + let pool = &tenant.spec.pools[0]; + + let volume_spec = tenant.rustfs_pool_volume_spec(pool, "https", "mse", "k8s.mse.cloud"); + + assert_eq!( + volume_spec, + "https://prod-rustfs-mse-nvme-500-{0...2}.prod-rustfs-hl.mse.svc.k8s.mse.cloud:9000/data/rustfs{0...0}" + ); + } + #[test] fn tls_statefulset_keeps_operator_managed_env_when_spec_env_conflicts() { let mut tenant = crate::tests::create_test_tenant(None, None);