diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a8a7021a..e321390b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -21,6 +21,11 @@ use stackable_operator::{ }, crd::{listener::v1alpha1::Listener, s3}, database_connections::drivers::jdbc::JdbcDatabaseConnectionDetails, + k8s_openapi::api::{ + apps::v1::StatefulSet, + core::v1::{ConfigMap, Service}, + policy::v1::PodDisruptionBudget, + }, kube::{ Resource, ResourceExt, api::ObjectMeta, @@ -49,12 +54,7 @@ use strum::EnumDiscriminants; use crate::{ OPERATOR_NAME, - controller::build::resource::{ - discovery, - listener::build_role_listener, - pdb::build_pdb, - service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, - }, + controller::build::resource::discovery, crd::{APP_NAME, HdfsConnection, HiveClusterStatus, HiveRole, MetaStoreConfig, v1alpha1}, }; @@ -68,30 +68,13 @@ pub struct Ctx { #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(strum::IntoStaticStr))] -#[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("failed to apply Service for role group {role_group}"))] - ApplyRoleGroupService { - source: stackable_operator::cluster_resources::Error, - role_group: RoleGroupName, - }, - - #[snafu(display("failed to build ConfigMap for role group {role_group}"))] - BuildRoleGroupConfigMap { - source: build::resource::config_map::Error, - role_group: RoleGroupName, - }, + #[snafu(display("failed to build the Kubernetes resources"))] + BuildResources { source: build::Error }, - #[snafu(display("failed to apply ConfigMap for role group {role_group}"))] - ApplyRoleGroupConfig { + #[snafu(display("failed to apply Kubernetes resource"))] + ApplyResource { source: stackable_operator::cluster_resources::Error, - role_group: RoleGroupName, - }, - - #[snafu(display("failed to apply StatefulSet for role group {role_group}"))] - ApplyRoleGroupStatefulSet { - source: stackable_operator::cluster_resources::Error, - role_group: RoleGroupName, }, #[snafu(display("failed to build discovery ConfigMap"))] @@ -127,11 +110,6 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("failed to apply PodDisruptionBudget"))] - ApplyPdb { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to get required Labels"))] GetRequiredLabels { source: @@ -143,11 +121,6 @@ pub enum Error { source: error_boundary::InvalidObject, }, - #[snafu(display("failed to apply group listener for {role}"))] - ApplyGroupListener { - source: stackable_operator::cluster_resources::Error, - role: String, - }, #[snafu(display("failed to dereference cluster resources"))] Dereference { source: crate::controller::dereference::Error, @@ -155,12 +128,6 @@ pub enum Error { #[snafu(display("failed to validate cluster configuration"))] Validate { source: validate::Error }, - - #[snafu(display("failed to build StatefulSet for role group {role_group}"))] - BuildRoleGroupStatefulSet { - source: build::resource::statefulset::Error, - role_group: RoleGroupName, - }, } type Result = std::result::Result; @@ -462,6 +429,19 @@ pub struct ValidatedRoleConfig { pub listener_class: ListenerClassName, } +/// Every Kubernetes resource produced by the client-free [`build`](build::build) step. +/// +/// The role-level discovery `ConfigMap` is deliberately absent: it is built from the *applied* +/// role [`Listener`]'s ingress addresses, so it is assembled in the reconcile step after the +/// Listener has been applied, not here. +pub struct KubernetesResources { + pub stateful_sets: Vec, + pub services: Vec, + pub listeners: Vec, + pub config_maps: Vec, + pub pod_disruption_budgets: Vec, +} + pub async fn reconcile_hive( hive: Arc>, ctx: Arc, @@ -515,98 +495,79 @@ pub async fn reconcile_hive( .await .context(ApplyRoleBindingSnafu)?; - let mut ss_cond_builder = StatefulSetConditionBuilder::default(); + // The ServiceAccount name is deterministic on the built object, so the client-free build step + // does not depend on the applied ServiceAccount. + let service_account_name = rbac_sa.name_any(); - for (hive_role, role_group_configs) in &validated_cluster.role_group_configs { - for (role_group_name, rg) in role_group_configs { - let rg_metrics_service = - build_rolegroup_metrics_service(&validated_cluster, role_group_name); + let resources = build::build( + &validated_cluster, + &client.kubernetes_cluster_info, + &service_account_name, + ) + .context(BuildResourcesSnafu)?; - let rg_headless_service = - build_rolegroup_headless_service(&validated_cluster, role_group_name); + let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - let rg_configmap = build::resource::config_map::build_metastore_rolegroup_config_map( - &validated_cluster, - &client.kubernetes_cluster_info, - role_group_name, - rg, - ) - .with_context(|_| BuildRoleGroupConfigMapSnafu { - role_group: role_group_name.clone(), - })?; - - let rg_statefulset = - build::resource::statefulset::build_metastore_rolegroup_statefulset( - hive_role, - &validated_cluster, - role_group_name, - rg, - &rbac_sa.name_any(), - ) - .with_context(|_| BuildRoleGroupStatefulSetSnafu { - role_group: role_group_name.clone(), - })?; + // Apply order: everything before StatefulSets, StatefulSets last. A StatefulSet must only be + // applied after all ConfigMaps and Secrets it mounts, to prevent unnecessary Pod restarts. + // See https://github.com/stackabletech/commons-operator/issues/111 for details. + for service in resources.services { + cluster_resources + .add(client, service) + .await + .context(ApplyResourceSnafu)?; + } + // The role Listener is applied before the discovery ConfigMap, which is built below from the + // applied Listener's ingress addresses. Hive has a single role Listener, so at most one is + // captured here. + let mut applied_role_listener: Option = None; + for listener in resources.listeners { + applied_role_listener = Some( cluster_resources - .add(client, rg_metrics_service) + .add(client, listener) .await - .context(ApplyRoleGroupServiceSnafu { - role_group: role_group_name.clone(), - })?; + .context(ApplyResourceSnafu)?, + ); + } + + for config_map in resources.config_maps { + cluster_resources + .add(client, config_map) + .await + .context(ApplyResourceSnafu)?; + } + for pdb in resources.pod_disruption_budgets { + cluster_resources + .add(client, pdb) + .await + .context(ApplyResourceSnafu)?; + } + + for statefulset in resources.stateful_sets { + ss_cond_builder.add( cluster_resources - .add(client, rg_headless_service) + .add(client, statefulset) .await - .context(ApplyRoleGroupServiceSnafu { - role_group: role_group_name.clone(), - })?; - - cluster_resources.add(client, rg_configmap).await.context( - ApplyRoleGroupConfigSnafu { - role_group: role_group_name.clone(), - }, - )?; - - // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it - // mounts to prevent unnecessary Pod restarts. - // See https://github.com/stackabletech/commons-operator/issues/111 for details. - ss_cond_builder.add( - cluster_resources - .add(client, rg_statefulset) - .await - .context(ApplyRoleGroupStatefulSetSnafu { - role_group: role_group_name.clone(), - })?, - ); - } + .context(ApplyResourceSnafu)?, + ); } + // The discovery ConfigMap is built from the *applied* role Listener's ingress addresses, so it + // is assembled here rather than in the client-free build step. Its applied resource version + // feeds the status discovery hash. // std's SipHasher is deprecated, and DefaultHasher is unstable across Rust releases. // We don't /need/ stability, but it's still nice to avoid spurious changes where possible. let mut discovery_hash = FnvHasher::with_key(0); - if let Some(role_config) = &validated_cluster.role_config { - if let Some(pdb) = build_pdb(&role_config.pdb, &validated_cluster, &HiveRole::MetaStore) { - cluster_resources - .add(client, pdb) - .await - .context(ApplyPdbSnafu)?; - } - - let role_listener: Listener = build_role_listener( + if let Some(role_listener) = applied_role_listener { + let discovery_cm = discovery::build_discovery_configmap( &validated_cluster, - &HiveRole::MetaStore, - &role_config.listener_class, - ); - let listener = cluster_resources.add(client, role_listener).await.context( - ApplyGroupListenerSnafu { - role: HiveRole::MetaStore.to_string(), - }, - )?; - - let discovery_cm = - discovery::build_discovery_configmap(&validated_cluster, HiveRole::MetaStore, listener) - .context(BuildDiscoveryConfigSnafu)?; + HiveRole::MetaStore, + role_listener, + ) + .context(BuildDiscoveryConfigSnafu)?; let discovery_cm = cluster_resources .add(client, discovery_cm) .await diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 124a7d7b..974a71f7 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -2,7 +2,25 @@ use std::str::FromStr; -use stackable_operator::v2::types::operator::{ProductVersion, RoleGroupName}; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + utils::cluster_info::KubernetesClusterInfo, + v2::types::operator::{ProductVersion, RoleGroupName}, +}; + +use crate::{ + controller::{ + KubernetesResources, ValidatedCluster, + build::resource::{ + config_map::build_metastore_rolegroup_config_map, + listener::build_role_listener, + pdb::build_pdb, + service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, + statefulset::build_metastore_rolegroup_statefulset, + }, + }, + crd::HiveRole, +}; // Placeholder role-group name used for the recommended labels of the role-level discovery // `ConfigMap` (which is not tied to a single role group). @@ -23,3 +41,146 @@ pub mod kerberos; pub mod opa; pub mod properties; pub mod resource; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build ConfigMap for role group {role_group}"))] + ConfigMap { + source: resource::config_map::Error, + role_group: RoleGroupName, + }, + + #[snafu(display("failed to build StatefulSet for role group {role_group}"))] + StatefulSet { + source: resource::statefulset::Error, + role_group: RoleGroupName, + }, +} + +/// Builds every Kubernetes resource for the given validated cluster. +/// +/// Does not need a Kubernetes client: every reference to another Kubernetes resource is already +/// dereferenced and validated by this point. Cluster configuration is likewise already validated, +/// so the errors returned here are resource-assembly failures only. +/// +/// The role-level discovery `ConfigMap` is *not* built here: it depends on the *applied* role +/// [`Listener`](stackable_operator::crd::listener::v1alpha1::Listener)'s ingress addresses and is +/// therefore assembled in the reconcile step after the Listener has been applied. +/// +/// `cluster_info` carries the Kubernetes cluster domain (needed by the Kerberos config); it is +/// static cluster metadata, not a live client, so the build step stays client-free. +/// +/// `service_account_name` is the name of the RBAC `ServiceAccount` the role-group Pods run under +/// (RBAC resources are built and applied separately, in the reconcile step). +pub fn build( + cluster: &ValidatedCluster, + cluster_info: &KubernetesClusterInfo, + service_account_name: &str, +) -> Result { + let mut stateful_sets = vec![]; + let mut services = vec![]; + let mut listeners = vec![]; + let mut config_maps = vec![]; + let mut pod_disruption_budgets = vec![]; + + // Role-level resources. Hive has the single `metastore` role; its PDB and Listener are built + // here, but the discovery ConfigMap (which needs the applied Listener) is built in reconcile. + if let Some(role_config) = &cluster.role_config { + pod_disruption_budgets.extend(build_pdb(&role_config.pdb, cluster, &HiveRole::MetaStore)); + listeners.push(build_role_listener( + cluster, + &HiveRole::MetaStore, + &role_config.listener_class, + )); + } + + for (hive_role, role_group_configs) in &cluster.role_group_configs { + for (role_group_name, rg) in role_group_configs { + services.push(build_rolegroup_headless_service(cluster, role_group_name)); + services.push(build_rolegroup_metrics_service(cluster, role_group_name)); + config_maps.push( + build_metastore_rolegroup_config_map(cluster, cluster_info, role_group_name, rg) + .context(ConfigMapSnafu { + role_group: role_group_name.clone(), + })?, + ); + stateful_sets.push( + build_metastore_rolegroup_statefulset( + hive_role, + cluster, + role_group_name, + rg, + service_account_name, + ) + .context(StatefulSetSnafu { + role_group: role_group_name.clone(), + })?, + ); + } + } + + Ok(KubernetesResources { + stateful_sets, + services, + listeners, + config_maps, + pod_disruption_budgets, + }) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_operator::{ + commons::networking::DomainName, kube::Resource, utils::cluster_info::KubernetesClusterInfo, + }; + + use super::build; + use crate::controller::test_support::{DERBY_YAML, minimal_hive, validated_cluster}; + + fn test_cluster_info() -> KubernetesClusterInfo { + KubernetesClusterInfo { + cluster_domain: DomainName::from_str("cluster.local").expect("valid cluster domain"), + } + } + + fn sorted_names(resources: &[impl Resource]) -> Vec { + let mut names: Vec = resources + .iter() + .filter_map(|resource| resource.meta().name.clone()) + .collect(); + names.sort(); + names + } + + #[test] + fn build_produces_expected_resource_names() { + let hive = minimal_hive(DERBY_YAML); + let cluster = validated_cluster(&hive); + + let resources = build(&cluster, &test_cluster_info(), "simple-hive-serviceaccount") + .expect("build succeeds"); + + assert_eq!( + sorted_names(&resources.stateful_sets), + ["simple-hive-metastore-default"] + ); + // One headless and one metrics Service per role group. + assert_eq!(resources.services.len(), 2); + assert_eq!( + sorted_names(&resources.config_maps), + ["simple-hive-metastore-default"] + ); + // The single metastore role has one role Listener. + assert_eq!( + sorted_names(&resources.listeners), + ["simple-hive-metastore"] + ); + // A default PDB for the metastore role. + assert_eq!( + sorted_names(&resources.pod_disruption_budgets), + ["simple-hive-metastore"] + ); + } +}